specweave 0.18.1 → 0.20.0

package/plugins/specweave-kafka/commands/monitor-setup.md

@@ -0,0 +1,96 @@
+---
+name: specweave-kafka:monitor-setup
+description: Set up comprehensive Kafka monitoring with Prometheus + Grafana. Configures JMX exporter, dashboards, and alerting rules.
+---
+
+# Set Up Kafka Monitoring
+
+Configure comprehensive monitoring for your Kafka cluster using Prometheus and Grafana.
+
+## What This Command Does
+
+1. **JMX Exporter Setup**: Configure Prometheus JMX exporter for Kafka brokers
+2. **Prometheus Configuration**: Add Kafka scrape targets
+3. **Grafana Dashboards**: Install 5 pre-built dashboards
+4. **Alerting Rules**: Configure 14 critical/high/warning alerts
+5. **Verification**: Test metrics collection and dashboard access
+
+## Interactive Workflow
+
+I'll detect your environment and guide setup:
+
+### Environment Detection
+- **Kubernetes** (Strimzi/Confluent Operator) → Use PodMonitor
+- **Docker Compose** → Add Prometheus + Grafana services
+- **VM/Bare Metal** → Configure JMX exporter JAR
+
+### Question 1: Where is Kafka running?
+- Kubernetes (Strimzi)
+- Docker Compose
+- VMs/EC2 instances
+
+### Question 2: Prometheus already installed?
+- Yes → Just add Kafka scrape config
+- No → Install Prometheus + Grafana stack
+
+## Example Usage
+
+```bash
+# Start monitoring setup wizard
+/specweave-kafka:monitor-setup
+
+# I'll activate kafka-observability skill and:
+# 1. Detect your environment
+# 2. Configure JMX exporter (port 7071)
+# 3. Set up Prometheus scraping
+# 4. Install 5 Grafana dashboards
+# 5. Configure 14 alerting rules
+# 6. Verify metrics collection
+```
+
+## What Gets Configured
+
+**JMX Exporter** (Kafka brokers):
+- Metrics endpoint on port 7071
+- 50+ critical Kafka metrics exported
+- Broker, topic, consumer lag, JVM metrics
+
+**Prometheus Scraping**:
+```yaml
+scrape_configs:
+  - job_name: 'kafka'
+    static_configs:
+      - targets: ['kafka-0:7071', 'kafka-1:7071', 'kafka-2:7071']
+```
+
+**5 Grafana Dashboards**:
+1. **Cluster Overview** - Health, throughput, ISR changes
+2. **Broker Metrics** - CPU, memory, network, request handlers
+3. **Consumer Lag** - Lag per group/topic, offset tracking
+4. **Topic Metrics** - Partition count, replication, log size
+5. **JVM Metrics** - Heap, GC, threads, file descriptors
+
+**14 Alerting Rules**:
+- CRITICAL: Under-replicated partitions, offline partitions, no controller
+- HIGH: Consumer lag, ISR shrinks, leader elections
+- WARNING: CPU, memory, GC time, disk usage
+
+## Prerequisites
+
+- Kafka cluster running (self-hosted or K8s)
+- Prometheus installed (or will be installed)
+- Grafana installed (or will be installed)
+
+## Post-Setup
+
+After setup completes, I'll:
+1. ✅ Provide Grafana URL and credentials
+2. ✅ Show how to access dashboards
+3. ✅ Explain critical alerts
+4. ✅ Suggest testing alerts by stopping a broker
+
+---
+
+**Skills Activated**: kafka-observability
+**Related Commands**: /specweave-kafka:deploy
+**Dashboard Locations**: `plugins/specweave-kafka/monitoring/grafana/dashboards/`

package/plugins/specweave-kafka/docker/kafka-local/docker-compose.yml

@@ -0,0 +1,187 @@
+# Apache Kafka Local Development Environment (KRaft Mode)
+#
+# Single-node Kafka cluster for local testing (no ZooKeeper)
+# Includes: Kafka, Schema Registry, Kafka UI, Prometheus, Grafana
+
+version: '3.8'
+
+services:
+  # ========================================
+  # Kafka Broker (KRaft Mode)
+  # ========================================
+  kafka:
+    image: apache/kafka:3.7.0
+    container_name: kafka
+    hostname: kafka
+    ports:
+      - "9092:9092"   # Kafka client connections
+      - "9093:9093"   # Controller
+      - "7071:7071"   # JMX exporter
+    environment:
+      # KRaft configuration
+      KAFKA_NODE_ID: 1
+      KAFKA_PROCESS_ROLES: broker,controller
+      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:9093
+      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
+
+      # Listeners
+      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+
+      # Log directories
+      KAFKA_LOG_DIRS: /var/lib/kafka/data
+
+      # Cluster ID (auto-generated on first start)
+      CLUSTER_ID: MkU3OEVBNTcwNTJENDM2Qk
+
+      # JVM settings
+      KAFKA_HEAP_OPTS: "-Xms1G -Xmx1G"
+      KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+UseG1GC -XX:MaxGCPauseMillis=20"
+
+      # JMX for monitoring
+      KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=9999 -javaagent:/opt/jmx_prometheus_javaagent.jar=7071:/opt/kafka-broker.yml"
+
+      # Performance tuning
+      KAFKA_NUM_NETWORK_THREADS: 3
+      KAFKA_NUM_IO_THREADS: 8
+      KAFKA_SOCKET_SEND_BUFFER_BYTES: 102400
+      KAFKA_SOCKET_RECEIVE_BUFFER_BYTES: 102400
+      KAFKA_SOCKET_REQUEST_MAX_BYTES: 104857600
+
+      # Replication
+      KAFKA_DEFAULT_REPLICATION_FACTOR: 1
+      KAFKA_MIN_INSYNC_REPLICAS: 1
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+
+      # Log retention
+      KAFKA_LOG_RETENTION_HOURS: 24
+      KAFKA_LOG_SEGMENT_BYTES: 1073741824
+      KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: 300000
+
+      # Compression
+      KAFKA_COMPRESSION_TYPE: lz4
+
+      # Topic management
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
+      KAFKA_DELETE_TOPIC_ENABLE: "true"
+    volumes:
+      - kafka-data:/var/lib/kafka/data
+      - ./config/jmx_prometheus_javaagent.jar:/opt/jmx_prometheus_javaagent.jar:ro
+      - ./config/kafka-broker.yml:/opt/kafka-broker.yml:ro
+    healthcheck:
+      test: ["CMD-SHELL", "kafka-broker-api-versions --bootstrap-server localhost:9092 || exit 1"]
+      interval: 10s
+      timeout: 10s
+      retries: 5
+      start_period: 30s
+    networks:
+      - kafka-network
+
+  # ========================================
+  # Schema Registry
+  # ========================================
+  schema-registry:
+    image: confluentinc/cp-schema-registry:7.6.0
+    container_name: schema-registry
+    hostname: schema-registry
+    ports:
+      - "8081:8081"
+    environment:
+      SCHEMA_REGISTRY_HOST_NAME: schema-registry
+      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092
+      SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
+      SCHEMA_REGISTRY_SCHEMA_REGISTRY_GROUP_ID: schema-registry
+      SCHEMA_REGISTRY_KAFKASTORE_TOPIC: _schemas
+      SCHEMA_REGISTRY_KAFKASTORE_TOPIC_REPLICATION_FACTOR: 1
+    depends_on:
+      kafka:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8081/subjects"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    networks:
+      - kafka-network
+
+  # ========================================
+  # Kafka UI
+  # ========================================
+  kafka-ui:
+    image: provectuslabs/kafka-ui:latest
+    container_name: kafka-ui
+    ports:
+      - "8080:8080"
+    environment:
+      KAFKA_CLUSTERS_0_NAME: local
+      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:9092
+      KAFKA_CLUSTERS_0_SCHEMAREGISTRY: http://schema-registry:8081
+      KAFKA_CLUSTERS_0_METRICS_PORT: 9999
+      DYNAMIC_CONFIG_ENABLED: "true"
+    depends_on:
+      - kafka
+      - schema-registry
+    networks:
+      - kafka-network
+
+  # ========================================
+  # Prometheus
+  # ========================================
+  prometheus:
+    image: prom/prometheus:latest
+    container_name: prometheus
+    ports:
+      - "9090:9090"
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.path=/prometheus'
+      - '--web.console.libraries=/etc/prometheus/console_libraries'
+      - '--web.console.templates=/etc/prometheus/consoles'
+    volumes:
+      - ./config/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - prometheus-data:/prometheus
+    networks:
+      - kafka-network
+
+  # ========================================
+  # Grafana
+  # ========================================
+  grafana:
+    image: grafana/grafana:latest
+    container_name: grafana
+    ports:
+      - "3000:3000"
+    environment:
+      GF_SECURITY_ADMIN_USER: admin
+      GF_SECURITY_ADMIN_PASSWORD: admin
+      GF_INSTALL_PLUGINS: grafana-piechart-panel
+    volumes:
+      - ./config/grafana/provisioning:/etc/grafana/provisioning:ro
+      - ./config/grafana/dashboards:/var/lib/grafana/dashboards:ro
+      - grafana-data:/var/lib/grafana
+    depends_on:
+      - prometheus
+    networks:
+      - kafka-network
+
+# ========================================
+# Networks
+# ========================================
+networks:
+  kafka-network:
+    driver: bridge
+
+# ========================================
+# Volumes
+# ========================================
+volumes:
+  kafka-data:
+    driver: local
+  prometheus-data:
+    driver: local
+  grafana-data:
+    driver: local

package/plugins/specweave-kafka/docker/redpanda/docker-compose.yml

@@ -0,0 +1,199 @@
+# Redpanda Local Development Environment
+#
+# Redpanda is a Kafka-compatible streaming platform (10x faster, C++)
+# Includes: Redpanda cluster, Console UI, Prometheus
+
+version: '3.8'
+
+services:
+  # ========================================
+  # Redpanda Broker 1
+  # ========================================
+  redpanda-1:
+    image: docker.redpanda.com/redpandadata/redpanda:v24.2.4
+    container_name: redpanda-1
+    hostname: redpanda-1
+    ports:
+      - "9092:9092"   # Kafka API
+      - "9644:9644"   # Admin API
+      - "8082:8082"   # HTTP Proxy
+    command:
+      - redpanda
+      - start
+      - --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:19092
+      - --advertise-kafka-addr internal://redpanda-1:9092,external://localhost:9092
+      - --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:18082
+      - --advertise-pandaproxy-addr internal://redpanda-1:8082,external://localhost:8082
+      - --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:18081
+      - --rpc-addr redpanda-1:33145
+      - --advertise-rpc-addr redpanda-1:33145
+      - --mode dev-container
+      - --smp 1
+      - --memory 1G
+    volumes:
+      - redpanda-1-data:/var/lib/redpanda/data
+    healthcheck:
+      test: ["CMD-SHELL", "rpk cluster health | grep 'Healthy:.*true' || exit 1"]
+      interval: 10s
+      timeout: 10s
+      retries: 5
+      start_period: 30s
+    networks:
+      - redpanda-network
+
+  # ========================================
+  # Redpanda Broker 2
+  # ========================================
+  redpanda-2:
+    image: docker.redpanda.com/redpandadata/redpanda:v24.2.4
+    container_name: redpanda-2
+    hostname: redpanda-2
+    ports:
+      - "9093:9092"
+      - "9645:9644"
+      - "8083:8082"
+    command:
+      - redpanda
+      - start
+      - --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:19093
+      - --advertise-kafka-addr internal://redpanda-2:9092,external://localhost:9093
+      - --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:18083
+      - --advertise-pandaproxy-addr internal://redpanda-2:8082,external://localhost:8083
+      - --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:18082
+      - --rpc-addr redpanda-2:33145
+      - --advertise-rpc-addr redpanda-2:33145
+      - --seeds redpanda-1:33145
+      - --mode dev-container
+      - --smp 1
+      - --memory 1G
+    volumes:
+      - redpanda-2-data:/var/lib/redpanda/data
+    healthcheck:
+      test: ["CMD-SHELL", "rpk cluster health | grep 'Healthy:.*true' || exit 1"]
+      interval: 10s
+      timeout: 10s
+      retries: 5
+      start_period: 30s
+    depends_on:
+      - redpanda-1
+    networks:
+      - redpanda-network
+
+  # ========================================
+  # Redpanda Broker 3
+  # ========================================
+  redpanda-3:
+    image: docker.redpanda.com/redpandadata/redpanda:v24.2.4
+    container_name: redpanda-3
+    hostname: redpanda-3
+    ports:
+      - "9094:9092"
+      - "9646:9644"
+      - "8084:8082"
+    command:
+      - redpanda
+      - start
+      - --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:19094
+      - --advertise-kafka-addr internal://redpanda-3:9092,external://localhost:9094
+      - --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:18084
+      - --advertise-pandaproxy-addr internal://redpanda-3:8082,external://localhost:8084
+      - --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:18083
+      - --rpc-addr redpanda-3:33145
+      - --advertise-rpc-addr redpanda-3:33145
+      - --seeds redpanda-1:33145
+      - --mode dev-container
+      - --smp 1
+      - --memory 1G
+    volumes:
+      - redpanda-3-data:/var/lib/redpanda/data
+    healthcheck:
+      test: ["CMD-SHELL", "rpk cluster health | grep 'Healthy:.*true' || exit 1"]
+      interval: 10s
+      timeout: 10s
+      retries: 5
+      start_period: 30s
+    depends_on:
+      - redpanda-1
+    networks:
+      - redpanda-network
+
+  # ========================================
+  # Redpanda Console (UI)
+  # ========================================
+  console:
+    image: docker.redpanda.com/redpandadata/console:v2.6.0
+    container_name: redpanda-console
+    ports:
+      - "8080:8080"
+    environment:
+      CONFIG_FILEPATH: /tmp/config.yml
+      KAFKA_BROKERS: redpanda-1:9092,redpanda-2:9092,redpanda-3:9092
+      KAFKA_SCHEMAREGISTRY_ENABLED: "true"
+      KAFKA_SCHEMAREGISTRY_URLS: http://redpanda-1:8081
+    volumes:
+      - ./config/console-config.yml:/tmp/config.yml:ro
+    depends_on:
+      - redpanda-1
+      - redpanda-2
+      - redpanda-3
+    networks:
+      - redpanda-network
+
+  # ========================================
+  # Prometheus (metrics)
+  # ========================================
+  prometheus:
+    image: prom/prometheus:latest
+    container_name: prometheus
+    ports:
+      - "9090:9090"
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.path=/prometheus'
+    volumes:
+      - ./config/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - prometheus-data:/prometheus
+    networks:
+      - redpanda-network
+
+  # ========================================
+  # Grafana (dashboards)
+  # ========================================
+  grafana:
+    image: grafana/grafana:latest
+    container_name: grafana
+    ports:
+      - "3000:3000"
+    environment:
+      GF_SECURITY_ADMIN_USER: admin
+      GF_SECURITY_ADMIN_PASSWORD: admin
+    volumes:
+      - ./config/grafana/provisioning:/etc/grafana/provisioning:ro
+      - ./config/grafana/dashboards:/var/lib/grafana/dashboards:ro
+      - grafana-data:/var/lib/grafana
+    depends_on:
+      - prometheus
+    networks:
+      - redpanda-network
+
+# ========================================
+# Networks
+# ========================================
+networks:
+  redpanda-network:
+    driver: bridge
+
+# ========================================
+# Volumes
+# ========================================
+volumes:
+  redpanda-1-data:
+    driver: local
+  redpanda-2-data:
+    driver: local
+  redpanda-3-data:
+    driver: local
+  prometheus-data:
+    driver: local
+  grafana-data:
+    driver: local

package/plugins/specweave-kafka/docker/templates/consumer-nodejs.js

@@ -0,0 +1,225 @@
+/**
+ * Kafka Consumer Example (Node.js)
+ *
+ * Install: npm install kafkajs
+ */
+
+const { Kafka, logLevel } = require('kafkajs');
+
+// ========================================
+// Configuration
+// ========================================
+
+const kafka = new Kafka({
+  clientId: 'my-app',
+  brokers: ['localhost:9092'],
+  logLevel: logLevel.INFO
+});
+
+const consumer = kafka.consumer({
+  groupId: 'my-consumer-group',
+
+  // Session timeout (broker detects consumer failure)
+  sessionTimeout: 30000,
+  heartbeatInterval: 3000,
+
+  // Rebalancing strategy
+  partitionAssigners: [
+    kafka.PartitionAssigners.roundRobin  // or cooperativeSticky
+  ],
+
+  // Auto-commit (disable for manual offset management)
+  autoCommit: true,
+  autoCommitInterval: 5000
+});
+
+// ========================================
+// Consumer Functions
+// ========================================
+
+async function consumeMessages(topic) {
+  await consumer.subscribe({
+    topic,
+    fromBeginning: false  // true = read from earliest offset
+  });
+
+  await consumer.run({
+    // Messages per partition to fetch
+    eachBatchAutoResolve: true,
+    partitionsConsumedConcurrently: 3,
+
+    // Process each message
+    eachMessage: async ({ topic, partition, message }) => {
+      try {
+        const value = JSON.parse(message.value.toString());
+        const headers = {};
+        for (const [key, val] of Object.entries(message.headers)) {
+          headers[key] = val.toString();
+        }
+
+        console.log({
+          topic,
+          partition,
+          offset: message.offset,
+          key: message.key?.toString(),
+          value,
+          headers,
+          timestamp: message.timestamp
+        });
+
+        // Process the message
+        await processMessage(value);
+
+        // Optional: Manual offset commit
+        // await consumer.commitOffsets([{
+        //   topic,
+        //   partition,
+        //   offset: (parseInt(message.offset) + 1).toString()
+        // }]);
+
+      } catch (error) {
+        console.error('❌ Error processing message:', error);
+        // Send to dead letter queue
+        // await sendToDLQ(message, error);
+      }
+    }
+  });
+}
+
+async function consumeBatch(topic) {
+  await consumer.subscribe({ topic });
+
+  await consumer.run({
+    // Process messages in batches
+    eachBatch: async ({
+      batch,
+      resolveOffset,
+      heartbeat,
+      commitOffsetsIfNecessary,
+      isRunning,
+      isStale
+    }) => {
+      console.log(`📦 Processing batch: ${batch.messages.length} messages`);
+
+      for (let message of batch.messages) {
+        if (!isRunning() || isStale()) {
+          break;  // Stop if consumer is shutting down
+        }
+
+        try {
+          const value = JSON.parse(message.value.toString());
+          await processMessage(value);
+
+          // Mark message as processed
+          resolveOffset(message.offset);
+
+          // Send heartbeat to avoid rebalance
+          await heartbeat();
+        } catch (error) {
+          console.error('❌ Error in batch:', error);
+        }
+      }
+
+      // Commit all offsets
+      await commitOffsetsIfNecessary();
+    }
+  });
+}
+
+// ========================================
+// Message Processing
+// ========================================
+
+async function processMessage(message) {
+  // Your business logic here
+  console.log('⚙️  Processing:', message);
+
+  // Simulate processing time
+  await new Promise(resolve => setTimeout(resolve, 100));
+
+  // Example: Handle different event types
+  switch (message.event) {
+    case 'user_login':
+      await handleUserLogin(message);
+      break;
+    case 'order_created':
+      await handleOrderCreated(message);
+      break;
+    default:
+      console.log('Unknown event type:', message.event);
+  }
+}
+
+async function handleUserLogin(message) {
+  console.log('👤 User login:', message.userId);
+  // Update user last login time
+  // Send welcome email
+  // etc.
+}
+
+async function handleOrderCreated(message) {
+  console.log('🛒 Order created:', message);
+  // Process order
+  // Send confirmation email
+  // Update inventory
+}
+
+// ========================================
+// Dead Letter Queue
+// ========================================
+
+async function sendToDLQ(message, error) {
+  const producer = kafka.producer();
+  await producer.connect();
+
+  await producer.send({
+    topic: 'my-topic-dlq',
+    messages: [{
+      key: message.key,
+      value: message.value,
+      headers: {
+        ...message.headers,
+        'error-message': error.message,
+        'error-stack': error.stack,
+        'failed-at': Date.now().toString(),
+        'original-topic': message.topic,
+        'original-partition': message.partition.toString(),
+        'original-offset': message.offset.toString()
+      }
+    }]
+  });
+
+  await producer.disconnect();
+  console.log('📮 Message sent to DLQ');
+}
+
+// ========================================
+// Main Function
+// ========================================
+
+async function main() {
+  await consumer.connect();
+  console.log('📡 Consumer connected');
+
+  // Start consuming
+  await consumeMessages('my-topic');
+
+  // Or use batch processing
+  // await consumeBatch('my-topic');
+}
+
+// ========================================
+// Run
+// ========================================
+
+main().catch(console.error);
+
+// Graceful shutdown
+const shutdown = async () => {
+  console.log('\n🛑 Shutting down consumer...');
+  await consumer.disconnect();
+  process.exit(0);
+};
+
+process.on('SIGINT', shutdown);
+process.on('SIGTERM', shutdown);