specweave 0.18.0 → 0.20.0

package/plugins/specweave-kafka/lib/security/kafka-security.ts

@@ -0,0 +1,494 @@
+/**
+ * Kafka Security Patterns
+ *
+ * TLS/SSL encryption, SASL authentication, and ACL management
+ *
+ * @module kafka-security
+ */
+
+import { Kafka, SASLOptions, ConnectionOptions } from 'kafkajs';
+import * as fs from 'fs';
+import * as path from 'path';
+
+/**
+ * TLS/SSL Configuration
+ */
+export interface TLSConfig {
+  /** Enable TLS */
+  enabled: boolean;
+  /** CA certificate (PEM format) */
+  ca?: string | Buffer;
+  /** Client certificate (PEM format) */
+  cert?: string | Buffer;
+  /** Client private key (PEM format) */
+  key?: string | Buffer;
+  /** Reject unauthorized certificates (default: true) */
+  rejectUnauthorized?: boolean;
+  /** Server name for SNI */
+  servername?: string;
+}
+
+/**
+ * SASL Authentication Configuration
+ */
+export interface SASLConfig {
+  /** SASL mechanism */
+  mechanism: 'plain' | 'scram-sha-256' | 'scram-sha-512' | 'aws' | 'oauthbearer';
+  /** Username */
+  username?: string;
+  /** Password */
+  password?: string;
+  /** AWS specific configuration */
+  aws?: {
+    accessKeyId: string;
+    secretAccessKey: string;
+    region: string;
+  };
+  /** OAuth specific configuration */
+  oauthBearer?: {
+    oauthBearerProvider: () => Promise<{ value: string }>;
+  };
+}
+
+/**
+ * ACL (Access Control List) Configuration
+ */
+export interface ACLConfig {
+  /** Principal (user) */
+  principal: string;
+  /** Resource type */
+  resourceType: 'TOPIC' | 'GROUP' | 'CLUSTER' | 'TRANSACTIONAL_ID';
+  /** Resource name (or '*' for all) */
+  resourceName: string;
+  /** Resource pattern type */
+  patternType: 'LITERAL' | 'PREFIXED';
+  /** Operation */
+  operation: 'READ' | 'WRITE' | 'CREATE' | 'DELETE' | 'ALTER' | 'DESCRIBE' | 'CLUSTER_ACTION' | 'DESCRIBE_CONFIGS' | 'ALTER_CONFIGS' | 'IDEMPOTENT_WRITE' | 'ALL';
+  /** Permission type */
+  permissionType: 'ALLOW' | 'DENY';
+  /** Host (or '*' for all) */
+  host: string;
+}
+
+/**
+ * Kafka Security Manager
+ */
+export class KafkaSecurityManager {
+  /**
+   * Create TLS configuration from files
+   */
+  static createTLSConfigFromFiles(options: {
+    caPath?: string;
+    certPath?: string;
+    keyPath?: string;
+    rejectUnauthorized?: boolean;
+    servername?: string;
+  }): TLSConfig {
+    const config: TLSConfig = {
+      enabled: true,
+      rejectUnauthorized: options.rejectUnauthorized !== false,
+      servername: options.servername,
+    };
+
+    if (options.caPath) {
+      config.ca = fs.readFileSync(path.resolve(options.caPath));
+    }
+
+    if (options.certPath) {
+      config.cert = fs.readFileSync(path.resolve(options.certPath));
+    }
+
+    if (options.keyPath) {
+      config.key = fs.readFileSync(path.resolve(options.keyPath));
+    }
+
+    return config;
+  }
+
+  /**
+   * Create SASL/PLAIN configuration
+   */
+  static createSASLPlainConfig(username: string, password: string): SASLConfig {
+    return {
+      mechanism: 'plain',
+      username,
+      password,
+    };
+  }
+
+  /**
+   * Create SASL/SCRAM-SHA-256 configuration
+   */
+  static createSASLScramSHA256Config(username: string, password: string): SASLConfig {
+    return {
+      mechanism: 'scram-sha-256',
+      username,
+      password,
+    };
+  }
+
+  /**
+   * Create SASL/SCRAM-SHA-512 configuration
+   */
+  static createSASLScramSHA512Config(username: string, password: string): SASLConfig {
+    return {
+      mechanism: 'scram-sha-512',
+      username,
+      password,
+    };
+  }
+
+  /**
+   * Create AWS IAM authentication configuration
+   */
+  static createSASLAWSConfig(
+    accessKeyId: string,
+    secretAccessKey: string,
+    region: string
+  ): SASLConfig {
+    return {
+      mechanism: 'aws',
+      aws: {
+        accessKeyId,
+        secretAccessKey,
+        region,
+      },
+    };
+  }
+
+  /**
+   * Create OAuth Bearer configuration
+   */
+  static createSASLOAuthBearerConfig(
+    tokenProvider: () => Promise<{ value: string }>
+  ): SASLConfig {
+    return {
+      mechanism: 'oauthbearer',
+      oauthBearer: {
+        oauthBearerProvider: tokenProvider,
+      },
+    };
+  }
+
+  /**
+   * Create Kafka client with security configuration
+   */
+  static createSecureKafkaClient(options: {
+    brokers: string[];
+    clientId: string;
+    tls?: TLSConfig;
+    sasl?: SASLConfig;
+  }): Kafka {
+    const connectionOptions: ConnectionOptions = {};
+
+    // TLS Configuration
+    if (options.tls?.enabled) {
+      connectionOptions.ssl = {
+        rejectUnauthorized: options.tls.rejectUnauthorized !== false,
+        ca: options.tls.ca ? [options.tls.ca] : undefined,
+        cert: options.tls.cert,
+        key: options.tls.key,
+        servername: options.tls.servername,
+      };
+    }
+
+    // SASL Configuration
+    if (options.sasl) {
+      let saslOptions: SASLOptions;
+
+      switch (options.sasl.mechanism) {
+        case 'plain':
+          saslOptions = {
+            mechanism: 'plain',
+            username: options.sasl.username!,
+            password: options.sasl.password!,
+          };
+          break;
+
+        case 'scram-sha-256':
+          saslOptions = {
+            mechanism: 'scram-sha-256',
+            username: options.sasl.username!,
+            password: options.sasl.password!,
+          };
+          break;
+
+        case 'scram-sha-512':
+          saslOptions = {
+            mechanism: 'scram-sha-512',
+            username: options.sasl.username!,
+            password: options.sasl.password!,
+          };
+          break;
+
+        case 'aws':
+          // @ts-ignore - AWS IAM auth typing
+          saslOptions = {
+            mechanism: 'aws',
+            authorizationIdentity: options.sasl.aws!.accessKeyId,
+            accessKeyId: options.sasl.aws!.accessKeyId,
+            secretAccessKey: options.sasl.aws!.secretAccessKey,
+          };
+          break;
+
+        case 'oauthbearer':
+          // @ts-ignore - OAuth bearer typing
+          saslOptions = {
+            mechanism: 'oauthbearer',
+            oauthBearerProvider: options.sasl.oauthBearer!.oauthBearerProvider,
+          };
+          break;
+
+        default:
+          throw new Error(`Unsupported SASL mechanism: ${options.sasl.mechanism}`);
+      }
+
+      connectionOptions.sasl = saslOptions;
+    }
+
+    return new Kafka({
+      clientId: options.clientId,
+      brokers: options.brokers,
+      ...connectionOptions,
+    });
+  }
+}
+
+/**
+ * ACL Manager (Kafka Admin CLI wrapper)
+ *
+ * Note: Requires kafka-acls.sh CLI tool and admin permissions
+ */
+export class KafkaACLManager {
+  private kafkaHome: string;
+
+  constructor(kafkaHome: string = process.env.KAFKA_HOME || '/opt/kafka') {
+    this.kafkaHome = kafkaHome;
+  }
+
+  /**
+   * Create ACL
+   */
+  async createACL(options: {
+    bootstrapServer: string;
+    acl: ACLConfig;
+    commandConfig?: string;
+  }): Promise<void> {
+    const args = [
+      '--bootstrap-server',
+      options.bootstrapServer,
+      '--add',
+      '--allow-principal',
+      options.acl.principal,
+      '--operation',
+      options.acl.operation,
+    ];
+
+    // Resource type
+    if (options.acl.resourceType === 'TOPIC') {
+      args.push('--topic', options.acl.resourceName);
+    } else if (options.acl.resourceType === 'GROUP') {
+      args.push('--group', options.acl.resourceName);
+    } else if (options.acl.resourceType === 'CLUSTER') {
+      args.push('--cluster');
+    } else if (options.acl.resourceType === 'TRANSACTIONAL_ID') {
+      args.push('--transactional-id', options.acl.resourceName);
+    }
+
+    // Pattern type
+    if (options.acl.patternType === 'PREFIXED') {
+      args.push('--resource-pattern-type', 'PREFIXED');
+    }
+
+    // Host
+    if (options.acl.host !== '*') {
+      args.push('--allow-host', options.acl.host);
+    }
+
+    // Command config (for TLS/SASL)
+    if (options.commandConfig) {
+      args.push('--command-config', options.commandConfig);
+    }
+
+    await this.runKafkaACLCommand(args);
+  }
+
+  /**
+   * List ACLs
+   */
+  async listACLs(options: {
+    bootstrapServer: string;
+    resourceType?: ACLConfig['resourceType'];
+    resourceName?: string;
+    principal?: string;
+    commandConfig?: string;
+  }): Promise<string> {
+    const args = [
+      '--bootstrap-server',
+      options.bootstrapServer,
+      '--list',
+    ];
+
+    if (options.resourceType) {
+      if (options.resourceType === 'TOPIC') {
+        args.push('--topic', options.resourceName || '*');
+      } else if (options.resourceType === 'GROUP') {
+        args.push('--group', options.resourceName || '*');
+      } else if (options.resourceType === 'CLUSTER') {
+        args.push('--cluster');
+      }
+    }
+
+    if (options.principal) {
+      args.push('--principal', options.principal);
+    }
+
+    if (options.commandConfig) {
+      args.push('--command-config', options.commandConfig);
+    }
+
+    return this.runKafkaACLCommand(args);
+  }
+
+  /**
+   * Delete ACL
+   */
+  async deleteACL(options: {
+    bootstrapServer: string;
+    acl: Partial<ACLConfig>;
+    commandConfig?: string;
+  }): Promise<void> {
+    const args = [
+      '--bootstrap-server',
+      options.bootstrapServer,
+      '--remove',
+    ];
+
+    if (options.acl.principal) {
+      args.push('--allow-principal', options.acl.principal);
+    }
+
+    if (options.acl.resourceType === 'TOPIC' && options.acl.resourceName) {
+      args.push('--topic', options.acl.resourceName);
+    }
+
+    if (options.acl.operation) {
+      args.push('--operation', options.acl.operation);
+    }
+
+    if (options.commandConfig) {
+      args.push('--command-config', options.commandConfig);
+    }
+
+    await this.runKafkaACLCommand(args);
+  }
+
+  /**
+   * Run kafka-acls.sh command
+   */
+  private async runKafkaACLCommand(args: string[]): Promise<string> {
+    const { exec } = require('child_process');
+    const { promisify } = require('util');
+    const execAsync = promisify(exec);
+
+    const command = `${this.kafkaHome}/bin/kafka-acls.sh ${args.join(' ')}`;
+    const { stdout, stderr } = await execAsync(command);
+
+    if (stderr) {
+      console.error('ACL command stderr:', stderr);
+    }
+
+    return stdout;
+  }
+}
+
+/**
+ * Example Usage: TLS + SASL/SCRAM-SHA-512
+ *
+ * ```typescript
+ * const tls = KafkaSecurityManager.createTLSConfigFromFiles({
+ *   caPath: './ca-cert.pem',
+ *   certPath: './client-cert.pem',
+ *   keyPath: './client-key.pem',
+ * });
+ *
+ * const sasl = KafkaSecurityManager.createSASLScramSHA512Config(
+ *   'kafka-user',
+ *   'secure-password'
+ * );
+ *
+ * const kafka = KafkaSecurityManager.createSecureKafkaClient({
+ *   brokers: ['broker1:9093', 'broker2:9093'],
+ *   clientId: 'secure-client',
+ *   tls,
+ *   sasl,
+ * });
+ * ```
+ */
+
+/**
+ * Example Usage: AWS MSK IAM Authentication
+ *
+ * ```typescript
+ * const sasl = KafkaSecurityManager.createSASLAWSConfig(
+ *   process.env.AWS_ACCESS_KEY_ID!,
+ *   process.env.AWS_SECRET_ACCESS_KEY!,
+ *   'us-east-1'
+ * );
+ *
+ * const kafka = KafkaSecurityManager.createSecureKafkaClient({
+ *   brokers: ['b-1.msk-cluster.amazonaws.com:9098'],
+ *   clientId: 'msk-client',
+ *   tls: { enabled: true },
+ *   sasl,
+ * });
+ * ```
+ */
+
+/**
+ * Example Usage: ACL Management
+ *
+ * ```typescript
+ * const aclManager = new KafkaACLManager('/opt/kafka');
+ *
+ * // Grant read access to topic
+ * await aclManager.createACL({
+ *   bootstrapServer: 'localhost:9092',
+ *   acl: {
+ *     principal: 'User:alice',
+ *     resourceType: 'TOPIC',
+ *     resourceName: 'orders',
+ *     patternType: 'LITERAL',
+ *     operation: 'READ',
+ *     permissionType: 'ALLOW',
+ *     host: '*',
+ *   },
+ * });
+ *
+ * // Grant write access with prefix matching
+ * await aclManager.createACL({
+ *   bootstrapServer: 'localhost:9092',
+ *   acl: {
+ *     principal: 'User:bob',
+ *     resourceType: 'TOPIC',
+ *     resourceName: 'analytics-',
+ *     patternType: 'PREFIXED',
+ *     operation: 'WRITE',
+ *     permissionType: 'ALLOW',
+ *     host: '*',
+ *   },
+ * });
+ *
+ * // List ACLs
+ * const acls = await aclManager.listACLs({
+ *   bootstrapServer: 'localhost:9092',
+ *   resourceType: 'TOPIC',
+ * });
+ * console.log(acls);
+ * ```
+ */
+
+export default {
+  KafkaSecurityManager,
+  KafkaACLManager,
+};

package/plugins/specweave-kafka/lib/utils/capacity-planner.js

@@ -0,0 +1,203 @@
+class KafkaCapacityPlanner {
+  constructor(constraints = {}) {
+    this.constraints = {
+      maxPartitionsPerBroker: constraints.maxPartitionsPerBroker || 4e3,
+      maxDiskUtilization: constraints.maxDiskUtilization || 70,
+      networkBandwidthMBps: constraints.networkBandwidthMBps || 125,
+      // 1Gbps
+      cpuCoresPerBroker: constraints.cpuCoresPerBroker || 8,
+      ramPerBrokerGB: constraints.ramPerBrokerGB || 32
+    };
+  }
+  /**
+   * Calculate optimal cluster sizing
+   */
+  calculateClusterSize(throughput, storage, topicCount = 1) {
+    const partitioning = this.calculatePartitionCount(throughput);
+    const storageCalc = this.calculateStorageRequirements(
+      throughput,
+      storage,
+      partitioning.partitionCount
+    );
+    const brokerCountOptions = {
+      throughput: this.calculateBrokersForThroughput(throughput),
+      storage: this.calculateBrokersForStorage(storageCalc.totalStorageGB),
+      partitions: this.calculateBrokersForPartitions(partitioning.partitionCount),
+      network: this.calculateBrokersForNetwork(throughput)
+    };
+    const brokerCount = Math.max(
+      brokerCountOptions.throughput,
+      brokerCountOptions.storage,
+      brokerCountOptions.partitions,
+      brokerCountOptions.network,
+      3
+      // Minimum for production (quorum)
+    );
+    const utilization = this.calculateUtilization(
+      brokerCount,
+      partitioning.partitionCount,
+      throughput,
+      storageCalc.totalStorageGB
+    );
+    const warnings = [];
+    if (utilization.cpu > 80) {
+      warnings.push(`High CPU utilization (${utilization.cpu.toFixed(1)}%) - consider more brokers`);
+    }
+    if (utilization.disk > this.constraints.maxDiskUtilization) {
+      warnings.push(`Disk utilization (${utilization.disk.toFixed(1)}%) exceeds ${this.constraints.maxDiskUtilization}% threshold`);
+    }
+    if (utilization.network > 70) {
+      warnings.push(`Network utilization (${utilization.network.toFixed(1)}%) is high - risk of bottleneck`);
+    }
+    if (utilization.partitions > 80) {
+      warnings.push(`Partition count (${partitioning.partitionCount}) is ${utilization.partitions.toFixed(1)}% of broker capacity`);
+    }
+    if (brokerCount < 3) {
+      warnings.push("Less than 3 brokers - not suitable for production (no fault tolerance)");
+    }
+    if (partitioning.partitionCount < brokerCount) {
+      warnings.push(`Partition count (${partitioning.partitionCount}) < broker count (${brokerCount}) - underutilized`);
+    }
+    return {
+      brokerCount,
+      partitionCount: partitioning.partitionCount,
+      storagePerBrokerGB: storageCalc.totalStorageGB / brokerCount,
+      totalStorageGB: storageCalc.totalStorageGB,
+      throughputHeadroom: this.calculateThroughputHeadroom(brokerCount, throughput),
+      utilization,
+      warnings
+    };
+  }
+  /**
+   * Calculate optimal partition count
+   */
+  calculatePartitionCount(throughput) {
+    const reasoning = [];
+    const maxThroughputPerPartition = 50;
+    const partitionsForProducerThroughput = Math.ceil(
+      throughput.producerThroughputMBps / maxThroughputPerPartition
+    );
+    reasoning.push(
+      `Producer throughput: ${throughput.producerThroughputMBps} MB/s \xF7 ${maxThroughputPerPartition} MB/s/partition = ${partitionsForProducerThroughput} partitions`
+    );
+    const partitionsForConsumerThroughput = Math.ceil(
+      throughput.consumerThroughputMBps / maxThroughputPerPartition
+    );
+    reasoning.push(
+      `Consumer throughput: ${throughput.consumerThroughputMBps} MB/s \xF7 ${maxThroughputPerPartition} MB/s/partition = ${partitionsForConsumerThroughput} partitions`
+    );
+    const targetPartitionSizeGB = 10;
+    const minPartitions = Math.max(
+      partitionsForProducerThroughput,
+      partitionsForConsumerThroughput,
+      1
+      // At least 1 partition
+    );
+    const partitionCount = this.nextPowerOfTwo(minPartitions);
+    reasoning.push(`Rounded to power of 2: ${partitionCount} partitions (for even distribution)`);
+    return {
+      partitionCount,
+      maxThroughputPerPartition,
+      consumerParallelism: partitionCount,
+      partitionSizeGB: targetPartitionSizeGB,
+      reasoning
+    };
+  }
+  /**
+   * Calculate storage requirements
+   */
+  calculateStorageRequirements(throughput, storage, partitionCount) {
+    const compressionRatio = storage.compressionRatio || 0.3;
+    const growthBuffer = storage.growthBuffer || 1.5;
+    const dataRateMBps = throughput.producerThroughputMBps;
+    const dataRateGBperHour = dataRateMBps * 3.6;
+    const rawStorageGB = dataRateGBperHour * storage.retentionHours;
+    const compressedStorageGB = rawStorageGB * compressionRatio;
+    const replicatedStorageGB = compressedStorageGB * storage.replicationFactor;
+    const totalStorageGB = replicatedStorageGB * growthBuffer;
+    const storagePerPartitionGB = totalStorageGB / partitionCount;
+    return {
+      totalStorageGB,
+      storagePerPartitionGB,
+      rawStorageGB,
+      compressedStorageGB
+    };
+  }
+  /**
+   * Calculate brokers needed for throughput
+   */
+  calculateBrokersForThroughput(throughput) {
+    const maxBrokerThroughputMBps = 100;
+    const totalThroughput = throughput.producerThroughputMBps + throughput.consumerThroughputMBps;
+    return Math.ceil(totalThroughput / maxBrokerThroughputMBps);
+  }
+  /**
+   * Calculate brokers needed for storage
+   */
+  calculateBrokersForStorage(totalStorageGB) {
+    const usableStoragePerBrokerGB = 2e3;
+    const maxUtilization = this.constraints.maxDiskUtilization / 100;
+    const effectiveStoragePerBroker = usableStoragePerBrokerGB * maxUtilization;
+    return Math.ceil(totalStorageGB / effectiveStoragePerBroker);
+  }
+  /**
+   * Calculate brokers needed for partition count
+   */
+  calculateBrokersForPartitions(partitionCount) {
+    return Math.ceil(partitionCount / this.constraints.maxPartitionsPerBroker);
+  }
+  /**
+   * Calculate brokers needed for network bandwidth
+   */
+  calculateBrokersForNetwork(throughput) {
+    const totalNetworkMBps = throughput.producerThroughputMBps + throughput.consumerThroughputMBps;
+    const maxNetworkUtilization = 0.7;
+    const effectiveBandwidth = this.constraints.networkBandwidthMBps * maxNetworkUtilization;
+    return Math.ceil(totalNetworkMBps / effectiveBandwidth);
+  }
+  /**
+   * Calculate resource utilization
+   */
+  calculateUtilization(brokerCount, partitionCount, throughput, totalStorageGB) {
+    const totalThroughput = throughput.producerThroughputMBps + throughput.consumerThroughputMBps;
+    const cpuPerBroker = totalThroughput / brokerCount / 10 * 5;
+    const cpu = Math.min(cpuPerBroker, 100);
+    const memoryPerBroker = (2 + partitionCount / brokerCount * 1e-3) / this.constraints.ramPerBrokerGB * 100;
+    const memory = Math.min(memoryPerBroker, 100);
+    const storagePerBroker = totalStorageGB / brokerCount;
+    const disk = storagePerBroker / 2e3 * 100;
+    const networkPerBroker = totalThroughput / brokerCount;
+    const network = networkPerBroker / this.constraints.networkBandwidthMBps * 100;
+    const partitionsPerBroker = partitionCount / brokerCount;
+    const partitions = partitionsPerBroker / this.constraints.maxPartitionsPerBroker * 100;
+    return {
+      cpu: Math.round(cpu * 10) / 10,
+      memory: Math.round(memory * 10) / 10,
+      disk: Math.round(disk * 10) / 10,
+      network: Math.round(network * 10) / 10,
+      partitions: Math.round(partitions * 10) / 10
+    };
+  }
+  /**
+   * Calculate throughput headroom
+   */
+  calculateThroughputHeadroom(brokerCount, throughput) {
+    const maxClusterThroughput = brokerCount * 100;
+    const actualThroughput = throughput.producerThroughputMBps + throughput.consumerThroughputMBps;
+    return Math.round((maxClusterThroughput - actualThroughput) / maxClusterThroughput * 100);
+  }
+  /**
+   * Round up to next power of 2
+   */
+  nextPowerOfTwo(n) {
+    if (n <= 1) return 1;
+    return Math.pow(2, Math.ceil(Math.log2(n)));
+  }
+}
+var capacity_planner_default = {
+  KafkaCapacityPlanner
+};
+export {
+  KafkaCapacityPlanner,
+  capacity_planner_default as default
+};