specweave 0.18.1 → 0.20.0

package/plugins/specweave-kafka/lib/adapters/platform-adapter.ts

@@ -0,0 +1,343 @@
+// ==================================================
+// Platform Adapter Interface
+// ==================================================
+// Provides unified API for Kafka operations across different platforms:
+// - Apache Kafka (self-hosted)
+// - AWS MSK (Managed Streaming for Kafka)
+// - Azure Event Hubs (Kafka API)
+// - Confluent Cloud
+
+export enum KafkaPlatform {
+  APACHE_KAFKA = 'apache-kafka',
+  AWS_MSK = 'aws-msk',
+  AZURE_EVENT_HUBS = 'azure-event-hubs',
+  CONFLUENT_CLOUD = 'confluent-cloud',
+}
+
+export interface ClusterInfo {
+  clusterId: string;
+  clusterName: string;
+  platform: KafkaPlatform;
+  version: string;
+  brokerCount: number;
+  bootstrapServers: string[];
+  endpoints: {
+    bootstrap: string;
+    metrics?: string;
+    admin?: string;
+  };
+}
+
+export interface TopicConfig {
+  name: string;
+  partitions: number;
+  replicationFactor: number;
+  config?: {
+    'retention.ms'?: number;
+    'retention.bytes'?: number;
+    'segment.bytes'?: number;
+    'compression.type'?: 'none' | 'gzip' | 'snappy' | 'lz4' | 'zstd';
+    'cleanup.policy'?: 'delete' | 'compact' | 'delete,compact';
+    'min.insync.replicas'?: number;
+    'max.message.bytes'?: number;
+    [key: string]: any;
+  };
+}
+
+export interface Topic {
+  name: string;
+  partitions: number;
+  replicationFactor: number;
+  isInternal: boolean;
+  config: Record<string, string>;
+}
+
+export interface Partition {
+  partition: number;
+  leader: number;
+  replicas: number[];
+  isr: number[];  // In-Sync Replicas
+  offlineReplicas: number[];
+}
+
+export interface TopicMetadata {
+  name: string;
+  partitions: Partition[];
+}
+
+export interface ConsumerGroupDescription {
+  groupId: string;
+  state: 'Stable' | 'Empty' | 'PreparingRebalance' | 'CompletingRebalance' | 'Dead';
+  protocolType: string;
+  protocol: string;
+  members: ConsumerGroupMember[];
+  coordinator: {
+    id: number;
+    host: string;
+    port: number;
+  };
+}
+
+export interface ConsumerGroupMember {
+  memberId: string;
+  clientId: string;
+  clientHost: string;
+  memberMetadata: any;
+  memberAssignment: {
+    topic: string;
+    partitions: number[];
+  }[];
+}
+
+export interface ConsumerGroupOffset {
+  topic: string;
+  partition: number;
+  currentOffset: number;
+  logEndOffset: number;
+  lag: number;
+  metadata: string;
+}
+
+export interface ClusterMetrics {
+  brokerCount: number;
+  topicCount: number;
+  partitionCount: number;
+  underReplicatedPartitions: number;
+  offlinePartitions: number;
+  activeController: number;
+  messagesInPerSec: number;
+  bytesInPerSec: number;
+  bytesOutPerSec: number;
+}
+
+export interface ConnectionConfig {
+  platform: KafkaPlatform;
+  brokers: string[];
+  ssl?: {
+    enabled: boolean;
+    truststore?: string;
+    keystore?: string;
+    password?: string;
+  };
+  sasl?: {
+    mechanism: 'PLAIN' | 'SCRAM-SHA-256' | 'SCRAM-SHA-512' | 'AWS_MSK_IAM' | 'OAUTHBEARER';
+    username?: string;
+    password?: string;
+    awsRegion?: string;  // For AWS MSK IAM
+    oauthConfig?: {      // For Confluent Cloud / OAUTHBEARER
+      tokenUrl: string;
+      clientId: string;
+      clientSecret: string;
+    };
+  };
+  clientId?: string;
+  connectionTimeout?: number;
+  requestTimeout?: number;
+}
+
+export interface AdminOperationResult {
+  success: boolean;
+  message: string;
+  details?: any;
+}
+
+/**
+ * Platform Adapter Interface
+ *
+ * Unified API for Kafka operations across different platforms.
+ * Implementations:
+ * - ApacheKafkaAdapter (self-hosted Kafka)
+ * - AwsMskAdapter (AWS MSK)
+ * - AzureEventHubsAdapter (Azure Event Hubs with Kafka API)
+ * - ConfluentCloudAdapter (Confluent Cloud)
+ */
+export interface PlatformAdapter {
+  readonly platform: KafkaPlatform;
+
+  /**
+   * Connect to Kafka cluster
+   */
+  connect(config: ConnectionConfig): Promise<void>;
+
+  /**
+   * Disconnect from Kafka cluster
+   */
+  disconnect(): Promise<void>;
+
+  /**
+   * Get cluster information
+   */
+  getClusterInfo(): Promise<ClusterInfo>;
+
+  /**
+   * Get cluster metrics (for monitoring)
+   */
+  getClusterMetrics(): Promise<ClusterMetrics>;
+
+  // ==================================================
+  // Topic Operations
+  // ==================================================
+
+  /**
+   * List all topics
+   */
+  listTopics(): Promise<string[]>;
+
+  /**
+   * Get topic metadata
+   */
+  getTopicMetadata(topic: string): Promise<TopicMetadata>;
+
+  /**
+   * Create topic
+   */
+  createTopic(config: TopicConfig): Promise<AdminOperationResult>;
+
+  /**
+   * Delete topic
+   */
+  deleteTopic(topic: string): Promise<AdminOperationResult>;
+
+  /**
+   * Update topic configuration
+   */
+  updateTopicConfig(topic: string, config: Record<string, string>): Promise<AdminOperationResult>;
+
+  /**
+   * Get topic configuration
+   */
+  getTopicConfig(topic: string): Promise<Record<string, string>>;
+
+  // ==================================================
+  // Consumer Group Operations
+  // ==================================================
+
+  /**
+   * List all consumer groups
+   */
+  listConsumerGroups(): Promise<string[]>;
+
+  /**
+   * Get consumer group description
+   */
+  describeConsumerGroup(groupId: string): Promise<ConsumerGroupDescription>;
+
+  /**
+   * Get consumer group offsets
+   */
+  getConsumerGroupOffsets(groupId: string): Promise<ConsumerGroupOffset[]>;
+
+  /**
+   * Delete consumer group
+   */
+  deleteConsumerGroup(groupId: string): Promise<AdminOperationResult>;
+
+  /**
+   * Reset consumer group offsets
+   */
+  resetConsumerGroupOffsets(
+    groupId: string,
+    topic: string,
+    options: {
+      toEarliest?: boolean;
+      toLatest?: boolean;
+      toDateTime?: Date;
+      toOffset?: number;
+    }
+  ): Promise<AdminOperationResult>;
+
+  // ==================================================
+  // ACL Operations (Security)
+  // ==================================================
+
+  /**
+   * Create ACL (Access Control List)
+   *
+   * Not supported on all platforms:
+   * - Apache Kafka: ✅ Supported
+   * - AWS MSK: ⚠️  Use IAM policies instead
+   * - Azure Event Hubs: ⚠️  Use Azure RBAC instead
+   * - Confluent Cloud: ✅ Supported
+   */
+  createAcl?(acl: {
+    resourceType: 'Topic' | 'Group' | 'Cluster';
+    resourceName: string;
+    principal: string;
+    operation: 'Read' | 'Write' | 'Create' | 'Delete' | 'Alter' | 'Describe' | 'All';
+    permissionType: 'Allow' | 'Deny';
+  }): Promise<AdminOperationResult>;
+
+  /**
+   * List ACLs
+   */
+  listAcls?(): Promise<any[]>;
+
+  // ==================================================
+  // Platform-Specific Operations
+  // ==================================================
+
+  /**
+   * Platform-specific operation (optional)
+   * Examples:
+   * - AWS MSK: Get IAM policy for topic access
+   * - Azure Event Hubs: Get Event Hub connection string
+   * - Confluent Cloud: Get Schema Registry URL
+   */
+  platformSpecific?(operation: string, params?: any): Promise<any>;
+}
+
+/**
+ * Platform Adapter Factory
+ *
+ * Creates appropriate adapter based on platform type
+ */
+export class PlatformAdapterFactory {
+  static create(platform: KafkaPlatform): PlatformAdapter {
+    switch (platform) {
+      case KafkaPlatform.APACHE_KAFKA:
+        // Lazy load to avoid circular dependencies
+        const { ApacheKafkaAdapter } = require('./apache-kafka-adapter');
+        return new ApacheKafkaAdapter();
+
+      case KafkaPlatform.AWS_MSK:
+        const { AwsMskAdapter } = require('./aws-msk-adapter');
+        return new AwsMskAdapter();
+
+      case KafkaPlatform.AZURE_EVENT_HUBS:
+        const { AzureEventHubsAdapter } = require('./azure-event-hubs-adapter');
+        return new AzureEventHubsAdapter();
+
+      case KafkaPlatform.CONFLUENT_CLOUD:
+        const { ConfluentCloudAdapter } = require('./confluent-cloud-adapter');
+        return new ConfluentCloudAdapter();
+
+      default:
+        throw new Error(`Unsupported platform: ${platform}`);
+    }
+  }
+
+  /**
+   * Auto-detect platform based on connection config
+   */
+  static detectPlatform(config: ConnectionConfig): KafkaPlatform {
+    const broker = config.brokers[0].toLowerCase();
+
+    // AWS MSK detection
+    if (broker.includes('.kafka.') && broker.includes('.amazonaws.com')) {
+      return KafkaPlatform.AWS_MSK;
+    }
+
+    // Azure Event Hubs detection
+    if (broker.includes('.servicebus.windows.net')) {
+      return KafkaPlatform.AZURE_EVENT_HUBS;
+    }
+
+    // Confluent Cloud detection
+    if (broker.includes('.confluent.cloud') || broker.includes('.gcp.confluent.cloud')) {
+      return KafkaPlatform.CONFLUENT_CLOUD;
+    }
+
+    // Default to Apache Kafka
+    return KafkaPlatform.APACHE_KAFKA;
+  }
+}

package/plugins/specweave-kafka/lib/cli/kcat-wrapper.js

@@ -0,0 +1,258 @@
+import { exec } from "child_process";
+import { promisify } from "util";
+const execAsync = promisify(exec);
+class KcatWrapper {
+  /**
+   * Produce a message to Kafka topic
+   */
+  async produce(message, options) {
+    const args = [
+      "kcat",
+      "-P",
+      // Producer mode
+      "-b",
+      options.brokers,
+      "-t",
+      options.topic
+    ];
+    if (options.key) {
+      args.push("-K", ":");
+    }
+    if (options.partition !== void 0) {
+      args.push("-p", options.partition.toString());
+    }
+    if (options.compression) {
+      args.push("-z", options.compression);
+    }
+    if (options.acks !== void 0) {
+      const acksValue = options.acks === "all" ? "-1" : options.acks.toString();
+      args.push("-X", `acks=${acksValue}`);
+    }
+    const command = args.join(" ");
+    const input = options.key ? `${options.key}:${message}` : message;
+    return this.execute(command, input);
+  }
+  /**
+   * Consume messages from Kafka topic
+   */
+  async consume(options) {
+    const args = [
+      "kcat",
+      "-C",
+      // Consumer mode
+      "-b",
+      options.brokers,
+      "-t",
+      options.topic,
+      "-f",
+      "%t:%p:%o:%k:%s\\n"
+      // Format: topic:partition:offset:key:value
+    ];
+    if (options.offset) {
+      args.push("-o", options.offset.toString());
+    }
+    if (options.partition !== void 0) {
+      args.push("-p", options.partition.toString());
+    }
+    if (options.count) {
+      args.push("-c", options.count.toString());
+    }
+    if (options.groupId) {
+      args.push("-G", options.groupId);
+    }
+    const command = args.join(" ");
+    const result = await this.execute(command);
+    if (!result.success) {
+      throw new Error(`kcat consume failed: ${result.error}`);
+    }
+    return this.parseConsumeOutput(result.output);
+  }
+  /**
+   * Get cluster metadata
+   */
+  async getMetadata(options) {
+    const args = [
+      "kcat",
+      "-L",
+      // Metadata mode
+      "-b",
+      options.brokers,
+      "-J"
+      // JSON output
+    ];
+    if (options.topic) {
+      args.push("-t", options.topic);
+    }
+    const command = args.join(" ");
+    const result = await this.execute(command);
+    if (!result.success) {
+      throw new Error(`kcat metadata failed: ${result.error}`);
+    }
+    return this.parseMetadataOutput(result.output);
+  }
+  /**
+   * Query topic offsets
+   */
+  async queryOffsets(options) {
+    const args = [
+      "kcat",
+      "-Q",
+      // Query mode
+      "-b",
+      options.brokers,
+      "-t",
+      options.topic
+    ];
+    if (options.partition !== void 0) {
+      args.push("-p", options.partition.toString());
+    }
+    const command = args.join(" ");
+    const result = await this.execute(command);
+    if (!result.success) {
+      throw new Error(`kcat query failed: ${result.error}`);
+    }
+    return this.parseQueryOutput(result.output);
+  }
+  /**
+   * Execute kcat command
+   */
+  async execute(command, input) {
+    const startTime = Date.now();
+    try {
+      const execOptions = {
+        maxBuffer: 10 * 1024 * 1024,
+        // 10MB buffer
+        timeout: 3e4
+        // 30 second timeout
+      };
+      if (input) {
+        execOptions.input = input;
+      }
+      const { stdout, stderr } = await execAsync(command, execOptions);
+      const duration = Date.now() - startTime;
+      return {
+        success: true,
+        output: stdout,
+        error: stderr || void 0,
+        exitCode: 0,
+        command,
+        duration
+      };
+    } catch (error) {
+      const duration = Date.now() - startTime;
+      return {
+        success: false,
+        output: error.stdout || "",
+        error: error.stderr || error.message,
+        exitCode: error.code || 1,
+        command,
+        duration
+      };
+    }
+  }
+  /**
+   * Parse consume output into messages
+   */
+  parseConsumeOutput(output) {
+    const lines = output.trim().split("\n").filter((line) => line.length > 0);
+    const messages = [];
+    for (const line of lines) {
+      const parts = line.split(":");
+      if (parts.length >= 5) {
+        messages.push({
+          topic: parts[0],
+          partition: parseInt(parts[1]),
+          offset: parseInt(parts[2]),
+          key: parts[3] || void 0,
+          value: parts.slice(4).join(":"),
+          // Rejoin in case value contains ':'
+          timestamp: Date.now()
+          // kcat doesn't provide timestamp in this format
+        });
+      }
+    }
+    return messages;
+  }
+  /**
+   * Parse metadata JSON output
+   */
+  parseMetadataOutput(output) {
+    const metadata = JSON.parse(output);
+    return {
+      clusterId: metadata.originatingBroker?.cluster || "unknown",
+      controllerId: metadata.controllerId || -1,
+      brokers: (metadata.brokers || []).map((b) => ({
+        id: b.id,
+        host: b.host,
+        port: b.port,
+        rack: b.rack
+      })),
+      topics: (metadata.topics || []).map((t) => ({
+        name: t.topic,
+        replicationFactor: this.calculateReplicationFactor(t.partitions),
+        configs: {},
+        partitions: (t.partitions || []).map((p) => ({
+          id: p.partition,
+          leader: p.leader,
+          replicas: p.replicas || [],
+          isr: p.isrs || [],
+          earliestOffset: 0,
+          // Not provided in metadata
+          latestOffset: 0
+          // Not provided in metadata
+        }))
+      }))
+    };
+  }
+  /**
+   * Calculate replication factor from partitions
+   */
+  calculateReplicationFactor(partitions) {
+    if (!partitions || partitions.length === 0) return 0;
+    return partitions[0].replicas?.length || 0;
+  }
+  /**
+   * Parse query offset output
+   */
+  parseQueryOutput(output) {
+    const lines = output.trim().split("\n");
+    const offsets = {};
+    for (const line of lines) {
+      const match = line.match(/(\d+)\s+\[(\d+)\]\s+\[(\d+)\]/);
+      if (match) {
+        const partition = parseInt(match[1]);
+        offsets[partition] = {
+          earliest: parseInt(match[2]),
+          latest: parseInt(match[3])
+        };
+      }
+    }
+    return offsets;
+  }
+  /**
+   * Check if kcat is installed
+   */
+  static async isInstalled() {
+    try {
+      const { stdout } = await execAsync("which kcat 2>/dev/null || which kafkacat 2>/dev/null");
+      return !!stdout.trim();
+    } catch {
+      return false;
+    }
+  }
+  /**
+   * Get kcat version
+   */
+  static async getVersion() {
+    try {
+      const { stdout } = await execAsync("kcat -V 2>&1 || kafkacat -V 2>&1");
+      const match = stdout.match(/kafkacat - Apache Kafka producer and consumer tool\s+(\S+)/);
+      return match ? match[1] : "unknown";
+    } catch {
+      return "unknown";
+    }
+  }
+}
+export {
+  KcatWrapper
+};