specweave 0.18.1 → 0.20.0

package/plugins/specweave-kafka/lib/cli/kcat-wrapper.ts

@@ -0,0 +1,298 @@
+/**
+ * kcat CLI Wrapper
+ *
+ * Type-safe TypeScript wrapper for kcat (kafkacat) CLI tool.
+ * Provides produce, consume, metadata, and query operations.
+ */
+
+import { exec } from 'child_process';
+import { promisify } from 'util';
+import {
+  KcatProduceOptions,
+  KcatConsumeOptions,
+  KcatMetadataOptions,
+  KcatQueryOptions,
+  ClusterMetadata,
+  ConsumerMessage,
+  CLIExecutionResult,
+  TopicMetadata,
+  PartitionMetadata
+} from './types';
+
+const execAsync = promisify(exec);
+
+export class KcatWrapper {
+  /**
+   * Produce a message to Kafka topic
+   */
+  async produce(message: string, options: KcatProduceOptions): Promise<CLIExecutionResult> {
+    const args: string[] = [
+      'kcat',
+      '-P', // Producer mode
+      '-b', options.brokers,
+      '-t', options.topic
+    ];
+
+    if (options.key) {
+      args.push('-K', ':'); // Key delimiter
+    }
+
+    if (options.partition !== undefined) {
+      args.push('-p', options.partition.toString());
+    }
+
+    if (options.compression) {
+      args.push('-z', options.compression);
+    }
+
+    if (options.acks !== undefined) {
+      const acksValue = options.acks === 'all' ? '-1' : options.acks.toString();
+      args.push('-X', `acks=${acksValue}`);
+    }
+
+    const command = args.join(' ');
+    const input = options.key ? `${options.key}:${message}` : message;
+
+    return this.execute(command, input);
+  }
+
+  /**
+   * Consume messages from Kafka topic
+   */
+  async consume(options: KcatConsumeOptions): Promise<ConsumerMessage[]> {
+    const args: string[] = [
+      'kcat',
+      '-C', // Consumer mode
+      '-b', options.brokers,
+      '-t', options.topic,
+      '-f', '%t:%p:%o:%k:%s\\n' // Format: topic:partition:offset:key:value
+    ];
+
+    if (options.offset) {
+      args.push('-o', options.offset.toString());
+    }
+
+    if (options.partition !== undefined) {
+      args.push('-p', options.partition.toString());
+    }
+
+    if (options.count) {
+      args.push('-c', options.count.toString());
+    }
+
+    if (options.groupId) {
+      args.push('-G', options.groupId);
+    }
+
+    const command = args.join(' ');
+    const result = await this.execute(command);
+
+    if (!result.success) {
+      throw new Error(`kcat consume failed: ${result.error}`);
+    }
+
+    return this.parseConsumeOutput(result.output);
+  }
+
+  /**
+   * Get cluster metadata
+   */
+  async getMetadata(options: KcatMetadataOptions): Promise<ClusterMetadata> {
+    const args: string[] = [
+      'kcat',
+      '-L', // Metadata mode
+      '-b', options.brokers,
+      '-J' // JSON output
+    ];
+
+    if (options.topic) {
+      args.push('-t', options.topic);
+    }
+
+    const command = args.join(' ');
+    const result = await this.execute(command);
+
+    if (!result.success) {
+      throw new Error(`kcat metadata failed: ${result.error}`);
+    }
+
+    return this.parseMetadataOutput(result.output);
+  }
+
+  /**
+   * Query topic offsets
+   */
+  async queryOffsets(options: KcatQueryOptions): Promise<Record<number, { earliest: number; latest: number }>> {
+    const args: string[] = [
+      'kcat',
+      '-Q', // Query mode
+      '-b', options.brokers,
+      '-t', options.topic
+    ];
+
+    if (options.partition !== undefined) {
+      args.push('-p', options.partition.toString());
+    }
+
+    const command = args.join(' ');
+    const result = await this.execute(command);
+
+    if (!result.success) {
+      throw new Error(`kcat query failed: ${result.error}`);
+    }
+
+    return this.parseQueryOutput(result.output);
+  }
+
+  /**
+   * Execute kcat command
+   */
+  private async execute(command: string, input?: string): Promise<CLIExecutionResult> {
+    const startTime = Date.now();
+
+    try {
+      const execOptions: any = {
+        maxBuffer: 10 * 1024 * 1024, // 10MB buffer
+        timeout: 30000 // 30 second timeout
+      };
+
+      if (input) {
+        execOptions.input = input;
+      }
+
+      const { stdout, stderr } = await execAsync(command, execOptions);
+      const duration = Date.now() - startTime;
+
+      return {
+        success: true,
+        output: stdout,
+        error: stderr || undefined,
+        exitCode: 0,
+        command,
+        duration
+      };
+    } catch (error: any) {
+      const duration = Date.now() - startTime;
+
+      return {
+        success: false,
+        output: error.stdout || '',
+        error: error.stderr || error.message,
+        exitCode: error.code || 1,
+        command,
+        duration
+      };
+    }
+  }
+
+  /**
+   * Parse consume output into messages
+   */
+  private parseConsumeOutput(output: string): ConsumerMessage[] {
+    const lines = output.trim().split('\n').filter(line => line.length > 0);
+    const messages: ConsumerMessage[] = [];
+
+    for (const line of lines) {
+      // Format: topic:partition:offset:key:value
+      const parts = line.split(':');
+      if (parts.length >= 5) {
+        messages.push({
+          topic: parts[0],
+          partition: parseInt(parts[1]),
+          offset: parseInt(parts[2]),
+          key: parts[3] || undefined,
+          value: parts.slice(4).join(':'), // Rejoin in case value contains ':'
+          timestamp: Date.now() // kcat doesn't provide timestamp in this format
+        });
+      }
+    }
+
+    return messages;
+  }
+
+  /**
+   * Parse metadata JSON output
+   */
+  private parseMetadataOutput(output: string): ClusterMetadata {
+    const metadata = JSON.parse(output);
+
+    return {
+      clusterId: metadata.originatingBroker?.cluster || 'unknown',
+      controllerId: metadata.controllerId || -1,
+      brokers: (metadata.brokers || []).map((b: any) => ({
+        id: b.id,
+        host: b.host,
+        port: b.port,
+        rack: b.rack
+      })),
+      topics: (metadata.topics || []).map((t: any) => ({
+        name: t.topic,
+        replicationFactor: this.calculateReplicationFactor(t.partitions),
+        configs: {},
+        partitions: (t.partitions || []).map((p: any) => ({
+          id: p.partition,
+          leader: p.leader,
+          replicas: p.replicas || [],
+          isr: p.isrs || [],
+          earliestOffset: 0, // Not provided in metadata
+          latestOffset: 0 // Not provided in metadata
+        }))
+      }))
+    };
+  }
+
+  /**
+   * Calculate replication factor from partitions
+   */
+  private calculateReplicationFactor(partitions: any[]): number {
+    if (!partitions || partitions.length === 0) return 0;
+    return partitions[0].replicas?.length || 0;
+  }
+
+  /**
+   * Parse query offset output
+   */
+  private parseQueryOutput(output: string): Record<number, { earliest: number; latest: number }> {
+    const lines = output.trim().split('\n');
+    const offsets: Record<number, { earliest: number; latest: number }> = {};
+
+    for (const line of lines) {
+      // Format: topic partition [earliest] [latest]
+      const match = line.match(/(\d+)\s+\[(\d+)\]\s+\[(\d+)\]/);
+      if (match) {
+        const partition = parseInt(match[1]);
+        offsets[partition] = {
+          earliest: parseInt(match[2]),
+          latest: parseInt(match[3])
+        };
+      }
+    }
+
+    return offsets;
+  }
+
+  /**
+   * Check if kcat is installed
+   */
+  static async isInstalled(): Promise<boolean> {
+    try {
+      const { stdout } = await execAsync('which kcat 2>/dev/null || which kafkacat 2>/dev/null');
+      return !!stdout.trim();
+    } catch {
+      return false;
+    }
+  }
+
+  /**
+   * Get kcat version
+   */
+  static async getVersion(): Promise<string> {
+    try {
+      const { stdout } = await execAsync('kcat -V 2>&1 || kafkacat -V 2>&1');
+      const match = stdout.match(/kafkacat - Apache Kafka producer and consumer tool\s+(\S+)/);
+      return match ? match[1] : 'unknown';
+    } catch {
+      return 'unknown';
+    }
+  }
+}

package/plugins/specweave-kafka/lib/cli/types.js

@@ -0,0 +1,10 @@
+var CLITool = /* @__PURE__ */ ((CLITool2) => {
+  CLITool2["KCAT"] = "kcat";
+  CLITool2["KCLI"] = "kcli";
+  CLITool2["KAF"] = "kaf";
+  CLITool2["KAFKACTL"] = "kafkactl";
+  return CLITool2;
+})(CLITool || {});
+export {
+  CLITool
+};

package/plugins/specweave-kafka/lib/cli/types.ts

@@ -0,0 +1,92 @@
+/**
+ * CLI Tool Types and Interfaces
+ *
+ * Types for kcat, kcli, kaf, kafkactl wrappers
+ */
+
+export enum CLITool {
+  KCAT = 'kcat',
+  KCLI = 'kcli',
+  KAF = 'kaf',
+  KAFKACTL = 'kafkactl'
+}
+
+export interface KcatProduceOptions {
+  topic: string;
+  brokers: string;
+  key?: string;
+  partition?: number;
+  headers?: Record<string, string>;
+  compression?: 'none' | 'gzip' | 'snappy' | 'lz4' | 'zstd';
+  acks?: 0 | 1 | 'all';
+}
+
+export interface KcatConsumeOptions {
+  topic: string;
+  brokers: string;
+  offset?: 'beginning' | 'end' | 'stored' | number;
+  partition?: number;
+  count?: number;
+  format?: 'json' | 'avro' | 'protobuf' | 'string';
+  groupId?: string;
+}
+
+export interface KcatMetadataOptions {
+  brokers: string;
+  topic?: string;
+}
+
+export interface KcatQueryOptions {
+  brokers: string;
+  topic: string;
+  partition?: number;
+}
+
+export interface TopicMetadata {
+  name: string;
+  partitions: PartitionMetadata[];
+  replicationFactor: number;
+  configs: Record<string, string>;
+}
+
+export interface PartitionMetadata {
+  id: number;
+  leader: number;
+  replicas: number[];
+  isr: number[]; // In-Sync Replicas
+  earliestOffset: number;
+  latestOffset: number;
+}
+
+export interface BrokerMetadata {
+  id: number;
+  host: string;
+  port: number;
+  rack?: string;
+}
+
+export interface ClusterMetadata {
+  brokers: BrokerMetadata[];
+  topics: TopicMetadata[];
+  clusterId: string;
+  controllerId: number;
+}
+
+export interface ConsumerMessage {
+  topic: string;
+  partition: number;
+  offset: number;
+  timestamp: number;
+  key?: string;
+  value: string;
+  headers?: Record<string, string>;
+}
+
+export interface CLIExecutionResult {
+  success: boolean;
+  output: string;
+  error?: string;
+  exitCode: number;
+  command: string;
+  duration: number;
+}

package/plugins/specweave-kafka/lib/connectors/connector-catalog.js

@@ -0,0 +1,305 @@
+var ConnectorCategory = /* @__PURE__ */ ((ConnectorCategory2) => {
+  ConnectorCategory2["DATABASE"] = "database";
+  ConnectorCategory2["CLOUD_STORAGE"] = "cloud-storage";
+  ConnectorCategory2["MESSAGE_QUEUE"] = "message-queue";
+  ConnectorCategory2["DATA_WAREHOUSE"] = "data-warehouse";
+  ConnectorCategory2["SEARCH"] = "search";
+  ConnectorCategory2["MONITORING"] = "monitoring";
+  ConnectorCategory2["FILE_SYSTEM"] = "file-system";
+  return ConnectorCategory2;
+})(ConnectorCategory || {});
+class ConnectorCatalog {
+  /**
+   * JDBC Source Connector (Database → Kafka)
+   *
+   * Use case: Stream database tables to Kafka (CDC alternative)
+   */
+  static jdbcSource(options) {
+    const config = {
+      name: `jdbc-source-${options.tableName}`,
+      "connector.class": "io.confluent.connect.jdbc.JdbcSourceConnector",
+      "tasks.max": "1",
+      "connection.url": options.connectionUrl,
+      "connection.user": options.user,
+      "connection.password": options.password,
+      "table.whitelist": options.tableName,
+      "mode": options.mode,
+      "topic.prefix": options.topicPrefix,
+      "poll.interval.ms": "5000"
+    };
+    if (options.mode === "incrementing" && options.incrementingColumn) {
+      config["incrementing.column.name"] = options.incrementingColumn;
+    }
+    if ((options.mode === "timestamp" || options.mode === "timestamp+incrementing") && options.timestampColumn) {
+      config["timestamp.column.name"] = options.timestampColumn;
+    }
+    return config;
+  }
+  /**
+   * JDBC Sink Connector (Kafka → Database)
+   *
+   * Use case: Write Kafka events to database tables
+   */
+  static jdbcSink(options) {
+    return {
+      name: `jdbc-sink-${options.topics.join("-")}`,
+      "connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector",
+      "tasks.max": "1",
+      "connection.url": options.connectionUrl,
+      "connection.user": options.user,
+      "connection.password": options.password,
+      "topics": options.topics.join(","),
+      "auto.create": options.autoCreate !== false ? "true" : "false",
+      "auto.evolve": options.autoEvolve !== false ? "true" : "false",
+      "insert.mode": options.insertMode || "insert",
+      "pk.mode": options.pkMode || "none",
+      "pk.fields": options.pkFields?.join(",") || ""
+    };
+  }
+  /**
+   * Debezium MySQL Source Connector (CDC)
+   *
+   * Use case: Capture all database changes in real-time
+   */
+  static debeziumMySQL(options) {
+    return {
+      name: `debezium-mysql-${options.databaseName}`,
+      "connector.class": "io.debezium.connector.mysql.MySqlConnector",
+      "tasks.max": "1",
+      "database.hostname": options.hostname,
+      "database.port": options.port.toString(),
+      "database.user": options.user,
+      "database.password": options.password,
+      "database.server.id": options.serverId.toString(),
+      "database.server.name": options.serverName,
+      "database.include.list": options.databaseName,
+      "table.include.list": options.tableIncludeList || `${options.databaseName}.*`,
+      "database.history.kafka.bootstrap.servers": "localhost:9092",
+      "database.history.kafka.topic": `dbhistory.${options.databaseName}`
+    };
+  }
+  /**
+   * Debezium PostgreSQL Source Connector (CDC)
+   */
+  static debeziumPostgreSQL(options) {
+    return {
+      name: `debezium-postgres-${options.databaseName}`,
+      "connector.class": "io.debezium.connector.postgresql.PostgresConnector",
+      "tasks.max": "1",
+      "database.hostname": options.hostname,
+      "database.port": options.port.toString(),
+      "database.user": options.user,
+      "database.password": options.password,
+      "database.dbname": options.databaseName,
+      "database.server.name": options.serverName,
+      "slot.name": options.slotName,
+      "publication.name": options.publicationName,
+      "plugin.name": "pgoutput"
+    };
+  }
+  /**
+   * S3 Sink Connector (Kafka → AWS S3)
+   *
+   * Use case: Archive Kafka data to S3 for analytics
+   */
+  static s3Sink(options) {
+    const formatClass = options.format === "avro" ? "io.confluent.connect.s3.format.avro.AvroFormat" : options.format === "parquet" ? "io.confluent.connect.s3.format.parquet.ParquetFormat" : "io.confluent.connect.s3.format.json.JsonFormat";
+    return {
+      name: `s3-sink-${options.topics.join("-")}`,
+      "connector.class": "io.confluent.connect.s3.S3SinkConnector",
+      "tasks.max": "1",
+      "topics": options.topics.join(","),
+      "s3.bucket.name": options.s3BucketName,
+      "s3.region": options.s3Region,
+      "format.class": formatClass,
+      "flush.size": (options.flushSize || 1e3).toString(),
+      "rotate.interval.ms": (options.rotateIntervalMs || 36e5).toString(),
+      "partitioner.class": options.partitionerClass || "io.confluent.connect.storage.partitioner.TimeBasedPartitioner",
+      "path.format": "'year'=YYYY/'month'=MM/'day'=dd/'hour'=HH",
+      "locale": "en-US",
+      "timezone": "UTC",
+      "timestamp.extractor": "Record"
+    };
+  }
+  /**
+   * Elasticsearch Sink Connector (Kafka → Elasticsearch)
+   *
+   * Use case: Index Kafka data for full-text search
+   */
+  static elasticsearchSink(options) {
+    return {
+      name: `elasticsearch-sink-${options.topics.join("-")}`,
+      "connector.class": "io.confluent.connect.elasticsearch.ElasticsearchSinkConnector",
+      "tasks.max": "1",
+      "topics": options.topics.join(","),
+      "connection.url": options.connectionUrl,
+      "type.name": options.typeName || "_doc",
+      "key.ignore": "true",
+      "schema.ignore": "false",
+      "batch.size": (options.batchSize || 2e3).toString(),
+      "max.buffered.records": "20000",
+      "linger.ms": "1000",
+      "flush.timeout.ms": "10000",
+      "max.in.flight.requests": "5",
+      "retry.backoff.ms": "100",
+      "max.retries": "10"
+    };
+  }
+  /**
+   * MongoDB Sink Connector (Kafka → MongoDB)
+   *
+   * Use case: Write Kafka events to MongoDB collections
+   */
+  static mongodbSink(options) {
+    return {
+      name: `mongodb-sink-${options.topics.join("-")}`,
+      "connector.class": "com.mongodb.kafka.connect.MongoSinkConnector",
+      "tasks.max": "1",
+      "topics": options.topics.join(","),
+      "connection.uri": options.connectionUri,
+      "database": options.databaseName,
+      "collection": options.collectionName || "kafka_data",
+      "max.num.retries": "3",
+      "retries.defer.timeout": "5000"
+    };
+  }
+  /**
+   * HTTP Sink Connector (Kafka → REST API)
+   *
+   * Use case: Send Kafka events to external APIs
+   */
+  static httpSink(options) {
+    const config = {
+      name: `http-sink-${options.topics.join("-")}`,
+      "connector.class": "io.confluent.connect.http.HttpSinkConnector",
+      "tasks.max": "1",
+      "topics": options.topics.join(","),
+      "http.api.url": options.httpApiUrl,
+      "request.method": options.httpMethod || "POST",
+      "batch.max.size": (options.batchSize || 10).toString(),
+      "retry.on.status.codes": "500-599",
+      "max.retries": "3",
+      "retry.backoff.ms": "1000"
+    };
+    if (options.headers) {
+      Object.entries(options.headers).forEach(([key, value], index) => {
+        config[`headers.${index}.name`] = key;
+        config[`headers.${index}.value`] = value;
+      });
+    }
+    return config;
+  }
+  /**
+   * HDFS Sink Connector (Kafka → Hadoop HDFS)
+   *
+   * Use case: Archive Kafka data to Hadoop for batch processing
+   */
+  static hdfsSink(options) {
+    const formatClass = options.format === "avro" ? "io.confluent.connect.hdfs.avro.AvroFormat" : options.format === "parquet" ? "io.confluent.connect.hdfs.parquet.ParquetFormat" : "io.confluent.connect.hdfs.json.JsonFormat";
+    return {
+      name: `hdfs-sink-${options.topics.join("-")}`,
+      "connector.class": "io.confluent.connect.hdfs.HdfsSinkConnector",
+      "tasks.max": "1",
+      "topics": options.topics.join(","),
+      "hdfs.url": options.hdfsUrl,
+      "format.class": formatClass,
+      "flush.size": (options.flushSize || 1e3).toString(),
+      "rotate.interval.ms": (options.rotateIntervalMs || 36e5).toString(),
+      "partitioner.class": "io.confluent.connect.storage.partitioner.TimeBasedPartitioner",
+      "path.format": "'year'=YYYY/'month'=MM/'day'=dd/'hour'=HH",
+      "locale": "en-US",
+      "timezone": "UTC"
+    };
+  }
+  /**
+   * Snowflake Sink Connector (Kafka → Snowflake)
+   *
+   * Use case: Stream Kafka data to Snowflake data warehouse
+   */
+  static snowflakeSink(options) {
+    return {
+      name: `snowflake-sink-${options.topics.join("-")}`,
+      "connector.class": "com.snowflake.kafka.connector.SnowflakeSinkConnector",
+      "tasks.max": "8",
+      "topics": options.topics.join(","),
+      "snowflake.url.name": options.snowflakeUrl,
+      "snowflake.user.name": options.snowflakeUser,
+      "snowflake.private.key": options.snowflakePrivateKey,
+      "snowflake.database.name": options.snowflakeDatabase,
+      "snowflake.schema.name": options.snowflakeSchema,
+      "buffer.count.records": "10000",
+      "buffer.flush.time": "60",
+      "buffer.size.bytes": "5000000"
+    };
+  }
+  /**
+   * BigQuery Sink Connector (Kafka → Google BigQuery)
+   *
+   * Use case: Stream Kafka data to BigQuery for analytics
+   */
+  static bigQuerySink(options) {
+    return {
+      name: `bigquery-sink-${options.topics.join("-")}`,
+      "connector.class": "com.wepay.kafka.connect.bigquery.BigQuerySinkConnector",
+      "tasks.max": "1",
+      "topics": options.topics.join(","),
+      "project": options.projectId,
+      "defaultDataset": options.datasetName,
+      "autoCreateTables": options.autoCreateTables !== false ? "true" : "false",
+      "autoUpdateSchemas": "true",
+      "sanitizeTopics": "true",
+      "allowNewBigQueryFields": "true",
+      "allowBigQueryRequiredFieldRelaxation": "true"
+    };
+  }
+}
+class ConnectorManager {
+  /**
+   * Deploy connector via REST API
+   */
+  static async deployConnector(connectUrl, config) {
+    const response = await fetch(`${connectUrl}/connectors`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ name: config.name, config })
+    });
+    if (!response.ok) {
+      throw new Error(`Failed to deploy connector: ${await response.text()}`);
+    }
+    console.log(`\u2705 Connector deployed: ${config.name}`);
+  }
+  /**
+   * List all connectors
+   */
+  static async listConnectors(connectUrl) {
+    const response = await fetch(`${connectUrl}/connectors`);
+    return response.json();
+  }
+  /**
+   * Get connector status
+   */
+  static async getConnectorStatus(connectUrl, connectorName) {
+    const response = await fetch(`${connectUrl}/connectors/${connectorName}/status`);
+    return response.json();
+  }
+  /**
+   * Delete connector
+   */
+  static async deleteConnector(connectUrl, connectorName) {
+    await fetch(`${connectUrl}/connectors/${connectorName}`, {
+      method: "DELETE"
+    });
+    console.log(`\u2705 Connector deleted: ${connectorName}`);
+  }
+}
+var connector_catalog_default = {
+  ConnectorCatalog,
+  ConnectorManager,
+  ConnectorCategory
+};
+export {
+  ConnectorCatalog,
+  ConnectorCategory,
+  ConnectorManager,
+  connector_catalog_default as default
+};