specweave 0.18.1 → 0.20.0

package/plugins/specweave-kafka/docker/templates/consumer-python.py

@@ -0,0 +1,220 @@
+"""
+Kafka Consumer Example (Python)
+
+Install: pip install kafka-python
+"""
+
+from kafka import KafkaConsumer
+from kafka.errors import KafkaError
+import json
+import signal
+import sys
+
+# ========================================
+# Configuration
+# ========================================
+
+consumer = KafkaConsumer(
+    'my-topic',  # Can subscribe to multiple topics
+    bootstrap_servers=['localhost:9092'],
+    client_id='my-app',
+    group_id='my-consumer-group',
+
+    # Deserialization
+    key_deserializer=lambda k: k.decode('utf-8') if k else None,
+    value_deserializer=lambda v: json.loads(v.decode('utf-8')),
+
+    # Offset management
+    auto_offset_reset='earliest',  # 'earliest', 'latest', 'none'
+    enable_auto_commit=True,
+    auto_commit_interval_ms=5000,
+
+    # Performance tuning
+    fetch_min_bytes=1,          # Minimum data to fetch
+    fetch_max_wait_ms=500,      # Max wait time
+    max_poll_records=500,       # Messages per poll
+    max_poll_interval_ms=300000,  # 5 minutes
+
+    # Session management
+    session_timeout_ms=30000,
+    heartbeat_interval_ms=3000,
+
+    # Optional: SASL/SSL (for production)
+    # security_protocol='SASL_SSL',
+    # sasl_mechanism='SCRAM-SHA-512',
+    # sasl_plain_username='user',
+    # sasl_plain_password='password'
+)
+
+# ========================================
+# Consumer Functions
+# ========================================
+
+def process_message(message):
+    """Process a single message"""
+    try:
+        print(f"""
+📨 Message received:
+  Topic: {message.topic}
+  Partition: {message.partition}
+  Offset: {message.offset}
+  Key: {message.key}
+  Value: {message.value}
+  Timestamp: {message.timestamp}
+  Headers: {dict(message.headers) if message.headers else {}}
+        """)
+
+        # Your business logic here
+        event_type = message.value.get('event')
+
+        if event_type == 'user_login':
+            handle_user_login(message.value)
+        elif event_type == 'order_created':
+            handle_order_created(message.value)
+        else:
+            print(f"Unknown event type: {event_type}")
+
+    except Exception as e:
+        print(f"❌ Error processing message: {e}")
+        # Send to dead letter queue
+        send_to_dlq(message, e)
+
+def handle_user_login(data):
+    """Handle user login event"""
+    print(f"👤 User login: {data.get('userId')}")
+    # Update user last login time
+    # Send welcome email
+    # etc.
+
+def handle_order_created(data):
+    """Handle order created event"""
+    print(f"🛒 Order created: {data}")
+    # Process order
+    # Send confirmation email
+    # Update inventory
+
+def send_to_dlq(message, error):
+    """Send failed message to dead letter queue"""
+    from kafka import KafkaProducer
+    import json
+
+    producer = KafkaProducer(
+        bootstrap_servers=['localhost:9092'],
+        value_serializer=lambda v: json.dumps(v).encode('utf-8')
+    )
+
+    dlq_message = {
+        'original_topic': message.topic,
+        'original_partition': message.partition,
+        'original_offset': message.offset,
+        'original_key': message.key,
+        'original_value': message.value,
+        'error_message': str(error),
+        'failed_at': int(time.time() * 1000)
+    }
+
+    producer.send('my-topic-dlq', value=dlq_message)
+    producer.flush()
+    producer.close()
+
+    print("📮 Message sent to DLQ")
+
+# ========================================
+# Consume Methods
+# ========================================
+
+def consume_simple():
+    """Simple consume loop"""
+    print("📡 Consumer connected (simple mode)")
+
+    try:
+        for message in consumer:
+            process_message(message)
+    except KeyboardInterrupt:
+        pass
+    finally:
+        consumer.close()
+        print("👋 Consumer disconnected")
+
+def consume_batch():
+    """Consume messages in batches"""
+    print("📡 Consumer connected (batch mode)")
+
+    try:
+        while True:
+            # Poll for messages (returns a dict of topic partition to records)
+            message_batch = consumer.poll(timeout_ms=1000, max_records=100)
+
+            if not message_batch:
+                continue
+
+            for topic_partition, messages in message_batch.items():
+                print(f"📦 Processing batch: {len(messages)} messages "
+                      f"from {topic_partition.topic} partition {topic_partition.partition}")
+
+                for message in messages:
+                    process_message(message)
+
+                # Commit offsets after processing batch
+                consumer.commit()
+
+    except KeyboardInterrupt:
+        pass
+    finally:
+        consumer.close()
+        print("👋 Consumer disconnected")
+
+def consume_manual_commit():
+    """Consume with manual offset management"""
+    print("📡 Consumer connected (manual commit)")
+
+    # Disable auto-commit
+    consumer._config['enable_auto_commit'] = False
+
+    try:
+        for message in consumer:
+            try:
+                process_message(message)
+
+                # Manual commit after successful processing
+                consumer.commit()
+
+            except Exception as e:
+                print(f"❌ Error processing, skipping commit: {e}")
+                # Don't commit offset, message will be redelivered
+
+    except KeyboardInterrupt:
+        pass
+    finally:
+        consumer.close()
+        print("👋 Consumer disconnected")
+
+# ========================================
+# Main Function
+# ========================================
+
+def main():
+    # Choose consumption method
+    # consume_simple()
+    # consume_batch()
+    consume_simple()
+
+# ========================================
+# Graceful Shutdown
+# ========================================
+
+def signal_handler(sig, frame):
+    print('\n🛑 Shutting down consumer...')
+    consumer.close()
+    sys.exit(0)
+
+signal.signal(signal.SIGINT, signal_handler)
+signal.signal(signal.SIGTERM, signal_handler)
+
+# ========================================
+# Run
+# ========================================
+
+if __name__ == '__main__':
+    import time
+    main()

package/plugins/specweave-kafka/docker/templates/producer-nodejs.js

@@ -0,0 +1,168 @@
+/**
+ * Kafka Producer Example (Node.js)
+ *
+ * Install: npm install kafkajs
+ */
+
+const { Kafka, CompressionTypes, logLevel } = require('kafkajs');
+
+// ========================================
+// Configuration
+// ========================================
+
+const kafka = new Kafka({
+  clientId: 'my-app',
+  brokers: ['localhost:9092'],
+
+  // Optional: Logging
+  logLevel: logLevel.INFO,
+
+  // Optional: SASL/SSL (for production)
+  // ssl: true,
+  // sasl: {
+  //   mechanism: 'scram-sha-512',
+  //   username: 'user',
+  //   password: 'password'
+  // }
+});
+
+const producer = kafka.producer({
+  // Performance tuning
+  allowAutoTopicCreation: true,
+  transactionTimeout: 30000,
+
+  // Batching for throughput
+  compression: CompressionTypes.LZ4,
+  batch: {
+    size: 16384,    // 16 KB
+    lingerMs: 10    // Wait 10ms to batch
+  },
+
+  // Idempotence (exactly-once semantics)
+  idempotent: true,
+  maxInFlightRequests: 5,
+
+  // Acknowledgment
+  acks: 1  // 0 = none, 1 = leader, -1/all = all replicas
+});
+
+// ========================================
+// Producer Functions
+// ========================================
+
+async function produceMessage(topic, key, value) {
+  try {
+    const result = await producer.send({
+      topic,
+      messages: [
+        {
+          key,
+          value: JSON.stringify(value),
+          headers: {
+            'source': 'my-app',
+            'timestamp': Date.now().toString()
+          }
+        }
+      ]
+    });
+
+    console.log('✅ Message sent:', result);
+    return result;
+  } catch (error) {
+    console.error('❌ Error producing message:', error);
+    throw error;
+  }
+}
+
+async function produceBatch(topic, messages) {
+  try {
+    const kafkaMessages = messages.map(msg => ({
+      key: msg.key,
+      value: JSON.stringify(msg.value),
+      headers: {
+        'source': 'my-app',
+        'timestamp': Date.now().toString()
+      }
+    }));
+
+    const result = await producer.send({
+      topic,
+      messages: kafkaMessages
+    });
+
+    console.log(`✅ ${messages.length} messages sent:`, result);
+    return result;
+  } catch (error) {
+    console.error('❌ Error producing batch:', error);
+    throw error;
+  }
+}
+
+// Transactional producer (exactly-once semantics)
+async function produceTransactional(topic, messages) {
+  const transaction = await producer.transaction();
+
+  try {
+    const kafkaMessages = messages.map(msg => ({
+      key: msg.key,
+      value: JSON.stringify(msg.value)
+    }));
+
+    await transaction.send({
+      topic,
+      messages: kafkaMessages
+    });
+
+    await transaction.commit();
+    console.log('✅ Transaction committed');
+  } catch (error) {
+    await transaction.abort();
+    console.error('❌ Transaction aborted:', error);
+    throw error;
+  }
+}
+
+// ========================================
+// Main Function
+// ========================================
+
+async function main() {
+  await producer.connect();
+  console.log('📡 Producer connected');
+
+  // Example 1: Single message
+  await produceMessage('my-topic', 'user-123', {
+    event: 'user_login',
+    userId: 'user-123',
+    timestamp: Date.now()
+  });
+
+  // Example 2: Batch of messages
+  await produceBatch('my-topic', [
+    { key: 'user-123', value: { event: 'page_view', page: '/home' } },
+    { key: 'user-124', value: { event: 'page_view', page: '/products' } },
+    { key: 'user-125', value: { event: 'page_view', page: '/checkout' } }
+  ]);
+
+  // Example 3: Transactional send
+  await produceTransactional('my-topic', [
+    { key: 'order-1', value: { event: 'order_created', amount: 99.99 } },
+    { key: 'order-1', value: { event: 'payment_processed', amount: 99.99 } }
+  ]);
+
+  await producer.disconnect();
+  console.log('👋 Producer disconnected');
+}
+
+// ========================================
+// Run
+// ========================================
+
+main().catch(console.error);
+
+// Graceful shutdown
+process.on('SIGINT', async () => {
+  console.log('\n🛑 Shutting down...');
+  await producer.disconnect();
+  process.exit(0);
+});

package/plugins/specweave-kafka/docker/templates/producer-python.py

@@ -0,0 +1,167 @@
+"""
+Kafka Producer Example (Python)
+
+Install: pip install kafka-python
+"""
+
+from kafka import KafkaProducer
+from kafka.errors import KafkaError
+import json
+import time
+import signal
+import sys
+
+# ========================================
+# Configuration
+# ========================================
+
+producer = KafkaProducer(
+    bootstrap_servers=['localhost:9092'],
+    client_id='my-app',
+
+    # Serialization
+    key_serializer=lambda k: k.encode('utf-8') if k else None,
+    value_serializer=lambda v: json.dumps(v).encode('utf-8'),
+
+    # Performance tuning
+    compression_type='lz4',  # 'gzip', 'snappy', 'lz4', 'zstd'
+    batch_size=16384,        # 16 KB
+    linger_ms=10,            # Wait 10ms to batch
+    buffer_memory=33554432,  # 32 MB
+
+    # Acknowledgment
+    acks=1,  # 0=none, 1=leader, all=all replicas
+
+    # Retries
+    retries=3,
+    max_in_flight_requests_per_connection=5,
+
+    # Optional: SASL/SSL (for production)
+    # security_protocol='SASL_SSL',
+    # sasl_mechanism='SCRAM-SHA-512',
+    # sasl_plain_username='user',
+    # sasl_plain_password='password'
+)
+
+# ========================================
+# Producer Functions
+# ========================================
+
+def produce_message(topic, key, value):
+    """Send a single message"""
+    try:
+        future = producer.send(
+            topic,
+            key=key,
+            value=value,
+            headers=[
+                ('source', b'my-app'),
+                ('timestamp', str(int(time.time() * 1000)).encode())
+            ]
+        )
+
+        # Block for 'synchronous' send
+        record_metadata = future.get(timeout=10)
+
+        print(f"✅ Message sent to {record_metadata.topic} "
+              f"partition {record_metadata.partition} "
+              f"offset {record_metadata.offset}")
+
+        return record_metadata
+
+    except KafkaError as e:
+        print(f"❌ Error producing message: {e}")
+        raise
+
+def produce_message_async(topic, key, value):
+    """Send message asynchronously with callback"""
+    def on_send_success(record_metadata):
+        print(f"✅ Message sent: partition={record_metadata.partition}, "
+              f"offset={record_metadata.offset}")
+
+    def on_send_error(excp):
+        print(f"❌ Error sending message: {excp}")
+
+    producer.send(topic, key=key, value=value) \\
+        .add_callback(on_send_success) \\
+        .add_errback(on_send_error)
+
+def produce_batch(topic, messages):
+    """Send multiple messages in a batch"""
+    try:
+        for msg in messages:
+            producer.send(
+                topic,
+                key=msg['key'],
+                value=msg['value'],
+                headers=[('source', b'my-app')]
+            )
+
+        # Flush to ensure all messages are sent
+        producer.flush()
+        print(f"✅ {len(messages)} messages sent")
+
+    except KafkaError as e:
+        print(f"❌ Error producing batch: {e}")
+        raise
+
+# ========================================
+# Main Function
+# ========================================
+
+def main():
+    print("📡 Producer connected")
+
+    # Example 1: Single message (synchronous)
+    produce_message(
+        'my-topic',
+        'user-123',
+        {
+            'event': 'user_login',
+            'userId': 'user-123',
+            'timestamp': int(time.time() * 1000)
+        }
+    )
+
+    # Example 2: Async message
+    produce_message_async(
+        'my-topic',
+        'user-124',
+        {'event': 'page_view', 'page': '/home'}
+    )
+
+    # Example 3: Batch of messages
+    produce_batch('my-topic', [
+        {'key': 'user-123', 'value': {'event': 'page_view', 'page': '/home'}},
+        {'key': 'user-124', 'value': {'event': 'page_view', 'page': '/products'}},
+        {'key': 'user-125', 'value': {'event': 'page_view', 'page': '/checkout'}}
+    ])
+
+    # Ensure all messages are sent
+    producer.flush()
+
+    print("👋 Producer disconnected")
+
+# ========================================
+# Graceful Shutdown
+# ========================================
+
+def signal_handler(sig, frame):
+    print('\n🛑 Shutting down producer...')
+    producer.flush()
+    producer.close()
+    sys.exit(0)
+
+signal.signal(signal.SIGINT, signal_handler)
+signal.signal(signal.SIGTERM, signal_handler)
+
+# ========================================
+# Run
+# ========================================
+
+if __name__ == '__main__':
+    try:
+        main()
+    except Exception as e:
+        print(f"Error: {e}")
+        producer.close()