kailash 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. kailash/__init__.py +31 -0
  2. kailash/__main__.py +11 -0
  3. kailash/cli/__init__.py +5 -0
  4. kailash/cli/commands.py +563 -0
  5. kailash/manifest.py +778 -0
  6. kailash/nodes/__init__.py +23 -0
  7. kailash/nodes/ai/__init__.py +26 -0
  8. kailash/nodes/ai/agents.py +417 -0
  9. kailash/nodes/ai/models.py +488 -0
  10. kailash/nodes/api/__init__.py +52 -0
  11. kailash/nodes/api/auth.py +567 -0
  12. kailash/nodes/api/graphql.py +480 -0
  13. kailash/nodes/api/http.py +598 -0
  14. kailash/nodes/api/rate_limiting.py +572 -0
  15. kailash/nodes/api/rest.py +665 -0
  16. kailash/nodes/base.py +1032 -0
  17. kailash/nodes/base_async.py +128 -0
  18. kailash/nodes/code/__init__.py +32 -0
  19. kailash/nodes/code/python.py +1021 -0
  20. kailash/nodes/data/__init__.py +125 -0
  21. kailash/nodes/data/readers.py +496 -0
  22. kailash/nodes/data/sharepoint_graph.py +623 -0
  23. kailash/nodes/data/sql.py +380 -0
  24. kailash/nodes/data/streaming.py +1168 -0
  25. kailash/nodes/data/vector_db.py +964 -0
  26. kailash/nodes/data/writers.py +529 -0
  27. kailash/nodes/logic/__init__.py +6 -0
  28. kailash/nodes/logic/async_operations.py +702 -0
  29. kailash/nodes/logic/operations.py +551 -0
  30. kailash/nodes/transform/__init__.py +5 -0
  31. kailash/nodes/transform/processors.py +379 -0
  32. kailash/runtime/__init__.py +6 -0
  33. kailash/runtime/async_local.py +356 -0
  34. kailash/runtime/docker.py +697 -0
  35. kailash/runtime/local.py +434 -0
  36. kailash/runtime/parallel.py +557 -0
  37. kailash/runtime/runner.py +110 -0
  38. kailash/runtime/testing.py +347 -0
  39. kailash/sdk_exceptions.py +307 -0
  40. kailash/tracking/__init__.py +7 -0
  41. kailash/tracking/manager.py +885 -0
  42. kailash/tracking/metrics_collector.py +342 -0
  43. kailash/tracking/models.py +535 -0
  44. kailash/tracking/storage/__init__.py +0 -0
  45. kailash/tracking/storage/base.py +113 -0
  46. kailash/tracking/storage/database.py +619 -0
  47. kailash/tracking/storage/filesystem.py +543 -0
  48. kailash/utils/__init__.py +0 -0
  49. kailash/utils/export.py +924 -0
  50. kailash/utils/templates.py +680 -0
  51. kailash/visualization/__init__.py +62 -0
  52. kailash/visualization/api.py +732 -0
  53. kailash/visualization/dashboard.py +951 -0
  54. kailash/visualization/performance.py +808 -0
  55. kailash/visualization/reports.py +1471 -0
  56. kailash/workflow/__init__.py +15 -0
  57. kailash/workflow/builder.py +245 -0
  58. kailash/workflow/graph.py +827 -0
  59. kailash/workflow/mermaid_visualizer.py +628 -0
  60. kailash/workflow/mock_registry.py +63 -0
  61. kailash/workflow/runner.py +302 -0
  62. kailash/workflow/state.py +238 -0
  63. kailash/workflow/visualization.py +588 -0
  64. kailash-0.1.0.dist-info/METADATA +710 -0
  65. kailash-0.1.0.dist-info/RECORD +69 -0
  66. kailash-0.1.0.dist-info/WHEEL +5 -0
  67. kailash-0.1.0.dist-info/entry_points.txt +2 -0
  68. kailash-0.1.0.dist-info/licenses/LICENSE +21 -0
  69. kailash-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1168 @@
1
+ """Streaming data nodes for the Kailash system.
2
+
3
+ This module provides nodes for handling streaming data sources and sinks.
4
+ Key features include:
5
+
6
+ - Real-time data ingestion (Kafka, RabbitMQ, WebSockets)
7
+ - Stream processing and transformation
8
+ - Event-driven architectures
9
+ - Backpressure handling
10
+ - Message acknowledgment
11
+
12
+ Design Philosophy:
13
+ - Support various streaming protocols
14
+ - Handle backpressure gracefully
15
+ - Provide reliable message delivery
16
+ - Enable stream transformations
17
+ - Support both push and pull models
18
+
19
+ Common Use Cases:
20
+ - Real-time data pipelines
21
+ - Event sourcing systems
22
+ - Log aggregation
23
+ - Monitoring and alerting
24
+ - Live data feeds
25
+
26
+ Example:
27
+ >>> # Consume from Kafka
28
+ >>> consumer = KafkaConsumerNode()
29
+ >>> consumer.configure({
30
+ ... "bootstrap_servers": "localhost:9092",
31
+ ... "topic": "events",
32
+ ... "group_id": "my-group",
33
+ ... "auto_offset_reset": "earliest"
34
+ ... })
35
+ >>> result = consumer.execute({"max_messages": 100})
36
+ >>>
37
+ >>> # Publish to stream
38
+ >>> publisher = StreamPublisherNode()
39
+ >>> publisher.configure({
40
+ ... "protocol": "websocket",
41
+ ... "url": "ws://localhost:8080/stream"
42
+ ... })
43
+ >>> publisher.execute({"messages": result["messages"]})
44
+ """
45
+
46
+ import time
47
+ from typing import Any, Dict, List
48
+
49
+ from kailash.nodes.base import Node, NodeMetadata, NodeParameter, register_node
50
+ from kailash.sdk_exceptions import NodeConfigurationError, NodeExecutionError
51
+
52
+
53
+ @register_node()
54
+ class KafkaConsumerNode(Node):
55
+ """Consumes messages from Apache Kafka topics.
56
+
57
+ This node provides a high-level interface for consuming messages from
58
+ Kafka topics with support for consumer groups, offset management, and
59
+ error handling.
60
+
61
+ Design Pattern:
62
+ - Observer pattern for message consumption
63
+ - Iterator pattern for message streaming
64
+ - Template method for processing
65
+
66
+ Features:
67
+ - Consumer group support
68
+ - Automatic offset management
69
+ - Message deserialization
70
+ - Error recovery
71
+ - Batch consumption
72
+ - SSL/SASL authentication
73
+
74
+ Common Usage Patterns:
75
+ - Event stream processing
76
+ - Log aggregation
77
+ - Real-time analytics
78
+ - Microservice communication
79
+ - Data synchronization
80
+
81
+ Upstream Dependencies:
82
+ - Configuration nodes
83
+ - Schema registry nodes
84
+ - Authentication nodes
85
+
86
+ Downstream Consumers:
87
+ - Processing nodes
88
+ - Storage nodes
89
+ - Analytics nodes
90
+ - Notification nodes
91
+
92
+ Configuration:
93
+ bootstrap_servers (str): Kafka broker addresses
94
+ topic (str): Topic to consume from
95
+ group_id (str): Consumer group ID
96
+ auto_offset_reset (str): Where to start reading
97
+ max_poll_records (int): Max messages per poll
98
+ enable_auto_commit (bool): Auto commit offsets
99
+ security_protocol (str): Security protocol
100
+ sasl_mechanism (str): SASL mechanism
101
+ sasl_username (str): SASL username
102
+ sasl_password (str): SASL password
103
+
104
+ Inputs:
105
+ max_messages (int): Maximum messages to consume
106
+ timeout_ms (int): Poll timeout in milliseconds
107
+
108
+ Outputs:
109
+ messages (List[Dict]): Consumed messages
110
+ metadata (Dict): Consumer metadata
111
+
112
+ Error Handling:
113
+ - Connection failures with retry
114
+ - Deserialization errors
115
+ - Offset management errors
116
+ - Authentication failures
117
+
118
+ Example:
119
+ >>> consumer = KafkaConsumerNode()
120
+ >>> consumer.configure({
121
+ ... "bootstrap_servers": "localhost:9092",
122
+ ... "topic": "user-events",
123
+ ... "group_id": "analytics-group",
124
+ ... "auto_offset_reset": "latest",
125
+ ... "max_poll_records": 500
126
+ ... })
127
+ >>> result = consumer.execute({
128
+ ... "max_messages": 1000,
129
+ ... "timeout_ms": 5000
130
+ ... })
131
+ >>> print(f"Consumed {len(result['messages'])} messages")
132
+ """
133
+
134
+ metadata = NodeMetadata(
135
+ name="KafkaConsumerNode",
136
+ description="Consumes messages from Kafka",
137
+ version="1.0.0",
138
+ tags={"streaming", "kafka", "consumer"},
139
+ )
140
+
141
+ def __init__(self):
142
+ """Initialize the Kafka consumer node.
143
+
144
+ Sets up the node and prepares for consumer initialization.
145
+ The actual consumer is created during configuration.
146
+ """
147
+ super().__init__()
148
+ self._consumer = None
149
+ self._topic = None
150
+
151
+ def get_parameters(self) -> Dict[str, NodeParameter]:
152
+ """Define parameters for the Kafka consumer node."""
153
+ return {
154
+ "bootstrap_servers": NodeParameter(
155
+ name="bootstrap_servers",
156
+ type=str,
157
+ description="Kafka broker addresses",
158
+ required=True,
159
+ ),
160
+ "topic": NodeParameter(
161
+ name="topic",
162
+ type=str,
163
+ description="Topic to consume from",
164
+ required=True,
165
+ ),
166
+ "group_id": NodeParameter(
167
+ name="group_id",
168
+ type=str,
169
+ description="Consumer group ID",
170
+ required=True,
171
+ ),
172
+ "auto_offset_reset": NodeParameter(
173
+ name="auto_offset_reset",
174
+ type=str,
175
+ description="Where to start reading",
176
+ required=False,
177
+ default="latest",
178
+ ),
179
+ "max_poll_records": NodeParameter(
180
+ name="max_poll_records",
181
+ type=int,
182
+ description="Max messages per poll",
183
+ required=False,
184
+ default=500,
185
+ ),
186
+ "enable_auto_commit": NodeParameter(
187
+ name="enable_auto_commit",
188
+ type=bool,
189
+ description="Auto commit offsets",
190
+ required=False,
191
+ default=True,
192
+ ),
193
+ "security_protocol": NodeParameter(
194
+ name="security_protocol",
195
+ type=str,
196
+ description="Security protocol",
197
+ required=False,
198
+ default="PLAINTEXT",
199
+ ),
200
+ "value_deserializer": NodeParameter(
201
+ name="value_deserializer",
202
+ type=str,
203
+ description="Message deserializer",
204
+ required=False,
205
+ default="json",
206
+ ),
207
+ }
208
+
209
+ def configure(self, config: Dict[str, Any]) -> None:
210
+ """Configure the Kafka consumer.
211
+
212
+ Creates and configures the Kafka consumer with the specified
213
+ settings including security, serialization, and consumer group.
214
+
215
+ Args:
216
+ config: Configuration dictionary
217
+
218
+ Raises:
219
+ NodeConfigurationError: If configuration fails
220
+ """
221
+ super().configure(config)
222
+
223
+ required_fields = ["bootstrap_servers", "topic", "group_id"]
224
+ for field in required_fields:
225
+ if not self.config.get(field):
226
+ raise NodeConfigurationError(f"{field} is required")
227
+
228
+ try:
229
+ # Placeholder for actual consumer creation
230
+ self._create_consumer()
231
+ except Exception as e:
232
+ raise NodeConfigurationError(f"Failed to create consumer: {str(e)}")
233
+
234
+ def _create_consumer(self) -> None:
235
+ """Create the Kafka consumer instance.
236
+
237
+ Initializes the consumer with all configured settings.
238
+ This is a placeholder for actual consumer creation.
239
+ """
240
+ # Placeholder for actual consumer creation
241
+ self._consumer = f"kafka_consumer_{self.config['group_id']}"
242
+ self._topic = self.config["topic"]
243
+
244
+ def run(self, **kwargs) -> Dict[str, Any]:
245
+ """Consume messages from Kafka.
246
+
247
+ Implementation of the abstract run method from the base Node class.
248
+
249
+ Args:
250
+ **kwargs: Keyword arguments for consumption
251
+
252
+ Returns:
253
+ Consumed messages and metadata
254
+ """
255
+ return self.execute(kwargs)
256
+
257
+ def execute(self, inputs: Dict[str, Any]) -> Dict[str, Any]:
258
+ """Consume messages from Kafka.
259
+
260
+ Polls for messages up to the specified limit or timeout.
261
+
262
+ Args:
263
+ inputs: Execution parameters
264
+
265
+ Returns:
266
+ Consumed messages and metadata
267
+
268
+ Raises:
269
+ NodeExecutionError: If consumption fails
270
+ """
271
+ try:
272
+ max_messages = inputs.get("max_messages", 100)
273
+ timeout_ms = inputs.get("timeout_ms", 1000)
274
+
275
+ messages = self._consume_messages(max_messages, timeout_ms)
276
+
277
+ return {
278
+ "messages": messages,
279
+ "metadata": {
280
+ "topic": self._topic,
281
+ "group_id": self.config["group_id"],
282
+ "message_count": len(messages),
283
+ "timestamp": time.time(),
284
+ },
285
+ }
286
+ except Exception as e:
287
+ raise NodeExecutionError(f"Failed to consume messages: {str(e)}")
288
+
289
+ def _consume_messages(self, max_messages: int, timeout_ms: int) -> List[Dict]:
290
+ """Consume messages from Kafka.
291
+
292
+ This is a placeholder for actual message consumption logic.
293
+
294
+ Args:
295
+ max_messages: Maximum messages to consume
296
+ timeout_ms: Poll timeout
297
+
298
+ Returns:
299
+ List of consumed messages
300
+ """
301
+ # Placeholder implementation
302
+ messages = []
303
+ for i in range(min(max_messages, 10)): # Simulate consuming up to 10 messages
304
+ messages.append(
305
+ {
306
+ "key": f"key_{i}",
307
+ "value": {"event": f"event_{i}", "timestamp": time.time()},
308
+ "partition": i % 3,
309
+ "offset": i,
310
+ "timestamp": time.time(),
311
+ }
312
+ )
313
+ return messages
314
+
315
+ def cleanup(self) -> None:
316
+ """Clean up consumer resources.
317
+
318
+ Closes the consumer connection and releases resources.
319
+ """
320
+ if self._consumer:
321
+ # Placeholder for actual cleanup
322
+ self._consumer = None
323
+ super().cleanup()
324
+
325
+
326
+ @register_node()
327
+ class StreamPublisherNode(Node):
328
+ """Publishes messages to various streaming platforms.
329
+
330
+ This node provides a unified interface for publishing messages to
331
+ different streaming platforms including Kafka, RabbitMQ, WebSockets,
332
+ and server-sent events (SSE).
333
+
334
+ Design Pattern:
335
+ - Adapter pattern for different protocols
336
+ - Producer pattern for message publishing
337
+ - Strategy pattern for serialization
338
+
339
+ Features:
340
+ - Multi-protocol support
341
+ - Message batching
342
+ - Retry logic
343
+ - Async publishing
344
+ - Compression support
345
+ - Dead letter queues
346
+
347
+ Common Usage Patterns:
348
+ - Event publishing
349
+ - Real-time notifications
350
+ - Data distribution
351
+ - Log forwarding
352
+ - Metric emission
353
+
354
+ Upstream Dependencies:
355
+ - Processing nodes
356
+ - Transformation nodes
357
+ - Aggregation nodes
358
+
359
+ Downstream Consumers:
360
+ - Other streaming systems
361
+ - Analytics platforms
362
+ - Storage systems
363
+ - Monitoring tools
364
+
365
+ Configuration:
366
+ protocol (str): Streaming protocol to use
367
+ endpoint (str): Server endpoint/URL
368
+ topic (str): Topic/channel to publish to
369
+ auth_type (str): Authentication type
370
+ batch_size (int): Message batch size
371
+ compression (str): Compression algorithm
372
+ retry_count (int): Number of retries
373
+
374
+ Inputs:
375
+ messages (List[Dict]): Messages to publish
376
+ headers (Dict): Optional message headers
377
+
378
+ Outputs:
379
+ published_count (int): Number of messages published
380
+ failed_messages (List[Dict]): Failed messages
381
+ metadata (Dict): Publishing metadata
382
+
383
+ Error Handling:
384
+ - Connection failures
385
+ - Serialization errors
386
+ - Rate limiting
387
+ - Authentication errors
388
+
389
+ Example:
390
+ >>> publisher = StreamPublisherNode()
391
+ >>> publisher.configure({
392
+ ... "protocol": "kafka",
393
+ ... "endpoint": "localhost:9092",
394
+ ... "topic": "processed-events",
395
+ ... "batch_size": 100,
396
+ ... "compression": "gzip"
397
+ ... })
398
+ >>> result = publisher.execute({
399
+ ... "messages": [
400
+ ... {"id": 1, "event": "user_login"},
401
+ ... {"id": 2, "event": "page_view"}
402
+ ... ]
403
+ ... })
404
+ >>> print(f"Published {result['published_count']} messages")
405
+ """
406
+
407
+ metadata = NodeMetadata(
408
+ name="StreamPublisherNode",
409
+ description="Publishes to streaming platforms",
410
+ version="1.0.0",
411
+ tags={"streaming", "publisher", "messaging"},
412
+ )
413
+
414
+ def __init__(self):
415
+ """Initialize the stream publisher node.
416
+
417
+ Sets up the node and prepares for publisher initialization.
418
+ """
419
+ super().__init__()
420
+ self._publisher = None
421
+ self._protocol = None
422
+
423
+ def get_parameters(self) -> Dict[str, NodeParameter]:
424
+ """Define parameters for the stream publisher node."""
425
+ return {
426
+ "protocol": NodeParameter(
427
+ name="protocol",
428
+ type=str,
429
+ description="Streaming protocol",
430
+ required=True,
431
+ ),
432
+ "endpoint": NodeParameter(
433
+ name="endpoint", type=str, description="Server endpoint", required=True
434
+ ),
435
+ "topic": NodeParameter(
436
+ name="topic", type=str, description="Topic/channel name", required=True
437
+ ),
438
+ "auth_type": NodeParameter(
439
+ name="auth_type",
440
+ type=str,
441
+ description="Authentication type",
442
+ required=False,
443
+ default="none",
444
+ ),
445
+ "batch_size": NodeParameter(
446
+ name="batch_size",
447
+ type=int,
448
+ description="Message batch size",
449
+ required=False,
450
+ default=100,
451
+ ),
452
+ "compression": NodeParameter(
453
+ name="compression",
454
+ type=str,
455
+ description="Compression algorithm",
456
+ required=False,
457
+ default="none",
458
+ ),
459
+ "retry_count": NodeParameter(
460
+ name="retry_count",
461
+ type=int,
462
+ description="Number of retries",
463
+ required=False,
464
+ default=3,
465
+ ),
466
+ }
467
+
468
+ def configure(self, config: Dict[str, Any]) -> None:
469
+ """Configure the stream publisher.
470
+
471
+ Creates the appropriate publisher based on the protocol.
472
+
473
+ Args:
474
+ config: Configuration dictionary
475
+
476
+ Raises:
477
+ NodeConfigurationError: If configuration fails
478
+ """
479
+ super().configure(config)
480
+
481
+ required_fields = ["protocol", "endpoint", "topic"]
482
+ for field in required_fields:
483
+ if not self.config.get(field):
484
+ raise NodeConfigurationError(f"{field} is required")
485
+
486
+ self._protocol = self.config["protocol"]
487
+
488
+ try:
489
+ # Placeholder for actual publisher creation
490
+ self._create_publisher()
491
+ except Exception as e:
492
+ raise NodeConfigurationError(f"Failed to create publisher: {str(e)}")
493
+
494
+ def _create_publisher(self) -> None:
495
+ """Create the appropriate publisher instance.
496
+
497
+ This is a placeholder for actual publisher creation.
498
+ """
499
+ # Placeholder for actual publisher creation
500
+ self._publisher = f"{self._protocol}_publisher"
501
+
502
+ def run(self, **kwargs) -> Dict[str, Any]:
503
+ """Publish messages to the streaming platform.
504
+
505
+ Implementation of the abstract run method from the base Node class.
506
+
507
+ Args:
508
+ **kwargs: Messages and optional headers
509
+
510
+ Returns:
511
+ Publishing results
512
+ """
513
+ return self.execute(kwargs)
514
+
515
+ def execute(self, inputs: Dict[str, Any]) -> Dict[str, Any]:
516
+ """Publish messages to the streaming platform.
517
+
518
+ Args:
519
+ inputs: Messages and optional headers
520
+
521
+ Returns:
522
+ Publishing results
523
+
524
+ Raises:
525
+ NodeExecutionError: If publishing fails
526
+ """
527
+ try:
528
+ messages = inputs.get("messages", [])
529
+ headers = inputs.get("headers", {})
530
+
531
+ if not messages:
532
+ return {
533
+ "published_count": 0,
534
+ "failed_messages": [],
535
+ "metadata": {"protocol": self._protocol},
536
+ }
537
+
538
+ results = self._publish_messages(messages, headers)
539
+
540
+ return results
541
+ except Exception as e:
542
+ raise NodeExecutionError(f"Failed to publish messages: {str(e)}")
543
+
544
+ def _publish_messages(self, messages: List[Dict], headers: Dict) -> Dict[str, Any]:
545
+ """Publish messages to the stream.
546
+
547
+ This is a placeholder for actual publishing logic.
548
+
549
+ Args:
550
+ messages: Messages to publish
551
+ headers: Optional headers
552
+
553
+ Returns:
554
+ Publishing results
555
+ """
556
+ # Placeholder implementation
557
+ batch_size = self.config.get("batch_size", 100)
558
+ published_count = 0
559
+ failed_messages = []
560
+
561
+ for i in range(0, len(messages), batch_size):
562
+ batch = messages[i : i + batch_size]
563
+ # Simulate publishing with 95% success rate
564
+ for msg in batch:
565
+ if hash(str(msg)) % 100 < 95: # Simulate 95% success
566
+ published_count += 1
567
+ else:
568
+ failed_messages.append(msg)
569
+
570
+ return {
571
+ "published_count": published_count,
572
+ "failed_messages": failed_messages,
573
+ "metadata": {
574
+ "protocol": self._protocol,
575
+ "topic": self.config["topic"],
576
+ "batch_size": batch_size,
577
+ "timestamp": time.time(),
578
+ },
579
+ }
580
+
581
+
582
+ @register_node()
583
+ class WebSocketNode(Node):
584
+ """Handles WebSocket connections for bidirectional streaming.
585
+
586
+ This node provides WebSocket client functionality for real-time
587
+ bidirectional communication with servers, supporting both sending
588
+ and receiving messages.
589
+
590
+ Design Pattern:
591
+ - Observer pattern for message events
592
+ - State pattern for connection states
593
+ - Command pattern for message handling
594
+
595
+ Features:
596
+ - Auto-reconnection
597
+ - Message queuing
598
+ - Event callbacks
599
+ - Heartbeat/ping-pong
600
+ - SSL/TLS support
601
+ - Custom headers
602
+
603
+ Common Usage Patterns:
604
+ - Real-time chat systems
605
+ - Live data feeds
606
+ - Collaborative applications
607
+ - Gaming backends
608
+ - IoT communication
609
+
610
+ Upstream Dependencies:
611
+ - Authentication nodes
612
+ - Message formatting nodes
613
+ - Configuration nodes
614
+
615
+ Downstream Consumers:
616
+ - UI update nodes
617
+ - Storage nodes
618
+ - Processing nodes
619
+ - Analytics nodes
620
+
621
+ Configuration:
622
+ url (str): WebSocket URL
623
+ headers (Dict): Connection headers
624
+ reconnect (bool): Auto-reconnect on disconnect
625
+ ping_interval (int): Heartbeat interval
626
+ ssl_verify (bool): Verify SSL certificates
627
+
628
+ Inputs:
629
+ action (str): Action to perform ("connect", "send", "receive", "disconnect")
630
+ message (Any): Message to send (for "send" action)
631
+ timeout (float): Receive timeout (for "receive" action)
632
+
633
+ Outputs:
634
+ status (str): Connection status
635
+ messages (List[Any]): Received messages
636
+ metadata (Dict): Connection metadata
637
+
638
+ Error Handling:
639
+ - Connection failures
640
+ - Message serialization
641
+ - Timeout handling
642
+ - SSL errors
643
+
644
+ Example:
645
+ >>> ws_node = WebSocketNode()
646
+ >>> ws_node.configure({
647
+ ... "url": "wss://example.com/socket",
648
+ ... "headers": {"Authorization": "Bearer token"},
649
+ ... "reconnect": True,
650
+ ... "ping_interval": 30
651
+ ... })
652
+ >>>
653
+ >>> # Connect
654
+ >>> ws_node.execute({"action": "connect"})
655
+ >>>
656
+ >>> # Send message
657
+ >>> ws_node.execute({
658
+ ... "action": "send",
659
+ ... "message": {"type": "subscribe", "channel": "updates"}
660
+ ... })
661
+ >>>
662
+ >>> # Receive messages
663
+ >>> result = ws_node.execute({
664
+ ... "action": "receive",
665
+ ... "timeout": 5.0
666
+ ... })
667
+ """
668
+
669
+ metadata = NodeMetadata(
670
+ name="WebSocketNode",
671
+ description="WebSocket client for streaming",
672
+ version="1.0.0",
673
+ tags={"streaming", "websocket", "data"},
674
+ )
675
+
676
+ def __init__(self):
677
+ """Initialize the WebSocket node.
678
+
679
+ Sets up the node and prepares for WebSocket connection.
680
+ """
681
+ super().__init__()
682
+ self._ws = None
683
+ self._connected = False
684
+ self._message_queue = []
685
+
686
+ def get_parameters(self) -> Dict[str, NodeParameter]:
687
+ """Get the parameters for this node.
688
+
689
+ Returns:
690
+ Parameter definitions for configuration and execution
691
+ """
692
+ return {
693
+ "url": NodeParameter(
694
+ name="url", type=str, description="WebSocket URL", required=True
695
+ ),
696
+ "headers": NodeParameter(
697
+ name="headers",
698
+ type=dict,
699
+ description="Connection headers",
700
+ required=False,
701
+ default={},
702
+ ),
703
+ "reconnect": NodeParameter(
704
+ name="reconnect",
705
+ type=bool,
706
+ description="Auto-reconnect on disconnect",
707
+ required=False,
708
+ default=True,
709
+ ),
710
+ "ping_interval": NodeParameter(
711
+ name="ping_interval",
712
+ type=int,
713
+ description="Heartbeat interval in seconds",
714
+ required=False,
715
+ default=30,
716
+ ),
717
+ "ssl_verify": NodeParameter(
718
+ name="ssl_verify",
719
+ type=bool,
720
+ description="Verify SSL certificates",
721
+ required=False,
722
+ default=True,
723
+ ),
724
+ "max_reconnect_attempts": NodeParameter(
725
+ name="max_reconnect_attempts",
726
+ type=int,
727
+ description="Maximum reconnection attempts",
728
+ required=False,
729
+ default=5,
730
+ ),
731
+ "action": NodeParameter(
732
+ name="action",
733
+ type=str,
734
+ description="Action to perform (connect, send, receive, disconnect)",
735
+ required=False,
736
+ default="receive",
737
+ ),
738
+ "message": NodeParameter(
739
+ name="message",
740
+ type=Any,
741
+ description="Message to send (for send action)",
742
+ required=False,
743
+ ),
744
+ "timeout": NodeParameter(
745
+ name="timeout",
746
+ type=float,
747
+ description="Receive timeout (for receive action)",
748
+ required=False,
749
+ default=1.0,
750
+ ),
751
+ }
752
+
753
+ def configure(self, config: Dict[str, Any]) -> None:
754
+ """Configure the WebSocket connection.
755
+
756
+ Validates the URL and prepares connection parameters.
757
+
758
+ Args:
759
+ config: Configuration dictionary
760
+
761
+ Raises:
762
+ NodeConfigurationError: If configuration is invalid
763
+ """
764
+ super().configure(config)
765
+
766
+ if not self.config.get("url"):
767
+ raise NodeConfigurationError("WebSocket URL is required")
768
+
769
+ url = self.config["url"]
770
+ if not url.startswith(("ws://", "wss://")):
771
+ raise NodeConfigurationError("URL must start with ws:// or wss://")
772
+
773
+ def run(self, **kwargs) -> Dict[str, Any]:
774
+ """Run the WebSocket node.
775
+
776
+ This method fulfills the abstract run method requirement from the base Node class.
777
+
778
+ Args:
779
+ **kwargs: Input parameters
780
+
781
+ Returns:
782
+ Operation results
783
+
784
+ Raises:
785
+ NodeExecutionError: If execution fails
786
+ """
787
+ return self.execute(kwargs)
788
+
789
+ def execute(self, inputs: Dict[str, Any]) -> Dict[str, Any]:
790
+ """Execute WebSocket operations.
791
+
792
+ Performs the requested action (connect, send, receive, disconnect).
793
+
794
+ Args:
795
+ inputs: Action and parameters
796
+
797
+ Returns:
798
+ Operation results
799
+
800
+ Raises:
801
+ NodeExecutionError: If operation fails
802
+ """
803
+ try:
804
+ action = inputs.get("action", "receive")
805
+
806
+ if action == "connect":
807
+ return self._connect()
808
+ elif action == "send":
809
+ return self._send_message(inputs.get("message"))
810
+ elif action == "receive":
811
+ return self._receive_messages(inputs.get("timeout", 1.0))
812
+ elif action == "disconnect":
813
+ return self._disconnect()
814
+ else:
815
+ raise ValueError(f"Unknown action: {action}")
816
+ except Exception as e:
817
+ raise NodeExecutionError(f"WebSocket operation failed: {str(e)}")
818
+
819
+ def _connect(self) -> Dict[str, Any]:
820
+ """Connect to the WebSocket server.
821
+
822
+ Returns:
823
+ Connection status
824
+ """
825
+ # Placeholder for actual connection
826
+ self._ws = f"websocket_to_{self.config['url']}"
827
+ self._connected = True
828
+
829
+ return {
830
+ "status": "connected",
831
+ "url": self.config["url"],
832
+ "metadata": {"timestamp": time.time()},
833
+ }
834
+
835
+ def _send_message(self, message: Any) -> Dict[str, Any]:
836
+ """Send a message through the WebSocket.
837
+
838
+ Args:
839
+ message: Message to send
840
+
841
+ Returns:
842
+ Send status
843
+ """
844
+ if not self._connected:
845
+ raise ValueError("Not connected to WebSocket")
846
+
847
+ # Placeholder for actual send
848
+ return {
849
+ "status": "sent",
850
+ "message": message,
851
+ "metadata": {"timestamp": time.time()},
852
+ }
853
+
854
+ def _receive_messages(self, timeout: float) -> Dict[str, Any]:
855
+ """Receive messages from the WebSocket.
856
+
857
+ Args:
858
+ timeout: Receive timeout
859
+
860
+ Returns:
861
+ Received messages
862
+ """
863
+ if not self._connected:
864
+ raise ValueError("Not connected to WebSocket")
865
+
866
+ # Placeholder for actual receive
867
+ messages = []
868
+ # Simulate receiving 1-3 messages
869
+ for i in range(hash(str(time.time())) % 3 + 1):
870
+ messages.append(
871
+ {"type": "update", "data": f"message_{i}", "timestamp": time.time()}
872
+ )
873
+
874
+ return {
875
+ "status": "received",
876
+ "messages": messages,
877
+ "metadata": {"count": len(messages), "timeout": timeout},
878
+ }
879
+
880
+ def _disconnect(self) -> Dict[str, Any]:
881
+ """Disconnect from the WebSocket server.
882
+
883
+ Returns:
884
+ Disconnection status
885
+ """
886
+ # Placeholder for actual disconnect
887
+ self._connected = False
888
+ self._ws = None
889
+
890
+ return {"status": "disconnected", "metadata": {"timestamp": time.time()}}
891
+
892
+ def cleanup(self) -> None:
893
+ """Clean up WebSocket resources.
894
+
895
+ Ensures the connection is closed and resources are released.
896
+ """
897
+ if self._connected:
898
+ self._disconnect()
899
+ super().cleanup()
900
+
901
+
902
+ @register_node()
903
+ class EventStreamNode(Node):
904
+ """Handles server-sent events (SSE) for unidirectional streaming.
905
+
906
+ This node provides SSE client functionality for receiving real-time
907
+ events from servers using the EventSource protocol.
908
+
909
+ Design Pattern:
910
+ - Observer pattern for event handling
911
+ - Iterator pattern for event streaming
912
+ - Decorator pattern for event processing
913
+
914
+ Features:
915
+ - Auto-reconnection
916
+ - Event type filtering
917
+ - Custom event handlers
918
+ - Last-Event-ID tracking
919
+ - Connection timeout
920
+
921
+ Common Usage Patterns:
922
+ - Live updates/notifications
923
+ - Stock price feeds
924
+ - News streams
925
+ - Progress monitoring
926
+ - Log streaming
927
+
928
+ Upstream Dependencies:
929
+ - Authentication nodes
930
+ - Configuration nodes
931
+
932
+ Downstream Consumers:
933
+ - Event processing nodes
934
+ - Storage nodes
935
+ - UI update nodes
936
+ - Analytics nodes
937
+
938
+ Configuration:
939
+ url (str): SSE endpoint URL
940
+ headers (Dict): Request headers
941
+ event_types (List[str]): Event types to listen for
942
+ reconnect_time (int): Reconnection delay
943
+ timeout (int): Connection timeout
944
+
945
+ Inputs:
946
+ action (str): Action to perform ("start", "stop", "receive")
947
+ max_events (int): Maximum events to receive
948
+
949
+ Outputs:
950
+ events (List[Dict]): Received events
951
+ status (str): Stream status
952
+ metadata (Dict): Stream metadata
953
+
954
+ Error Handling:
955
+ - Connection failures
956
+ - Timeout handling
957
+ - Invalid event data
958
+ - Authentication errors
959
+
960
+ Example:
961
+ >>> sse_node = EventStreamNode()
962
+ >>> sse_node.configure({
963
+ ... "url": "https://api.example.com/events",
964
+ ... "headers": {"Authorization": "Bearer token"},
965
+ ... "event_types": ["update", "notification"],
966
+ ... "reconnect_time": 3000
967
+ ... })
968
+ >>>
969
+ >>> # Start listening
970
+ >>> sse_node.execute({"action": "start"})
971
+ >>>
972
+ >>> # Receive events
973
+ >>> result = sse_node.execute({
974
+ ... "action": "receive",
975
+ ... "max_events": 10
976
+ ... })
977
+ >>> print(f"Received {len(result['events'])} events")
978
+ """
979
+
980
+ metadata = NodeMetadata(
981
+ name="EventStreamNode",
982
+ description="Server-sent events client",
983
+ version="1.0.0",
984
+ tags={"streaming", "sse", "data"},
985
+ )
986
+
987
+ def __init__(self):
988
+ """Initialize the EventStream node.
989
+
990
+ Sets up the node and prepares for SSE connection.
991
+ """
992
+ super().__init__()
993
+ self._stream = None
994
+ self._connected = False
995
+ self._last_event_id = None
996
+
997
+ def get_parameters(self) -> Dict[str, NodeParameter]:
998
+ """Get the parameters for this node.
999
+
1000
+ Returns:
1001
+ Parameter definitions for configuration and execution
1002
+ """
1003
+ return {
1004
+ "url": NodeParameter(
1005
+ name="url", type=str, description="SSE endpoint URL", required=True
1006
+ ),
1007
+ "headers": NodeParameter(
1008
+ name="headers",
1009
+ type=dict,
1010
+ description="Request headers",
1011
+ required=False,
1012
+ default={},
1013
+ ),
1014
+ "event_types": NodeParameter(
1015
+ name="event_types",
1016
+ type=list,
1017
+ description="Event types to listen for",
1018
+ required=False,
1019
+ default=[],
1020
+ ),
1021
+ "reconnect_time": NodeParameter(
1022
+ name="reconnect_time",
1023
+ type=int,
1024
+ description="Reconnection delay in ms",
1025
+ required=False,
1026
+ default=3000,
1027
+ ),
1028
+ "timeout": NodeParameter(
1029
+ name="timeout",
1030
+ type=int,
1031
+ description="Connection timeout in seconds",
1032
+ required=False,
1033
+ default=60,
1034
+ ),
1035
+ "action": NodeParameter(
1036
+ name="action",
1037
+ type=str,
1038
+ description="Action to perform (start, stop, receive)",
1039
+ required=False,
1040
+ default="receive",
1041
+ ),
1042
+ "max_events": NodeParameter(
1043
+ name="max_events",
1044
+ type=int,
1045
+ description="Maximum events to receive",
1046
+ required=False,
1047
+ default=10,
1048
+ ),
1049
+ }
1050
+
1051
+ def run(self, **kwargs) -> Dict[str, Any]:
1052
+ """Run the EventStream node.
1053
+
1054
+ This method fulfills the abstract run method requirement from the base Node class.
1055
+
1056
+ Args:
1057
+ **kwargs: Input parameters
1058
+
1059
+ Returns:
1060
+ Operation results
1061
+
1062
+ Raises:
1063
+ NodeExecutionError: If execution fails
1064
+ """
1065
+ return self.execute(kwargs)
1066
+
1067
+ def execute(self, inputs: Dict[str, Any]) -> Dict[str, Any]:
1068
+ """Execute EventStream operations.
1069
+
1070
+ Args:
1071
+ inputs: Action and parameters
1072
+
1073
+ Returns:
1074
+ Operation results
1075
+
1076
+ Raises:
1077
+ NodeExecutionError: If operation fails
1078
+ """
1079
+ try:
1080
+ action = inputs.get("action", "receive")
1081
+
1082
+ if action == "start":
1083
+ return self._start_stream()
1084
+ elif action == "stop":
1085
+ return self._stop_stream()
1086
+ elif action == "receive":
1087
+ return self._receive_events(inputs.get("max_events", 10))
1088
+ else:
1089
+ raise ValueError(f"Unknown action: {action}")
1090
+ except Exception as e:
1091
+ raise NodeExecutionError(f"EventStream operation failed: {str(e)}")
1092
+
1093
+ def _start_stream(self) -> Dict[str, Any]:
1094
+ """Start the event stream connection.
1095
+
1096
+ Returns:
1097
+ Connection status
1098
+ """
1099
+ # Placeholder for actual connection
1100
+ self._stream = f"sse_stream_{self.config['url']}"
1101
+ self._connected = True
1102
+
1103
+ return {
1104
+ "status": "streaming",
1105
+ "url": self.config["url"],
1106
+ "metadata": {"timestamp": time.time()},
1107
+ }
1108
+
1109
+ def _stop_stream(self) -> Dict[str, Any]:
1110
+ """Stop the event stream connection.
1111
+
1112
+ Returns:
1113
+ Disconnection status
1114
+ """
1115
+ # Placeholder for actual disconnection
1116
+ self._connected = False
1117
+ self._stream = None
1118
+
1119
+ return {
1120
+ "status": "stopped",
1121
+ "metadata": {
1122
+ "timestamp": time.time(),
1123
+ "last_event_id": self._last_event_id,
1124
+ },
1125
+ }
1126
+
1127
+ def _receive_events(self, max_events: int) -> Dict[str, Any]:
1128
+ """Receive events from the stream.
1129
+
1130
+ Args:
1131
+ max_events: Maximum events to receive
1132
+
1133
+ Returns:
1134
+ Received events
1135
+ """
1136
+ if not self._connected:
1137
+ raise ValueError("Not connected to event stream")
1138
+
1139
+ # Placeholder for actual event reception
1140
+ events = []
1141
+ event_types = self.config.get("event_types", [])
1142
+
1143
+ # Simulate receiving events
1144
+ for i in range(min(max_events, 5)):
1145
+ event_type = event_types[i % len(event_types)] if event_types else "message"
1146
+ event = {
1147
+ "id": f"event_{time.time()}_{i}",
1148
+ "type": event_type,
1149
+ "data": {"content": f"Event data {i}"},
1150
+ "timestamp": time.time(),
1151
+ }
1152
+ events.append(event)
1153
+ self._last_event_id = event["id"]
1154
+
1155
+ return {
1156
+ "status": "received",
1157
+ "events": events,
1158
+ "metadata": {"count": len(events), "last_event_id": self._last_event_id},
1159
+ }
1160
+
1161
+ def cleanup(self) -> None:
1162
+ """Clean up EventStream resources.
1163
+
1164
+ Ensures the stream is closed and resources are released.
1165
+ """
1166
+ if self._connected:
1167
+ self._stop_stream()
1168
+ super().cleanup()