pynode-flow 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (227) hide show
  1. pynode/__init__.py +37 -0
  2. pynode/__main__.py +9 -0
  3. pynode/_perf.py +65 -0
  4. pynode/_version.py +24 -0
  5. pynode/api/__init__.py +15 -0
  6. pynode/api/helpers.py +79 -0
  7. pynode/api/nodes.py +423 -0
  8. pynode/api/services.py +124 -0
  9. pynode/api/sse.py +58 -0
  10. pynode/api/uploads.py +59 -0
  11. pynode/api/workflows.py +466 -0
  12. pynode/config.py +93 -0
  13. pynode/install_nodes.py +143 -0
  14. pynode/main.py +118 -0
  15. pynode/node_registry.py +156 -0
  16. pynode/nodes/AutoSyncNode/auto_sync_node.py +151 -0
  17. pynode/nodes/BBoxMetricsNode/__init__.py +3 -0
  18. pynode/nodes/BBoxMetricsNode/bbox_metrics_node.py +250 -0
  19. pynode/nodes/BBoxMetricsNode/requirements.txt +1 -0
  20. pynode/nodes/BatchNode/__init__.py +3 -0
  21. pynode/nodes/BatchNode/batch_node.py +86 -0
  22. pynode/nodes/BatchNode/requirements.txt +1 -0
  23. pynode/nodes/CameraNode/__init__.py +4 -0
  24. pynode/nodes/CameraNode/camera_node.py +204 -0
  25. pynode/nodes/CameraNode/requirements.txt +2 -0
  26. pynode/nodes/ChangeNode/__init__.py +1 -0
  27. pynode/nodes/ChangeNode/change_node.py +255 -0
  28. pynode/nodes/ChangeNode/requirements.txt +1 -0
  29. pynode/nodes/ConfidenceFilterNode/__init__.py +4 -0
  30. pynode/nodes/ConfidenceFilterNode/confidence_filter_node.py +208 -0
  31. pynode/nodes/ConfidenceFilterNode/requirements.txt +1 -0
  32. pynode/nodes/CounterNode/__init__.py +3 -0
  33. pynode/nodes/CounterNode/counter_node.py +191 -0
  34. pynode/nodes/CropNode/__init__.py +4 -0
  35. pynode/nodes/CropNode/crop_node.py +317 -0
  36. pynode/nodes/CropNode/requirements.txt +2 -0
  37. pynode/nodes/DebugNode/__init__.py +4 -0
  38. pynode/nodes/DebugNode/debug_node.py +169 -0
  39. pynode/nodes/DebugNode/requirements.txt +1 -0
  40. pynode/nodes/DelayNode/delay_node.py +218 -0
  41. pynode/nodes/DelayNode/requirements.txt +1 -0
  42. pynode/nodes/DrawPredictionsNode/__init__.py +3 -0
  43. pynode/nodes/DrawPredictionsNode/draw_predictions_node.py +365 -0
  44. pynode/nodes/DrawPredictionsNode/requirements.txt +2 -0
  45. pynode/nodes/ErrorNode/error_node.py +94 -0
  46. pynode/nodes/ErrorNode/requirements.txt +1 -0
  47. pynode/nodes/FilterNode/__init__.py +3 -0
  48. pynode/nodes/FilterNode/filter_node.py +102 -0
  49. pynode/nodes/FilterNode/requirements.txt +1 -0
  50. pynode/nodes/FrameSourceNode/__init__.py +4 -0
  51. pynode/nodes/FrameSourceNode/frame_source_node.py +319 -0
  52. pynode/nodes/FrameSourceNode/requirements.txt +1 -0
  53. pynode/nodes/FunctionNode/__init__.py +3 -0
  54. pynode/nodes/FunctionNode/function_node.py +168 -0
  55. pynode/nodes/FunctionNode/requirements.txt +1 -0
  56. pynode/nodes/GateNode/gate_node.py +60 -0
  57. pynode/nodes/GateNode/requirements.txt +1 -0
  58. pynode/nodes/GetiUploadNode/__init__.py +3 -0
  59. pynode/nodes/GetiUploadNode/geti_upload_node.py +278 -0
  60. pynode/nodes/GetiUploadNode/requirements.txt +1 -0
  61. pynode/nodes/ImageFormatNode/__init__.py +4 -0
  62. pynode/nodes/ImageFormatNode/image_format_node.py +310 -0
  63. pynode/nodes/ImageFormatNode/requirements.txt +2 -0
  64. pynode/nodes/ImageUploadNode/image_upload_node.py +96 -0
  65. pynode/nodes/ImageViewerNode/__init__.py +4 -0
  66. pynode/nodes/ImageViewerNode/image_viewer_node.py +162 -0
  67. pynode/nodes/ImageViewerNode/requirements.txt +2 -0
  68. pynode/nodes/ImageWriterNode/__init__.py +3 -0
  69. pynode/nodes/ImageWriterNode/imagewriter_node.py +352 -0
  70. pynode/nodes/ImageWriterNode/requirements.txt +1 -0
  71. pynode/nodes/InferenceNode/InferenceEngine/__init__.py +15 -0
  72. pynode/nodes/InferenceNode/InferenceEngine/device_detection.py +149 -0
  73. pynode/nodes/InferenceNode/InferenceEngine/engines/base_engine.py +155 -0
  74. pynode/nodes/InferenceNode/InferenceEngine/engines/example_engine_template.py +272 -0
  75. pynode/nodes/InferenceNode/InferenceEngine/engines/geti_engine.py +309 -0
  76. pynode/nodes/InferenceNode/InferenceEngine/engines/onnx_engine.py +346 -0
  77. pynode/nodes/InferenceNode/InferenceEngine/engines/pass_engine.py +86 -0
  78. pynode/nodes/InferenceNode/InferenceEngine/engines/ultralytics_engine.py +863 -0
  79. pynode/nodes/InferenceNode/InferenceEngine/inference_engine_factory.py +533 -0
  80. pynode/nodes/InferenceNode/InferenceEngine/readme.md +107 -0
  81. pynode/nodes/InferenceNode/InferenceEngine/result_converters.py +121 -0
  82. pynode/nodes/InferenceNode/inference_node.py +451 -0
  83. pynode/nodes/InferenceNode/requirements.txt +13 -0
  84. pynode/nodes/InjectNode/inject_node.py +245 -0
  85. pynode/nodes/InjectNode/requirements.txt +1 -0
  86. pynode/nodes/JoinNode/__init__.py +3 -0
  87. pynode/nodes/JoinNode/join_node.py +156 -0
  88. pynode/nodes/JoinNode/requirements.txt +1 -0
  89. pynode/nodes/LabelFilterNode/__init__.py +3 -0
  90. pynode/nodes/LabelFilterNode/label_filter_node.py +188 -0
  91. pynode/nodes/LabelFilterNode/requirements.txt +1 -0
  92. pynode/nodes/MQTTNode/__init__.py +8 -0
  93. pynode/nodes/MQTTNode/mqtt_in_node.py +148 -0
  94. pynode/nodes/MQTTNode/mqtt_out_node.py +157 -0
  95. pynode/nodes/MQTTNode/mqtt_service.py +367 -0
  96. pynode/nodes/MQTTNode/requirements.txt +1 -0
  97. pynode/nodes/MessageWriterNode/__init__.py +7 -0
  98. pynode/nodes/MessageWriterNode/messagereader_node.py +568 -0
  99. pynode/nodes/MessageWriterNode/messagewriter_node.py +505 -0
  100. pynode/nodes/OmronCameraNode/__init__.py +3 -0
  101. pynode/nodes/OmronCameraNode/omron_camera_node.py +652 -0
  102. pynode/nodes/OmronCameraNode/requirements.txt +4 -0
  103. pynode/nodes/OpenCV/__init__.py +1 -0
  104. pynode/nodes/OpenCV/arithmetic_node.py +376 -0
  105. pynode/nodes/OpenCV/bitwise_node.py +127 -0
  106. pynode/nodes/OpenCV/blend_node.py +127 -0
  107. pynode/nodes/OpenCV/blob_detector_node.py +233 -0
  108. pynode/nodes/OpenCV/blur_node.py +141 -0
  109. pynode/nodes/OpenCV/color_space_node.py +139 -0
  110. pynode/nodes/OpenCV/colormap_node.py +262 -0
  111. pynode/nodes/OpenCV/contours_node.py +258 -0
  112. pynode/nodes/OpenCV/denormalize_coords_node.py +365 -0
  113. pynode/nodes/OpenCV/draw_node.py +312 -0
  114. pynode/nodes/OpenCV/edge_detector_node.py +177 -0
  115. pynode/nodes/OpenCV/equirectangular_node.py +447 -0
  116. pynode/nodes/OpenCV/fft_node.py +102 -0
  117. pynode/nodes/OpenCV/focus_estimation_node.py +189 -0
  118. pynode/nodes/OpenCV/frequency_filter_node.py +241 -0
  119. pynode/nodes/OpenCV/histogram_node.py +160 -0
  120. pynode/nodes/OpenCV/hough_circles_node.py +224 -0
  121. pynode/nodes/OpenCV/hough_lines_node.py +243 -0
  122. pynode/nodes/OpenCV/in_range_node.py +186 -0
  123. pynode/nodes/OpenCV/morphology_node.py +136 -0
  124. pynode/nodes/OpenCV/normalize_coords_node.py +344 -0
  125. pynode/nodes/OpenCV/paste_node.py +244 -0
  126. pynode/nodes/OpenCV/perspective_node.py +174 -0
  127. pynode/nodes/OpenCV/realsense_depth_node.py +253 -0
  128. pynode/nodes/OpenCV/requirements.txt +2 -0
  129. pynode/nodes/OpenCV/resize_node.py +175 -0
  130. pynode/nodes/OpenCV/rotate_node.py +159 -0
  131. pynode/nodes/OpenCV/template_match_node.py +227 -0
  132. pynode/nodes/OpenCV/threshold_node.py +167 -0
  133. pynode/nodes/PointInShapeNode/__init__.py +3 -0
  134. pynode/nodes/PointInShapeNode/point_in_shape_node.py +313 -0
  135. pynode/nodes/PointInShapeNode/requirements.txt +2 -0
  136. pynode/nodes/PolygonMetricsNode/__init__.py +3 -0
  137. pynode/nodes/PolygonMetricsNode/polygon_metrics_node.py +331 -0
  138. pynode/nodes/PolygonMetricsNode/requirements.txt +2 -0
  139. pynode/nodes/QueueLengthProbeNode/__init__.py +3 -0
  140. pynode/nodes/QueueLengthProbeNode/queue_length_probe_node.py +116 -0
  141. pynode/nodes/Qwen3VLMNode/__init__.py +3 -0
  142. pynode/nodes/Qwen3VLMNode/qwen3_vlm_node.py +519 -0
  143. pynode/nodes/Qwen3VLMNode/requirements.txt +5 -0
  144. pynode/nodes/RESTEndpointNode/__init__.py +3 -0
  145. pynode/nodes/RESTEndpointNode/rest_endpoint_node.py +491 -0
  146. pynode/nodes/RangeNode/__init__.py +3 -0
  147. pynode/nodes/RangeNode/range_node.py +115 -0
  148. pynode/nodes/RangeNode/requirements.txt +1 -0
  149. pynode/nodes/RateProbeNode/__init__.py +3 -0
  150. pynode/nodes/RateProbeNode/rate_probe_node.py +183 -0
  151. pynode/nodes/RateProbeNode/requirements.txt +1 -0
  152. pynode/nodes/RoboflowUploadNode/__init__.py +3 -0
  153. pynode/nodes/RoboflowUploadNode/requirements.txt +1 -0
  154. pynode/nodes/RoboflowUploadNode/roboflow_upload_node.py +229 -0
  155. pynode/nodes/SliceNode/README.md +149 -0
  156. pynode/nodes/SliceNode/__init__.py +5 -0
  157. pynode/nodes/SliceNode/merge_predictions_node.py +444 -0
  158. pynode/nodes/SliceNode/requirements.txt +3 -0
  159. pynode/nodes/SliceNode/slice_collector_node.py +370 -0
  160. pynode/nodes/SliceNode/slice_image_node.py +380 -0
  161. pynode/nodes/SplitNode/__init__.py +3 -0
  162. pynode/nodes/SplitNode/requirements.txt +1 -0
  163. pynode/nodes/SplitNode/split_node.py +155 -0
  164. pynode/nodes/SwitchNode/requirements.txt +1 -0
  165. pynode/nodes/SwitchNode/switch_node.py +259 -0
  166. pynode/nodes/SyncNode/__init__.py +3 -0
  167. pynode/nodes/SyncNode/sync_node.py +287 -0
  168. pynode/nodes/TemplateNode/requirements.txt +1 -0
  169. pynode/nodes/TemplateNode/template_node.py +86 -0
  170. pynode/nodes/TrackerNode/__init__.py +2 -0
  171. pynode/nodes/TrackerNode/supervision_tracker_node.py +248 -0
  172. pynode/nodes/TrackerNode/tracker_node.py +745 -0
  173. pynode/nodes/UltralyticsNode/__init__.py +4 -0
  174. pynode/nodes/UltralyticsNode/requirements.txt +2 -0
  175. pynode/nodes/UltralyticsNode/ultralytics_node.py +364 -0
  176. pynode/nodes/UnknownNode/__init__.py +93 -0
  177. pynode/nodes/VLLMNode/__init__.py +3 -0
  178. pynode/nodes/VLLMNode/vllm_node.py +345 -0
  179. pynode/nodes/VideoReaderNode/__init__.py +3 -0
  180. pynode/nodes/VideoReaderNode/video_reader_node.py +425 -0
  181. pynode/nodes/VideoWriterNode/__init__.py +3 -0
  182. pynode/nodes/VideoWriterNode/video_writer_node.py +441 -0
  183. pynode/nodes/WebhookNode/__init__.py +3 -0
  184. pynode/nodes/WebhookNode/webhook_node.py +591 -0
  185. pynode/nodes/__init__.py +109 -0
  186. pynode/nodes/base_node.py +562 -0
  187. pynode/nodes/image_utils.py +291 -0
  188. pynode/nodes/info.py +109 -0
  189. pynode/nodes/mDNSNode/__init__.py +13 -0
  190. pynode/nodes/mDNSNode/mdns_broadcast_node.py +214 -0
  191. pynode/nodes/mDNSNode/mdns_discovery_node.py +362 -0
  192. pynode/nodes/mDNSNode/mdns_manager.py +399 -0
  193. pynode/nodes/mDNSNode/requirements.txt +1 -0
  194. pynode/nodes/messages.py +81 -0
  195. pynode/server.py +187 -0
  196. pynode/static/images/favicon4.png +0 -0
  197. pynode/static/index.html +200 -0
  198. pynode/static/js/README.md +139 -0
  199. pynode/static/js/auth.js +89 -0
  200. pynode/static/js/clipboard.js +152 -0
  201. pynode/static/js/config.js +14 -0
  202. pynode/static/js/connections.js +844 -0
  203. pynode/static/js/debug.js +524 -0
  204. pynode/static/js/events.js +775 -0
  205. pynode/static/js/history.js +199 -0
  206. pynode/static/js/main.js +55 -0
  207. pynode/static/js/minimap.js +242 -0
  208. pynode/static/js/nodes.js +995 -0
  209. pynode/static/js/palette.js +255 -0
  210. pynode/static/js/properties.js +1270 -0
  211. pynode/static/js/selection.js +241 -0
  212. pynode/static/js/state.js +211 -0
  213. pynode/static/js/ui-utils.js +28 -0
  214. pynode/static/js/viewport.js +204 -0
  215. pynode/static/js/workflow.js +469 -0
  216. pynode/static/js/workflows.js +344 -0
  217. pynode/static/style.css +2751 -0
  218. pynode/workflow_engine.py +483 -0
  219. pynode/workflow_manager.py +440 -0
  220. pynode_flow-0.2.0.dist-info/METADATA +520 -0
  221. pynode_flow-0.2.0.dist-info/RECORD +227 -0
  222. pynode_flow-0.2.0.dist-info/WHEEL +5 -0
  223. pynode_flow-0.2.0.dist-info/entry_points.txt +3 -0
  224. pynode_flow-0.2.0.dist-info/licenses/LICENSE +21 -0
  225. pynode_flow-0.2.0.dist-info/scm_file_list.json +270 -0
  226. pynode_flow-0.2.0.dist-info/scm_version.json +8 -0
  227. pynode_flow-0.2.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,568 @@
1
+ """
2
+ Message Reader Node - reads messages or data from disk files.
3
+ Supports reading various formats written by MessageWriterNode.
4
+ """
5
+
6
+ import os
7
+ import json
8
+ import pickle
9
+ import base64
10
+ import glob
11
+ import time
12
+ from datetime import datetime
13
+ from typing import Any, Dict, List, Optional
14
+ from pynode.nodes.base_node import BaseNode, Info, MessageKeys
15
+
16
+ _info = Info()
17
+ _info.add_text("Reads messages or data from disk files. Supports JSON, binary, and text formats with batch reading capabilities.")
18
+ _info.add_header("Inputs")
19
+ _info.add_bullets(
20
+ ("Input 0:", "Optional trigger message. If provided, reads files based on msg.path or configured directory.")
21
+ )
22
+ _info.add_header("Outputs")
23
+ _info.add_bullets(
24
+ ("Output 0:", "Message containing the read data. For batch reading, outputs multiple messages.")
25
+ )
26
+ _info.add_header("Message Properties")
27
+ _info.add_bullets(
28
+ ("msg.path:", "Specific file path to read (overrides directory setting)"),
29
+ ("msg.pattern:", "File pattern for batch reading (e.g., '*.json')")
30
+ )
31
+ _info.add_header("Reading Modes")
32
+ _info.add_bullets(
33
+ ("single_file:", "Read one specific file"),
34
+ ("batch_pattern:", "Read multiple files matching a pattern"),
35
+ ("watch_directory:", "Monitor directory for new files"),
36
+ ("latest_file:", "Read the most recently modified file")
37
+ )
38
+ _info.add_header("Input Formats")
39
+ _info.add_bullets(
40
+ ("auto:", "Auto-detect format from file extension"),
41
+ ("json:", "JSON format (with numpy array reconstruction)"),
42
+ ("pickle:", "Python pickle format"),
43
+ ("numpy:", "NumPy .npy binary format"),
44
+ ("text:", "Plain text format")
45
+ )
46
+
47
+
48
+ class MessageReaderNode(BaseNode):
49
+ """
50
+ Message Reader node - reads messages or data from disk files.
51
+ Companion to MessageWriterNode with format reconstruction capabilities.
52
+ """
53
+ display_name = 'Message Reader'
54
+ icon = '📖'
55
+ category = 'input'
56
+ color = '#6B9BD2'
57
+ border_color = '#4A7BA7'
58
+ text_color = '#FFFFFF'
59
+ input_count = 1
60
+ output_count = 1
61
+ ui_component = 'button'
62
+ ui_component_config = {
63
+ 'icon': '▶',
64
+ 'action': 'read_files',
65
+ 'tooltip': 'Read Files'
66
+ }
67
+ info = str(_info)
68
+
69
+ # UI-triggerable actions (see BaseNode.actions)
70
+ actions = ['read_files']
71
+
72
+ DEFAULT_CONFIG = {
73
+ 'directory': './output',
74
+ 'filename_pattern': '*.json',
75
+ 'reading_mode': 'single_file',
76
+ 'specific_file': '',
77
+ 'input_format': 'auto',
78
+ 'sort_order': 'name',
79
+ 'max_files': '10',
80
+ 'include_metadata': 'true',
81
+ 'recursive': 'false',
82
+ 'output_structure': 'auto_detect',
83
+ 'allow_pickle': False
84
+ }
85
+
86
+ properties = [
87
+ {
88
+ 'name': 'directory',
89
+ 'label': 'Directory',
90
+ 'type': 'text',
91
+ 'default': DEFAULT_CONFIG['directory'],
92
+ 'help': 'Directory to read files from'
93
+ },
94
+ {
95
+ 'name': 'reading_mode',
96
+ 'label': 'Reading Mode',
97
+ 'type': 'select',
98
+ 'options': [
99
+ {'value': 'single_file', 'label': 'Single File'},
100
+ {'value': 'batch_pattern', 'label': 'Batch (Pattern)'},
101
+ {'value': 'latest_file', 'label': 'Latest File'},
102
+ {'value': 'all_files', 'label': 'All Files'}
103
+ ],
104
+ 'default': DEFAULT_CONFIG['reading_mode'],
105
+ 'help': 'How to select files to read'
106
+ },
107
+ {
108
+ 'name': 'specific_file',
109
+ 'label': 'Specific File',
110
+ 'type': 'text',
111
+ 'default': DEFAULT_CONFIG['specific_file'],
112
+ 'help': f'Specific filename when reading_mode is "single_file" - leave blank to use {MessageKeys.MSG}.{MessageKeys.PAYLOAD}.filename',
113
+ 'showIf': {'reading_mode': 'single_file'}
114
+ },
115
+ {
116
+ 'name': 'filename_pattern',
117
+ 'label': 'Filename Pattern',
118
+ 'type': 'text',
119
+ 'default': DEFAULT_CONFIG['filename_pattern'],
120
+ 'help': 'Pattern to match files (e.g., *.json, message_*.txt)',
121
+ 'showIf': {'reading_mode': ['batch_pattern', 'latest_file', 'all_files']}
122
+ },
123
+ {
124
+ 'name': 'input_format',
125
+ 'label': 'Input Format',
126
+ 'type': 'select',
127
+ 'options': [
128
+ {'value': 'auto', 'label': 'Auto-detect'},
129
+ {'value': 'json', 'label': 'JSON'},
130
+ {'value': 'pickle', 'label': 'Pickle'},
131
+ {'value': 'numpy', 'label': 'NumPy (.npy)'},
132
+ {'value': 'text', 'label': 'Text'},
133
+ {'value': 'raw', 'label': 'Raw bytes'}
134
+ ],
135
+ 'default': DEFAULT_CONFIG['input_format'],
136
+ 'help': 'Format of the input files'
137
+ },
138
+ {
139
+ 'name': 'allow_pickle',
140
+ 'label': 'Allow Pickle Loading',
141
+ 'type': 'checkbox',
142
+ 'default': False,
143
+ 'help': 'SECURITY WARNING: pickle files can execute arbitrary code when '
144
+ 'loaded. Only enable this for files from sources you fully trust.'
145
+ },
146
+ {
147
+ 'name': 'sort_order',
148
+ 'label': 'Sort Order',
149
+ 'type': 'select',
150
+ 'options': [
151
+ {'value': 'name', 'label': 'Name'},
152
+ {'value': 'modified', 'label': 'Modified Time'},
153
+ {'value': 'size', 'label': 'File Size'}
154
+ ],
155
+ 'default': DEFAULT_CONFIG['sort_order'],
156
+ 'help': 'How to sort files for batch reading',
157
+ 'showIf': {'reading_mode': ['batch_pattern', 'latest_file', 'all_files']}
158
+ },
159
+ {
160
+ 'name': 'max_files',
161
+ 'label': 'Max Files',
162
+ 'type': 'text',
163
+ 'default': DEFAULT_CONFIG['max_files'],
164
+ 'help': 'Maximum number of files to read in batch mode',
165
+ 'showIf': {'reading_mode': ['batch_pattern', 'all_files']}
166
+ },
167
+ {
168
+ 'name': 'include_metadata',
169
+ 'label': 'Include File Metadata',
170
+ 'type': 'select',
171
+ 'options': [
172
+ {'value': 'true', 'label': 'Yes'},
173
+ {'value': 'false', 'label': 'No'}
174
+ ],
175
+ 'default': DEFAULT_CONFIG['include_metadata'],
176
+ 'help': 'Include file metadata (size, modified time) in output'
177
+ },
178
+ {
179
+ 'name': 'recursive',
180
+ 'label': 'Recursive Search',
181
+ 'type': 'select',
182
+ 'options': [
183
+ {'value': 'false', 'label': 'No'},
184
+ {'value': 'true', 'label': 'Yes'}
185
+ ],
186
+ 'default': DEFAULT_CONFIG['recursive'],
187
+ 'help': 'Search subdirectories recursively'
188
+ },
189
+ {
190
+ 'name': 'output_structure',
191
+ 'label': 'Output Structure',
192
+ 'type': 'select',
193
+ 'options': [
194
+ {'value': 'auto_detect', 'label': 'Auto-detect (smart reconstruction)'},
195
+ {'value': 'wrap_in_data', 'label': f'Always wrap in {MessageKeys.MSG}.{MessageKeys.PAYLOAD}.data'},
196
+ {'value': 'direct_payload', 'label': f'Direct to {MessageKeys.MSG}.{MessageKeys.PAYLOAD}'},
197
+ {'value': 'reconstruct_message', 'label': 'Reconstruct whole message'}
198
+ ],
199
+ 'default': DEFAULT_CONFIG['output_structure'],
200
+ 'help': 'How to structure the read data in the output message'
201
+ }
202
+ ]
203
+
204
+ def __init__(self, node_id=None, name="message reader"):
205
+ super().__init__(node_id, name)
206
+ self._last_read_time = None
207
+ self._file_cache = {}
208
+
209
+ def _create_output_message(self, data, original_filename=None):
210
+ """Create output message based on output_structure setting."""
211
+ output_structure = self.config.get('output_structure', 'auto_detect')
212
+
213
+ if output_structure == 'auto_detect':
214
+ # Try to detect if this was a whole message or just payload
215
+ if (isinstance(data, dict) and
216
+ (MessageKeys.PAYLOAD in data or MessageKeys.TOPIC in data or MessageKeys.MSGID in data) and
217
+ not ('data' in data and len(data) == 1)): # Not just wrapped data
218
+ # Looks like a whole message - reconstruct it
219
+ msg = data.copy()
220
+ if original_filename and 'filename' not in msg:
221
+ msg['filename'] = original_filename
222
+ return msg
223
+ else:
224
+ # Looks like payload data - put directly in msg.payload
225
+ return self.create_message(payload=data, filename=original_filename)
226
+
227
+ elif output_structure == 'reconstruct_message':
228
+ # Always try to use data as whole message
229
+ if isinstance(data, dict):
230
+ msg = data.copy()
231
+ if original_filename and 'filename' not in msg:
232
+ msg['filename'] = original_filename
233
+ return msg
234
+ else:
235
+ return self.create_message(payload=data, filename=original_filename)
236
+
237
+ elif output_structure == 'direct_payload':
238
+ # Always put data directly in payload
239
+ return self.create_message(payload=data, filename=original_filename)
240
+
241
+ else: # wrap_in_data (original behavior)
242
+ # Always wrap in msg.payload.data
243
+ return self.create_message(
244
+ payload={'data': data, 'filename': original_filename}
245
+ )
246
+
247
+ def on_input(self, msg: Dict[str, Any], input_index: int = 0):
248
+ """
249
+ Read files based on configuration or message content.
250
+ """
251
+ try:
252
+ # Get directory and file info from message or config
253
+ if 'path' in msg:
254
+ # Specific file path provided in message
255
+ file_path = msg['path']
256
+ if os.path.isfile(file_path):
257
+ self._read_single_file(file_path, msg)
258
+ else:
259
+ self.report_error(f"File not found: {file_path}")
260
+ else:
261
+ # Use configured reading mode
262
+ self._read_files_by_mode(msg)
263
+
264
+ except Exception as e:
265
+ self.report_error(f"Failed to read files: {str(e)}")
266
+ output_msg = self.create_message(
267
+ payload={
268
+ 'status': 'error',
269
+ 'error': str(e)
270
+ },
271
+ topic=msg.get(MessageKeys.TOPIC, 'MessageReader/error')
272
+ )
273
+ self.send(output_msg)
274
+
275
+ def read_files(self):
276
+ """Action method for UI button - read files without input message."""
277
+ empty_msg = self.create_message(payload={}, topic='MessageReader/action')
278
+ self._read_files_by_mode(empty_msg)
279
+
280
+ def _read_files_by_mode(self, trigger_msg: Dict[str, Any]):
281
+ """Read files based on configured reading mode."""
282
+ reading_mode = self.config.get('reading_mode', 'single_file')
283
+ directory = self.config.get('directory', './output')
284
+
285
+ if not os.path.exists(directory):
286
+ self.report_error(f"Directory does not exist: {directory}")
287
+ return
288
+
289
+ if reading_mode == 'single_file':
290
+ filename = self.config.get('specific_file', '')
291
+
292
+ # If filename is blank, try to get it from incoming message
293
+ if not filename:
294
+ payload = trigger_msg.get(MessageKeys.PAYLOAD, {})
295
+ if isinstance(payload, dict) and 'filename' in payload:
296
+ filename = payload['filename']
297
+ else:
298
+ filename = 'message_0001.json' # Default fallback
299
+
300
+ file_path = os.path.join(directory, filename)
301
+ self._read_single_file(file_path, trigger_msg)
302
+
303
+ elif reading_mode == 'batch_pattern':
304
+ self._read_batch_files(directory, trigger_msg)
305
+
306
+ elif reading_mode == 'latest_file':
307
+ self._read_latest_file(directory, trigger_msg)
308
+
309
+ elif reading_mode == 'all_files':
310
+ self._read_all_files(directory, trigger_msg)
311
+
312
+ def _read_single_file(self, file_path: str, trigger_msg: Dict[str, Any]):
313
+ """Read a single file and send as message."""
314
+ try:
315
+ if not os.path.exists(file_path):
316
+ self.report_error(f"File not found: {file_path}")
317
+ return
318
+
319
+ data = self._read_file_data(file_path)
320
+ filename = os.path.basename(file_path)
321
+
322
+ # Use helper to create properly structured message
323
+ output_msg = self._create_output_message(data, filename)
324
+
325
+ # Add metadata if requested
326
+ if self.get_config_bool('include_metadata', True):
327
+ metadata = self._get_file_metadata(file_path)
328
+ if MessageKeys.PAYLOAD in output_msg:
329
+ if isinstance(output_msg[MessageKeys.PAYLOAD], dict):
330
+ output_msg[MessageKeys.PAYLOAD]['metadata'] = metadata
331
+ else:
332
+ output_msg['metadata'] = metadata
333
+ else:
334
+ output_msg['metadata'] = metadata
335
+
336
+ # Preserve topic and add source info
337
+ output_msg[MessageKeys.TOPIC] = trigger_msg.get(MessageKeys.TOPIC, 'MessageReader')
338
+ output_msg['source_file'] = file_path
339
+ output_msg['reading_mode'] = 'single_file'
340
+
341
+ self.send(output_msg)
342
+
343
+ except Exception as e:
344
+ self.report_error(f"Failed to read file {file_path}: {str(e)}")
345
+
346
+ def _read_batch_files(self, directory: str, trigger_msg: Dict[str, Any]):
347
+ """Read multiple files matching pattern."""
348
+ pattern = self.config.get('filename_pattern', '*.json')
349
+ max_files = self.get_config_int('max_files', 10)
350
+
351
+ file_list = self._get_file_list(directory, pattern)
352
+
353
+ if not file_list:
354
+ output_msg = self.create_message(
355
+ payload={
356
+ 'status': 'no_files',
357
+ 'pattern': pattern,
358
+ 'directory': directory
359
+ },
360
+ topic=trigger_msg.get(MessageKeys.TOPIC, 'MessageReader')
361
+ )
362
+ self.send(output_msg)
363
+ return
364
+
365
+ # Limit number of files
366
+ files_to_read = file_list[:max_files]
367
+
368
+ for i, file_path in enumerate(files_to_read):
369
+ try:
370
+ data = self._read_file_data(file_path)
371
+ filename = os.path.basename(file_path)
372
+
373
+ # Use helper to create properly structured message
374
+ output_msg = self._create_output_message(data, filename)
375
+
376
+ # Add batch-specific metadata
377
+ if MessageKeys.PAYLOAD in output_msg:
378
+ if isinstance(output_msg[MessageKeys.PAYLOAD], dict):
379
+ output_msg[MessageKeys.PAYLOAD]['batch_index'] = i
380
+ output_msg[MessageKeys.PAYLOAD]['batch_total'] = len(files_to_read)
381
+ if self.get_config_bool('include_metadata', True):
382
+ output_msg[MessageKeys.PAYLOAD]['metadata'] = self._get_file_metadata(file_path)
383
+ else:
384
+ # For direct payload mode, add batch info as top-level properties
385
+ output_msg['batch_index'] = i
386
+ output_msg['batch_total'] = len(files_to_read)
387
+ if self.get_config_bool('include_metadata', True):
388
+ output_msg['metadata'] = self._get_file_metadata(file_path)
389
+
390
+ # Preserve topic and add source info
391
+ output_msg[MessageKeys.TOPIC] = trigger_msg.get(MessageKeys.TOPIC, 'MessageReader')
392
+ output_msg['source_file'] = file_path
393
+ output_msg['reading_mode'] = 'batch_pattern'
394
+
395
+ self.send(output_msg)
396
+
397
+ except Exception as e:
398
+ self.report_error(f"Failed to read file {file_path}: {str(e)}")
399
+
400
+ def _read_latest_file(self, directory: str, trigger_msg: Dict[str, Any]):
401
+ """Read the most recently modified file."""
402
+ pattern = self.config.get('filename_pattern', '*.json')
403
+ file_list = self._get_file_list(directory, pattern, sort_by='modified')
404
+
405
+ if not file_list:
406
+ output_msg = self.create_message(
407
+ payload={
408
+ 'status': 'no_files',
409
+ 'pattern': pattern,
410
+ 'directory': directory
411
+ },
412
+ topic=trigger_msg.get(MessageKeys.TOPIC, 'MessageReader')
413
+ )
414
+ self.send(output_msg)
415
+ return
416
+
417
+ latest_file = file_list[0] # First in list is most recent
418
+ self._read_single_file(latest_file, trigger_msg)
419
+
420
+ def _read_all_files(self, directory: str, trigger_msg: Dict[str, Any]):
421
+ """Read all files in directory (respecting max_files limit)."""
422
+ self.config['filename_pattern'] = '*' # Override pattern for all files
423
+ self._read_batch_files(directory, trigger_msg)
424
+
425
+ def _get_file_list(self, directory: str, pattern: str, sort_by: Optional[str] = None) -> List[str]:
426
+ """Get list of files matching pattern, sorted by specified criteria."""
427
+ recursive = self.get_config_bool('recursive', False)
428
+
429
+ if recursive:
430
+ # Recursive search
431
+ search_pattern = os.path.join(directory, '**', pattern)
432
+ file_list = glob.glob(search_pattern, recursive=True)
433
+ else:
434
+ # Non-recursive search
435
+ search_pattern = os.path.join(directory, pattern)
436
+ file_list = glob.glob(search_pattern)
437
+
438
+ # Filter to only files (not directories)
439
+ file_list = [f for f in file_list if os.path.isfile(f)]
440
+
441
+ # Sort files
442
+ sort_order = sort_by or self.config.get('sort_order', 'name')
443
+
444
+ if sort_order == 'modified':
445
+ file_list.sort(key=lambda x: os.path.getmtime(x), reverse=True)
446
+ elif sort_order == 'size':
447
+ file_list.sort(key=lambda x: os.path.getsize(x), reverse=True)
448
+ else: # name
449
+ file_list.sort()
450
+
451
+ return file_list
452
+
453
+ def _read_file_data(self, file_path: str) -> Any:
454
+ """Read and parse file data based on format."""
455
+ input_format = self.config.get('input_format', 'auto')
456
+
457
+ # Auto-detect format from extension
458
+ if input_format == 'auto':
459
+ _, ext = os.path.splitext(file_path)
460
+ ext = ext.lower().lstrip('.')
461
+
462
+ if ext == 'json':
463
+ input_format = 'json'
464
+ elif ext in ['pkl', 'pickle']:
465
+ input_format = 'pickle'
466
+ elif ext == 'npy':
467
+ input_format = 'numpy'
468
+ elif ext in ['txt', 'log', 'csv']:
469
+ input_format = 'text'
470
+ else:
471
+ input_format = 'raw'
472
+
473
+ # Read based on format
474
+ if input_format == 'json':
475
+ return self._read_json_file(file_path)
476
+ elif input_format == 'pickle':
477
+ return self._read_pickle_file(file_path)
478
+ elif input_format == 'numpy':
479
+ return self._read_numpy_file(file_path)
480
+ elif input_format == 'text':
481
+ return self._read_text_file(file_path)
482
+ elif input_format == 'raw':
483
+ return self._read_raw_file(file_path)
484
+ else:
485
+ return self._read_text_file(file_path) # Default fallback
486
+
487
+ def _read_json_file(self, file_path: str) -> Any:
488
+ """Read JSON file with numpy array reconstruction."""
489
+ with open(file_path, 'r', encoding='utf-8') as f:
490
+ data = json.load(f)
491
+
492
+ # Recursively reconstruct numpy arrays
493
+ return self._reconstruct_numpy_arrays(data)
494
+
495
+ def _read_pickle_file(self, file_path: str) -> Any:
496
+ """Read pickle file (only if explicitly allowed by config).
497
+
498
+ Unpickling can execute arbitrary code, so it is gated behind the
499
+ 'allow_pickle' setting (default off). This covers both explicit
500
+ 'pickle' format and auto-detected .pkl/.pickle files, since all
501
+ pickle reads funnel through this method.
502
+ """
503
+ if not self.get_config_bool('allow_pickle', False):
504
+ self.report_error(
505
+ "Pickle loading disabled; enable 'Allow Pickle Loading' in node "
506
+ "settings to load pickle files. Warning: pickle files can execute "
507
+ f"arbitrary code — only load trusted files. ({file_path})"
508
+ )
509
+ return None
510
+ with open(file_path, 'rb') as f:
511
+ return pickle.load(f)
512
+
513
+ def _read_numpy_file(self, file_path: str) -> Any:
514
+ """Read numpy .npy file."""
515
+ try:
516
+ import numpy as np
517
+ return np.load(file_path)
518
+ except ImportError:
519
+ # Fallback to raw bytes if numpy not available
520
+ return self._read_raw_file(file_path)
521
+
522
+ def _read_text_file(self, file_path: str) -> str:
523
+ """Read text file."""
524
+ with open(file_path, 'r', encoding='utf-8') as f:
525
+ return f.read()
526
+
527
+ def _read_raw_file(self, file_path: str) -> bytes:
528
+ """Read raw bytes."""
529
+ with open(file_path, 'rb') as f:
530
+ return f.read()
531
+
532
+ def _reconstruct_numpy_arrays(self, data: Any) -> Any:
533
+ """Recursively reconstruct numpy arrays from JSON serialization."""
534
+ if isinstance(data, dict):
535
+ if data.get('_numpy_array'):
536
+ # This is a serialized numpy array
537
+ try:
538
+ import numpy as np
539
+ array_bytes = base64.b64decode(data['data'])
540
+ dtype = np.dtype(data['dtype'])
541
+ shape = tuple(data['shape'])
542
+ return np.frombuffer(array_bytes, dtype=dtype).reshape(shape)
543
+ except ImportError:
544
+ # Return as dict if numpy not available
545
+ return data
546
+ except Exception as e:
547
+ self.report_error(f"Failed to reconstruct numpy array: {str(e)}")
548
+ return data
549
+ else:
550
+ # Recursively process dictionary
551
+ return {key: self._reconstruct_numpy_arrays(value) for key, value in data.items()}
552
+ elif isinstance(data, list):
553
+ # Recursively process list
554
+ return [self._reconstruct_numpy_arrays(item) for item in data]
555
+ else:
556
+ # Return as-is for other types
557
+ return data
558
+
559
+ def _get_file_metadata(self, file_path: str) -> Dict[str, Any]:
560
+ """Get file metadata."""
561
+ stat = os.stat(file_path)
562
+ return {
563
+ 'size': stat.st_size,
564
+ 'modified': stat.st_mtime,
565
+ 'modified_iso': datetime.fromtimestamp(stat.st_mtime).isoformat(),
566
+ 'basename': os.path.basename(file_path),
567
+ 'extension': os.path.splitext(file_path)[1]
568
+ }