pynode-flow 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pynode/__init__.py +37 -0
- pynode/__main__.py +9 -0
- pynode/_perf.py +65 -0
- pynode/_version.py +24 -0
- pynode/api/__init__.py +15 -0
- pynode/api/helpers.py +79 -0
- pynode/api/nodes.py +423 -0
- pynode/api/services.py +124 -0
- pynode/api/sse.py +58 -0
- pynode/api/uploads.py +59 -0
- pynode/api/workflows.py +466 -0
- pynode/config.py +93 -0
- pynode/install_nodes.py +143 -0
- pynode/main.py +118 -0
- pynode/node_registry.py +156 -0
- pynode/nodes/AutoSyncNode/auto_sync_node.py +151 -0
- pynode/nodes/BBoxMetricsNode/__init__.py +3 -0
- pynode/nodes/BBoxMetricsNode/bbox_metrics_node.py +250 -0
- pynode/nodes/BBoxMetricsNode/requirements.txt +1 -0
- pynode/nodes/BatchNode/__init__.py +3 -0
- pynode/nodes/BatchNode/batch_node.py +86 -0
- pynode/nodes/BatchNode/requirements.txt +1 -0
- pynode/nodes/CameraNode/__init__.py +4 -0
- pynode/nodes/CameraNode/camera_node.py +204 -0
- pynode/nodes/CameraNode/requirements.txt +2 -0
- pynode/nodes/ChangeNode/__init__.py +1 -0
- pynode/nodes/ChangeNode/change_node.py +255 -0
- pynode/nodes/ChangeNode/requirements.txt +1 -0
- pynode/nodes/ConfidenceFilterNode/__init__.py +4 -0
- pynode/nodes/ConfidenceFilterNode/confidence_filter_node.py +208 -0
- pynode/nodes/ConfidenceFilterNode/requirements.txt +1 -0
- pynode/nodes/CounterNode/__init__.py +3 -0
- pynode/nodes/CounterNode/counter_node.py +191 -0
- pynode/nodes/CropNode/__init__.py +4 -0
- pynode/nodes/CropNode/crop_node.py +317 -0
- pynode/nodes/CropNode/requirements.txt +2 -0
- pynode/nodes/DebugNode/__init__.py +4 -0
- pynode/nodes/DebugNode/debug_node.py +169 -0
- pynode/nodes/DebugNode/requirements.txt +1 -0
- pynode/nodes/DelayNode/delay_node.py +218 -0
- pynode/nodes/DelayNode/requirements.txt +1 -0
- pynode/nodes/DrawPredictionsNode/__init__.py +3 -0
- pynode/nodes/DrawPredictionsNode/draw_predictions_node.py +365 -0
- pynode/nodes/DrawPredictionsNode/requirements.txt +2 -0
- pynode/nodes/ErrorNode/error_node.py +94 -0
- pynode/nodes/ErrorNode/requirements.txt +1 -0
- pynode/nodes/FilterNode/__init__.py +3 -0
- pynode/nodes/FilterNode/filter_node.py +102 -0
- pynode/nodes/FilterNode/requirements.txt +1 -0
- pynode/nodes/FrameSourceNode/__init__.py +4 -0
- pynode/nodes/FrameSourceNode/frame_source_node.py +319 -0
- pynode/nodes/FrameSourceNode/requirements.txt +1 -0
- pynode/nodes/FunctionNode/__init__.py +3 -0
- pynode/nodes/FunctionNode/function_node.py +168 -0
- pynode/nodes/FunctionNode/requirements.txt +1 -0
- pynode/nodes/GateNode/gate_node.py +60 -0
- pynode/nodes/GateNode/requirements.txt +1 -0
- pynode/nodes/GetiUploadNode/__init__.py +3 -0
- pynode/nodes/GetiUploadNode/geti_upload_node.py +278 -0
- pynode/nodes/GetiUploadNode/requirements.txt +1 -0
- pynode/nodes/ImageFormatNode/__init__.py +4 -0
- pynode/nodes/ImageFormatNode/image_format_node.py +310 -0
- pynode/nodes/ImageFormatNode/requirements.txt +2 -0
- pynode/nodes/ImageUploadNode/image_upload_node.py +96 -0
- pynode/nodes/ImageViewerNode/__init__.py +4 -0
- pynode/nodes/ImageViewerNode/image_viewer_node.py +162 -0
- pynode/nodes/ImageViewerNode/requirements.txt +2 -0
- pynode/nodes/ImageWriterNode/__init__.py +3 -0
- pynode/nodes/ImageWriterNode/imagewriter_node.py +352 -0
- pynode/nodes/ImageWriterNode/requirements.txt +1 -0
- pynode/nodes/InferenceNode/InferenceEngine/__init__.py +15 -0
- pynode/nodes/InferenceNode/InferenceEngine/device_detection.py +149 -0
- pynode/nodes/InferenceNode/InferenceEngine/engines/base_engine.py +155 -0
- pynode/nodes/InferenceNode/InferenceEngine/engines/example_engine_template.py +272 -0
- pynode/nodes/InferenceNode/InferenceEngine/engines/geti_engine.py +309 -0
- pynode/nodes/InferenceNode/InferenceEngine/engines/onnx_engine.py +346 -0
- pynode/nodes/InferenceNode/InferenceEngine/engines/pass_engine.py +86 -0
- pynode/nodes/InferenceNode/InferenceEngine/engines/ultralytics_engine.py +863 -0
- pynode/nodes/InferenceNode/InferenceEngine/inference_engine_factory.py +533 -0
- pynode/nodes/InferenceNode/InferenceEngine/readme.md +107 -0
- pynode/nodes/InferenceNode/InferenceEngine/result_converters.py +121 -0
- pynode/nodes/InferenceNode/inference_node.py +451 -0
- pynode/nodes/InferenceNode/requirements.txt +13 -0
- pynode/nodes/InjectNode/inject_node.py +245 -0
- pynode/nodes/InjectNode/requirements.txt +1 -0
- pynode/nodes/JoinNode/__init__.py +3 -0
- pynode/nodes/JoinNode/join_node.py +156 -0
- pynode/nodes/JoinNode/requirements.txt +1 -0
- pynode/nodes/LabelFilterNode/__init__.py +3 -0
- pynode/nodes/LabelFilterNode/label_filter_node.py +188 -0
- pynode/nodes/LabelFilterNode/requirements.txt +1 -0
- pynode/nodes/MQTTNode/__init__.py +8 -0
- pynode/nodes/MQTTNode/mqtt_in_node.py +148 -0
- pynode/nodes/MQTTNode/mqtt_out_node.py +157 -0
- pynode/nodes/MQTTNode/mqtt_service.py +367 -0
- pynode/nodes/MQTTNode/requirements.txt +1 -0
- pynode/nodes/MessageWriterNode/__init__.py +7 -0
- pynode/nodes/MessageWriterNode/messagereader_node.py +568 -0
- pynode/nodes/MessageWriterNode/messagewriter_node.py +505 -0
- pynode/nodes/OmronCameraNode/__init__.py +3 -0
- pynode/nodes/OmronCameraNode/omron_camera_node.py +652 -0
- pynode/nodes/OmronCameraNode/requirements.txt +4 -0
- pynode/nodes/OpenCV/__init__.py +1 -0
- pynode/nodes/OpenCV/arithmetic_node.py +376 -0
- pynode/nodes/OpenCV/bitwise_node.py +127 -0
- pynode/nodes/OpenCV/blend_node.py +127 -0
- pynode/nodes/OpenCV/blob_detector_node.py +233 -0
- pynode/nodes/OpenCV/blur_node.py +141 -0
- pynode/nodes/OpenCV/color_space_node.py +139 -0
- pynode/nodes/OpenCV/colormap_node.py +262 -0
- pynode/nodes/OpenCV/contours_node.py +258 -0
- pynode/nodes/OpenCV/denormalize_coords_node.py +365 -0
- pynode/nodes/OpenCV/draw_node.py +312 -0
- pynode/nodes/OpenCV/edge_detector_node.py +177 -0
- pynode/nodes/OpenCV/equirectangular_node.py +447 -0
- pynode/nodes/OpenCV/fft_node.py +102 -0
- pynode/nodes/OpenCV/focus_estimation_node.py +189 -0
- pynode/nodes/OpenCV/frequency_filter_node.py +241 -0
- pynode/nodes/OpenCV/histogram_node.py +160 -0
- pynode/nodes/OpenCV/hough_circles_node.py +224 -0
- pynode/nodes/OpenCV/hough_lines_node.py +243 -0
- pynode/nodes/OpenCV/in_range_node.py +186 -0
- pynode/nodes/OpenCV/morphology_node.py +136 -0
- pynode/nodes/OpenCV/normalize_coords_node.py +344 -0
- pynode/nodes/OpenCV/paste_node.py +244 -0
- pynode/nodes/OpenCV/perspective_node.py +174 -0
- pynode/nodes/OpenCV/realsense_depth_node.py +253 -0
- pynode/nodes/OpenCV/requirements.txt +2 -0
- pynode/nodes/OpenCV/resize_node.py +175 -0
- pynode/nodes/OpenCV/rotate_node.py +159 -0
- pynode/nodes/OpenCV/template_match_node.py +227 -0
- pynode/nodes/OpenCV/threshold_node.py +167 -0
- pynode/nodes/PointInShapeNode/__init__.py +3 -0
- pynode/nodes/PointInShapeNode/point_in_shape_node.py +313 -0
- pynode/nodes/PointInShapeNode/requirements.txt +2 -0
- pynode/nodes/PolygonMetricsNode/__init__.py +3 -0
- pynode/nodes/PolygonMetricsNode/polygon_metrics_node.py +331 -0
- pynode/nodes/PolygonMetricsNode/requirements.txt +2 -0
- pynode/nodes/QueueLengthProbeNode/__init__.py +3 -0
- pynode/nodes/QueueLengthProbeNode/queue_length_probe_node.py +116 -0
- pynode/nodes/Qwen3VLMNode/__init__.py +3 -0
- pynode/nodes/Qwen3VLMNode/qwen3_vlm_node.py +519 -0
- pynode/nodes/Qwen3VLMNode/requirements.txt +5 -0
- pynode/nodes/RESTEndpointNode/__init__.py +3 -0
- pynode/nodes/RESTEndpointNode/rest_endpoint_node.py +491 -0
- pynode/nodes/RangeNode/__init__.py +3 -0
- pynode/nodes/RangeNode/range_node.py +115 -0
- pynode/nodes/RangeNode/requirements.txt +1 -0
- pynode/nodes/RateProbeNode/__init__.py +3 -0
- pynode/nodes/RateProbeNode/rate_probe_node.py +183 -0
- pynode/nodes/RateProbeNode/requirements.txt +1 -0
- pynode/nodes/RoboflowUploadNode/__init__.py +3 -0
- pynode/nodes/RoboflowUploadNode/requirements.txt +1 -0
- pynode/nodes/RoboflowUploadNode/roboflow_upload_node.py +229 -0
- pynode/nodes/SliceNode/README.md +149 -0
- pynode/nodes/SliceNode/__init__.py +5 -0
- pynode/nodes/SliceNode/merge_predictions_node.py +444 -0
- pynode/nodes/SliceNode/requirements.txt +3 -0
- pynode/nodes/SliceNode/slice_collector_node.py +370 -0
- pynode/nodes/SliceNode/slice_image_node.py +380 -0
- pynode/nodes/SplitNode/__init__.py +3 -0
- pynode/nodes/SplitNode/requirements.txt +1 -0
- pynode/nodes/SplitNode/split_node.py +155 -0
- pynode/nodes/SwitchNode/requirements.txt +1 -0
- pynode/nodes/SwitchNode/switch_node.py +259 -0
- pynode/nodes/SyncNode/__init__.py +3 -0
- pynode/nodes/SyncNode/sync_node.py +287 -0
- pynode/nodes/TemplateNode/requirements.txt +1 -0
- pynode/nodes/TemplateNode/template_node.py +86 -0
- pynode/nodes/TrackerNode/__init__.py +2 -0
- pynode/nodes/TrackerNode/supervision_tracker_node.py +248 -0
- pynode/nodes/TrackerNode/tracker_node.py +745 -0
- pynode/nodes/UltralyticsNode/__init__.py +4 -0
- pynode/nodes/UltralyticsNode/requirements.txt +2 -0
- pynode/nodes/UltralyticsNode/ultralytics_node.py +364 -0
- pynode/nodes/UnknownNode/__init__.py +93 -0
- pynode/nodes/VLLMNode/__init__.py +3 -0
- pynode/nodes/VLLMNode/vllm_node.py +345 -0
- pynode/nodes/VideoReaderNode/__init__.py +3 -0
- pynode/nodes/VideoReaderNode/video_reader_node.py +425 -0
- pynode/nodes/VideoWriterNode/__init__.py +3 -0
- pynode/nodes/VideoWriterNode/video_writer_node.py +441 -0
- pynode/nodes/WebhookNode/__init__.py +3 -0
- pynode/nodes/WebhookNode/webhook_node.py +591 -0
- pynode/nodes/__init__.py +109 -0
- pynode/nodes/base_node.py +562 -0
- pynode/nodes/image_utils.py +291 -0
- pynode/nodes/info.py +109 -0
- pynode/nodes/mDNSNode/__init__.py +13 -0
- pynode/nodes/mDNSNode/mdns_broadcast_node.py +214 -0
- pynode/nodes/mDNSNode/mdns_discovery_node.py +362 -0
- pynode/nodes/mDNSNode/mdns_manager.py +399 -0
- pynode/nodes/mDNSNode/requirements.txt +1 -0
- pynode/nodes/messages.py +81 -0
- pynode/server.py +187 -0
- pynode/static/images/favicon4.png +0 -0
- pynode/static/index.html +200 -0
- pynode/static/js/README.md +139 -0
- pynode/static/js/auth.js +89 -0
- pynode/static/js/clipboard.js +152 -0
- pynode/static/js/config.js +14 -0
- pynode/static/js/connections.js +844 -0
- pynode/static/js/debug.js +524 -0
- pynode/static/js/events.js +775 -0
- pynode/static/js/history.js +199 -0
- pynode/static/js/main.js +55 -0
- pynode/static/js/minimap.js +242 -0
- pynode/static/js/nodes.js +995 -0
- pynode/static/js/palette.js +255 -0
- pynode/static/js/properties.js +1270 -0
- pynode/static/js/selection.js +241 -0
- pynode/static/js/state.js +211 -0
- pynode/static/js/ui-utils.js +28 -0
- pynode/static/js/viewport.js +204 -0
- pynode/static/js/workflow.js +469 -0
- pynode/static/js/workflows.js +344 -0
- pynode/static/style.css +2751 -0
- pynode/workflow_engine.py +483 -0
- pynode/workflow_manager.py +440 -0
- pynode_flow-0.2.0.dist-info/METADATA +520 -0
- pynode_flow-0.2.0.dist-info/RECORD +227 -0
- pynode_flow-0.2.0.dist-info/WHEEL +5 -0
- pynode_flow-0.2.0.dist-info/entry_points.txt +3 -0
- pynode_flow-0.2.0.dist-info/licenses/LICENSE +21 -0
- pynode_flow-0.2.0.dist-info/scm_file_list.json +270 -0
- pynode_flow-0.2.0.dist-info/scm_version.json +8 -0
- pynode_flow-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,568 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Message Reader Node - reads messages or data from disk files.
|
|
3
|
+
Supports reading various formats written by MessageWriterNode.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import os
|
|
7
|
+
import json
|
|
8
|
+
import pickle
|
|
9
|
+
import base64
|
|
10
|
+
import glob
|
|
11
|
+
import time
|
|
12
|
+
from datetime import datetime
|
|
13
|
+
from typing import Any, Dict, List, Optional
|
|
14
|
+
from pynode.nodes.base_node import BaseNode, Info, MessageKeys
|
|
15
|
+
|
|
16
|
+
_info = Info()
|
|
17
|
+
_info.add_text("Reads messages or data from disk files. Supports JSON, binary, and text formats with batch reading capabilities.")
|
|
18
|
+
_info.add_header("Inputs")
|
|
19
|
+
_info.add_bullets(
|
|
20
|
+
("Input 0:", "Optional trigger message. If provided, reads files based on msg.path or configured directory.")
|
|
21
|
+
)
|
|
22
|
+
_info.add_header("Outputs")
|
|
23
|
+
_info.add_bullets(
|
|
24
|
+
("Output 0:", "Message containing the read data. For batch reading, outputs multiple messages.")
|
|
25
|
+
)
|
|
26
|
+
_info.add_header("Message Properties")
|
|
27
|
+
_info.add_bullets(
|
|
28
|
+
("msg.path:", "Specific file path to read (overrides directory setting)"),
|
|
29
|
+
("msg.pattern:", "File pattern for batch reading (e.g., '*.json')")
|
|
30
|
+
)
|
|
31
|
+
_info.add_header("Reading Modes")
|
|
32
|
+
_info.add_bullets(
|
|
33
|
+
("single_file:", "Read one specific file"),
|
|
34
|
+
("batch_pattern:", "Read multiple files matching a pattern"),
|
|
35
|
+
("watch_directory:", "Monitor directory for new files"),
|
|
36
|
+
("latest_file:", "Read the most recently modified file")
|
|
37
|
+
)
|
|
38
|
+
_info.add_header("Input Formats")
|
|
39
|
+
_info.add_bullets(
|
|
40
|
+
("auto:", "Auto-detect format from file extension"),
|
|
41
|
+
("json:", "JSON format (with numpy array reconstruction)"),
|
|
42
|
+
("pickle:", "Python pickle format"),
|
|
43
|
+
("numpy:", "NumPy .npy binary format"),
|
|
44
|
+
("text:", "Plain text format")
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class MessageReaderNode(BaseNode):
|
|
49
|
+
"""
|
|
50
|
+
Message Reader node - reads messages or data from disk files.
|
|
51
|
+
Companion to MessageWriterNode with format reconstruction capabilities.
|
|
52
|
+
"""
|
|
53
|
+
display_name = 'Message Reader'
|
|
54
|
+
icon = '📖'
|
|
55
|
+
category = 'input'
|
|
56
|
+
color = '#6B9BD2'
|
|
57
|
+
border_color = '#4A7BA7'
|
|
58
|
+
text_color = '#FFFFFF'
|
|
59
|
+
input_count = 1
|
|
60
|
+
output_count = 1
|
|
61
|
+
ui_component = 'button'
|
|
62
|
+
ui_component_config = {
|
|
63
|
+
'icon': '▶',
|
|
64
|
+
'action': 'read_files',
|
|
65
|
+
'tooltip': 'Read Files'
|
|
66
|
+
}
|
|
67
|
+
info = str(_info)
|
|
68
|
+
|
|
69
|
+
# UI-triggerable actions (see BaseNode.actions)
|
|
70
|
+
actions = ['read_files']
|
|
71
|
+
|
|
72
|
+
DEFAULT_CONFIG = {
|
|
73
|
+
'directory': './output',
|
|
74
|
+
'filename_pattern': '*.json',
|
|
75
|
+
'reading_mode': 'single_file',
|
|
76
|
+
'specific_file': '',
|
|
77
|
+
'input_format': 'auto',
|
|
78
|
+
'sort_order': 'name',
|
|
79
|
+
'max_files': '10',
|
|
80
|
+
'include_metadata': 'true',
|
|
81
|
+
'recursive': 'false',
|
|
82
|
+
'output_structure': 'auto_detect',
|
|
83
|
+
'allow_pickle': False
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
properties = [
|
|
87
|
+
{
|
|
88
|
+
'name': 'directory',
|
|
89
|
+
'label': 'Directory',
|
|
90
|
+
'type': 'text',
|
|
91
|
+
'default': DEFAULT_CONFIG['directory'],
|
|
92
|
+
'help': 'Directory to read files from'
|
|
93
|
+
},
|
|
94
|
+
{
|
|
95
|
+
'name': 'reading_mode',
|
|
96
|
+
'label': 'Reading Mode',
|
|
97
|
+
'type': 'select',
|
|
98
|
+
'options': [
|
|
99
|
+
{'value': 'single_file', 'label': 'Single File'},
|
|
100
|
+
{'value': 'batch_pattern', 'label': 'Batch (Pattern)'},
|
|
101
|
+
{'value': 'latest_file', 'label': 'Latest File'},
|
|
102
|
+
{'value': 'all_files', 'label': 'All Files'}
|
|
103
|
+
],
|
|
104
|
+
'default': DEFAULT_CONFIG['reading_mode'],
|
|
105
|
+
'help': 'How to select files to read'
|
|
106
|
+
},
|
|
107
|
+
{
|
|
108
|
+
'name': 'specific_file',
|
|
109
|
+
'label': 'Specific File',
|
|
110
|
+
'type': 'text',
|
|
111
|
+
'default': DEFAULT_CONFIG['specific_file'],
|
|
112
|
+
'help': f'Specific filename when reading_mode is "single_file" - leave blank to use {MessageKeys.MSG}.{MessageKeys.PAYLOAD}.filename',
|
|
113
|
+
'showIf': {'reading_mode': 'single_file'}
|
|
114
|
+
},
|
|
115
|
+
{
|
|
116
|
+
'name': 'filename_pattern',
|
|
117
|
+
'label': 'Filename Pattern',
|
|
118
|
+
'type': 'text',
|
|
119
|
+
'default': DEFAULT_CONFIG['filename_pattern'],
|
|
120
|
+
'help': 'Pattern to match files (e.g., *.json, message_*.txt)',
|
|
121
|
+
'showIf': {'reading_mode': ['batch_pattern', 'latest_file', 'all_files']}
|
|
122
|
+
},
|
|
123
|
+
{
|
|
124
|
+
'name': 'input_format',
|
|
125
|
+
'label': 'Input Format',
|
|
126
|
+
'type': 'select',
|
|
127
|
+
'options': [
|
|
128
|
+
{'value': 'auto', 'label': 'Auto-detect'},
|
|
129
|
+
{'value': 'json', 'label': 'JSON'},
|
|
130
|
+
{'value': 'pickle', 'label': 'Pickle'},
|
|
131
|
+
{'value': 'numpy', 'label': 'NumPy (.npy)'},
|
|
132
|
+
{'value': 'text', 'label': 'Text'},
|
|
133
|
+
{'value': 'raw', 'label': 'Raw bytes'}
|
|
134
|
+
],
|
|
135
|
+
'default': DEFAULT_CONFIG['input_format'],
|
|
136
|
+
'help': 'Format of the input files'
|
|
137
|
+
},
|
|
138
|
+
{
|
|
139
|
+
'name': 'allow_pickle',
|
|
140
|
+
'label': 'Allow Pickle Loading',
|
|
141
|
+
'type': 'checkbox',
|
|
142
|
+
'default': False,
|
|
143
|
+
'help': 'SECURITY WARNING: pickle files can execute arbitrary code when '
|
|
144
|
+
'loaded. Only enable this for files from sources you fully trust.'
|
|
145
|
+
},
|
|
146
|
+
{
|
|
147
|
+
'name': 'sort_order',
|
|
148
|
+
'label': 'Sort Order',
|
|
149
|
+
'type': 'select',
|
|
150
|
+
'options': [
|
|
151
|
+
{'value': 'name', 'label': 'Name'},
|
|
152
|
+
{'value': 'modified', 'label': 'Modified Time'},
|
|
153
|
+
{'value': 'size', 'label': 'File Size'}
|
|
154
|
+
],
|
|
155
|
+
'default': DEFAULT_CONFIG['sort_order'],
|
|
156
|
+
'help': 'How to sort files for batch reading',
|
|
157
|
+
'showIf': {'reading_mode': ['batch_pattern', 'latest_file', 'all_files']}
|
|
158
|
+
},
|
|
159
|
+
{
|
|
160
|
+
'name': 'max_files',
|
|
161
|
+
'label': 'Max Files',
|
|
162
|
+
'type': 'text',
|
|
163
|
+
'default': DEFAULT_CONFIG['max_files'],
|
|
164
|
+
'help': 'Maximum number of files to read in batch mode',
|
|
165
|
+
'showIf': {'reading_mode': ['batch_pattern', 'all_files']}
|
|
166
|
+
},
|
|
167
|
+
{
|
|
168
|
+
'name': 'include_metadata',
|
|
169
|
+
'label': 'Include File Metadata',
|
|
170
|
+
'type': 'select',
|
|
171
|
+
'options': [
|
|
172
|
+
{'value': 'true', 'label': 'Yes'},
|
|
173
|
+
{'value': 'false', 'label': 'No'}
|
|
174
|
+
],
|
|
175
|
+
'default': DEFAULT_CONFIG['include_metadata'],
|
|
176
|
+
'help': 'Include file metadata (size, modified time) in output'
|
|
177
|
+
},
|
|
178
|
+
{
|
|
179
|
+
'name': 'recursive',
|
|
180
|
+
'label': 'Recursive Search',
|
|
181
|
+
'type': 'select',
|
|
182
|
+
'options': [
|
|
183
|
+
{'value': 'false', 'label': 'No'},
|
|
184
|
+
{'value': 'true', 'label': 'Yes'}
|
|
185
|
+
],
|
|
186
|
+
'default': DEFAULT_CONFIG['recursive'],
|
|
187
|
+
'help': 'Search subdirectories recursively'
|
|
188
|
+
},
|
|
189
|
+
{
|
|
190
|
+
'name': 'output_structure',
|
|
191
|
+
'label': 'Output Structure',
|
|
192
|
+
'type': 'select',
|
|
193
|
+
'options': [
|
|
194
|
+
{'value': 'auto_detect', 'label': 'Auto-detect (smart reconstruction)'},
|
|
195
|
+
{'value': 'wrap_in_data', 'label': f'Always wrap in {MessageKeys.MSG}.{MessageKeys.PAYLOAD}.data'},
|
|
196
|
+
{'value': 'direct_payload', 'label': f'Direct to {MessageKeys.MSG}.{MessageKeys.PAYLOAD}'},
|
|
197
|
+
{'value': 'reconstruct_message', 'label': 'Reconstruct whole message'}
|
|
198
|
+
],
|
|
199
|
+
'default': DEFAULT_CONFIG['output_structure'],
|
|
200
|
+
'help': 'How to structure the read data in the output message'
|
|
201
|
+
}
|
|
202
|
+
]
|
|
203
|
+
|
|
204
|
+
def __init__(self, node_id=None, name="message reader"):
|
|
205
|
+
super().__init__(node_id, name)
|
|
206
|
+
self._last_read_time = None
|
|
207
|
+
self._file_cache = {}
|
|
208
|
+
|
|
209
|
+
def _create_output_message(self, data, original_filename=None):
|
|
210
|
+
"""Create output message based on output_structure setting."""
|
|
211
|
+
output_structure = self.config.get('output_structure', 'auto_detect')
|
|
212
|
+
|
|
213
|
+
if output_structure == 'auto_detect':
|
|
214
|
+
# Try to detect if this was a whole message or just payload
|
|
215
|
+
if (isinstance(data, dict) and
|
|
216
|
+
(MessageKeys.PAYLOAD in data or MessageKeys.TOPIC in data or MessageKeys.MSGID in data) and
|
|
217
|
+
not ('data' in data and len(data) == 1)): # Not just wrapped data
|
|
218
|
+
# Looks like a whole message - reconstruct it
|
|
219
|
+
msg = data.copy()
|
|
220
|
+
if original_filename and 'filename' not in msg:
|
|
221
|
+
msg['filename'] = original_filename
|
|
222
|
+
return msg
|
|
223
|
+
else:
|
|
224
|
+
# Looks like payload data - put directly in msg.payload
|
|
225
|
+
return self.create_message(payload=data, filename=original_filename)
|
|
226
|
+
|
|
227
|
+
elif output_structure == 'reconstruct_message':
|
|
228
|
+
# Always try to use data as whole message
|
|
229
|
+
if isinstance(data, dict):
|
|
230
|
+
msg = data.copy()
|
|
231
|
+
if original_filename and 'filename' not in msg:
|
|
232
|
+
msg['filename'] = original_filename
|
|
233
|
+
return msg
|
|
234
|
+
else:
|
|
235
|
+
return self.create_message(payload=data, filename=original_filename)
|
|
236
|
+
|
|
237
|
+
elif output_structure == 'direct_payload':
|
|
238
|
+
# Always put data directly in payload
|
|
239
|
+
return self.create_message(payload=data, filename=original_filename)
|
|
240
|
+
|
|
241
|
+
else: # wrap_in_data (original behavior)
|
|
242
|
+
# Always wrap in msg.payload.data
|
|
243
|
+
return self.create_message(
|
|
244
|
+
payload={'data': data, 'filename': original_filename}
|
|
245
|
+
)
|
|
246
|
+
|
|
247
|
+
def on_input(self, msg: Dict[str, Any], input_index: int = 0):
|
|
248
|
+
"""
|
|
249
|
+
Read files based on configuration or message content.
|
|
250
|
+
"""
|
|
251
|
+
try:
|
|
252
|
+
# Get directory and file info from message or config
|
|
253
|
+
if 'path' in msg:
|
|
254
|
+
# Specific file path provided in message
|
|
255
|
+
file_path = msg['path']
|
|
256
|
+
if os.path.isfile(file_path):
|
|
257
|
+
self._read_single_file(file_path, msg)
|
|
258
|
+
else:
|
|
259
|
+
self.report_error(f"File not found: {file_path}")
|
|
260
|
+
else:
|
|
261
|
+
# Use configured reading mode
|
|
262
|
+
self._read_files_by_mode(msg)
|
|
263
|
+
|
|
264
|
+
except Exception as e:
|
|
265
|
+
self.report_error(f"Failed to read files: {str(e)}")
|
|
266
|
+
output_msg = self.create_message(
|
|
267
|
+
payload={
|
|
268
|
+
'status': 'error',
|
|
269
|
+
'error': str(e)
|
|
270
|
+
},
|
|
271
|
+
topic=msg.get(MessageKeys.TOPIC, 'MessageReader/error')
|
|
272
|
+
)
|
|
273
|
+
self.send(output_msg)
|
|
274
|
+
|
|
275
|
+
def read_files(self):
|
|
276
|
+
"""Action method for UI button - read files without input message."""
|
|
277
|
+
empty_msg = self.create_message(payload={}, topic='MessageReader/action')
|
|
278
|
+
self._read_files_by_mode(empty_msg)
|
|
279
|
+
|
|
280
|
+
def _read_files_by_mode(self, trigger_msg: Dict[str, Any]):
|
|
281
|
+
"""Read files based on configured reading mode."""
|
|
282
|
+
reading_mode = self.config.get('reading_mode', 'single_file')
|
|
283
|
+
directory = self.config.get('directory', './output')
|
|
284
|
+
|
|
285
|
+
if not os.path.exists(directory):
|
|
286
|
+
self.report_error(f"Directory does not exist: {directory}")
|
|
287
|
+
return
|
|
288
|
+
|
|
289
|
+
if reading_mode == 'single_file':
|
|
290
|
+
filename = self.config.get('specific_file', '')
|
|
291
|
+
|
|
292
|
+
# If filename is blank, try to get it from incoming message
|
|
293
|
+
if not filename:
|
|
294
|
+
payload = trigger_msg.get(MessageKeys.PAYLOAD, {})
|
|
295
|
+
if isinstance(payload, dict) and 'filename' in payload:
|
|
296
|
+
filename = payload['filename']
|
|
297
|
+
else:
|
|
298
|
+
filename = 'message_0001.json' # Default fallback
|
|
299
|
+
|
|
300
|
+
file_path = os.path.join(directory, filename)
|
|
301
|
+
self._read_single_file(file_path, trigger_msg)
|
|
302
|
+
|
|
303
|
+
elif reading_mode == 'batch_pattern':
|
|
304
|
+
self._read_batch_files(directory, trigger_msg)
|
|
305
|
+
|
|
306
|
+
elif reading_mode == 'latest_file':
|
|
307
|
+
self._read_latest_file(directory, trigger_msg)
|
|
308
|
+
|
|
309
|
+
elif reading_mode == 'all_files':
|
|
310
|
+
self._read_all_files(directory, trigger_msg)
|
|
311
|
+
|
|
312
|
+
def _read_single_file(self, file_path: str, trigger_msg: Dict[str, Any]):
|
|
313
|
+
"""Read a single file and send as message."""
|
|
314
|
+
try:
|
|
315
|
+
if not os.path.exists(file_path):
|
|
316
|
+
self.report_error(f"File not found: {file_path}")
|
|
317
|
+
return
|
|
318
|
+
|
|
319
|
+
data = self._read_file_data(file_path)
|
|
320
|
+
filename = os.path.basename(file_path)
|
|
321
|
+
|
|
322
|
+
# Use helper to create properly structured message
|
|
323
|
+
output_msg = self._create_output_message(data, filename)
|
|
324
|
+
|
|
325
|
+
# Add metadata if requested
|
|
326
|
+
if self.get_config_bool('include_metadata', True):
|
|
327
|
+
metadata = self._get_file_metadata(file_path)
|
|
328
|
+
if MessageKeys.PAYLOAD in output_msg:
|
|
329
|
+
if isinstance(output_msg[MessageKeys.PAYLOAD], dict):
|
|
330
|
+
output_msg[MessageKeys.PAYLOAD]['metadata'] = metadata
|
|
331
|
+
else:
|
|
332
|
+
output_msg['metadata'] = metadata
|
|
333
|
+
else:
|
|
334
|
+
output_msg['metadata'] = metadata
|
|
335
|
+
|
|
336
|
+
# Preserve topic and add source info
|
|
337
|
+
output_msg[MessageKeys.TOPIC] = trigger_msg.get(MessageKeys.TOPIC, 'MessageReader')
|
|
338
|
+
output_msg['source_file'] = file_path
|
|
339
|
+
output_msg['reading_mode'] = 'single_file'
|
|
340
|
+
|
|
341
|
+
self.send(output_msg)
|
|
342
|
+
|
|
343
|
+
except Exception as e:
|
|
344
|
+
self.report_error(f"Failed to read file {file_path}: {str(e)}")
|
|
345
|
+
|
|
346
|
+
def _read_batch_files(self, directory: str, trigger_msg: Dict[str, Any]):
|
|
347
|
+
"""Read multiple files matching pattern."""
|
|
348
|
+
pattern = self.config.get('filename_pattern', '*.json')
|
|
349
|
+
max_files = self.get_config_int('max_files', 10)
|
|
350
|
+
|
|
351
|
+
file_list = self._get_file_list(directory, pattern)
|
|
352
|
+
|
|
353
|
+
if not file_list:
|
|
354
|
+
output_msg = self.create_message(
|
|
355
|
+
payload={
|
|
356
|
+
'status': 'no_files',
|
|
357
|
+
'pattern': pattern,
|
|
358
|
+
'directory': directory
|
|
359
|
+
},
|
|
360
|
+
topic=trigger_msg.get(MessageKeys.TOPIC, 'MessageReader')
|
|
361
|
+
)
|
|
362
|
+
self.send(output_msg)
|
|
363
|
+
return
|
|
364
|
+
|
|
365
|
+
# Limit number of files
|
|
366
|
+
files_to_read = file_list[:max_files]
|
|
367
|
+
|
|
368
|
+
for i, file_path in enumerate(files_to_read):
|
|
369
|
+
try:
|
|
370
|
+
data = self._read_file_data(file_path)
|
|
371
|
+
filename = os.path.basename(file_path)
|
|
372
|
+
|
|
373
|
+
# Use helper to create properly structured message
|
|
374
|
+
output_msg = self._create_output_message(data, filename)
|
|
375
|
+
|
|
376
|
+
# Add batch-specific metadata
|
|
377
|
+
if MessageKeys.PAYLOAD in output_msg:
|
|
378
|
+
if isinstance(output_msg[MessageKeys.PAYLOAD], dict):
|
|
379
|
+
output_msg[MessageKeys.PAYLOAD]['batch_index'] = i
|
|
380
|
+
output_msg[MessageKeys.PAYLOAD]['batch_total'] = len(files_to_read)
|
|
381
|
+
if self.get_config_bool('include_metadata', True):
|
|
382
|
+
output_msg[MessageKeys.PAYLOAD]['metadata'] = self._get_file_metadata(file_path)
|
|
383
|
+
else:
|
|
384
|
+
# For direct payload mode, add batch info as top-level properties
|
|
385
|
+
output_msg['batch_index'] = i
|
|
386
|
+
output_msg['batch_total'] = len(files_to_read)
|
|
387
|
+
if self.get_config_bool('include_metadata', True):
|
|
388
|
+
output_msg['metadata'] = self._get_file_metadata(file_path)
|
|
389
|
+
|
|
390
|
+
# Preserve topic and add source info
|
|
391
|
+
output_msg[MessageKeys.TOPIC] = trigger_msg.get(MessageKeys.TOPIC, 'MessageReader')
|
|
392
|
+
output_msg['source_file'] = file_path
|
|
393
|
+
output_msg['reading_mode'] = 'batch_pattern'
|
|
394
|
+
|
|
395
|
+
self.send(output_msg)
|
|
396
|
+
|
|
397
|
+
except Exception as e:
|
|
398
|
+
self.report_error(f"Failed to read file {file_path}: {str(e)}")
|
|
399
|
+
|
|
400
|
+
def _read_latest_file(self, directory: str, trigger_msg: Dict[str, Any]):
|
|
401
|
+
"""Read the most recently modified file."""
|
|
402
|
+
pattern = self.config.get('filename_pattern', '*.json')
|
|
403
|
+
file_list = self._get_file_list(directory, pattern, sort_by='modified')
|
|
404
|
+
|
|
405
|
+
if not file_list:
|
|
406
|
+
output_msg = self.create_message(
|
|
407
|
+
payload={
|
|
408
|
+
'status': 'no_files',
|
|
409
|
+
'pattern': pattern,
|
|
410
|
+
'directory': directory
|
|
411
|
+
},
|
|
412
|
+
topic=trigger_msg.get(MessageKeys.TOPIC, 'MessageReader')
|
|
413
|
+
)
|
|
414
|
+
self.send(output_msg)
|
|
415
|
+
return
|
|
416
|
+
|
|
417
|
+
latest_file = file_list[0] # First in list is most recent
|
|
418
|
+
self._read_single_file(latest_file, trigger_msg)
|
|
419
|
+
|
|
420
|
+
def _read_all_files(self, directory: str, trigger_msg: Dict[str, Any]):
|
|
421
|
+
"""Read all files in directory (respecting max_files limit)."""
|
|
422
|
+
self.config['filename_pattern'] = '*' # Override pattern for all files
|
|
423
|
+
self._read_batch_files(directory, trigger_msg)
|
|
424
|
+
|
|
425
|
+
def _get_file_list(self, directory: str, pattern: str, sort_by: Optional[str] = None) -> List[str]:
|
|
426
|
+
"""Get list of files matching pattern, sorted by specified criteria."""
|
|
427
|
+
recursive = self.get_config_bool('recursive', False)
|
|
428
|
+
|
|
429
|
+
if recursive:
|
|
430
|
+
# Recursive search
|
|
431
|
+
search_pattern = os.path.join(directory, '**', pattern)
|
|
432
|
+
file_list = glob.glob(search_pattern, recursive=True)
|
|
433
|
+
else:
|
|
434
|
+
# Non-recursive search
|
|
435
|
+
search_pattern = os.path.join(directory, pattern)
|
|
436
|
+
file_list = glob.glob(search_pattern)
|
|
437
|
+
|
|
438
|
+
# Filter to only files (not directories)
|
|
439
|
+
file_list = [f for f in file_list if os.path.isfile(f)]
|
|
440
|
+
|
|
441
|
+
# Sort files
|
|
442
|
+
sort_order = sort_by or self.config.get('sort_order', 'name')
|
|
443
|
+
|
|
444
|
+
if sort_order == 'modified':
|
|
445
|
+
file_list.sort(key=lambda x: os.path.getmtime(x), reverse=True)
|
|
446
|
+
elif sort_order == 'size':
|
|
447
|
+
file_list.sort(key=lambda x: os.path.getsize(x), reverse=True)
|
|
448
|
+
else: # name
|
|
449
|
+
file_list.sort()
|
|
450
|
+
|
|
451
|
+
return file_list
|
|
452
|
+
|
|
453
|
+
def _read_file_data(self, file_path: str) -> Any:
|
|
454
|
+
"""Read and parse file data based on format."""
|
|
455
|
+
input_format = self.config.get('input_format', 'auto')
|
|
456
|
+
|
|
457
|
+
# Auto-detect format from extension
|
|
458
|
+
if input_format == 'auto':
|
|
459
|
+
_, ext = os.path.splitext(file_path)
|
|
460
|
+
ext = ext.lower().lstrip('.')
|
|
461
|
+
|
|
462
|
+
if ext == 'json':
|
|
463
|
+
input_format = 'json'
|
|
464
|
+
elif ext in ['pkl', 'pickle']:
|
|
465
|
+
input_format = 'pickle'
|
|
466
|
+
elif ext == 'npy':
|
|
467
|
+
input_format = 'numpy'
|
|
468
|
+
elif ext in ['txt', 'log', 'csv']:
|
|
469
|
+
input_format = 'text'
|
|
470
|
+
else:
|
|
471
|
+
input_format = 'raw'
|
|
472
|
+
|
|
473
|
+
# Read based on format
|
|
474
|
+
if input_format == 'json':
|
|
475
|
+
return self._read_json_file(file_path)
|
|
476
|
+
elif input_format == 'pickle':
|
|
477
|
+
return self._read_pickle_file(file_path)
|
|
478
|
+
elif input_format == 'numpy':
|
|
479
|
+
return self._read_numpy_file(file_path)
|
|
480
|
+
elif input_format == 'text':
|
|
481
|
+
return self._read_text_file(file_path)
|
|
482
|
+
elif input_format == 'raw':
|
|
483
|
+
return self._read_raw_file(file_path)
|
|
484
|
+
else:
|
|
485
|
+
return self._read_text_file(file_path) # Default fallback
|
|
486
|
+
|
|
487
|
+
def _read_json_file(self, file_path: str) -> Any:
|
|
488
|
+
"""Read JSON file with numpy array reconstruction."""
|
|
489
|
+
with open(file_path, 'r', encoding='utf-8') as f:
|
|
490
|
+
data = json.load(f)
|
|
491
|
+
|
|
492
|
+
# Recursively reconstruct numpy arrays
|
|
493
|
+
return self._reconstruct_numpy_arrays(data)
|
|
494
|
+
|
|
495
|
+
def _read_pickle_file(self, file_path: str) -> Any:
|
|
496
|
+
"""Read pickle file (only if explicitly allowed by config).
|
|
497
|
+
|
|
498
|
+
Unpickling can execute arbitrary code, so it is gated behind the
|
|
499
|
+
'allow_pickle' setting (default off). This covers both explicit
|
|
500
|
+
'pickle' format and auto-detected .pkl/.pickle files, since all
|
|
501
|
+
pickle reads funnel through this method.
|
|
502
|
+
"""
|
|
503
|
+
if not self.get_config_bool('allow_pickle', False):
|
|
504
|
+
self.report_error(
|
|
505
|
+
"Pickle loading disabled; enable 'Allow Pickle Loading' in node "
|
|
506
|
+
"settings to load pickle files. Warning: pickle files can execute "
|
|
507
|
+
f"arbitrary code — only load trusted files. ({file_path})"
|
|
508
|
+
)
|
|
509
|
+
return None
|
|
510
|
+
with open(file_path, 'rb') as f:
|
|
511
|
+
return pickle.load(f)
|
|
512
|
+
|
|
513
|
+
def _read_numpy_file(self, file_path: str) -> Any:
|
|
514
|
+
"""Read numpy .npy file."""
|
|
515
|
+
try:
|
|
516
|
+
import numpy as np
|
|
517
|
+
return np.load(file_path)
|
|
518
|
+
except ImportError:
|
|
519
|
+
# Fallback to raw bytes if numpy not available
|
|
520
|
+
return self._read_raw_file(file_path)
|
|
521
|
+
|
|
522
|
+
def _read_text_file(self, file_path: str) -> str:
|
|
523
|
+
"""Read text file."""
|
|
524
|
+
with open(file_path, 'r', encoding='utf-8') as f:
|
|
525
|
+
return f.read()
|
|
526
|
+
|
|
527
|
+
def _read_raw_file(self, file_path: str) -> bytes:
|
|
528
|
+
"""Read raw bytes."""
|
|
529
|
+
with open(file_path, 'rb') as f:
|
|
530
|
+
return f.read()
|
|
531
|
+
|
|
532
|
+
def _reconstruct_numpy_arrays(self, data: Any) -> Any:
|
|
533
|
+
"""Recursively reconstruct numpy arrays from JSON serialization."""
|
|
534
|
+
if isinstance(data, dict):
|
|
535
|
+
if data.get('_numpy_array'):
|
|
536
|
+
# This is a serialized numpy array
|
|
537
|
+
try:
|
|
538
|
+
import numpy as np
|
|
539
|
+
array_bytes = base64.b64decode(data['data'])
|
|
540
|
+
dtype = np.dtype(data['dtype'])
|
|
541
|
+
shape = tuple(data['shape'])
|
|
542
|
+
return np.frombuffer(array_bytes, dtype=dtype).reshape(shape)
|
|
543
|
+
except ImportError:
|
|
544
|
+
# Return as dict if numpy not available
|
|
545
|
+
return data
|
|
546
|
+
except Exception as e:
|
|
547
|
+
self.report_error(f"Failed to reconstruct numpy array: {str(e)}")
|
|
548
|
+
return data
|
|
549
|
+
else:
|
|
550
|
+
# Recursively process dictionary
|
|
551
|
+
return {key: self._reconstruct_numpy_arrays(value) for key, value in data.items()}
|
|
552
|
+
elif isinstance(data, list):
|
|
553
|
+
# Recursively process list
|
|
554
|
+
return [self._reconstruct_numpy_arrays(item) for item in data]
|
|
555
|
+
else:
|
|
556
|
+
# Return as-is for other types
|
|
557
|
+
return data
|
|
558
|
+
|
|
559
|
+
def _get_file_metadata(self, file_path: str) -> Dict[str, Any]:
|
|
560
|
+
"""Get file metadata."""
|
|
561
|
+
stat = os.stat(file_path)
|
|
562
|
+
return {
|
|
563
|
+
'size': stat.st_size,
|
|
564
|
+
'modified': stat.st_mtime,
|
|
565
|
+
'modified_iso': datetime.fromtimestamp(stat.st_mtime).isoformat(),
|
|
566
|
+
'basename': os.path.basename(file_path),
|
|
567
|
+
'extension': os.path.splitext(file_path)[1]
|
|
568
|
+
}
|