synapse-sdk 1.0.0b23__py3-none-any.whl → 2025.9.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of synapse-sdk might be problematic. Click here for more details.
- synapse_sdk/clients/agent/ray.py +50 -0
- synapse_sdk/devtools/docs/docs/api/clients/ray.md +24 -3
- synapse_sdk/devtools/docs/docs/api/index.md +5 -5
- synapse_sdk/devtools/docs/docs/features/utils/file.md +415 -0
- synapse_sdk/devtools/docs/docs/{api → features}/utils/network.md +1 -1
- synapse_sdk/devtools/docs/docs/plugins/export-plugins.md +140 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/clients/ray.md +24 -3
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/index.md +5 -5
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/features/index.md +5 -5
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/features/utils/file.md +415 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/{api → features}/utils/network.md +1 -1
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/export-plugins.md +138 -0
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/plugins.md +48 -2
- synapse_sdk/devtools/docs/sidebars.ts +10 -10
- synapse_sdk/plugins/categories/export/templates/plugin/__init__.py +17 -2
- synapse_sdk/utils/file/__init__.py +39 -0
- synapse_sdk/utils/file/archive.py +32 -0
- synapse_sdk/utils/file/checksum.py +56 -0
- synapse_sdk/utils/file/chunking.py +31 -0
- synapse_sdk/utils/file/download.py +124 -0
- synapse_sdk/utils/file/encoding.py +40 -0
- synapse_sdk/utils/file/io.py +22 -0
- synapse_sdk/utils/file/video/__init__.py +29 -0
- synapse_sdk/utils/file/video/transcode.py +307 -0
- {synapse_sdk-1.0.0b23.dist-info → synapse_sdk-2025.9.1.dist-info}/METADATA +3 -2
- {synapse_sdk-1.0.0b23.dist-info → synapse_sdk-2025.9.1.dist-info}/RECORD +35 -26
- synapse_sdk/devtools/docs/docs/api/utils/file.md +0 -195
- synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/utils/file.md +0 -195
- /synapse_sdk/devtools/docs/docs/{api → features}/utils/storage.md +0 -0
- /synapse_sdk/devtools/docs/docs/{api → features}/utils/types.md +0 -0
- /synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/{api → features}/utils/storage.md +0 -0
- /synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/{api → features}/utils/types.md +0 -0
- /synapse_sdk/utils/{file.py → file.py.backup} +0 -0
- {synapse_sdk-1.0.0b23.dist-info → synapse_sdk-2025.9.1.dist-info}/WHEEL +0 -0
- {synapse_sdk-1.0.0b23.dist-info → synapse_sdk-2025.9.1.dist-info}/entry_points.txt +0 -0
- {synapse_sdk-1.0.0b23.dist-info → synapse_sdk-2025.9.1.dist-info}/licenses/LICENSE +0 -0
- {synapse_sdk-1.0.0b23.dist-info → synapse_sdk-2025.9.1.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,307 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import shutil
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Callable, Optional
|
|
6
|
+
|
|
7
|
+
import ffmpeg
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
# Exception classes
|
|
11
|
+
class VideoTranscodeError(Exception):
|
|
12
|
+
"""Base exception for video transcoding errors."""
|
|
13
|
+
|
|
14
|
+
pass
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class UnsupportedFormatError(VideoTranscodeError):
|
|
18
|
+
"""Raised when input format is not supported."""
|
|
19
|
+
|
|
20
|
+
pass
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class FFmpegNotFoundError(VideoTranscodeError):
|
|
24
|
+
"""Raised when FFmpeg is not installed or not in PATH."""
|
|
25
|
+
|
|
26
|
+
pass
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class TranscodingFailedError(VideoTranscodeError):
|
|
30
|
+
"""Raised when FFmpeg transcoding process fails."""
|
|
31
|
+
|
|
32
|
+
pass
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass
|
|
36
|
+
class TranscodeConfig:
|
|
37
|
+
"""Video transcoding configuration."""
|
|
38
|
+
|
|
39
|
+
vcodec: str = 'libx264' # Video codec
|
|
40
|
+
preset: str = 'medium' # Encoding preset (ultrafast to veryslow)
|
|
41
|
+
crf: int = 28 # Constant Rate Factor (0-51, lower=better quality)
|
|
42
|
+
acodec: str = 'aac' # Audio codec
|
|
43
|
+
audio_bitrate: str = '128k' # Audio bitrate
|
|
44
|
+
movflags: str = '+faststart' # MP4 optimization flags
|
|
45
|
+
resolution: Optional[str] = None # Target resolution (e.g., '1920x1080')
|
|
46
|
+
fps: Optional[int] = None # Target frame rate
|
|
47
|
+
start_time: Optional[float] = None # Trim start time in seconds
|
|
48
|
+
duration: Optional[float] = None # Trim duration in seconds
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
# Supported input formats
|
|
52
|
+
SUPPORTED_FORMATS = {'.mp4', '.avi', '.mov', '.mkv', '.webm', '.flv', '.wmv', '.mpeg', '.mpg', '.m4v', '.3gp', '.ogv'}
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _check_ffmpeg_available():
|
|
56
|
+
"""Check if FFmpeg is available in PATH."""
|
|
57
|
+
if not shutil.which('ffmpeg'):
|
|
58
|
+
raise FFmpegNotFoundError(
|
|
59
|
+
'FFmpeg is not installed or not found in PATH. Please install FFmpeg to use video transcoding features.'
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def validate_video_format(video_path: str | Path) -> bool:
|
|
64
|
+
"""
|
|
65
|
+
Check if video format is supported for transcoding.
|
|
66
|
+
|
|
67
|
+
Args:
|
|
68
|
+
video_path (str | Path): Path to the video file
|
|
69
|
+
|
|
70
|
+
Returns:
|
|
71
|
+
bool: True if format is supported, False otherwise
|
|
72
|
+
"""
|
|
73
|
+
path = Path(video_path)
|
|
74
|
+
return path.suffix.lower() in SUPPORTED_FORMATS
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def get_video_info(video_path: str | Path) -> dict:
|
|
78
|
+
"""
|
|
79
|
+
Extract video metadata (resolution, duration, codecs, etc.).
|
|
80
|
+
|
|
81
|
+
Args:
|
|
82
|
+
video_path (str | Path): Path to the video file
|
|
83
|
+
|
|
84
|
+
Returns:
|
|
85
|
+
dict: Video metadata information
|
|
86
|
+
|
|
87
|
+
Raises:
|
|
88
|
+
VideoTranscodeError: If unable to probe video file
|
|
89
|
+
"""
|
|
90
|
+
_check_ffmpeg_available()
|
|
91
|
+
|
|
92
|
+
try:
|
|
93
|
+
probe = ffmpeg.probe(str(video_path))
|
|
94
|
+
|
|
95
|
+
video_info = {}
|
|
96
|
+
|
|
97
|
+
# Get format information
|
|
98
|
+
if 'format' in probe:
|
|
99
|
+
format_info = probe['format']
|
|
100
|
+
video_info['duration'] = float(format_info.get('duration', 0))
|
|
101
|
+
video_info['size'] = int(format_info.get('size', 0))
|
|
102
|
+
video_info['bitrate'] = int(format_info.get('bit_rate', 0))
|
|
103
|
+
|
|
104
|
+
# Get stream information
|
|
105
|
+
video_streams = [stream for stream in probe['streams'] if stream['codec_type'] == 'video']
|
|
106
|
+
audio_streams = [stream for stream in probe['streams'] if stream['codec_type'] == 'audio']
|
|
107
|
+
|
|
108
|
+
if video_streams:
|
|
109
|
+
video_stream = video_streams[0]
|
|
110
|
+
video_info['width'] = int(video_stream.get('width', 0))
|
|
111
|
+
video_info['height'] = int(video_stream.get('height', 0))
|
|
112
|
+
video_info['video_codec'] = video_stream.get('codec_name', '')
|
|
113
|
+
video_info['fps'] = eval(video_stream.get('r_frame_rate', '0/1'))
|
|
114
|
+
|
|
115
|
+
if audio_streams:
|
|
116
|
+
audio_stream = audio_streams[0]
|
|
117
|
+
video_info['audio_codec'] = audio_stream.get('codec_name', '')
|
|
118
|
+
video_info['channels'] = int(audio_stream.get('channels', 0))
|
|
119
|
+
video_info['sample_rate'] = int(audio_stream.get('sample_rate', 0))
|
|
120
|
+
|
|
121
|
+
return video_info
|
|
122
|
+
|
|
123
|
+
except Exception as e:
|
|
124
|
+
raise VideoTranscodeError(f'Failed to probe video file: {str(e)}')
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _build_ffmpeg_stream(input_path: str | Path, output_path: str | Path, config: TranscodeConfig):
|
|
128
|
+
"""Build FFmpeg stream with configuration."""
|
|
129
|
+
stream = ffmpeg.input(str(input_path))
|
|
130
|
+
|
|
131
|
+
# Apply start time and duration trimming
|
|
132
|
+
if config.start_time is not None or config.duration is not None:
|
|
133
|
+
kwargs = {}
|
|
134
|
+
if config.start_time is not None:
|
|
135
|
+
kwargs['ss'] = config.start_time
|
|
136
|
+
if config.duration is not None:
|
|
137
|
+
kwargs['t'] = config.duration
|
|
138
|
+
stream = ffmpeg.input(str(input_path), **kwargs)
|
|
139
|
+
|
|
140
|
+
# Apply video filters
|
|
141
|
+
if config.resolution or config.fps:
|
|
142
|
+
if config.resolution:
|
|
143
|
+
width, height = config.resolution.split('x')
|
|
144
|
+
stream = ffmpeg.filter(stream, 'scale', width, height)
|
|
145
|
+
if config.fps:
|
|
146
|
+
stream = ffmpeg.filter(stream, 'fps', fps=config.fps)
|
|
147
|
+
|
|
148
|
+
# Build output with encoding parameters
|
|
149
|
+
output_kwargs = {
|
|
150
|
+
'vcodec': config.vcodec,
|
|
151
|
+
'preset': config.preset,
|
|
152
|
+
'crf': config.crf,
|
|
153
|
+
'acodec': config.acodec,
|
|
154
|
+
'audio_bitrate': config.audio_bitrate,
|
|
155
|
+
'movflags': config.movflags,
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
return ffmpeg.output(stream, str(output_path), **output_kwargs)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def transcode_video(
|
|
162
|
+
input_path: str | Path,
|
|
163
|
+
output_path: str | Path,
|
|
164
|
+
config: Optional[TranscodeConfig] = None,
|
|
165
|
+
progress_callback: Optional[Callable[[float], None]] = None,
|
|
166
|
+
) -> Path:
|
|
167
|
+
"""
|
|
168
|
+
Transcode video with specified configuration.
|
|
169
|
+
|
|
170
|
+
Args:
|
|
171
|
+
input_path (str | Path): Path to input video file
|
|
172
|
+
output_path (str | Path): Path to output video file
|
|
173
|
+
config (Optional[TranscodeConfig]): Transcoding configuration
|
|
174
|
+
progress_callback (Optional[Callable[[float], None]]): Progress callback function
|
|
175
|
+
|
|
176
|
+
Returns:
|
|
177
|
+
Path: Path to the transcoded video file
|
|
178
|
+
|
|
179
|
+
Raises:
|
|
180
|
+
UnsupportedFormatError: If input format is not supported
|
|
181
|
+
FFmpegNotFoundError: If FFmpeg is not available
|
|
182
|
+
TranscodingFailedError: If transcoding fails
|
|
183
|
+
"""
|
|
184
|
+
_check_ffmpeg_available()
|
|
185
|
+
|
|
186
|
+
input_path = Path(input_path)
|
|
187
|
+
output_path = Path(output_path)
|
|
188
|
+
|
|
189
|
+
if not validate_video_format(input_path):
|
|
190
|
+
raise UnsupportedFormatError(f'Unsupported video format: {input_path.suffix}')
|
|
191
|
+
|
|
192
|
+
if config is None:
|
|
193
|
+
config = TranscodeConfig()
|
|
194
|
+
|
|
195
|
+
# Ensure output directory exists
|
|
196
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
197
|
+
|
|
198
|
+
try:
|
|
199
|
+
# Build FFmpeg command
|
|
200
|
+
stream = _build_ffmpeg_stream(input_path, output_path, config)
|
|
201
|
+
|
|
202
|
+
# Run FFmpeg
|
|
203
|
+
if progress_callback:
|
|
204
|
+
# Get video duration for progress calculation
|
|
205
|
+
video_info = get_video_info(input_path)
|
|
206
|
+
total_duration = video_info.get('duration', 0)
|
|
207
|
+
|
|
208
|
+
# Run with progress monitoring
|
|
209
|
+
process = ffmpeg.run_async(stream, pipe_stderr=True, overwrite_output=True)
|
|
210
|
+
|
|
211
|
+
while True:
|
|
212
|
+
output = process.stderr.readline()
|
|
213
|
+
if output == b'' and process.poll() is not None:
|
|
214
|
+
break
|
|
215
|
+
if output:
|
|
216
|
+
line = output.decode('utf-8')
|
|
217
|
+
# Parse progress from FFmpeg output
|
|
218
|
+
if 'time=' in line and total_duration > 0:
|
|
219
|
+
try:
|
|
220
|
+
time_str = line.split('time=')[1].split()[0]
|
|
221
|
+
hours, minutes, seconds = time_str.split(':')
|
|
222
|
+
current_time = int(hours) * 3600 + int(minutes) * 60 + float(seconds)
|
|
223
|
+
progress = min(current_time / total_duration, 1.0)
|
|
224
|
+
progress_callback(progress)
|
|
225
|
+
except (ValueError, IndexError):
|
|
226
|
+
pass
|
|
227
|
+
|
|
228
|
+
if process.returncode != 0:
|
|
229
|
+
raise TranscodingFailedError('FFmpeg process failed')
|
|
230
|
+
else:
|
|
231
|
+
# Run without progress monitoring
|
|
232
|
+
ffmpeg.run(stream, overwrite_output=True, quiet=True)
|
|
233
|
+
|
|
234
|
+
return output_path
|
|
235
|
+
|
|
236
|
+
except ffmpeg.Error as e:
|
|
237
|
+
error_message = e.stderr.decode('utf-8') if e.stderr else str(e)
|
|
238
|
+
raise TranscodingFailedError(f'Transcoding failed: {error_message}')
|
|
239
|
+
except Exception as e:
|
|
240
|
+
raise VideoTranscodeError(f'Unexpected error during transcoding: {str(e)}')
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def optimize_for_web(video_path: str | Path, output_path: str | Path) -> Path:
|
|
244
|
+
"""
|
|
245
|
+
Quick optimization for web streaming with default settings.
|
|
246
|
+
|
|
247
|
+
Args:
|
|
248
|
+
video_path (str | Path): Path to input video file
|
|
249
|
+
output_path (str | Path): Path to output video file
|
|
250
|
+
|
|
251
|
+
Returns:
|
|
252
|
+
Path: Path to the optimized video file
|
|
253
|
+
"""
|
|
254
|
+
config = TranscodeConfig(
|
|
255
|
+
preset='fast', # Faster encoding for web optimization
|
|
256
|
+
crf=23, # Better quality for web
|
|
257
|
+
movflags='+faststart+frag_keyframe+empty_moov', # Advanced web optimization
|
|
258
|
+
)
|
|
259
|
+
return transcode_video(video_path, output_path, config)
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
async def atranscode_video(
|
|
263
|
+
input_path: str | Path, output_path: str | Path, config: Optional[TranscodeConfig] = None
|
|
264
|
+
) -> Path:
|
|
265
|
+
"""
|
|
266
|
+
Async version of transcode_video.
|
|
267
|
+
|
|
268
|
+
Args:
|
|
269
|
+
input_path (str | Path): Path to input video file
|
|
270
|
+
output_path (str | Path): Path to output video file
|
|
271
|
+
config (Optional[TranscodeConfig]): Transcoding configuration
|
|
272
|
+
|
|
273
|
+
Returns:
|
|
274
|
+
Path: Path to the transcoded video file
|
|
275
|
+
"""
|
|
276
|
+
loop = asyncio.get_event_loop()
|
|
277
|
+
return await loop.run_in_executor(None, transcode_video, input_path, output_path, config)
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def transcode_batch(
|
|
281
|
+
video_paths: list[Path], output_dir: Path, config: Optional[TranscodeConfig] = None, max_workers: int = 4
|
|
282
|
+
) -> list[Path]:
|
|
283
|
+
"""
|
|
284
|
+
Process multiple videos concurrently.
|
|
285
|
+
|
|
286
|
+
Args:
|
|
287
|
+
video_paths (list[Path]): List of input video file paths
|
|
288
|
+
output_dir (Path): Directory for output files
|
|
289
|
+
config (Optional[TranscodeConfig]): Transcoding configuration
|
|
290
|
+
max_workers (int): Maximum number of concurrent workers
|
|
291
|
+
|
|
292
|
+
Returns:
|
|
293
|
+
list[Path]: List of paths to transcoded video files
|
|
294
|
+
"""
|
|
295
|
+
import concurrent.futures
|
|
296
|
+
|
|
297
|
+
output_dir = Path(output_dir)
|
|
298
|
+
output_dir.mkdir(parents=True, exist_ok=True)
|
|
299
|
+
|
|
300
|
+
def process_video(video_path):
|
|
301
|
+
output_path = output_dir / f'{video_path.stem}_transcoded.mp4'
|
|
302
|
+
return transcode_video(video_path, output_path, config)
|
|
303
|
+
|
|
304
|
+
with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
|
|
305
|
+
results = list(executor.map(process_video, video_paths))
|
|
306
|
+
|
|
307
|
+
return results
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: synapse-sdk
|
|
3
|
-
Version:
|
|
3
|
+
Version: 2025.9.1
|
|
4
4
|
Summary: synapse sdk
|
|
5
5
|
Author-email: datamaker <developer@datamaker.io>
|
|
6
6
|
License: MIT
|
|
@@ -23,8 +23,9 @@ Requires-Dist: fsspec[gcs,s3,sftp]
|
|
|
23
23
|
Requires-Dist: inquirer
|
|
24
24
|
Requires-Dist: pillow
|
|
25
25
|
Requires-Dist: websockets
|
|
26
|
+
Requires-Dist: ffmpeg-python==0.2.0
|
|
26
27
|
Provides-Extra: all
|
|
27
|
-
Requires-Dist: ray[all]==2.
|
|
28
|
+
Requires-Dist: ray[all]==2.49.1; extra == "all"
|
|
28
29
|
Requires-Dist: python-nmap; extra == "all"
|
|
29
30
|
Requires-Dist: hyperopt; extra == "all"
|
|
30
31
|
Requires-Dist: bayesian-optimization==1.4.3; extra == "all"
|
|
@@ -29,7 +29,7 @@ synapse_sdk/clients/exceptions.py,sha256=ylv7x10eOp4aA3a48jwonnvqvkiYwzJYXjkVkRT
|
|
|
29
29
|
synapse_sdk/clients/utils.py,sha256=8pPJTdzHiRPSbZMoQYHAgR2BAMO6u_R_jMV6a2p34iQ,392
|
|
30
30
|
synapse_sdk/clients/agent/__init__.py,sha256=FqYbtzMJdzRfuU2SA-Yxdc0JKmVP1wxH6OlUNmB4lH8,2230
|
|
31
31
|
synapse_sdk/clients/agent/core.py,sha256=aeMSzf8BF7LjVcmHaL8zC7ofBZUff8kIeqkW1xUJ6Sk,745
|
|
32
|
-
synapse_sdk/clients/agent/ray.py,sha256=
|
|
32
|
+
synapse_sdk/clients/agent/ray.py,sha256=xs-X5ObKPbDN23gmDxs2lhI8h6K6iocGk7PAN9hBTjc,13990
|
|
33
33
|
synapse_sdk/clients/agent/service.py,sha256=s7KuPK_DB1nr2VHrigttV1WyFonaGHNrPvU8loRxHcE,478
|
|
34
34
|
synapse_sdk/clients/backend/__init__.py,sha256=9FzjQn0ljRhtdaoG3n38Mdgte7GFwIh4OtEmoqVg2_E,2098
|
|
35
35
|
synapse_sdk/clients/backend/annotation.py,sha256=t9KnSdCJ7NvDRkAOKn4lm5hgoawbLCwIHMeo45QiIlQ,1249
|
|
@@ -53,7 +53,7 @@ synapse_sdk/devtools/docs/README.md,sha256=yBzWf0K1ef4oymFXDaHo0nYWEgMQJqsOyrkNh
|
|
|
53
53
|
synapse_sdk/devtools/docs/docusaurus.config.ts,sha256=K1b002RS0x0tsOyib_6CSgUlASEULC9vAX8YDpbsRN4,3229
|
|
54
54
|
synapse_sdk/devtools/docs/package-lock.json,sha256=dtepgbPC8gq5uz-hdcac4hIU-Cs209tX0sfBuB7RfEQ,705694
|
|
55
55
|
synapse_sdk/devtools/docs/package.json,sha256=8veqayA4U3OLpdQaz6AzB59RQwTU-5soeYlYYWIxq28,1187
|
|
56
|
-
synapse_sdk/devtools/docs/sidebars.ts,sha256=
|
|
56
|
+
synapse_sdk/devtools/docs/sidebars.ts,sha256=1Gwh74epym2d_ebV9bcz8Yg8fKJTq_Fk6T1eG-Kl4TM,1573
|
|
57
57
|
synapse_sdk/devtools/docs/tsconfig.json,sha256=O9BNlRPjPiaVHW2_boShMbmTnh0Z2k0KQO6Alf9FMVY,215
|
|
58
58
|
synapse_sdk/devtools/docs/blog/2019-05-28-first-blog-post.md,sha256=iP7gl_FPqo-qX13lkSRcRoT6ayJNmCkXoyvlm7GH248,312
|
|
59
59
|
synapse_sdk/devtools/docs/blog/2019-05-29-long-blog-post.md,sha256=cM-dhhTeurEWMcdn0Kx-NpNts2YUUraSI_XFk_gVHEE,3122
|
|
@@ -71,23 +71,23 @@ synapse_sdk/devtools/docs/docs/installation.md,sha256=TUXPQ9zhWW68WdimH85GihGMBG
|
|
|
71
71
|
synapse_sdk/devtools/docs/docs/introduction.md,sha256=qaZaU9E8raOm3Eqf7Zxpi9qG26gyS91OK1pWHsnPPMA,1581
|
|
72
72
|
synapse_sdk/devtools/docs/docs/quickstart.md,sha256=7h6JvNwkEuUuWE8cD3s0UhNeKG6U2_Svbq5XYhx_Zmo,1520
|
|
73
73
|
synapse_sdk/devtools/docs/docs/troubleshooting.md,sha256=vFxo_ejj-fIW-GgmhEHgCmgWUf8KxFD4xyYRKlaVe0M,11137
|
|
74
|
-
synapse_sdk/devtools/docs/docs/api/index.md,sha256=
|
|
74
|
+
synapse_sdk/devtools/docs/docs/api/index.md,sha256=l5EA65asRKZpPLKp1agn8jIQjzURzaOxlo2HNAzazio,1381
|
|
75
75
|
synapse_sdk/devtools/docs/docs/api/clients/agent.md,sha256=QbtdFQA-7rZhtnOh9tGSE9tKQQFeYGE7Z7RmxjVJzuw,786
|
|
76
76
|
synapse_sdk/devtools/docs/docs/api/clients/backend.md,sha256=j0NYx82rEiydlMtNQvqkWIXLVHjxckRaZOh-JiVr4LM,1453
|
|
77
77
|
synapse_sdk/devtools/docs/docs/api/clients/base.md,sha256=IxRGjYFI9KjZU3TFkjdWVdw9sgH40WAtOgIPJjORysc,720
|
|
78
|
-
synapse_sdk/devtools/docs/docs/api/clients/ray.md,sha256=
|
|
78
|
+
synapse_sdk/devtools/docs/docs/api/clients/ray.md,sha256=dnrqIT3MeCUnCH-9yUH59RSm46ZUtdRuncpfjFoPMXY,8089
|
|
79
79
|
synapse_sdk/devtools/docs/docs/api/plugins/categories.md,sha256=8zRHyyl3JQ2p0zKU86-nE-NDeWZLMwp0bh8ZF-PSx9M,839
|
|
80
80
|
synapse_sdk/devtools/docs/docs/api/plugins/models.md,sha256=XRMTD2DBilZwmO6AmPhLgdzNfBpcFkAdrBg9mgOMTUs,1060
|
|
81
81
|
synapse_sdk/devtools/docs/docs/api/plugins/utils.md,sha256=39soJ1JLROm_mygR9UWG6XqZ1i5ESmTeZGijrAfUh-o,8389
|
|
82
|
-
synapse_sdk/devtools/docs/docs/api/utils/file.md,sha256=wmYX9KUIWupMpSUKATLtUM9UetsNMAc51iK97xlLYZM,5596
|
|
83
|
-
synapse_sdk/devtools/docs/docs/api/utils/network.md,sha256=1FMLDjJkZVF3Ni1kjbtEP6shnpWM4GMLxWy1WHnveqg,9727
|
|
84
|
-
synapse_sdk/devtools/docs/docs/api/utils/storage.md,sha256=uIpc37trKUm4XHe1fd4auw56vMq3DvAMjJveSfN4rqU,1000
|
|
85
|
-
synapse_sdk/devtools/docs/docs/api/utils/types.md,sha256=l84J1Ooa40xBYdvK1YpeKBT9kaqaWQUMNIFPUiC2e_U,1046
|
|
86
82
|
synapse_sdk/devtools/docs/docs/concepts/index.md,sha256=ezn67P568PvFRPVE4T74j5MuEikgVOX7DV6MxIQBOHo,760
|
|
87
83
|
synapse_sdk/devtools/docs/docs/examples/index.md,sha256=IZnF8veO7PqJYtAQevL1VHt1rBzieCSaXoWTQoXS4lA,504
|
|
88
84
|
synapse_sdk/devtools/docs/docs/features/index.md,sha256=FD2hUzTzFgKa5pcDWPtA6eVw9pLLiRajYi4xA8aJTHg,963
|
|
89
85
|
synapse_sdk/devtools/docs/docs/features/converters/index.md,sha256=W5d5UO8nbOLWs1G9IgdTutlMl-d6M0hDLMJbg8DrCQ8,12713
|
|
90
|
-
synapse_sdk/devtools/docs/docs/
|
|
86
|
+
synapse_sdk/devtools/docs/docs/features/utils/file.md,sha256=H4vzxZiJ0X-US8OCfgfvUeXTp56BTg7NnmXU5wSTfb0,11869
|
|
87
|
+
synapse_sdk/devtools/docs/docs/features/utils/network.md,sha256=IOYMhC7bM6BwpvaJosj9vxHXr0Vc5_qP09HP8OVuM64,9734
|
|
88
|
+
synapse_sdk/devtools/docs/docs/features/utils/storage.md,sha256=uIpc37trKUm4XHe1fd4auw56vMq3DvAMjJveSfN4rqU,1000
|
|
89
|
+
synapse_sdk/devtools/docs/docs/features/utils/types.md,sha256=l84J1Ooa40xBYdvK1YpeKBT9kaqaWQUMNIFPUiC2e_U,1046
|
|
90
|
+
synapse_sdk/devtools/docs/docs/plugins/export-plugins.md,sha256=GDdAxcFyxoy96pMkGmodUXrKTxfFerk1Dt10QNHXTNA,32103
|
|
91
91
|
synapse_sdk/devtools/docs/docs/plugins/plugins.md,sha256=2WMjbTKjIXTAdeODeTvJkDSsR9PiA58_VnCFuBnWRw4,24358
|
|
92
92
|
synapse_sdk/devtools/docs/docs/plugins/upload-plugins.md,sha256=6-IE6QNimgggIJF5BxeGBUhKfV76WPglGH1-olU_R8Q,18489
|
|
93
93
|
synapse_sdk/devtools/docs/docs/tutorial-basics/_category_.json,sha256=qQVHZ1p0CxHg5Gb4CYNxUSeqZ3LVo4nXN_N0yUMVpDM,180
|
|
@@ -113,21 +113,21 @@ synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/install
|
|
|
113
113
|
synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/introduction.md,sha256=TE1XUd4axGRqodFduAUxAI0h72PDToWWE2WeTz-D0bg,1646
|
|
114
114
|
synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/quickstart.md,sha256=SjQD0q6L16zIwk6LE7HDXlbZdcnZbnwSmGZz-0KWiL8,1683
|
|
115
115
|
synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/troubleshooting.md,sha256=OvXUtf_CkQlCZ6PzTSUM_YdWFTknEkgvttO8Ywy1qTo,11793
|
|
116
|
-
synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/index.md,sha256=
|
|
116
|
+
synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/index.md,sha256=60fhoQsnjuhdjFVVKCNd2qu6cY8f9A4PeZNx08obcdo,1524
|
|
117
117
|
synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/clients/agent.md,sha256=UVllVNexQYmiHAk6dVVvd0MPQt6S9vBcK5Ur-BeLP2M,827
|
|
118
118
|
synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/clients/backend.md,sha256=3XUZv_LYBuFfmY60t-0ZNeH4U61EzHrmI9ggfs7vS_8,1505
|
|
119
119
|
synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/clients/base.md,sha256=cn8B5Py1TQjQc3a1B8wT_UuOGmb9eCk2IrKWSZnPlmA,804
|
|
120
|
-
synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/clients/ray.md,sha256=
|
|
121
|
-
synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/utils/file.md,sha256=mPfds1ChrMR2yTcLTbBSQHSI7wit_76dI4_m6JIyebY,6107
|
|
122
|
-
synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/utils/network.md,sha256=pIboIn7vB7VZbtvtFvJT-9rnP5fHu2kNVvZKbZ3i3wk,10341
|
|
123
|
-
synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/utils/storage.md,sha256=V-IIUSVJjwh9n6KVZI5DxrU_T0l3AOtcCrAC2PRs43Q,1048
|
|
124
|
-
synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/utils/types.md,sha256=JdDMC2FKovAU55pPEBoIUPPniU5mKAvA5w1Zb36u9AE,1207
|
|
120
|
+
synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/api/clients/ray.md,sha256=m7yxYeDMnRdYPrzTPuSk9EHuDRjEMZRY6q-ty4-HmGs,8931
|
|
125
121
|
synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/concepts/index.md,sha256=6o-8H7khfiMhQSXBAvkYpf6jzOEndPje1QAGdlGfskw,843
|
|
126
122
|
synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/examples/index.md,sha256=pMAc1VLUXnHCv2xvvx-GBqVQ1Uh4h-u4nrco5UZ2z-Y,613
|
|
127
|
-
synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/features/index.md,sha256=
|
|
123
|
+
synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/features/index.md,sha256=X0haPkqUY89K5R3LiGLQX9Boh_BpLxUsHA13skTTAp8,1015
|
|
128
124
|
synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/features/converters/index.md,sha256=WpXcLBqC-xlvuoh-AhNOtFSo4qmnBIbR_wqLlazT7zo,1366
|
|
129
|
-
synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/
|
|
130
|
-
synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/
|
|
125
|
+
synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/features/utils/file.md,sha256=1H-gv5w55BDjANFDrT7yIx2o-xM4g0E4At7hXFTCX5s,12546
|
|
126
|
+
synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/features/utils/network.md,sha256=_jYDA25anaQ7FdfJmapACjqDvw3D2J8wj0RWCnXXpLY,10348
|
|
127
|
+
synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/features/utils/storage.md,sha256=V-IIUSVJjwh9n6KVZI5DxrU_T0l3AOtcCrAC2PRs43Q,1048
|
|
128
|
+
synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/features/utils/types.md,sha256=JdDMC2FKovAU55pPEBoIUPPniU5mKAvA5w1Zb36u9AE,1207
|
|
129
|
+
synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/export-plugins.md,sha256=DSnhvWk54V6qlMXaeaiwgTJ9l0DVzkc36p0rOyqBjlE,33567
|
|
130
|
+
synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/plugins.md,sha256=wUuuOmQQY1dURZzJ0HSpiABaJWqKbxQn31Xfn-wUf8E,4947
|
|
131
131
|
synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/upload-plugins.md,sha256=E-xplOw6ZEgB0rtarshmSqRIFpYL12Ax5mXULI4VWXo,24740
|
|
132
132
|
synapse_sdk/devtools/docs/i18n/ko/docusaurus-theme-classic/footer.json,sha256=SCLmysIhJ6XEEfnL7cOQb3s08EG4LA-ColdRRxecIH8,1601
|
|
133
133
|
synapse_sdk/devtools/docs/i18n/ko/docusaurus-theme-classic/navbar.json,sha256=M-ly81cWnesYYtCRcfMvOsKl5P0z52K9kiw0T6GIQ60,429
|
|
@@ -185,7 +185,7 @@ synapse_sdk/plugins/categories/export/enums.py,sha256=gtyngvQ1DKkos9iKGcbecwTVQQ
|
|
|
185
185
|
synapse_sdk/plugins/categories/export/actions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
186
186
|
synapse_sdk/plugins/categories/export/actions/export.py,sha256=BO8N1LT8X9ImDmsqMdrbCUePsMEtOhazCuVtwp_ZmXs,13706
|
|
187
187
|
synapse_sdk/plugins/categories/export/templates/config.yaml,sha256=fAqi-9dH7qEkFDSDFvt0BcZ72d2qbyt1P83L2ti6ELM,58
|
|
188
|
-
synapse_sdk/plugins/categories/export/templates/plugin/__init__.py,sha256=
|
|
188
|
+
synapse_sdk/plugins/categories/export/templates/plugin/__init__.py,sha256=9pEo-TUCDugi-UVL5x9iz8gnMqGabsDTaKIm-SeQf7s,16042
|
|
189
189
|
synapse_sdk/plugins/categories/export/templates/plugin/export.py,sha256=3dZUEdhj-4YySJ-cPyNAqQvj5xcAXIX4KmXKcacTtd0,2754
|
|
190
190
|
synapse_sdk/plugins/categories/neural_net/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
191
191
|
synapse_sdk/plugins/categories/neural_net/actions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -257,7 +257,7 @@ synapse_sdk/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU
|
|
|
257
257
|
synapse_sdk/utils/dataset.py,sha256=zWTzFmv589izFr62BDuApi3r5FpTsdm-5AmriC0AEdM,1865
|
|
258
258
|
synapse_sdk/utils/debug.py,sha256=F7JlUwYjTFZAMRbBqKm6hxOIz-_IXYA8lBInOS4jbS4,100
|
|
259
259
|
synapse_sdk/utils/encryption.py,sha256=KMARrAk5aIHfBLC8CvdXiSIuaGvxljluubjF9PVLf7c,5100
|
|
260
|
-
synapse_sdk/utils/file.py,sha256=lDyvSdw8_lV1mIwBWzO0DiCjPcv0BgC4qJNsr-ymbDQ,9162
|
|
260
|
+
synapse_sdk/utils/file.py.backup,sha256=lDyvSdw8_lV1mIwBWzO0DiCjPcv0BgC4qJNsr-ymbDQ,9162
|
|
261
261
|
synapse_sdk/utils/http.py,sha256=yRxYfru8tMnBVeBK-7S0Ga13yOf8oRHquG5e8K_FWcI,4759
|
|
262
262
|
synapse_sdk/utils/module_loading.py,sha256=chHpU-BZjtYaTBD_q0T7LcKWtqKvYBS4L0lPlKkoMQ8,1020
|
|
263
263
|
synapse_sdk/utils/network.py,sha256=4zWUbuEPcMXb_NVVeuUJ8zSl10ZAlWVHfzqbR25aez8,11978
|
|
@@ -275,6 +275,15 @@ synapse_sdk/utils/converters/pascal/to_dm.py,sha256=bQNUepahOCot4J23LCPOlFOhIZJ8
|
|
|
275
275
|
synapse_sdk/utils/converters/yolo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
276
276
|
synapse_sdk/utils/converters/yolo/from_dm.py,sha256=-JDCQLk4g1_FIVoOwZ1Tcs2kWFkhXRCAPVLKLXz6sLU,16180
|
|
277
277
|
synapse_sdk/utils/converters/yolo/to_dm.py,sha256=XcSkyPickvdPR2JZ-PMMPEf1-uWQL76_pO78-4aPYcQ,10614
|
|
278
|
+
synapse_sdk/utils/file/__init__.py,sha256=1FFrQVddZ8GgjcXkpWh4YcKLdzzCIB0p8bGmE3PPE5o,967
|
|
279
|
+
synapse_sdk/utils/file/archive.py,sha256=Wdn4f7vkqePNCE2JPMzZnyIfLrQ33LuPRvEGwB4Ieg4,1107
|
|
280
|
+
synapse_sdk/utils/file/checksum.py,sha256=9TArSECBHVwb0-NmtlJyNjZXXXMMOtwGxXW1p350B1g,1570
|
|
281
|
+
synapse_sdk/utils/file/chunking.py,sha256=IC1FPMqRwtuDFTZOZDvhCuUfo2LBMIC4Tf-LeLB0A1Q,1088
|
|
282
|
+
synapse_sdk/utils/file/download.py,sha256=Zsd0ik7ajRexwcRoQvx7AlZbe4uD9hhurZYs3eGw_1s,3931
|
|
283
|
+
synapse_sdk/utils/file/encoding.py,sha256=wcmWJuqhOnO94Umz03z_7znhtKwgpKIK9CsEagNdIwc,1124
|
|
284
|
+
synapse_sdk/utils/file/io.py,sha256=fl_PvRrdnAkGMmxcDth6nshudv94nalibkPOaI7KIOk,447
|
|
285
|
+
synapse_sdk/utils/file/video/__init__.py,sha256=mlmTtmx22-XZmrf5acPTOo2Ra2MMtVxUkxkDdQht4UE,614
|
|
286
|
+
synapse_sdk/utils/file/video/transcode.py,sha256=KKn2Mi9rP2ew33q4P-cn9O5JSVQODR5rg1FDPU6fjxo,10375
|
|
278
287
|
synapse_sdk/utils/pydantic/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
279
288
|
synapse_sdk/utils/pydantic/config.py,sha256=1vYOcUI35GslfD1rrqhFkNXXJOXt4IDqOPSx9VWGfNE,123
|
|
280
289
|
synapse_sdk/utils/pydantic/errors.py,sha256=0v0T12eQBr1KrFiEOBu6KMaPK4aPEGEC6etPJGoR5b4,1061
|
|
@@ -287,9 +296,9 @@ synapse_sdk/utils/storage/providers/gcp.py,sha256=i2BQCu1Kej1If9SuNr2_lEyTcr5M_n
|
|
|
287
296
|
synapse_sdk/utils/storage/providers/http.py,sha256=2DhIulND47JOnS5ZY7MZUex7Su3peAPksGo1Wwg07L4,5828
|
|
288
297
|
synapse_sdk/utils/storage/providers/s3.py,sha256=ZmqekAvIgcQBdRU-QVJYv1Rlp6VHfXwtbtjTSphua94,2573
|
|
289
298
|
synapse_sdk/utils/storage/providers/sftp.py,sha256=_8s9hf0JXIO21gvm-JVS00FbLsbtvly4c-ETLRax68A,1426
|
|
290
|
-
synapse_sdk-
|
|
291
|
-
synapse_sdk-
|
|
292
|
-
synapse_sdk-
|
|
293
|
-
synapse_sdk-
|
|
294
|
-
synapse_sdk-
|
|
295
|
-
synapse_sdk-
|
|
299
|
+
synapse_sdk-2025.9.1.dist-info/licenses/LICENSE,sha256=bKzmC5YAg4V1Fhl8OO_tqY8j62hgdncAkN7VrdjmrGk,1101
|
|
300
|
+
synapse_sdk-2025.9.1.dist-info/METADATA,sha256=ZIeAof3DTKvJG72NhD8WlTJXhHpkXAZexGaH0i5lFrM,3781
|
|
301
|
+
synapse_sdk-2025.9.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
302
|
+
synapse_sdk-2025.9.1.dist-info/entry_points.txt,sha256=VNptJoGoNJI8yLXfBmhgUefMsmGI0m3-0YoMvrOgbxo,48
|
|
303
|
+
synapse_sdk-2025.9.1.dist-info/top_level.txt,sha256=ytgJMRK1slVOKUpgcw3LEyHHP7S34J6n_gJzdkcSsw8,12
|
|
304
|
+
synapse_sdk-2025.9.1.dist-info/RECORD,,
|
|
@@ -1,195 +0,0 @@
|
|
|
1
|
-
---
|
|
2
|
-
id: file
|
|
3
|
-
title: File Utilities
|
|
4
|
-
sidebar_position: 1
|
|
5
|
-
---
|
|
6
|
-
|
|
7
|
-
# File Utilities
|
|
8
|
-
|
|
9
|
-
File operations and handling utilities.
|
|
10
|
-
|
|
11
|
-
## File Operations
|
|
12
|
-
|
|
13
|
-
### Archive Functions
|
|
14
|
-
|
|
15
|
-
Functions for creating and extracting plugin archives.
|
|
16
|
-
|
|
17
|
-
### Download Functions
|
|
18
|
-
|
|
19
|
-
Utilities for downloading files from URLs.
|
|
20
|
-
|
|
21
|
-
```python
|
|
22
|
-
from synapse_sdk.utils.file import download_file
|
|
23
|
-
|
|
24
|
-
local_path = download_file(url, destination)
|
|
25
|
-
```
|
|
26
|
-
|
|
27
|
-
### Upload Functions
|
|
28
|
-
|
|
29
|
-
File upload utilities with chunked upload support.
|
|
30
|
-
|
|
31
|
-
## Chunked File Operations
|
|
32
|
-
|
|
33
|
-
### read_file_in_chunks
|
|
34
|
-
|
|
35
|
-
Read files in chunks for efficient memory usage, particularly useful for large files or when processing files in chunks for uploading or hashing.
|
|
36
|
-
|
|
37
|
-
```python
|
|
38
|
-
from synapse_sdk.utils.file import read_file_in_chunks
|
|
39
|
-
|
|
40
|
-
# Read a file in default 50MB chunks
|
|
41
|
-
for chunk in read_file_in_chunks('/path/to/large_file.bin'):
|
|
42
|
-
process_chunk(chunk)
|
|
43
|
-
|
|
44
|
-
# Read with custom chunk size (10MB)
|
|
45
|
-
for chunk in read_file_in_chunks('/path/to/file.bin', chunk_size=1024*1024*10):
|
|
46
|
-
upload_chunk(chunk)
|
|
47
|
-
```
|
|
48
|
-
|
|
49
|
-
**Parameters:**
|
|
50
|
-
|
|
51
|
-
- `file_path` (str | Path): Path to the file to read
|
|
52
|
-
- `chunk_size` (int, optional): Size of each chunk in bytes. Defaults to 50MB (52,428,800 bytes)
|
|
53
|
-
|
|
54
|
-
**Returns:**
|
|
55
|
-
|
|
56
|
-
- Generator yielding file content chunks as bytes
|
|
57
|
-
|
|
58
|
-
**Raises:**
|
|
59
|
-
|
|
60
|
-
- `FileNotFoundError`: If the file doesn't exist
|
|
61
|
-
- `PermissionError`: If the file can't be read due to permissions
|
|
62
|
-
- `OSError`: If there's an OS-level error reading the file
|
|
63
|
-
|
|
64
|
-
### Use Cases
|
|
65
|
-
|
|
66
|
-
**Large File Processing**: Efficiently process files that are too large to fit in memory:
|
|
67
|
-
|
|
68
|
-
```python
|
|
69
|
-
import hashlib
|
|
70
|
-
|
|
71
|
-
def calculate_hash_for_large_file(file_path):
|
|
72
|
-
hash_md5 = hashlib.md5()
|
|
73
|
-
for chunk in read_file_in_chunks(file_path):
|
|
74
|
-
hash_md5.update(chunk)
|
|
75
|
-
return hash_md5.hexdigest()
|
|
76
|
-
```
|
|
77
|
-
|
|
78
|
-
**Chunked Upload Integration**: The function integrates seamlessly with the `CoreClientMixin.create_chunked_upload` method:
|
|
79
|
-
|
|
80
|
-
```python
|
|
81
|
-
from synapse_sdk.clients.backend.core import CoreClientMixin
|
|
82
|
-
|
|
83
|
-
client = CoreClientMixin(base_url='https://api.example.com')
|
|
84
|
-
result = client.create_chunked_upload('/path/to/large_file.zip')
|
|
85
|
-
```
|
|
86
|
-
|
|
87
|
-
**Best Practices:**
|
|
88
|
-
|
|
89
|
-
- Use default chunk size (50MB) for optimal upload performance
|
|
90
|
-
- Adjust chunk size based on available memory and network conditions
|
|
91
|
-
- For very large files (>1GB), consider using smaller chunks for better progress tracking
|
|
92
|
-
- Always handle exceptions when working with file operations
|
|
93
|
-
|
|
94
|
-
## Checksum Functions
|
|
95
|
-
|
|
96
|
-
### get_checksum_from_file
|
|
97
|
-
|
|
98
|
-
Calculate checksum for file-like objects without requiring Django dependencies. This function works with any file-like object that has a `read()` method, making it compatible with Django's File objects, BytesIO, StringIO, and regular file objects.
|
|
99
|
-
|
|
100
|
-
```python
|
|
101
|
-
import hashlib
|
|
102
|
-
from io import BytesIO
|
|
103
|
-
from synapse_sdk.utils.file import get_checksum_from_file
|
|
104
|
-
|
|
105
|
-
# Basic usage with BytesIO (defaults to SHA1)
|
|
106
|
-
data = BytesIO(b'Hello, world!')
|
|
107
|
-
checksum = get_checksum_from_file(data)
|
|
108
|
-
print(checksum) # SHA1 hash as hexadecimal string
|
|
109
|
-
|
|
110
|
-
# Using different hash algorithms
|
|
111
|
-
checksum_md5 = get_checksum_from_file(data, digest_mod=hashlib.md5)
|
|
112
|
-
checksum_sha256 = get_checksum_from_file(data, digest_mod=hashlib.sha256)
|
|
113
|
-
|
|
114
|
-
# With real file objects
|
|
115
|
-
with open('/path/to/file.txt', 'rb') as f:
|
|
116
|
-
checksum = get_checksum_from_file(f)
|
|
117
|
-
|
|
118
|
-
# With StringIO (text files)
|
|
119
|
-
from io import StringIO
|
|
120
|
-
text_data = StringIO('Hello, world!')
|
|
121
|
-
checksum = get_checksum_from_file(text_data) # Automatically UTF-8 encoded
|
|
122
|
-
```
|
|
123
|
-
|
|
124
|
-
**Parameters:**
|
|
125
|
-
|
|
126
|
-
- `file` (IO[Any]): File-like object with read() method that supports reading in chunks
|
|
127
|
-
- `digest_mod` (Callable[[], Any], optional): Hash algorithm from hashlib. Defaults to `hashlib.sha1`
|
|
128
|
-
|
|
129
|
-
**Returns:**
|
|
130
|
-
|
|
131
|
-
- `str`: Hexadecimal digest of the file contents
|
|
132
|
-
|
|
133
|
-
**Key Features:**
|
|
134
|
-
|
|
135
|
-
- **Memory Efficient**: Reads files in 4KB chunks to handle large files
|
|
136
|
-
- **Automatic File Pointer Reset**: Resets to beginning if the file object supports seeking
|
|
137
|
-
- **Text/Binary Agnostic**: Handles both text (StringIO) and binary (BytesIO) file objects
|
|
138
|
-
- **No Django Dependency**: Works without Django while being compatible with Django File objects
|
|
139
|
-
- **Flexible Hash Algorithms**: Supports any hashlib algorithm (SHA1, SHA256, MD5, etc.)
|
|
140
|
-
|
|
141
|
-
**Use Cases:**
|
|
142
|
-
|
|
143
|
-
**Django File Object Compatibility**: Works with Django's File objects without requiring Django:
|
|
144
|
-
|
|
145
|
-
```python
|
|
146
|
-
# Simulating Django File-like behavior
|
|
147
|
-
class FileWrapper:
|
|
148
|
-
def __init__(self, data):
|
|
149
|
-
self._data = data
|
|
150
|
-
self._pos = 0
|
|
151
|
-
|
|
152
|
-
def read(self, size=None):
|
|
153
|
-
if size is None:
|
|
154
|
-
result = self._data[self._pos:]
|
|
155
|
-
self._pos = len(self._data)
|
|
156
|
-
else:
|
|
157
|
-
result = self._data[self._pos:self._pos + size]
|
|
158
|
-
self._pos += len(result)
|
|
159
|
-
return result
|
|
160
|
-
|
|
161
|
-
file_obj = FileWrapper(b'File content')
|
|
162
|
-
checksum = get_checksum_from_file(file_obj)
|
|
163
|
-
```
|
|
164
|
-
|
|
165
|
-
**Large File Processing**: Efficiently calculate checksums for large files:
|
|
166
|
-
|
|
167
|
-
```python
|
|
168
|
-
# Large file processing with memory efficiency
|
|
169
|
-
with open('/path/to/large_file.bin', 'rb') as large_file:
|
|
170
|
-
checksum = get_checksum_from_file(large_file, digest_mod=hashlib.sha256)
|
|
171
|
-
```
|
|
172
|
-
|
|
173
|
-
**Multiple Hash Algorithms**: Calculate different checksums for the same file:
|
|
174
|
-
|
|
175
|
-
```python
|
|
176
|
-
algorithms = [
|
|
177
|
-
('MD5', hashlib.md5),
|
|
178
|
-
('SHA1', hashlib.sha1),
|
|
179
|
-
('SHA256', hashlib.sha256),
|
|
180
|
-
]
|
|
181
|
-
|
|
182
|
-
with open('/path/to/file.bin', 'rb') as f:
|
|
183
|
-
checksums = {}
|
|
184
|
-
for name, algo in algorithms:
|
|
185
|
-
f.seek(0) # Reset file pointer
|
|
186
|
-
checksums[name] = get_checksum_from_file(f, digest_mod=algo)
|
|
187
|
-
```
|
|
188
|
-
|
|
189
|
-
## Path Utilities
|
|
190
|
-
|
|
191
|
-
Functions for path manipulation and validation.
|
|
192
|
-
|
|
193
|
-
## Temporary Files
|
|
194
|
-
|
|
195
|
-
Utilities for managing temporary files and cleanup.
|