synapse-sdk 1.0.0b23__py3-none-any.whl → 2025.9.1__py3-none-any.whl

synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/export-plugins.md

@@ -213,6 +213,7 @@ class Exporter(BaseExporter):
 - **before_convert()**: 변환 전 데이터 전처리
 - **after_convert()**: 변환 후 데이터 후처리
 - **process_file_saving()**: 사용자 정의 파일 저장 로직
+- **additional_file_saving()**: 모든 export 항목 처리 후 추가 파일 저장
 
 ### 헬퍼 메서드
 
@@ -225,6 +226,143 @@ class Exporter(BaseExporter):
 - `self.run.log_message()` 및 기타 run 메서드를 통한 로깅
 - run 메서드를 통한 오류 처리 및 메트릭 수집
 
+## 추가 파일 저장 (Additional File Saving)
+
+`additional_file_saving()` 메서드는 모든 export 항목이 처리된 후에 호출되며, 모든 처리된 항목의 집합적 데이터에 의존하는 파일을 저장하기 위해 설계되었습니다. 다음과 같은 용도로 유용합니다:
+
+- 메타데이터 파일 (예: 데이터셋 통계, 클래스 매핑)
+- 설정 파일 (예: YOLO용 dataset.yaml, classes.txt)
+- 요약 파일 (예: export 보고서, 처리 로그)
+- 인덱스 파일 (예: 파일 목록, 디렉토리 구조)
+
+### 메서드 시그니처
+
+```python
+def additional_file_saving(self, unique_export_path):
+    """모든 export 항목 처리 후 추가 파일 저장.
+    
+    이 메서드는 주 export 루프가 완료된 후 호출되며, 모든 처리된 export 항목의
+    집합적 데이터를 기반으로 생성되어야 하는 파일들(예: 메타데이터 파일, 
+    설정 파일, 요약 파일 등)을 저장하기 위한 것입니다.
+    
+    Args:
+        unique_export_path (str): 추가 파일이 저장될 고유한 export 디렉토리 경로.
+    """
+    pass
+```
+
+### 사용 예시
+
+```python
+class YOLOExporter(BaseExporter):
+    def __init__(self, run, export_items, path_root, **params):
+        super().__init__(run, export_items, path_root, **params)
+        self.class_names = set()
+        self.dataset_stats = {
+            'total_images': 0,
+            'total_annotations': 0,
+            'class_distribution': {}
+        }
+    
+    def convert_data(self, data):
+        # 변환 중 클래스와 통계 추적
+        for annotation in data.get('annotations', []):
+            class_name = annotation['class_name']
+            self.class_names.add(class_name)
+            self.dataset_stats['class_distribution'][class_name] = \
+                self.dataset_stats['class_distribution'].get(class_name, 0) + 1
+        
+        self.dataset_stats['total_images'] += 1
+        self.dataset_stats['total_annotations'] += len(data.get('annotations', []))
+        
+        return data  # ... 나머지 변환 로직
+    
+    def additional_file_saving(self, unique_export_path):
+        """YOLO 설정 및 메타데이터 파일 저장."""
+        data_dir = Path(unique_export_path) / 'data'
+        data_dir.mkdir(exist_ok=True)
+        
+        # 1. classes.txt 파일 저장
+        classes_file = data_dir / 'classes.txt'
+        with classes_file.open('w') as f:
+            for class_name in sorted(self.class_names):
+                f.write(f"{class_name}\n")
+        self.run.log_message(f"클래스 파일 저장: {classes_file}")
+        
+        # 2. dataset.yaml 파일 저장
+        dataset_config = {
+            'path': str(unique_export_path),
+            'train': 'images',
+            'val': 'images',
+            'names': {i: name for i, name in enumerate(sorted(self.class_names))}
+        }
+        
+        dataset_file = data_dir / 'dataset.yaml'
+        with dataset_file.open('w') as f:
+            yaml.dump(dataset_config, f, default_flow_style=False)
+        self.run.log_message(f"데이터셋 설정 저장: {dataset_file}")
+        
+        # 3. export 통계 저장
+        stats_file = data_dir / 'export_stats.json'
+        with stats_file.open('w') as f:
+            json.dump(self.dataset_stats, f, indent=2)
+        self.run.log_message(f"export 통계 저장: {stats_file}")
+```
+
+### 일반적인 사용 사례
+
+#### 1. 데이터셋 설정 파일
+```python
+def additional_file_saving(self, unique_export_path):
+    # 훈련 프레임워크용 데이터셋 설정 생성
+    config = {
+        'dataset_name': self.params.get('name'),
+        'created_at': datetime.now().isoformat(),
+        'total_samples': len(self.processed_items),
+        'classes': list(self.class_mapping.keys())
+    }
+    
+    config_file = Path(unique_export_path) / 'dataset_config.json'
+    with config_file.open('w') as f:
+        json.dump(config, f, indent=2)
+```
+
+#### 2. Export 요약 보고서
+```python
+def additional_file_saving(self, unique_export_path):
+    # export 요약 생성
+    summary = {
+        'export_info': {
+            'plugin_name': self.__class__.__name__,
+            'export_time': datetime.now().isoformat(),
+            'export_path': str(unique_export_path)
+        },
+        'statistics': self.get_export_statistics(),
+        'errors': self.get_error_summary()
+    }
+    
+    summary_file = Path(unique_export_path) / 'export_summary.json'
+    with summary_file.open('w') as f:
+        json.dump(summary, f, indent=2)
+```
+
+#### 3. 인덱스 및 매니페스트 파일
+```python
+def additional_file_saving(self, unique_export_path):
+    # 처리된 항목들에 대한 파일 인덱스 생성
+    file_index = []
+    for item in self.processed_items:
+        file_index.append({
+            'original_file': item['original_filename'],
+            'json_file': f"{item['stem']}.json",
+            'processed_at': item['timestamp']
+        })
+    
+    index_file = Path(unique_export_path) / 'file_index.json'
+    with index_file.open('w') as f:
+        json.dump(file_index, f, indent=2)
+```
+
 ## 주요 특징
 
 - **진행률 추적**: `run.set_progress()`로 내장 진행률 모니터링

synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/plugins.md

@@ -67,5 +67,51 @@ Synapse 플랫폼에서 주석이 달린 데이터, 그라운드 트루스 데
 - `ground_truth` - 그라운드 트루스 데이터셋 버전 내보내기
 - `task` - 관련 주석이 있는 작업 데이터 내보내기
 
-Export 플러그인에 대한 자세한 정보, BaseExporter 클래스 구조, 구현 예시 및 모범 사례는 [Export 플러그인](../export-plugins) 문서를 참조하세요.
-
+Export 플러그인에 대한 자세한 정보, BaseExporter 클래스 구조, 구현 예시 및 모범 사례는 [Export 플러그인](./export-plugins) 문서를 참조하세요.
+
+## 플러그인 구성
+
+플러그인은 YAML 구성 파일을 사용하여 구성되며, 이 파일은 메타데이터, 종속성, 패키지 관리 옵션 및 액션을 정의합니다.
+
+### 패키지 관리
+
+```yaml
+# 플러그인 메타데이터 (필수)
+name: "my-ml-plugin"          # 필수: 플러그인 이름
+version: "1.0.0"              # 필수: 버전
+category: "neural_net"        # 필수: 카테고리
+description: "사용자 정의 ML 플러그인"  # 필수: 설명
+
+# 패키지 관리 (선택사항)
+package_manager: "pip"        # 선택사항: "pip" 또는 "uv" (기본값: "pip")
+
+# 패키지 매니저 옵션 (선택사항)
+# uv만 지원 pip_install_options: 지원 예정
+# uv의 경우 기본값은 ['--no-cache']입니다
+package_manager_options: ["--no-cache", "--quiet"]  # 선택사항: 설치 옵션
+
+# 액션 정의 (필수)
+actions:                      # 필수: 최소 하나의 액션 필요
+  train:
+    entrypoint: "plugin.train.TrainAction"  # 필수
+    method: "job"             # 필수: 실행 방법
+```
+
+**구성 옵션 설명:**
+
+**필수 필드:**
+- `name`: 플러그인 이름
+- `version`: 플러그인 버전
+- `category`: 플러그인 카테고리 (`neural_net`, `export` 등)
+- `description`: 플러그인 설명
+- `actions`: 플러그인 액션 정의 (최소 하나 필요)
+  - `entrypoint`: 액션의 진입점 클래스
+  - `method`: 실행 방법 (`job`, `task`, `serve` 등)
+
+**선택사항 필드:**
+- `package_manager`: 종속성에 사용할 패키지 매니저 (기본값: `"pip"`)
+  - 지원 값: `"pip"` 또는 `"uv"`
+- `package_manager_options`: 패키지 설치 명령에 대한 사용자 제공 옵션
+  - 현재 Ray 2.44.1에서는 `uv`만 지원됩니다
+  - `uv`의 경우 기본값은 `["--no-cache"]`입니다
+  - 사용자 정의 옵션이 기본값과 병합됩니다

synapse_sdk/devtools/docs/sidebars.ts

@@ -25,6 +25,16 @@ const sidebars: SidebarsConfig = {
         'features/converters/converters',
       ],
     },
+    {
+      type: 'category',
+      label: 'Utilities',
+      items: [
+        'features/utils/file',
+        'features/utils/network',
+        'features/utils/storage',
+        'features/utils/types',
+      ],
+    },
     {
       type: 'category',
       label: 'Plugin System',
@@ -49,16 +59,6 @@ const sidebars: SidebarsConfig = {
             'api/clients/base',
           ],
         },
-        {
-          type: 'category',
-          label: 'Utilities',
-          items: [
-            'api/utils/file',
-            'api/utils/network',
-            'api/utils/storage',
-            'api/utils/types',
-          ],
-        },
       ],
     },
     'configuration',

synapse_sdk/plugins/categories/export/templates/plugin/__init__.py

@@ -306,6 +306,20 @@ class BaseExporter:
 
         return True
 
+    def additional_file_saving(self, unique_export_path):
+        """Save additional files after processing all export items.
+
+        This method is called after the main export loop completes and is intended
+        for saving files that need to be created based on the collective data from
+        all processed export items (e.g., metadata files, configuration files,
+        summary files, etc.).
+
+        Args:
+            unique_export_path (str): The unique export directory path where
+                additional files should be saved.
+        """
+        pass
+
     def export(self, export_items=None, results=None, **_kwargs) -> dict:
         """Main export method that can be overridden by subclasses for custom logic.
 
@@ -340,8 +354,8 @@ class BaseExporter:
 
         total = self.params['count']
 
-        original_file_metrics_record = self.run.MetricsRecord(stand_by=total)
-        data_file_metrics_record = self.run.MetricsRecord(stand_by=total)
+        original_file_metrics_record = self.run.MetricsRecord(stand_by=total, success=0, failed=0)
+        data_file_metrics_record = self.run.MetricsRecord(stand_by=total, success=0, failed=0)
 
         # progress init
         self.run.set_progress(0, total, category='dataset_conversion')
@@ -367,6 +381,7 @@ class BaseExporter:
             if not should_continue:
                 continue
 
+        self.additional_file_saving(unique_export_path)
         self.run.end_log()
 
         # Save error list files

synapse_sdk/utils/file/__init__.py

@@ -0,0 +1,39 @@
+# File utilities module
+# Maintains backward compatibility by re-exporting all functions
+
+from .archive import archive, unarchive
+from .checksum import calculate_checksum, get_checksum_from_file
+from .chunking import read_file_in_chunks
+from .download import (
+    adownload_file,
+    afiles_url_to_path,
+    afiles_url_to_path_from_objs,
+    download_file,
+    files_url_to_path,
+    files_url_to_path_from_objs,
+)
+from .encoding import convert_file_to_base64
+from .io import get_dict_from_file, get_temp_path
+
+__all__ = [
+    # Chunking
+    'read_file_in_chunks',
+    # Download
+    'download_file',
+    'adownload_file',
+    'files_url_to_path',
+    'afiles_url_to_path',
+    'files_url_to_path_from_objs',
+    'afiles_url_to_path_from_objs',
+    # Checksum
+    'calculate_checksum',
+    'get_checksum_from_file',
+    # Archive
+    'archive',
+    'unarchive',
+    # Encoding
+    'convert_file_to_base64',
+    # I/O
+    'get_dict_from_file',
+    'get_temp_path',
+]

synapse_sdk/utils/file/archive.py

@@ -0,0 +1,32 @@
+import zipfile
+from pathlib import Path
+
+
+def archive(input_path, output_path, append=False):
+    input_path = Path(input_path)
+    output_path = Path(output_path)
+
+    mode = 'a' if append and output_path.exists() else 'w'
+    with zipfile.ZipFile(output_path, mode=mode, compression=zipfile.ZIP_DEFLATED) as zipf:
+        if input_path.is_file():
+            zipf.write(input_path, input_path.name)
+        else:
+            for file_path in input_path.rglob('*'):
+                if file_path.is_file():  # Only add files, skip directories
+                    arcname = file_path.relative_to(input_path.parent)
+                    zipf.write(file_path, arcname)
+
+
+def unarchive(file_path, output_path):
+    """
+    Unarchives a ZIP file to a given directory.
+
+    Parameters:
+        file_path (str | Path): The path to the ZIP file.
+        output_path (str): The directory where the files will be extracted.
+    """
+    output_path = Path(output_path)
+    output_path.mkdir(parents=True, exist_ok=True)
+
+    with zipfile.ZipFile(str(file_path), 'r') as zip_ref:
+        zip_ref.extractall(output_path)

synapse_sdk/utils/file/checksum.py

@@ -0,0 +1,56 @@
+import hashlib
+from typing import IO, Any, Callable
+
+
+def calculate_checksum(file_path, prefix=''):
+    md5_hash = hashlib.md5()
+    with open(file_path, 'rb') as f:
+        for byte_block in iter(lambda: f.read(4096), b''):
+            md5_hash.update(byte_block)
+    checksum = md5_hash.hexdigest()
+    if prefix:
+        return f'dev-{checksum}'
+    return checksum
+
+
+def get_checksum_from_file(file: IO[Any], digest_mod: Callable[[], Any] = hashlib.sha1) -> str:
+    """
+    Calculate checksum for a file-like object.
+
+    Args:
+        file (IO[Any]): File-like object with read() method that supports reading in chunks
+        digest_mod (Callable[[], Any]): Hash algorithm from hashlib (defaults to hashlib.sha1)
+
+    Returns:
+        str: Hexadecimal digest of the file contents
+
+    Example:
+        ```python
+        import hashlib
+        from io import BytesIO
+        from synapse_sdk.utils.file import get_checksum_from_file
+
+        # With BytesIO
+        data = BytesIO(b'Hello, world!')
+        checksum = get_checksum_from_file(data)
+
+        # With different hash algorithm
+        checksum = get_checksum_from_file(data, digest_mod=hashlib.sha256)
+        ```
+    """
+    digest = digest_mod()
+    chunk_size = 4096
+
+    # Reset file pointer to beginning if possible
+    if hasattr(file, 'seek'):
+        file.seek(0)
+
+    while True:
+        chunk = file.read(chunk_size)
+        if not chunk:
+            break
+        if isinstance(chunk, str):
+            chunk = chunk.encode('utf-8')
+        digest.update(chunk)
+
+    return digest.hexdigest()

synapse_sdk/utils/file/chunking.py

@@ -0,0 +1,31 @@
+def read_file_in_chunks(file_path, chunk_size=1024 * 1024 * 50):
+    """
+    Read a file in chunks for efficient memory usage during file processing.
+
+    This function is particularly useful for large files or when you need to process
+    files in chunks, such as for uploading or hashing.
+
+    Args:
+        file_path (str | Path): Path to the file to read
+        chunk_size (int, optional): Size of each chunk in bytes. Defaults to 50MB (1024 * 1024 * 50)
+
+    Yields:
+        bytes: File content chunks
+
+    Raises:
+        FileNotFoundError: If the file doesn't exist
+        PermissionError: If the file can't be read due to permissions
+        OSError: If there's an OS-level error reading the file
+
+    Example:
+        ```python
+        from synapse_sdk.utils.file import read_file_in_chunks
+
+        # Read a file in 10MB chunks
+        for chunk in read_file_in_chunks('large_file.bin', chunk_size=1024*1024*10):
+            process_chunk(chunk)
+        ```
+    """
+    with open(file_path, 'rb') as file:
+        while chunk := file.read(chunk_size):
+            yield chunk

synapse_sdk/utils/file/download.py

@@ -0,0 +1,124 @@
+import asyncio
+import operator
+from functools import reduce
+from pathlib import Path
+
+import aiohttp
+import requests
+
+from synapse_sdk.utils.network import clean_url
+from synapse_sdk.utils.string import hash_text
+
+from .io import get_temp_path
+
+
+def download_file(url, path_download, name=None, coerce=None, use_cached=True):
+    chunk_size = 1024 * 1024 * 50
+    cleaned_url = clean_url(url)  # remove query params and fragment
+
+    if name:
+        use_cached = False
+    else:
+        name = hash_text(cleaned_url)
+
+    name += Path(cleaned_url).suffix
+
+    path = Path(path_download) / name
+
+    if not use_cached or not path.is_file():
+        response = requests.get(url, allow_redirects=True, stream=True)
+        response.raise_for_status()
+
+        with path.open('wb') as file:
+            for chunk in response.iter_content(chunk_size=chunk_size):
+                file.write(chunk)
+
+    if coerce:
+        path = coerce(path)
+
+    return path
+
+
+def files_url_to_path(files, coerce=None, file_field=None):
+    path_download = get_temp_path('media')
+    path_download.mkdir(parents=True, exist_ok=True)
+    if file_field:
+        files[file_field] = download_file(files[file_field], path_download, coerce=coerce)
+    else:
+        for file_name in files:
+            if isinstance(files[file_name], str):
+                files[file_name] = download_file(files[file_name], path_download, coerce=coerce)
+            else:
+                files[file_name]['path'] = download_file(files[file_name].pop('url'), path_download, coerce=coerce)
+
+
+def files_url_to_path_from_objs(objs, files_fields, coerce=None, is_list=False, is_async=False):
+    if is_async:
+        asyncio.run(afiles_url_to_path_from_objs(objs, files_fields, coerce=coerce, is_list=is_list))
+    else:
+        if not is_list:
+            objs = [objs]
+
+        for obj in objs:
+            for files_field in files_fields:
+                try:
+                    files = reduce(operator.getitem, files_field.split('.'), obj)
+                    if isinstance(files, str):
+                        files_url_to_path(obj, coerce=coerce, file_field=files_field)
+                    else:
+                        files_url_to_path(files, coerce=coerce)
+                except KeyError:
+                    pass
+
+
+async def adownload_file(url, path_download, name=None, coerce=None, use_cached=True):
+    chunk_size = 1024 * 1024 * 50
+    cleaned_url = clean_url(url)  # remove query params and fragment
+
+    if name:
+        use_cached = False
+    else:
+        name = hash_text(cleaned_url)
+
+    name += Path(cleaned_url).suffix
+
+    path = Path(path_download) / name
+
+    if not use_cached or not path.is_file():
+        async with aiohttp.ClientSession() as session:
+            async with session.get(url) as response:
+                with path.open('wb') as file:
+                    while chunk := await response.content.read(chunk_size):
+                        file.write(chunk)
+
+    if coerce:
+        path = coerce(path)
+
+    return path
+
+
+async def afiles_url_to_path(files, coerce=None):
+    path_download = get_temp_path('media')
+    path_download.mkdir(parents=True, exist_ok=True)
+    for file_name in files:
+        if isinstance(files[file_name], str):
+            files[file_name] = await adownload_file(files[file_name], path_download, coerce=coerce)
+        else:
+            files[file_name]['path'] = await adownload_file(files[file_name].pop('url'), path_download, coerce=coerce)
+
+
+async def afiles_url_to_path_from_objs(objs, files_fields, coerce=None, is_list=False):
+    if not is_list:
+        objs = [objs]
+
+    tasks = []
+
+    for obj in objs:
+        for files_field in files_fields:
+            try:
+                files = reduce(operator.getitem, files_field.split('.'), obj)
+                tasks.append(afiles_url_to_path(files, coerce=coerce))
+            except KeyError:
+                pass
+
+    await asyncio.gather(*tasks)

synapse_sdk/utils/file/encoding.py

@@ -0,0 +1,40 @@
+import base64
+import mimetypes
+from pathlib import Path
+
+
+def convert_file_to_base64(file_path):
+    """
+    Convert a file to base64 using pathlib.
+
+    Args:
+        file_path (str): Path to the file to convert
+
+    Returns:
+        str: Base64 encoded string of the file contents
+    """
+    # FIXME base64 is sent sometimes.
+    if file_path.startswith('data:'):
+        return file_path
+
+    # Convert string path to Path object
+    path = Path(file_path)
+
+    try:
+        # Read binary content of the file
+        binary_content = path.read_bytes()
+
+        # Convert to base64
+        base64_encoded = base64.b64encode(binary_content).decode('utf-8')
+
+        # Get the MIME type of the file
+        mime_type, _ = mimetypes.guess_type(path)
+        assert mime_type is not None, 'MIME type cannot be guessed'
+
+        # Convert bytes to string for readable output
+        return f'data:{mime_type};base64,{base64_encoded}'
+
+    except FileNotFoundError:
+        raise FileNotFoundError(f'File not found: {file_path}')
+    except Exception as e:
+        raise Exception(f'Error converting file to base64: {str(e)}')

synapse_sdk/utils/file/io.py

@@ -0,0 +1,22 @@
+import json
+from pathlib import Path
+
+import yaml
+
+
+def get_dict_from_file(file_path):
+    if isinstance(file_path, str):
+        file_path = Path(file_path)
+
+    with open(file_path) as f:
+        if file_path.suffix == '.yaml':
+            return yaml.safe_load(f)
+        else:
+            return json.load(f)
+
+
+def get_temp_path(sub_path=None):
+    path = Path('/tmp/datamaker')
+    if sub_path:
+        path = path / sub_path
+    return path

synapse_sdk/utils/file/video/__init__.py

@@ -0,0 +1,29 @@
+# Video processing utilities
+
+from .transcode import (
+    FFmpegNotFoundError,
+    TranscodeConfig,
+    TranscodingFailedError,
+    UnsupportedFormatError,
+    VideoTranscodeError,
+    atranscode_video,
+    get_video_info,
+    optimize_for_web,
+    transcode_batch,
+    transcode_video,
+    validate_video_format,
+)
+
+__all__ = [
+    'TranscodeConfig',
+    'VideoTranscodeError',
+    'UnsupportedFormatError',
+    'FFmpegNotFoundError',
+    'TranscodingFailedError',
+    'transcode_video',
+    'atranscode_video',
+    'get_video_info',
+    'validate_video_format',
+    'optimize_for_web',
+    'transcode_batch',
+]