synapse-sdk 1.0.0a73__tar.gz → 1.0.0a75__tar.gz

synapse_sdk-1.0.0a75/.github/workflows/test.yml

@@ -0,0 +1,35 @@
+name: Tests
+
+on:
+  pull_request:
+    types: [opened, reopened, synchronize]
+    branches: [main]
+  push:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: runner-set
+    strategy:
+      matrix:
+        python-version: ['3.12']
+      fail-fast: false
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v4
+      with:
+        version: "latest"
+
+    - name: Set up Python ${{ matrix.python-version }}
+      run: uv python install ${{ matrix.python-version }}
+
+    - name: Install dependencies
+      run: |
+        uv sync --extra test
+
+    - name: Run test with coverage
+      run: |
+        make test-coverage

{synapse_sdk-1.0.0a73 → synapse_sdk-1.0.0a75}/.gitignore

@@ -40,4 +40,13 @@ config.yaml*
 
 synapse-sample-plugin/
 
-CLAUDE.md
+
+# Local LLM Rule Files
+CLAUDE.local.md
+GEMINI.local.md
+AGENT.local.md
+
+
+# Test
+.coverage
+coverage.xml

synapse_sdk-1.0.0a75/Makefile

@@ -0,0 +1,54 @@
+.PHONY: test test-unit test-integration test-coverage test-lint clean
+
+# Run all tests
+test:
+	uv run python -m pytest tests/ -v
+
+# Run only unit tests (fast)
+test-unit:
+	uv run python -m pytest tests/ -m "unit" -v
+
+# Run only integration tests
+test-integration:
+	uv run python -m pytest tests/ -m "integration" -v
+
+# Run tests with coverage report
+test-coverage:
+	uv run python -m pytest tests/ --cov=synapse_sdk --cov-report=term-missing --cov-report=html --cov-report=xml
+
+# Run tests by category
+test-cli:
+	uv run python -m pytest tests/ -m "cli" -v
+
+test-storage:
+	uv run python -m pytest tests/ -m "storage" -v
+
+test-client:
+	uv run python -m pytest tests/ -v tests/clients/
+
+test-logger:
+	uv run python -m pytest tests/ -m "logger" -v
+
+test-plugin:
+	uv run python -m pytest tests/ -m "plugin" -v
+
+# Run slow tests
+test-slow:
+	uv run python -m pytest tests/ -m "slow" -v
+
+# Run tests with detailed output
+test-verbose:
+	uv run python -m pytest tests/ -vv --tb=long
+
+# Run tests and stop on first failure
+test-failfast:
+	uv run python -m pytest tests/ -x
+
+# Clean up test artifacts
+clean:
+	rm -rf .pytest_cache/
+	rm -rf .coverage
+	rm -rf htmlcov/
+	rm -rf .ruff_cache/
+	find . -type d -name __pycache__ -exec rm -rf {} +
+	find . -name "*.pyc" -delete

synapse_sdk-1.0.0a75/PKG-INFO

@@ -0,0 +1,72 @@
+Metadata-Version: 2.4
+Name: synapse-sdk
+Version: 1.0.0a75
+Summary: synapse sdk
+Author-email: datamaker <developer@datamaker.io>
+License: MIT
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: boto3
+Requires-Dist: click
+Requires-Dist: cookiecutter
+Requires-Dist: requests
+Requires-Dist: tqdm
+Requires-Dist: python-dotenv
+Requires-Dist: pyyaml
+Requires-Dist: pydantic
+Requires-Dist: pyjwt
+Requires-Dist: universal-pathlib
+Requires-Dist: fsspec[gcs,s3,sftp]
+Requires-Dist: inquirer
+Requires-Dist: pillow
+Provides-Extra: all
+Requires-Dist: ray[all]; extra == "all"
+Requires-Dist: python-nmap; extra == "all"
+Requires-Dist: hyperopt; extra == "all"
+Requires-Dist: bayesian-optimization==1.4.3; extra == "all"
+Provides-Extra: dashboard
+Requires-Dist: fastapi>=0.115.13; extra == "dashboard"
+Requires-Dist: uvicorn[standard]>=0.34.3; extra == "dashboard"
+Requires-Dist: websockets>=15.0.1; extra == "dashboard"
+Requires-Dist: aiofiles>=24.1.0; extra == "dashboard"
+Provides-Extra: test
+Requires-Dist: pytest>=7.0.0; extra == "test"
+Requires-Dist: pytest-cov>=4.0.0; extra == "test"
+Requires-Dist: pytest-mock>=3.10.0; extra == "test"
+Requires-Dist: pytest-timeout>=2.1.0; extra == "test"
+Requires-Dist: pytest-xdist>=3.0.0; extra == "test"
+Requires-Dist: pytest-html>=3.1.0; extra == "test"
+Requires-Dist: pytest-json-report>=1.5.0; extra == "test"
+Requires-Dist: requests-mock>=1.10.0; extra == "test"
+Requires-Dist: responses>=0.25.0; extra == "test"
+Provides-Extra: dev
+Requires-Dist: pre-commit; extra == "dev"
+Dynamic: license-file
+
+# 🧠 Synapse SDK
+
+![lint workflow](https://github.com/datamaker-kr/synapse-sdk/actions/workflows/lint.yml/badge.svg)
+![test workflow](https://github.com/datamaker-kr/synapse-sdk/actions/workflows/test.yml/badge.svg)
+
+A Python SDK for building and managing ML plugins, data annotation workflows, and AI agents.
+
+## ✨ Features
+
+- **🔌 Plugin System**: Create and manage ML plugins with categories like neural networks, data validation, and export tools
+- **🤖 Agent Management**: Backend and Ray-based agent clients for distributed AI workflows  
+- **🔄 Data Converters**: Convert between formats (COCO, Pascal VOC, YOLO) and annotation schemas
+- **🛠️ Development Tools**: Interactive web dashboard for monitoring and debugging
+- **⚡ CLI Interface**: Command-line tool for configuration, plugin management, and development
+
+## 🚀 Quick Start
+
+```bash
+pip install synapse-sdk
+synapse --help
+```
+
+## 📚 Documentation
+
+*Docs [https://docs.synapse.sh](https://docs.synapse.sh)*

synapse_sdk-1.0.0a75/README.md

@@ -0,0 +1,25 @@
+# 🧠 Synapse SDK
+
+![lint workflow](https://github.com/datamaker-kr/synapse-sdk/actions/workflows/lint.yml/badge.svg)
+![test workflow](https://github.com/datamaker-kr/synapse-sdk/actions/workflows/test.yml/badge.svg)
+
+A Python SDK for building and managing ML plugins, data annotation workflows, and AI agents.
+
+## ✨ Features
+
+- **🔌 Plugin System**: Create and manage ML plugins with categories like neural networks, data validation, and export tools
+- **🤖 Agent Management**: Backend and Ray-based agent clients for distributed AI workflows  
+- **🔄 Data Converters**: Convert between formats (COCO, Pascal VOC, YOLO) and annotation schemas
+- **🛠️ Development Tools**: Interactive web dashboard for monitoring and debugging
+- **⚡ CLI Interface**: Command-line tool for configuration, plugin management, and development
+
+## 🚀 Quick Start
+
+```bash
+pip install synapse-sdk
+synapse --help
+```
+
+## 📚 Documentation
+
+*Docs [https://docs.synapse.sh](https://docs.synapse.sh)*

synapse_sdk-1.0.0a75/docs/features/converters/index.md

@@ -0,0 +1,402 @@
+---
+id: converters
+title: Converters
+sidebar_position: 1
+---
+
+# Converters
+
+The Synapse SDK provides comprehensive data format conversion utilities for computer vision datasets. These converters enable seamless transformation between different annotation formats commonly used in machine learning workflows.
+
+## Overview
+
+The converter system supports bidirectional conversion between:
+
+- **DM Format** - Synapse Data Manager's native annotation format
+- **COCO Format** - Microsoft Common Objects in Context format
+- **Pascal VOC Format** - Visual Object Classes XML format
+- **YOLO Format** - You Only Look Once text-based format
+
+All converters support both categorized datasets (with train/valid/test splits) and non-categorized datasets.
+
+## Supported Annotation Types
+
+| Annotation Type | DM | COCO | Pascal VOC | YOLO |
+|----------------|----|----|-----------|------|
+| Bounding Boxes | ✅ | ✅ | ✅ | ✅ |
+| Polygons/Segmentation | ✅ | ✅ | ❌ | ✅ |
+| Keypoints | ✅ | ✅ | ❌ | ✅ |
+| Classifications | ✅ | ✅ | ❌ | ❌ |
+
+## Pascal VOC Converters
+
+### FromDMToPascalConverter
+
+Converts DM format annotations to Pascal VOC XML format.
+
+**Features:**
+- Converts bounding box annotations only
+- Creates standard Pascal VOC directory structure
+- Generates `classes.txt` file automatically
+- Supports both categorized and non-categorized datasets
+
+**Usage:**
+```python
+from synapse_sdk.utils.converters.pascal.from_dm import FromDMToPascalConverter
+
+# Convert categorized dataset
+converter = FromDMToPascalConverter(
+    root_dir='/path/to/dm/dataset',
+    is_categorized_dataset=True
+)
+converted_data = converter.convert()
+converter.save_to_folder('/output/pascal/dataset')
+
+# Convert non-categorized dataset
+converter = FromDMToPascalConverter(
+    root_dir='/path/to/dm/dataset',
+    is_categorized_dataset=False
+)
+converted_data = converter.convert()
+converter.save_to_folder('/output/pascal/dataset')
+```
+
+**Input Structure (Categorized):**
+```
+dm_dataset/
+├── train/
+│   ├── json/
+│   │   ├── image1.json
+│   │   └── image2.json
+│   └── original_files/
+│       ├── image1.jpg
+│       └── image2.jpg
+├── valid/
+│   ├── json/
+│   └── original_files/
+└── test/ (optional)
+    ├── json/
+    └── original_files/
+```
+
+**Output Structure:**
+```
+pascal_dataset/
+├── train/
+│   ├── Annotations/
+│   │   ├── image1.xml
+│   │   └── image2.xml
+│   └── Images/
+│       ├── image1.jpg
+│       └── image2.jpg
+├── valid/
+├── test/ (if present)
+└── classes.txt
+```
+
+### PascalToDMConverter
+
+Converts Pascal VOC XML annotations to DM format.
+
+**Features:**
+- Parses Pascal VOC XML files
+- Flexible directory naming (supports Annotations/annotations, Images/images/JPEGImages)
+- Extracts bounding box annotations only
+- Maintains class information
+
+**Usage:**
+```python
+from synapse_sdk.utils.converters.pascal.to_dm import PascalToDMConverter
+
+# Convert Pascal VOC dataset
+converter = PascalToDMConverter(
+    root_dir='/path/to/pascal/dataset',
+    is_categorized_dataset=True
+)
+converted_data = converter.convert()
+converter.save_to_folder('/output/dm/dataset')
+```
+
+## YOLO Converters
+
+### FromDMToYOLOConverter
+
+Converts DM format annotations to YOLO format with comprehensive annotation support.
+
+**Features:**
+- Supports bounding boxes, polygons, and keypoints
+- Creates `dataset.yaml` configuration file
+- Normalizes coordinates automatically
+- Handles keypoint visibility flags
+
+**Usage:**
+```python
+from synapse_sdk.utils.converters.yolo.from_dm import FromDMToYOLOConverter
+
+# Convert with all annotation types
+converter = FromDMToYOLOConverter(
+    root_dir='/path/to/dm/dataset',
+    is_categorized_dataset=True
+)
+converted_data = converter.convert()
+converter.save_to_folder('/output/yolo/dataset')
+```
+
+**Output Structure:**
+```
+yolo_dataset/
+├── train/
+│   ├── images/
+│   │   ├── image1.jpg
+│   │   └── image2.jpg
+│   └── labels/
+│       ├── image1.txt
+│       └── image2.txt
+├── valid/
+├── test/ (if present)
+├── dataset.yaml
+└── classes.txt
+```
+
+**YOLO Label Format Examples:**
+```
+# Bounding box: class_id center_x center_y width height
+0 0.5 0.5 0.3 0.4
+
+# Polygon: class_id x1 y1 x2 y2 x3 y3 x4 y4 ...
+0 0.1 0.1 0.9 0.1 0.9 0.9 0.1 0.9
+
+# Keypoints: class_id center_x center_y width height kp1_x kp1_y kp1_v kp2_x kp2_y kp2_v ...
+0 0.5 0.5 0.3 0.4 0.45 0.3 2 0.55 0.3 2 0.5 0.7 1
+```
+
+### YOLOToDMConverter
+
+Converts YOLO format annotations back to DM format.
+
+**Features:**
+- Intelligent parsing of different YOLO annotation types
+- Requires `dataset.yaml` for class name mapping
+- Handles bounding boxes, polygons, and keypoints
+- Automatically detects image dimensions
+
+**Usage:**
+```python
+from synapse_sdk.utils.converters.yolo.to_dm import YOLOToDMConverter
+
+converter = YOLOToDMConverter(
+    root_dir='/path/to/yolo/dataset',
+    is_categorized_dataset=True
+)
+converted_data = converter.convert()
+converter.save_to_folder('/output/dm/dataset')
+```
+
+## COCO Converters
+
+### FromDMToCOCOConverter
+
+Converts DM format to COCO format with full metadata support.
+
+**Features:**
+- Comprehensive COCO metadata (info, licenses, categories)
+- Supports bounding boxes, segmentation, and keypoints
+- Dynamic category management
+- Extensible for different data types
+
+**Usage:**
+```python
+from synapse_sdk.utils.converters.coco.from_dm import FromDMToCOCOConverter
+
+# Basic conversion
+converter = FromDMToCOCOConverter(
+    root_dir='/path/to/dm/dataset',
+    is_categorized_dataset=True
+)
+converted_data = converter.convert()
+converter.save_to_folder('/output/coco/dataset')
+
+# With custom metadata
+info_dict = {
+    "description": "My Custom Dataset",
+    "version": "1.0",
+    "contributor": "My Organization"
+}
+
+licenses_list = [{
+    "id": 1,
+    "name": "Custom License",
+    "url": "https://example.com/license"
+}]
+
+converter = FromDMToCOCOConverter(
+    root_dir='/path/to/dm/dataset',
+    info_dict=info_dict,
+    licenses_list=licenses_list,
+    is_categorized_dataset=True
+)
+```
+
+**Output Structure:**
+```
+coco_dataset/
+├── train/
+│   ├── annotations.json
+│   ├── image1.jpg
+│   └── image2.jpg
+├── valid/
+│   ├── annotations.json
+│   └── images...
+└── test/ (if present)
+```
+
+### COCOToDMConverter
+
+Converts COCO format annotations to DM format.
+
+**Features:**
+- Parses COCO JSON annotations
+- Handles image datasets
+- Maintains keypoint groupings through DM groups
+- Supports bounding boxes and keypoints
+
+**Usage:**
+```python
+from synapse_sdk.utils.converters.coco.to_dm import COCOToDMConverter
+
+converter = COCOToDMConverter(
+    root_dir='/path/to/coco/dataset',
+    is_categorized_dataset=True
+)
+converted_data = converter.convert()
+converter.save_to_folder('/output/dm/dataset')
+```
+
+## DM Version Converter
+
+### DMV1ToV2Converter
+
+Migrates legacy DM v1 datasets to the current v2 format.
+
+**Features:**
+- Comprehensive migration for all annotation types
+- Handles data structure changes between versions
+- Supports images and videos
+- Maintains annotation tool integrity
+
+**Supported Tools:**
+- Bounding boxes
+- Named entities
+- Classifications with attributes
+- Polylines and polygons
+- Keypoints
+- 3D bounding boxes
+- Segmentation
+- Relations and groups
+
+**Usage:**
+```python
+from synapse_sdk.utils.converters.dm.from_v1 import DMV1ToV2Converter
+
+converter = DMV1ToV2Converter(
+    root_dir='/path/to/dm/v1/dataset',
+    is_categorized_dataset=True
+)
+converted_data = converter.convert()
+converter.save_to_folder('/output/dm/v2/dataset')
+```
+
+### DMV2ToV1Converter
+
+Converts DM v2 datasets back to the legacy v1 format for compatibility with older systems.
+
+**Features:**
+- Reverse migration from DM v2 to v1 format
+- Preserves all annotation types and metadata
+- Maintains coordinate integrity across formats
+- Generates appropriate v1 structure with annotations and annotationsData
+
+**Supported Tools:**
+- Bounding boxes
+- Named entities  
+- Classifications with attributes
+- Polylines and polygons
+- Keypoints
+- 3D bounding boxes
+- Segmentation
+- Relations and groups
+
+**Usage:**
+```python
+from synapse_sdk.utils.converters.dm.to_v1 import DMV2ToV1Converter
+
+# Load v2 data and convert to v1
+with open('dm_v2_data.json', 'r') as f:
+    v2_data = json.load(f)
+
+converter = DMV2ToV1Converter(v2_data)
+v1_data = converter.convert()
+
+# Save or use the converted v1 data
+with open('dm_v1_data.json', 'w') as f:
+    json.dump(v1_data, f, indent=2)
+```
+
+## Common Parameters
+
+All converters share these common parameters:
+
+### `root_dir` (str)
+Path to the root directory containing the dataset.
+
+### `is_categorized_dataset` (bool)
+- `True`: Dataset has train/valid/test splits in separate subdirectories
+- `False`: Dataset is in a single directory without splits
+
+### Common Methods
+
+#### `convert()`
+Performs in-memory conversion and returns the converted data structure.
+
+#### `save_to_folder(output_dir)`
+Saves the converted data to the specified output directory, creating the appropriate file structure for the target format.
+
+## Error Handling
+
+All converters include robust error handling:
+
+- **File Validation**: Checks for required files and directories
+- **Format Validation**: Validates annotation format correctness
+- **Graceful Degradation**: Warns about unsupported annotations instead of failing
+- **Progress Tracking**: Shows progress for large dataset conversions
+
+## Best Practices
+
+1. **Backup Original Data**: Always keep backups before conversion
+2. **Validate Results**: Check converted annotations for accuracy
+3. **Test on Small Datasets**: Test conversion on small samples first
+4. **Check Requirements**: Ensure all required files (dataset.yaml, annotations.json, etc.) are present
+5. **Monitor Warnings**: Pay attention to conversion warnings for data quality issues
+
+## Examples
+
+### Converting a Complete Workflow
+
+```python
+# 1. Convert Pascal VOC to DM
+pascal_converter = PascalToDMConverter('/data/pascal', True)
+dm_data = pascal_converter.convert()
+pascal_converter.save_to_folder('/data/dm_intermediate')
+
+# 2. Convert DM to YOLO
+yolo_converter = FromDMToYOLOConverter('/data/dm_intermediate', True)
+yolo_data = yolo_converter.convert()
+yolo_converter.save_to_folder('/data/yolo_final')
+
+# 3. Convert DM to COCO for evaluation
+coco_converter = FromDMToCOCOConverter('/data/dm_intermediate', True)
+coco_data = coco_converter.convert()
+coco_converter.save_to_folder('/data/coco_eval')
+```
+
+This converter system provides a complete solution for dataset format transformation, enabling seamless integration between different machine learning workflows and annotation tools.

synapse_sdk-1.0.0a75/docs/features/index.md

@@ -0,0 +1,16 @@
+---
+id: features
+title: Features
+sidebar_position: 5
+---
+
+# Features
+
+This section covers the key features and functionality provided by the Synapse SDK.
+
+## [Data Converters](./converters/index.md)
+
+Comprehensive data format conversion utilities for computer vision datasets.
+
+- **[Format Converters](./converters/index.md)** - Convert between DM, COCO, Pascal VOC, and YOLO formats
+- **[Version Migration](./converters/index.md#dm-version-converter)** - Migrate DM datasets between versions