synapse-sdk 1.0.0a71__tar.gz → 1.0.0a73__tar.gz

{synapse_sdk-1.0.0a71 → synapse_sdk-1.0.0a73}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synapse-sdk
-Version: 1.0.0a71
+Version: 1.0.0a73
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT

{synapse_sdk-1.0.0a71 → synapse_sdk-1.0.0a73}/synapse_sdk/clients/backend/__init__.py

@@ -23,6 +23,7 @@ class BackendClient(
         authorization_token (str): The authorization token for the synapse backend API.
         tenant_token (str): The tenant token for the synapse backend API.
         agent_token (str): The agent token for the backend API.
+        timeout (Dict): Set reasonable default timeouts for better UX. It can receive keys called 'connect' and 'read'.
     """
 
     name = 'Backend'
@@ -31,8 +32,8 @@ class BackendClient(
     tenant_token = None
     agent_token = None
 
-    def __init__(self, base_url, access_token=None, token=None, tenant=None, agent_token=None, **kwargs):
-        super().__init__(base_url)
+    def __init__(self, base_url, access_token=None, token=None, tenant=None, agent_token=None, timeout=None, **kwargs):
+        super().__init__(base_url, timeout=timeout)
         self.access_token = access_token
         self.authorization_token = token
         self.tenant_token = tenant

{synapse_sdk-1.0.0a71 → synapse_sdk-1.0.0a73}/synapse_sdk/plugins/categories/pre_annotation/actions/to_task.py

@@ -67,9 +67,11 @@ class ToTaskParams(BaseModel):
         name (str): The name of the action.
         description (str | None): The description of the action.
         project (int): The project ID.
+        agent (int): The agent ID.
         task_filters (dict): The filters of tasks.
         method (AnnotationMethod): The method of annotation.
         target_specification_name (str | None): The name of the target specification.
+        model (int): The model ID.
         pre_processor (int | None): The pre processor ID.
         pre_processor_params (dict): The params of the pre processor.
     """
@@ -77,9 +79,11 @@ class ToTaskParams(BaseModel):
     name: Annotated[str, AfterValidator(non_blank)]
     description: Optional[str] = None
     project: int
+    agent: int
     task_filters: Dict[str, Any]
     method: Optional[AnnotationMethod] = None
     target_specification_name: Optional[str] = None
+    model: Optional[int] = None
     pre_processor: Optional[int] = None
     pre_processor_params: Dict[str, Any]
 

{synapse_sdk-1.0.0a71 → synapse_sdk-1.0.0a73}/synapse_sdk/utils/converters/__init__.py

@@ -48,7 +48,7 @@ class BaseConverter:
         else:
             required_dirs = {
                 'json': os.path.join(self.root_dir, 'json'),
-                'original_file': os.path.join(self.root_dir, 'original_file'),
+                'original_files': os.path.join(self.root_dir, 'original_files'),
             }
             self._validate_required_dirs(required_dirs)
             splits['root'] = self.root_dir
@@ -60,10 +60,10 @@ class BaseConverter:
         if split:
             split_dir = os.path.join(self.root_dir, split)
             self.json_dir = os.path.join(split_dir, 'json')
-            self.original_file_dir = os.path.join(split_dir, 'original_file')
+            self.original_file_dir = os.path.join(split_dir, 'original_files')
         else:
             self.json_dir = os.path.join(self.root_dir, 'json')
-            self.original_file_dir = os.path.join(self.root_dir, 'original_file')
+            self.original_file_dir = os.path.join(self.root_dir, 'original_files')
 
 
 class FromDMConverter(BaseConverter):
@@ -185,7 +185,7 @@ class ToDMConverter(BaseConverter):
             for split, img_dict in self.converted_data.items():
                 split_dir = os.path.join(output_dir, split)
                 json_dir = os.path.join(split_dir, 'json')
-                original_file_dir = os.path.join(split_dir, 'original_file')
+                original_file_dir = os.path.join(split_dir, 'original_files')
                 self.ensure_dir(json_dir)
                 self.ensure_dir(original_file_dir)
                 for img_filename, (dm_json, img_src_path) in img_dict.items():
@@ -196,7 +196,7 @@ class ToDMConverter(BaseConverter):
                         shutil.copy(img_src_path, os.path.join(original_file_dir, img_filename))
         else:
             json_dir = os.path.join(output_dir, 'json')
-            original_file_dir = os.path.join(output_dir, 'original_file')
+            original_file_dir = os.path.join(output_dir, 'original_files')
             self.ensure_dir(json_dir)
             self.ensure_dir(original_file_dir)
             for img_filename, (dm_json, img_src_path) in self.converted_data.items():
@@ -205,4 +205,5 @@ class ToDMConverter(BaseConverter):
                     json.dump(dm_json, jf, indent=2, ensure_ascii=False)
                 if img_src_path and os.path.exists(img_src_path):
                     shutil.copy(img_src_path, os.path.join(original_file_dir, img_filename))
+
         print(f'[DM] Data exported to {output_dir}')

{synapse_sdk-1.0.0a71 → synapse_sdk-1.0.0a73}/synapse_sdk/utils/converters/coco/from_dm.py

@@ -11,7 +11,7 @@ from tqdm import tqdm
 from synapse_sdk.utils.converters import FromDMConverter
 
 
-class DMToCOCOConverter(FromDMConverter):
+class FromDMToCOCOConverter(FromDMConverter):
     """Convert DM (Data Manager) format annotations to COCO format.
     Designed for easy future extensibility to handle various data types.
     """
@@ -242,11 +242,11 @@ class DMToCOCOConverter(FromDMConverter):
             for split, coco_data in self.converted_data.items():
                 split_output_dir = os.path.join(output_dir, split)
                 self._save_annotations_and_images(
-                    coco_data, split_output_dir, os.path.join(self.root_dir, split, 'original_file')
+                    coco_data, split_output_dir, os.path.join(self.root_dir, split, 'original_files')
                 )
         else:
             self._save_annotations_and_images(
-                self.converted_data, output_dir, os.path.join(self.root_dir, 'original_file')
+                self.converted_data, output_dir, os.path.join(self.root_dir, 'original_files')
             )
 
     def _save_annotations_and_images(self, coco_data, output_dir, original_file_dir):

synapse_sdk-1.0.0a73/synapse_sdk/utils/converters/yolo/from_dm.py

@@ -0,0 +1,296 @@
+import json
+import os
+import shutil
+from glob import glob
+from typing import Any, Dict, List, Optional, Union
+
+from PIL import Image
+
+from synapse_sdk.utils.converters import FromDMConverter
+
+
+class FromDMToYOLOConverter(FromDMConverter):
+    """Convert DM dataset format to YOLO format."""
+
+    IMG_EXTENSIONS = ['.jpg', '.jpeg', '.png', '.bmp']
+
+    def __init__(self, root_dir: str, is_categorized_dataset: bool = False):
+        super().__init__(root_dir, is_categorized_dataset)
+        self.class_names: List[str] = []
+        self.class_map: Dict[str, int] = {}
+        self.dataset_yaml_content: str = ''
+
+    @staticmethod
+    def get_all_classes(list_of_dirs: List[str]) -> List[str]:
+        """Collect all unique class names from all splits or the root."""
+        classes = set()
+        for d in list_of_dirs:
+            if not d or not os.path.isdir(d):
+                continue
+            json_dir = os.path.join(d, 'json') if os.path.isdir(os.path.join(d, 'json')) else d
+            for jfile in glob(os.path.join(json_dir, '*.json')):
+                with open(jfile, encoding='utf-8') as jf:
+                    data = json.load(jf)
+                for img_ann in data['images']:
+                    for k in ['bounding_box', 'polygon', 'keypoint']:
+                        if k in img_ann:
+                            for ann in img_ann[k]:
+                                classes.add(ann['classification'])
+        return sorted(list(classes))
+
+    @staticmethod
+    def get_image_size(image_path: str):
+        with Image.open(image_path) as img:
+            return img.size
+
+    @staticmethod
+    def polygon_to_bbox(polygon: list):
+        """Convert polygon points to bounding box [cx, cy, w, h]."""
+        if not polygon or len(polygon) == 0:
+            return None
+        xs = [p[0] for p in polygon]
+        ys = [p[1] for p in polygon]
+        x_min, y_min = min(xs), min(ys)
+        x_max, y_max = max(xs), max(ys)
+        cx = (x_min + x_max) / 2
+        cy = (y_min + y_max) / 2
+        w = x_max - x_min
+        h = y_max - y_min
+        return [cx, cy, w, h]
+
+    @staticmethod
+    def keypoints_to_yolo_string(keypoints: list, width: int, height: int):
+        """Convert keypoints to normalized YOLO keypoint format string (x1 y1 v1 x2 y2 v2 ...)."""
+        kp_strs = []
+        for kp in keypoints:
+            # kp: [x, y, visible]
+            x, y, v = kp
+            x = x / width
+            y = y / height
+            kp_strs.extend([f'{x:.6f}', f'{y:.6f}', str(v)])
+        return ' '.join(kp_strs)
+
+    def _convert_split_dir(self, split_dir: str, split_name: str) -> List[Dict[str, Any]]:
+        """Convert one split folder to YOLO format."""
+        if not self.class_map:
+            raise ValueError('class_map is not initialized. Ensure get_all_classes() is called before this method.')
+
+        json_dir = os.path.join(split_dir, 'json')
+        img_dir = os.path.join(split_dir, 'original_files')
+        entries = []
+        for jfile in glob(os.path.join(json_dir, '*.json')):
+            base = os.path.splitext(os.path.basename(jfile))[0]
+            found_img = None
+            for ext in self.IMG_EXTENSIONS:
+                img_path = os.path.join(img_dir, base + ext)
+                if os.path.exists(img_path):
+                    found_img = img_path
+                    break
+            if not found_img:
+                print(f'[{split_name}] Image for {base} not found, skipping.')
+                continue
+            width, height = self.get_image_size(found_img)
+            with open(jfile, encoding='utf-8') as jf:
+                data = json.load(jf)
+            img_ann = data['images'][0]
+            label_lines = []
+
+            # bbox
+            if 'bounding_box' in img_ann:
+                for box in img_ann['bounding_box']:
+                    cidx = self.class_map[box['classification']]
+                    x, y, w, h = box['data']
+                    cx = x + w / 2
+                    cy = y + h / 2
+                    cx /= width
+                    cy /= height
+                    w /= width
+                    h /= height
+                    label_lines.append(f'{cidx} {cx:.6f} {cy:.6f} {w:.6f} {h:.6f}')
+
+            # polygon
+            if 'polygon' in img_ann:
+                for poly in img_ann['polygon']:
+                    cidx = self.class_map[poly['classification']]
+                    bbox = self.polygon_to_bbox(poly['data'])
+                    if bbox is None:
+                        print(f'[{split_name}] Polygon for {base} is empty, skipping this polygon.')
+                        continue
+                    cx, cy, w, h = bbox
+                    cx /= width
+                    cy /= height
+                    w /= width
+                    h /= height
+                    label_lines.append(f'{cidx} {cx:.6f} {cy:.6f} {w:.6f} {h:.6f}')
+
+            # keypoint
+            if 'keypoint' in img_ann:
+                for kp in img_ann['keypoint']:
+                    cidx = self.class_map[kp['classification']]
+                    # Assume bounding box exists for keypoint, or fallback to full image
+                    if 'bounding_box' in kp:
+                        x, y, w, h = kp['bounding_box']
+                        cx = x + w / 2
+                        cy = y + h / 2
+                        cx /= width
+                        cy /= height
+                        w /= width
+                        h /= height
+                    else:
+                        # fallback to the whole image
+                        cx, cy, w, h = 0.5, 0.5, 1.0, 1.0
+                    kp_str = self.keypoints_to_yolo_string(kp['data'], width, height)
+                    label_lines.append(f'{cidx} {cx:.6f} {cy:.6f} {w:.6f} {h:.6f} {kp_str}')
+
+            entries.append({
+                'img_path': found_img,
+                'img_name': os.path.basename(found_img),
+                'label_name': base + '.txt',
+                'label_lines': label_lines,
+            })
+        return entries
+
+    def _convert_root_dir(self) -> List[Dict[str, Any]]:
+        """Convert non-categorized dataset to YOLO format."""
+        json_dir = os.path.join(self.root_dir, 'json')
+        img_dir = os.path.join(self.root_dir, 'original_files')
+        entries = []
+        for jfile in glob(os.path.join(json_dir, '*.json')):
+            base = os.path.splitext(os.path.basename(jfile))[0]
+            found_img = None
+            for ext in self.IMG_EXTENSIONS:
+                img_path = os.path.join(img_dir, base + ext)
+                if os.path.exists(img_path):
+                    found_img = img_path
+                    break
+            if not found_img:
+                print(f'[single] Image for {base} not found, skipping.')
+                continue
+            width, height = self.get_image_size(found_img)
+            with open(jfile, encoding='utf-8') as jf:
+                data = json.load(jf)
+            img_ann = data['images'][0]
+            label_lines = []
+
+            # bbox
+            if 'bounding_box' in img_ann:
+                for box in img_ann['bounding_box']:
+                    cidx = self.class_map[box['classification']]
+                    x, y, w, h = box['data']
+                    cx = x + w / 2
+                    cy = y + h / 2
+                    cx /= width
+                    cy /= height
+                    w /= width
+                    h /= height
+                    label_lines.append(f'{cidx} {cx:.6f} {cy:.6f} {w:.6f} {h:.6f}')
+
+            # polygon
+            if 'polygon' in img_ann:
+                for poly in img_ann['polygon']:
+                    cidx = self.class_map[poly['classification']]
+                    bbox = self.polygon_to_bbox(poly['data'])
+                    if bbox is None:
+                        print(f'[single] Polygon for {base} is empty, skipping this polygon.')
+                        continue
+                    cx, cy, w, h = bbox
+                    cx /= width
+                    cy /= height
+                    w /= width
+                    h /= height
+                    label_lines.append(f'{cidx} {cx:.6f} {cy:.6f} {w:.6f} {h:.6f}')
+
+            # keypoint
+            if 'keypoint' in img_ann:
+                for kp in img_ann['keypoint']:
+                    cidx = self.class_map[kp['classification']]
+                    if 'bounding_box' in kp:
+                        x, y, w, h = kp['bounding_box']
+                        cx = x + w / 2
+                        cy = y + h / 2
+                        cx /= width
+                        cy /= height
+                        w /= width
+                        h /= height
+                    else:
+                        cx, cy, w, h = 0.5, 0.5, 1.0, 1.0
+                    kp_str = self.keypoints_to_yolo_string(kp['data'], width, height)
+                    label_lines.append(f'{cidx} {cx:.6f} {cy:.6f} {w:.6f} {h:.6f} {kp_str}')
+
+            entries.append({
+                'img_path': found_img,
+                'img_name': os.path.basename(found_img),
+                'label_name': base + '.txt',
+                'label_lines': label_lines,
+            })
+        return entries
+
+    def convert(self) -> Union[Dict[str, List[Dict[str, Any]]], List[Dict[str, Any]]]:
+        """Convert DM format to YOLO format (categorized split or not).
+
+        Returns:
+            - If categorized: dict {split: list of entries}
+            - If not: list of entries
+        """
+        # Prepare dataset.yaml content (for save_to_folder)
+        yaml_lines = [
+            'path: ' + self.root_dir,
+        ]
+
+        if self.is_categorized_dataset:
+            splits = self._validate_splits(required_splits=['train', 'valid'], optional_splits=['test'])
+            self.class_names = self.get_all_classes(list(splits.values()))
+            self.class_map = {name: idx for idx, name in enumerate(self.class_names)}
+            result = {}
+            for split, split_dir in splits.items():
+                result[split] = self._convert_split_dir(split_dir, split)
+            self.converted_data = result
+
+            yaml_lines.append('train: train/images')
+            yaml_lines.append('val: valid/images')
+            if 'test' in splits:
+                yaml_lines.append('test: test/images')
+        else:
+            self._validate_splits(required_splits=[], optional_splits=[])
+            self.class_names = self.get_all_classes([self.root_dir])
+            self.class_map = {name: idx for idx, name in enumerate(self.class_names)}
+            result = self._convert_root_dir()
+            self.converted_data = result
+
+        yaml_lines += ['', f'nc: {len(self.class_names)}', f'names: {self.class_names}', '']
+        self.dataset_yaml_content = '\n'.join(yaml_lines)
+        return result
+
+    def save_to_folder(self, output_dir: Optional[str] = None) -> None:
+        """Save converted YOLO data to the specified folder."""
+        output_dir = output_dir or self.root_dir
+        self.ensure_dir(output_dir)
+        if self.converted_data is None:
+            self.converted_data = self.convert()
+
+        if self.is_categorized_dataset:
+            for split, entries in self.converted_data.items():
+                split_imgs = os.path.join(output_dir, split, 'images')
+                split_labels = os.path.join(output_dir, split, 'labels')
+                self.ensure_dir(split_imgs)
+                self.ensure_dir(split_labels)
+                for entry in entries:
+                    shutil.copy(entry['img_path'], os.path.join(split_imgs, entry['img_name']))
+                    with open(os.path.join(split_labels, entry['label_name']), 'w', encoding='utf-8') as f:
+                        f.write('\n'.join(entry['label_lines']))
+        else:
+            imgs_dir = os.path.join(output_dir, 'images')
+            labels_dir = os.path.join(output_dir, 'labels')
+            self.ensure_dir(imgs_dir)
+            self.ensure_dir(labels_dir)
+            for entry in self.converted_data:
+                shutil.copy(entry['img_path'], os.path.join(imgs_dir, entry['img_name']))
+                with open(os.path.join(labels_dir, entry['label_name']), 'w', encoding='utf-8') as f:
+                    f.write('\n'.join(entry['label_lines']))
+
+        with open(os.path.join(output_dir, 'dataset.yaml'), 'w', encoding='utf-8') as f:
+            f.write(self.dataset_yaml_content)
+        with open(os.path.join(output_dir, 'classes.txt'), 'w', encoding='utf-8') as f:
+            for c in self.class_names:
+                f.write(f'{c}\n')
+        print(f'YOLO data exported to {output_dir}')

{synapse_sdk-1.0.0a71 → synapse_sdk-1.0.0a73}/synapse_sdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synapse-sdk
-Version: 1.0.0a71
+Version: 1.0.0a73
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT

{synapse_sdk-1.0.0a71 → synapse_sdk-1.0.0a73}/synapse_sdk.egg-info/SOURCES.txt

@@ -253,6 +253,8 @@ synapse_sdk/utils/converters/__init__.py
 synapse_sdk/utils/converters/coco/__init__.py
 synapse_sdk/utils/converters/coco/from_dm.py
 synapse_sdk/utils/converters/coco/to_dm.py
+synapse_sdk/utils/converters/yolo/__init__.py
+synapse_sdk/utils/converters/yolo/from_dm.py
 synapse_sdk/utils/pydantic/__init__.py
 synapse_sdk/utils/pydantic/config.py
 synapse_sdk/utils/pydantic/errors.py
@@ -279,6 +281,7 @@ tests/utils/test_debug.py
 tests/utils/converters/__init__.py
 tests/utils/converters/conftest.py
 tests/utils/converters/test_coco_from_dm.py
+tests/utils/converters/test_yolo_from_dm.py
 tests/utils/converters/fixtures/__init__.py
 tests/utils/converters/fixtures/dm_schema.py
 tests/utils/converters/fixtures/data_types/image/categorized/test/json/dm_1.json

synapse_sdk-1.0.0a73/tests/utils/converters/fixtures/data_types/image/categorized/test/json/dm_3.json

@@ -0,0 +1,19 @@
+{
+    "images": [
+        {
+            "bounding_box": [
+                {
+                    "id": "mt3bxoZmo_",
+                    "classification": "a",
+                    "attrs": [],
+                    "data": [
+                        971,
+                        357,
+                        439,
+                        326
+                    ]
+                }
+            ]
+        }
+    ]
+}

synapse_sdk-1.0.0a73/tests/utils/converters/fixtures/data_types/image/not_categorized/json/dm_1.json

@@ -0,0 +1,19 @@
+{
+    "images": [
+        {
+            "bounding_box": [
+                {
+                    "id": "dZL-g7SYsR",
+                    "classification": "a",
+                    "attrs": [],
+                    "data": [
+                        348,
+                        196,
+                        321,
+                        244
+                    ]
+                }
+            ]
+        }
+    ]
+}

synapse_sdk-1.0.0a73/tests/utils/converters/fixtures/data_types/image/not_categorized/json/dm_2.json

@@ -0,0 +1,30 @@
+{
+    "images": [
+        {
+            "bounding_box": [
+                {
+                    "id": "y0-rEC1Uir",
+                    "classification": "a",
+                    "attrs": [],
+                    "data": [
+                        430,
+                        200,
+                        342,
+                        273
+                    ]
+                },
+                {
+                    "id": "KuZcgzsc_s",
+                    "classification": "a",
+                    "attrs": [],
+                    "data": [
+                        1208,
+                        504,
+                        232,
+                        209
+                    ]
+                }
+            ]
+        }
+    ]
+}

synapse_sdk-1.0.0a73/tests/utils/converters/fixtures/data_types/image/not_categorized/json/dm_3.json

@@ -0,0 +1,41 @@
+{
+    "images": [
+        {
+            "bounding_box": [
+                {
+                    "id": "r91yP2YJvJ",
+                    "classification": "a",
+                    "attrs": [],
+                    "data": [
+                        109,
+                        166,
+                        223,
+                        178
+                    ]
+                },
+                {
+                    "id": "B31gzHZQ_T",
+                    "classification": "a",
+                    "attrs": [],
+                    "data": [
+                        690,
+                        371,
+                        230,
+                        239
+                    ]
+                },
+                {
+                    "id": "Ao-Brlj6dd",
+                    "classification": "a",
+                    "attrs": [],
+                    "data": [
+                        1269,
+                        760,
+                        210,
+                        143
+                    ]
+                }
+            ]
+        }
+    ]
+}

{synapse_sdk-1.0.0a71 → synapse_sdk-1.0.0a73}/tests/utils/converters/test_coco_from_dm.py

@@ -241,18 +241,18 @@ class TestDMToCOCOConverter:
 
         # Create test files
         os.makedirs(os.path.join(temp_output_dir, 'json'), exist_ok=True)
-        os.makedirs(os.path.join(temp_output_dir, 'original_file'), exist_ok=True)
+        os.makedirs(os.path.join(temp_output_dir, 'original_files'), exist_ok=True)
 
         # Create matching files
         with open(os.path.join(temp_output_dir, 'json', 'test1.json'), 'w') as f:
             f.write('{}')
-        with open(os.path.join(temp_output_dir, 'original_file', 'test1.jpg'), 'w') as f:
+        with open(os.path.join(temp_output_dir, 'original_files', 'test1.jpg'), 'w') as f:
             f.write('test')
 
         # Create non-matching files
         with open(os.path.join(temp_output_dir, 'json', 'test2.json'), 'w') as f:
             f.write('{}')
-        with open(os.path.join(temp_output_dir, 'original_file', 'test3.jpg'), 'w') as f:
+        with open(os.path.join(temp_output_dir, 'original_files', 'test3.jpg'), 'w') as f:
             f.write('test')
 
         pairs = converter._find_json_file_pairs()
@@ -282,18 +282,18 @@ class TestDMToCOCOConverterIntegration:
 
         # Create invalid JSON file
         os.makedirs(os.path.join(temp_output_dir, 'json'), exist_ok=True)
-        os.makedirs(os.path.join(temp_output_dir, 'original_file'), exist_ok=True)
+        os.makedirs(os.path.join(temp_output_dir, 'original_files'), exist_ok=True)
 
         with open(os.path.join(temp_output_dir, 'json', 'invalid.json'), 'w') as f:
             f.write('invalid json content')
-        with open(os.path.join(temp_output_dir, 'original_file', 'invalid.jpg'), 'w') as f:
+        with open(os.path.join(temp_output_dir, 'original_files', 'invalid.jpg'), 'w') as f:
             f.write('test')
 
         with patch.object(converter, '_find_json_file_pairs') as mock_find_pairs:
             mock_find_pairs.return_value = [
                 (
                     os.path.join(temp_output_dir, 'json', 'invalid.json'),
-                    os.path.join(temp_output_dir, 'original_file', 'invalid.jpg'),
+                    os.path.join(temp_output_dir, 'original_files', 'invalid.jpg'),
                 )
             ]