synapse-sdk 2025.10.3__py3-none-any.whl → 2025.10.4__py3-none-any.whl

synapse_sdk/devtools/docs/sidebars.ts

@@ -54,6 +54,20 @@ const sidebars: SidebarsConfig = {
             'plugins/categories/upload-plugins/upload-plugin-template',
           ],
         },
+        {
+          type: 'category',
+          label: 'Pre-annotation Plugins',
+          link: {
+            type: 'doc',
+            id: 'plugins/categories/pre-annotation-plugins/pre-annotation-plugin-overview',
+          },
+          items: [
+            'plugins/categories/pre-annotation-plugins/pre-annotation-plugin-overview',
+            'plugins/categories/pre-annotation-plugins/to-task-overview',
+            'plugins/categories/pre-annotation-plugins/to-task-action-development',
+            'plugins/categories/pre-annotation-plugins/to-task-template-development',
+          ],
+        },
       ],
     },
     {

synapse_sdk/plugins/categories/export/actions/export/action.py

@@ -95,11 +95,11 @@ class ExportAction(Action):
             PydanticCustomError: If data retrieval fails
         """
         try:
-            result_list = handler.get_results(self.client, filters)
+            result_list = handler.get_results(self.client, filters, run=self.run)
             results = result_list[0]
             count = result_list[1]
         except ClientError:
-            raise PydanticCustomError('client_error', _('Unable to get Ground Truth dataset.'))
+            raise PydanticCustomError('client_error', _('Unable to get dataset.'))
         return results, count
 
     def start(self) -> Dict[str, Any]:
@@ -116,7 +116,12 @@ class ExportAction(Action):
         """
         self.run.log_message_with_code(LogCode.EXPORT_STARTED)
 
-        filters = {'expand': 'data', **self.params['filter']}
+        # Get expand setting from config, default to True (expand data)
+        filters = {**self.params['filter']}
+        data_expand = self.config.get('data_expand', True)
+        if data_expand:
+            filters['expand'] = 'data'
+
         target = self.params['target']
         handler = TargetHandlerFactory.get_handler(target)
 

synapse_sdk/plugins/categories/export/actions/export/utils.py

@@ -1,10 +1,12 @@
 from abc import ABC, abstractmethod
-from typing import Any
+from typing import Any, Optional
+import time
 
 from pydantic_core import PydanticCustomError
 
 from synapse_sdk.clients.exceptions import ClientError
 from synapse_sdk.i18n import gettext as _
+from synapse_sdk.shared.enums import Context
 
 
 class ExportTargetHandler(ABC):
@@ -15,6 +17,103 @@ class ExportTargetHandler(ABC):
     of methods to validate filters, retrieve results, and process collections of results.
     """
 
+    # TODO: This is a temporary workaround and needs improvement in the future
+    def _get_results_chunked(self, list_method, filters, chunk_size=100, max_retries=3, retry_delay=1, run=None):
+        """
+        Retrieve results in chunks to avoid memory and response size limits.
+
+        Args:
+            list_method: The client method to call (e.g., client.list_assignments)
+            filters (dict): The filter criteria to apply
+            chunk_size (int): Number of items to fetch per chunk
+            max_retries (int): Maximum number of retries for failed requests
+            retry_delay (int): Delay in seconds between retries
+
+        Returns:
+            tuple: A tuple containing the results generator and the total count
+        """
+        filters = filters.copy()
+        filters['page_size'] = chunk_size
+
+        page = 1
+        results = []
+        total_count = 0
+
+        try:
+            while True:
+                filters['page'] = page
+
+                # Retry logic for handling temporary server issues
+                for attempt in range(max_retries + 1):
+                    try:
+                        response = list_method(params=filters, list_all=False)
+                        break
+                    except ClientError as e:
+                        error_msg = str(e)
+
+                        # Use log_dev_event for better debugging and monitoring
+                        if run:
+                            run.log_dev_event(
+                                'Chunked data retrieval error',
+                                {
+                                    'page': page,
+                                    'attempt': attempt + 1,
+                                    'error_message': error_msg,
+                                    'chunk_size': chunk_size,
+                                },
+                                level=Context.WARNING,
+                            )
+
+                        # Check for JSON decode errors specifically
+                        if 'Expecting value' in error_msg or 'JSONDecodeError' in error_msg:
+                            if run:
+                                run.log_dev_event(
+                                    'JSON parsing error - skipping page',
+                                    {'page': page, 'error_type': 'JSON_DECODE_ERROR', 'error_details': error_msg},
+                                    level=Context.DANGER,
+                                )
+                            # Skip this page and continue with next
+                            page += 1
+                            break
+                        elif attempt < max_retries and ('503' in error_msg or 'connection' in error_msg.lower()):
+                            retry_delay_seconds = retry_delay * (2**attempt)
+                            if run:
+                                run.log_dev_event(
+                                    'Server issue - retrying with backoff',
+                                    {
+                                        'page': page,
+                                        'retry_attempt': attempt + 1,
+                                        'max_retries': max_retries,
+                                        'retry_delay_seconds': retry_delay_seconds,
+                                        'error_type': 'SERVER_ISSUE',
+                                    },
+                                    level=Context.INFO,
+                                )
+                            time.sleep(retry_delay_seconds)  # Exponential backoff
+                            continue
+                        else:
+                            raise
+
+                if page == 1:
+                    total_count = response['count']
+
+                current_results = response.get('results', [])
+                results.extend(current_results)
+
+                # Check if we've got all results or if there are no more results
+                if len(current_results) < chunk_size or not response.get('next'):
+                    break
+
+                page += 1
+
+                # Small delay between pages to avoid overwhelming the server
+                time.sleep(0.1)
+
+            return results, total_count
+        except Exception:
+            # Re-raise the exception to be handled by the calling method
+            raise
+
     @abstractmethod
     def validate_filter(self, value: dict, client: Any):
         """
@@ -33,13 +132,14 @@ class ExportTargetHandler(ABC):
         pass
 
     @abstractmethod
-    def get_results(self, client: Any, filters: dict):
+    def get_results(self, client: Any, filters: dict, run=None):
         """
         Retrieve original data from target sources.
 
         Args:
             client (Any): The client used to retrieve the results.
             filters (dict): The filter criteria to apply.
+            run: Optional ExportRun instance for logging.
 
         Returns:
             tuple: A tuple containing the results and the total count of results.
@@ -76,8 +176,8 @@ class AssignmentExportTargetHandler(ExportTargetHandler):
             raise PydanticCustomError('client_error', _('Unable to get Assignment.'))
         return value
 
-    def get_results(self, client: Any, filters: dict):
-        return client.list_assignments(params=filters, list_all=True)
+    def get_results(self, client: Any, filters: dict, run=None):
+        return self._get_results_chunked(client.list_assignments, filters, run=run)
 
     def get_export_item(self, results):
         for result in results:
@@ -104,9 +204,9 @@ class GroundTruthExportTargetHandler(ExportTargetHandler):
             raise PydanticCustomError('client_error', _('Unable to get Ground Truth dataset version.'))
         return value
 
-    def get_results(self, client: Any, filters: dict):
+    def get_results(self, client: Any, filters: dict, run=None):
         filters['ground_truth_dataset_versions'] = filters.pop('ground_truth_dataset_version')
-        return client.list_ground_truth_events(params=filters, list_all=True)
+        return self._get_results_chunked(client.list_ground_truth_events, filters, run=run)
 
     def get_export_item(self, results):
         for result in results:
@@ -134,9 +234,9 @@ class TaskExportTargetHandler(ExportTargetHandler):
             raise PydanticCustomError('client_error', _('Unable to get Task.'))
         return value
 
-    def get_results(self, client: Any, filters: dict):
+    def get_results(self, client: Any, filters: dict, run=None):
         filters['expand'] = ['data_unit', 'assignment', 'workshop']
-        return client.list_tasks(params=filters, list_all=True)
+        return self._get_results_chunked(client.list_tasks, filters, run=run)
 
     def get_export_item(self, results):
         for result in results:

{synapse_sdk-2025.10.3.dist-info → synapse_sdk-2025.10.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synapse-sdk
-Version: 2025.10.3
+Version: 2025.10.4
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT

{synapse_sdk-2025.10.3.dist-info → synapse_sdk-2025.10.4.dist-info}/RECORD

@@ -53,7 +53,7 @@ synapse_sdk/devtools/docs/README.md,sha256=yBzWf0K1ef4oymFXDaHo0nYWEgMQJqsOyrkNh
 synapse_sdk/devtools/docs/docusaurus.config.ts,sha256=K1b002RS0x0tsOyib_6CSgUlASEULC9vAX8YDpbsRN4,3229
 synapse_sdk/devtools/docs/package-lock.json,sha256=dtepgbPC8gq5uz-hdcac4hIU-Cs209tX0sfBuB7RfEQ,705694
 synapse_sdk/devtools/docs/package.json,sha256=8veqayA4U3OLpdQaz6AzB59RQwTU-5soeYlYYWIxq28,1187
-synapse_sdk/devtools/docs/sidebars.ts,sha256=6MuXgPLBmk-URt37Jnv1uJjiL98xiqSCiHvu-utCXq4,2427
+synapse_sdk/devtools/docs/sidebars.ts,sha256=A0LdvAgIBOfZTfcGcc7QI4pvEEYoC--7N1g2KZPkG8k,3031
 synapse_sdk/devtools/docs/tsconfig.json,sha256=O9BNlRPjPiaVHW2_boShMbmTnh0Z2k0KQO6Alf9FMVY,215
 synapse_sdk/devtools/docs/blog/2019-05-28-first-blog-post.md,sha256=iP7gl_FPqo-qX13lkSRcRoT6ayJNmCkXoyvlm7GH248,312
 synapse_sdk/devtools/docs/blog/2019-05-29-long-blog-post.md,sha256=cM-dhhTeurEWMcdn0Kx-NpNts2YUUraSI_XFk_gVHEE,3122
@@ -96,6 +96,10 @@ synapse_sdk/devtools/docs/docs/features/utils/storage.md,sha256=uIpc37trKUm4XHe1
 synapse_sdk/devtools/docs/docs/features/utils/types.md,sha256=l84J1Ooa40xBYdvK1YpeKBT9kaqaWQUMNIFPUiC2e_U,1046
 synapse_sdk/devtools/docs/docs/plugins/export-plugins.md,sha256=Wajf3CZRQiytRx6SS1lIBlGO5o4F2vOw-xog51JiHJU,37074
 synapse_sdk/devtools/docs/docs/plugins/plugins.md,sha256=G6y3XZZHuC65ApfKE0fYHCAC5TOFnZqzE7VR-KOExVk,25074
+synapse_sdk/devtools/docs/docs/plugins/categories/pre-annotation-plugins/pre-annotation-plugin-overview.md,sha256=Dhnlqd7okB3SoOZ5RYRYqeBBZnGfctZrlCYo1A7GWyA,6832
+synapse_sdk/devtools/docs/docs/plugins/categories/pre-annotation-plugins/to-task-action-development.md,sha256=itkXVOV-pZNsqX0PIA9YhCJNp8iNgxVdQvM0sVahHJ4,47810
+synapse_sdk/devtools/docs/docs/plugins/categories/pre-annotation-plugins/to-task-overview.md,sha256=J-z2KkLt5WWTgbJGRxvqrZQxfHvur_QtCGA6iT5a0Og,17134
+synapse_sdk/devtools/docs/docs/plugins/categories/pre-annotation-plugins/to-task-template-development.md,sha256=ARirm2jq5X2fVXWpZFX6ezh0aSIK5PCWvYC9nvsXIwY,41159
 synapse_sdk/devtools/docs/docs/plugins/categories/upload-plugins/upload-plugin-action.md,sha256=GmTZYLyPDuUC9WzjNjf_zNh5FkN-VYzLuSo_dQzewWw,30796
 synapse_sdk/devtools/docs/docs/plugins/categories/upload-plugins/upload-plugin-overview.md,sha256=E42ycC5Jfb19DBdLZ2QnCiABUuXkJW4a0ZEQwnXi2ZY,15740
 synapse_sdk/devtools/docs/docs/plugins/categories/upload-plugins/upload-plugin-template.md,sha256=HpqGQ9yUYy4673gkdPNsqFZRdAXnDv3xbrC87brG50o,22288
@@ -145,6 +149,10 @@ synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/feature
 synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/features/utils/types.md,sha256=JdDMC2FKovAU55pPEBoIUPPniU5mKAvA5w1Zb36u9AE,1207
 synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/export-plugins.md,sha256=XcvZuio6NUDuDSe_XhYq1ACDaAJThmekP3anEzOCRks,38804
 synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/plugins.md,sha256=wUuuOmQQY1dURZzJ0HSpiABaJWqKbxQn31Xfn-wUf8E,4947
+synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/categories/pre-annotation-plugins/pre-annotation-plugin-overview.md,sha256=JlPxHFzeyXx0xi2vpGXc-GTkFeg9R-Smpap0OjCyE7g,7592
+synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/categories/pre-annotation-plugins/to-task-action-development.md,sha256=UowD_fqWFVD81ubJKguX-O9fSf7L07i6Pft8DFdfVn0,48803
+synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/categories/pre-annotation-plugins/to-task-overview.md,sha256=InL4EI0BkVXgjs-j7cPRYaRXDVxxMgBYYLs_9FdyKxk,18408
+synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/categories/pre-annotation-plugins/to-task-template-development.md,sha256=Ab8EeSYMWZclb_qxvBaLaz4IdCXySrKN2o68s24VT6I,42441
 synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/categories/upload-plugins/upload-plugin-action.md,sha256=zaodG2yGwNQiPLrI59dhShB0WxZ1EMNdXnQx9E5Kzlk,32523
 synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/categories/upload-plugins/upload-plugin-overview.md,sha256=mcqq_bbC13KAgNZX6OUEbHxt03QhGMN9vF2MmU0WW-A,17111
 synapse_sdk/devtools/docs/i18n/ko/docusaurus-plugin-content-docs/current/plugins/categories/upload-plugins/upload-plugin-template.md,sha256=Rldxjo5DKmHfEBMoF86yLkgYGoip99gVkF7gBSkqNxM,23603
@@ -202,12 +210,12 @@ synapse_sdk/plugins/categories/data_validation/templates/plugin/validation.py,sh
 synapse_sdk/plugins/categories/export/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/export/actions/__init__.py,sha256=crCFQ3tTUFjI3UhszDBpsE7eYi7zpswSHjQgTh3tccI,61
 synapse_sdk/plugins/categories/export/actions/export/__init__.py,sha256=6y0WRhcqyhxbZIfbRCF0ZI6CQgqSXmqCRLP1hTkdBPc,727
-synapse_sdk/plugins/categories/export/actions/export/action.py,sha256=-jlPIkm8BGxouq6rr0Q8nBSCIFAGEmMOlmmbX4DZxYI,6125
+synapse_sdk/plugins/categories/export/actions/export/action.py,sha256=hh1_to4xPmY6Dv-4Ge24oGbyj3J92Wy95Dl0XIaS36Q,6303
 synapse_sdk/plugins/categories/export/actions/export/enums.py,sha256=XO4CF8jWt0c9lcUenKcjaKb0iXqQBFLyVMHmLm2_Nv8,3601
 synapse_sdk/plugins/categories/export/actions/export/exceptions.py,sha256=cDXVQetJvSlTbijN3elNwDLACELn2UxjyGnM2ALlKOk,1753
 synapse_sdk/plugins/categories/export/actions/export/models.py,sha256=NM9mSJAhcFFREUVlCy1LRbmq4Tm8rH_FGyZimS2aXho,2779
 synapse_sdk/plugins/categories/export/actions/export/run.py,sha256=bmilGnxfMWtIIvdIoY9sHkCPkQf_1-myur10I5TUEFY,6912
-synapse_sdk/plugins/categories/export/actions/export/utils.py,sha256=rbqoOIcgAOjlHV9Vn-2czHk6nzsxQUNu_lJzjkU9XfM,6367
+synapse_sdk/plugins/categories/export/actions/export/utils.py,sha256=rr-PajbTmFagVCDBJzPgeh34jgPzjW6eDwS70Wwp76o,10902
 synapse_sdk/plugins/categories/export/templates/config.yaml,sha256=fkbSPp7ISZ9NJ1Vvcz1dBSWoYjseVp1eJKyu5PPoboM,516
 synapse_sdk/plugins/categories/export/templates/plugin/__init__.py,sha256=oC8hczDebLaWoTaxANw0n9uMjVR5sydBs7pgTw78rbM,16057
 synapse_sdk/plugins/categories/export/templates/plugin/export.py,sha256=q9dbJIM3iCNu8W7J6HfIqQ5zeCx1QfyUaj7adhBvshI,6892
@@ -368,9 +376,9 @@ synapse_sdk/utils/storage/providers/gcp.py,sha256=i2BQCu1Kej1If9SuNr2_lEyTcr5M_n
 synapse_sdk/utils/storage/providers/http.py,sha256=2DhIulND47JOnS5ZY7MZUex7Su3peAPksGo1Wwg07L4,5828
 synapse_sdk/utils/storage/providers/s3.py,sha256=ZmqekAvIgcQBdRU-QVJYv1Rlp6VHfXwtbtjTSphua94,2573
 synapse_sdk/utils/storage/providers/sftp.py,sha256=_8s9hf0JXIO21gvm-JVS00FbLsbtvly4c-ETLRax68A,1426
-synapse_sdk-2025.10.3.dist-info/licenses/LICENSE,sha256=bKzmC5YAg4V1Fhl8OO_tqY8j62hgdncAkN7VrdjmrGk,1101
-synapse_sdk-2025.10.3.dist-info/METADATA,sha256=wvNWIgxu6dCxIudxnaO9yKxC-mg0xApM6CRul_s95m0,4186
-synapse_sdk-2025.10.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-synapse_sdk-2025.10.3.dist-info/entry_points.txt,sha256=VNptJoGoNJI8yLXfBmhgUefMsmGI0m3-0YoMvrOgbxo,48
-synapse_sdk-2025.10.3.dist-info/top_level.txt,sha256=ytgJMRK1slVOKUpgcw3LEyHHP7S34J6n_gJzdkcSsw8,12
-synapse_sdk-2025.10.3.dist-info/RECORD,,
+synapse_sdk-2025.10.4.dist-info/licenses/LICENSE,sha256=bKzmC5YAg4V1Fhl8OO_tqY8j62hgdncAkN7VrdjmrGk,1101
+synapse_sdk-2025.10.4.dist-info/METADATA,sha256=PlcCqkFByX8o9NHv9QZQiCy79fkfz_kUL-jMCTCPd9I,4186
+synapse_sdk-2025.10.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+synapse_sdk-2025.10.4.dist-info/entry_points.txt,sha256=VNptJoGoNJI8yLXfBmhgUefMsmGI0m3-0YoMvrOgbxo,48
+synapse_sdk-2025.10.4.dist-info/top_level.txt,sha256=ytgJMRK1slVOKUpgcw3LEyHHP7S34J6n_gJzdkcSsw8,12
+synapse_sdk-2025.10.4.dist-info/RECORD,,