synapse-sdk 1.0.0b5__py3-none-any.whl → 2025.12.3__py3-none-any.whl

synapse_sdk/clients/base.py

@@ -1,5 +1,4 @@
 import json
-from pathlib import Path
 
 import requests
 from requests.adapters import HTTPAdapter
@@ -7,6 +6,12 @@ from urllib3.util.retry import Retry
 
 from synapse_sdk.clients.exceptions import ClientError
 from synapse_sdk.utils.file import files_url_to_path_from_objs
+from synapse_sdk.utils.file.upload import (
+    FileProcessingError,
+    FileValidationError,
+    close_file_handles,
+    process_files_for_upload,
+)
 
 
 class BaseClient:
@@ -22,27 +27,58 @@ class BaseClient:
             'read': 15,  # Read timeout: 15 seconds
         }
 
-        # Create session with retry strategy
-        requests_session = requests.Session()
+        # Session is created on first use
+        self._session = None
+
+        # Store retry configuration for creating sessions
+        self._retry_config = {
+            'total': 3,  # Total retries
+            'backoff_factor': 1,  # Backoff factor between retries
+            'status_forcelist': [502, 503, 504],  # HTTP status codes to retry
+            'allowed_methods': ['GET', 'POST', 'PUT', 'DELETE', 'PATCH'],
+        }
+
+    def _create_session(self):
+        """Create a new requests session with retry strategy."""
+        session = requests.Session()
 
         # Configure retry strategy for transient failures
-        retry_strategy = Retry(
-            total=3,  # Total retries
-            backoff_factor=1,  # Backoff factor between retries
-            status_forcelist=[502, 503, 504],  # HTTP status codes to retry
-            allowed_methods=['GET', 'POST', 'PUT', 'DELETE', 'PATCH'],
-        )
+        retry_strategy = Retry(**self._retry_config)
 
         adapter = HTTPAdapter(max_retries=retry_strategy)
-        requests_session.mount('http://', adapter)
-        requests_session.mount('https://', adapter)
+        session.mount('http://', adapter)
+        session.mount('https://', adapter)
 
-        self.requests_session = requests_session
+        return session
+
+    @property
+    def requests_session(self):
+        """Get the requests session.
+
+        Returns a session instance, creating one if it doesn't exist.
+        """
+        if self._session is None:
+            self._session = self._create_session()
+        return self._session
 
-    def _get_url(self, path):
-        if not path.startswith('http'):
-            return f'{self.base_url}/{path.lstrip("/")}'
-        return path
+    def _get_url(self, path, trailing_slash=False):
+        """Construct a full URL from a path.
+
+        Args:
+            path (str): URL path or full URL
+            trailing_slash (bool): Whether to ensure URL ends with trailing slash
+
+        Returns:
+            str: Complete URL
+        """
+        # Use the path as-is if it's already a full URL, otherwise construct from base_url and path
+        url = path if path.startswith(('http://', 'https://')) else f'{self.base_url}/{path.lstrip("/")}'
+
+        # Add trailing slash if requested and not present
+        if trailing_slash and not url.endswith('/'):
+            url += '/'
+
+        return url
 
     def _get_headers(self):
         return {}
@@ -69,33 +105,35 @@ class BaseClient:
         # List to store opened files to close after request
         opened_files = []
 
-        if method in ['post', 'put', 'patch']:
-            # If files are included in the request, open them as binary files
-            if kwargs.get('files') is not None:
-                for name, file in kwargs['files'].items():
-                    # Handle both string and Path object cases
-                    if isinstance(file, str):
-                        file = Path(file)
-                    if isinstance(file, Path):
-                        opened_file = file.open(mode='rb')
-                        kwargs['files'][name] = (file.name, opened_file)
-                        opened_files.append(opened_file)
-                if 'data' in kwargs:
-                    for name, value in kwargs['data'].items():
-                        if isinstance(value, dict):
-                            kwargs['data'][name] = json.dumps(value)
-            else:
-                headers['Content-Type'] = 'application/json'
-                if 'data' in kwargs:
-                    kwargs['data'] = json.dumps(kwargs['data'])
-
         try:
+            if method in ['post', 'put', 'patch']:
+                # Process files if present using the utility function
+                # TODO: File handling logic using 'files' key is naive. Need to establish and document
+                #       a clear convention for including file information in request bodies across Synapse SDK.
+                if kwargs.get('files') is not None:
+                    kwargs['files'], opened_files = process_files_for_upload(kwargs['files'])
+
+                    # Handle data serialization when files are present
+                    if 'data' in kwargs:
+                        for name, value in kwargs['data'].items():
+                            if isinstance(value, dict):
+                                kwargs['data'][name] = json.dumps(value)
+                else:
+                    # No files - use JSON content type
+                    headers['Content-Type'] = 'application/json'
+                    if 'data' in kwargs:
+                        kwargs['data'] = json.dumps(kwargs['data'])
+
             # Send request
             response = getattr(self.requests_session, method)(url, headers=headers, **kwargs)
             if not response.ok:
                 raise ClientError(
                     response.status_code, response.json() if response.status_code == 400 else response.reason
                 )
+
+        except (FileValidationError, FileProcessingError) as e:
+            # Catch file validation and processing errors from the utility
+            raise ClientError(400, str(e)) from e
         except requests.exceptions.ConnectTimeout:
             raise ClientError(408, f'{self.name} connection timeout (>{self.timeout["connect"]}s)')
         except requests.exceptions.ReadTimeout:
@@ -111,10 +149,9 @@ class BaseClient:
         except requests.exceptions.RequestException as e:
             # Catch all other requests exceptions
             raise ClientError(500, f'{self.name} request failed: {str(e)[:100]}')
-
-        # Close all opened files
-        for opened_file in opened_files:
-            opened_file.close()
+        finally:
+            # Always close opened files, even if an exception occurred
+            close_file_handles(opened_files)
 
         return self._post_response(response)
 
@@ -125,8 +162,7 @@ class BaseClient:
             return response.text
 
     def _get(self, path, url_conversion=None, response_model=None, **kwargs):
-        """
-        Perform a GET request and optionally convert response to a pydantic model.
+        """Perform a GET request and optionally convert response to a pydantic model.
 
         Args:
             path (str): URL path to request.
@@ -152,8 +188,7 @@ class BaseClient:
         return response
 
     def _post(self, path, request_model=None, response_model=None, **kwargs):
-        """
-        Perform a POST request and optionally convert response to a pydantic model.
+        """Perform a POST request and optionally convert response to a pydantic model.
 
         Args:
             path (str): URL path to request.
@@ -173,8 +208,7 @@ class BaseClient:
             return response
 
     def _put(self, path, request_model=None, response_model=None, **kwargs):
-        """
-        Perform a PUT request to the specified path.
+        """Perform a PUT request to the specified path.
 
         Args:
             path (str): The URL path for the request.
@@ -196,8 +230,7 @@ class BaseClient:
             return response
 
     def _patch(self, path, request_model=None, response_model=None, **kwargs):
-        """
-        Perform a PATCH HTTP request to the specified path.
+        """Perform a PATCH HTTP request to the specified path.
 
         Args:
             path (str): The API endpoint path to make the request to.
@@ -210,7 +243,6 @@ class BaseClient:
             Union[dict, BaseModel]: If response_model is provided, returns an instance of that model.
                 Otherwise, returns the raw response data.
         """
-
         if kwargs.get('data') and request_model:
             kwargs['data'] = self._validate_request_body_with_pydantic_model(kwargs['data'], request_model)
         response = self._request('patch', path, **kwargs)
@@ -220,8 +252,7 @@ class BaseClient:
             return response
 
     def _delete(self, path, request_model=None, response_model=None, **kwargs):
-        """
-        Performs a DELETE request to the specified path.
+        """Performs a DELETE request to the specified path.
 
         Args:
             path (str): The API endpoint path to send the DELETE request to.
@@ -234,7 +265,6 @@ class BaseClient:
             Union[dict, BaseModel]: If response_model is provided, returns an instance of that model.
                                    Otherwise, returns the raw response data as a dictionary.
         """
-
         if kwargs.get('data') and request_model:
             kwargs['data'] = self._validate_request_body_with_pydantic_model(kwargs['data'], request_model)
         response = self._request('delete', path, **kwargs)
@@ -243,19 +273,139 @@ class BaseClient:
         else:
             return response
 
-    def _list(self, path, url_conversion=None, list_all=False, **kwargs):
-        response = self._get(path, **kwargs)
+    def _list(self, path, url_conversion=None, list_all=False, params=None, **kwargs):
+        """List resources from a paginated API endpoint.
+
+        Args:
+            path (str): URL path to request.
+            url_conversion (dict, optional): Configuration for URL to path conversion.
+                Used to convert file URLs to local paths in the response.
+                Example: {'files_fields': ['files'], 'is_list': True}
+                This will convert file URLs in the 'files' field of each result.
+            list_all (bool): If True, returns a generator yielding all results across all pages.
+                Default is False, which returns only the first page.
+            params (dict, optional): Query parameters to pass to the request.
+                Example: {'status': 'active', 'project': 123}
+            **kwargs: Additional keyword arguments to pass to the request.
+
+        Returns:
+            If list_all is False: dict response from the API containing:
+                - 'results': list of items on the current page
+                - 'count': total number of items
+                - 'next': URL to the next page (or None)
+                - 'previous': URL to the previous page (or None)
+            If list_all is True: tuple of (generator, count) where:
+                - generator: yields individual items from all pages
+                - count: total number of items across all pages
+
+        Examples:
+            Get first page only:
+            >>> response = client._list('api/tasks/')
+            >>> tasks = response['results']  # List of tasks on first page
+            >>> total_count = response['count']  # Total number of tasks
+
+            Get all results across all pages:
+            >>> generator, count = client._list('api/tasks/', list_all=True)
+            >>> all_tasks = list(generator)  # Fetches all pages
+
+            With filters and url_conversion:
+            >>> url_conversion = {'files_fields': ['files'], 'is_list': True}
+            >>> params = {'status': 'active'}
+            >>> generator, count = client._list(
+            ...     'api/data_units/',
+            ...     url_conversion=url_conversion,
+            ...     list_all=True,
+            ...     params=params
+            ... )
+            >>> active_units = list(generator)  # All active units with file URLs converted
+        """
+        if params is None:
+            params = {}
+
         if list_all:
-            return self._list_all(path, url_conversion, **kwargs), response['count']
+            response = self._get(path, params=params, **kwargs)
+            return self._list_all(path, url_conversion, params=params, **kwargs), response.get('count')
         else:
+            response = self._get(path, params=params, **kwargs)
             return response
 
-    def _list_all(self, path, url_conversion=None, params={}, **kwargs):
-        params['page_size'] = self.page_size
-        response = self._get(path, url_conversion, params=params, **kwargs)
-        yield from response['results']
-        if response['next']:
-            yield from self._list_all(response['next'], url_conversion, **kwargs)
+    def _list_all(self, path, url_conversion=None, params=None, **kwargs):
+        """Generator that yields all results from a paginated API endpoint.
+
+        This method handles pagination automatically by following the 'next' URLs
+        returned by the API until all pages have been fetched. It uses an iterative
+        approach (while loop) instead of recursion to avoid stack overflow with
+        deep pagination.
+
+        Args:
+            path (str): Initial URL path to request.
+            url_conversion (dict, optional): Configuration for URL to path conversion.
+                Applied to all pages. Common structure:
+                - 'files_fields': List of field names containing file URLs
+                - 'is_list': Whether the response is a list (True for paginated results)
+                Example: {'files_fields': ['files', 'images'], 'is_list': True}
+            params (dict, optional): Query parameters for the first request only.
+                Subsequent requests use the 'next' URL which already includes
+                all necessary parameters. If 'page_size' is not specified,
+                it defaults to self.page_size (100).
+                Example: {'status': 'active', 'page_size': 50}
+            **kwargs: Additional keyword arguments to pass to requests.
+                Example: timeout, headers, etc.
+
+        Yields:
+            dict: Individual result items from all pages. Each item is yielded
+                as soon as it's fetched, allowing for memory-efficient processing
+                of large datasets.
+
+        Examples:
+            Basic usage - fetch all tasks:
+            >>> for task in client._list_all('api/tasks/'):
+            ...     process_task(task)
+
+            With filters:
+            >>> params = {'status': 'pending', 'priority': 'high'}
+            >>> for task in client._list_all('api/tasks/', params=params):
+            ...     print(task['id'])
+
+            With url_conversion for file fields:
+            >>> url_conversion = {'files_fields': ['files'], 'is_list': True}
+            >>> for data_unit in client._list_all('api/data_units/', url_conversion):
+            ...     # File URLs in 'files' field are converted to local paths
+            ...     print(data_unit['files'])
+
+            Collecting results into a list:
+            >>> all_tasks = list(client._list_all('api/tasks/'))
+            >>> print(f"Total tasks: {len(all_tasks)}")
+
+        Note:
+            - This is a generator function, so results are fetched lazily as you iterate
+            - The first page is fetched with the provided params
+            - Subsequent pages use the 'next' URL from the API response
+            - No duplicate page_size parameters are added to subsequent requests
+            - Memory efficient: processes one item at a time rather than loading all at once
+        """
+        if params is None:
+            params = {}
+
+        # Set page_size only if not already specified by user
+        request_params = params.copy()
+        if 'page_size' not in request_params:
+            request_params['page_size'] = self.page_size
+
+        next_url = path
+        is_first_request = True
+
+        while next_url:
+            # First request uses params, subsequent requests use next URL directly
+            if is_first_request:
+                response = self._get(next_url, url_conversion, params=request_params, **kwargs)
+                is_first_request = False
+            else:
+                # next URL already contains all necessary query parameters
+                response = self._get(next_url, url_conversion, **kwargs)
+
+            yield from response['results']
+            next_url = response.get('next')
 
     def exists(self, api, *args, **kwargs):
         return getattr(self, api)(*args, **kwargs)['count'] > 0

synapse_sdk/loggers.py

@@ -64,6 +64,37 @@ class BaseLogger:
         else:
             self.progress_record.update(current_progress)
 
+    def set_progress_failed(self, category: str | None = None):
+        """Mark progress as failed with elapsed time but no completion.
+
+        This method should be called when an operation fails to indicate that
+        no progress was made, but still track how long the operation ran before failing.
+
+        Args:
+            category(str | None): progress category
+        """
+        assert category is not None or 'categories' not in self.progress_record
+
+        # Calculate elapsed time if start time was recorded
+        elapsed_time = None
+        if category in self.time_begin_per_category:
+            elapsed_time = time.time() - self.time_begin_per_category[category]
+            elapsed_time = round(elapsed_time, 2)
+
+        # Progress is 0% (not completed), no time remaining, but track elapsed time
+        failed_progress = {
+            'percent': 0.0,
+            'time_remaining': None,
+            'elapsed_time': elapsed_time,
+            'status': 'failed',
+        }
+
+        if category:
+            self.current_progress_category = category
+            self.progress_record['categories'][category].update(failed_progress)
+        else:
+            self.progress_record.update(failed_progress)
+
     def get_current_progress(self):
         categories = self.progress_record.get('categories')
 
@@ -121,6 +152,10 @@ class ConsoleLogger(BaseLogger):
         super().set_progress(current, total, category=category)
         print(self.get_current_progress())
 
+    def set_progress_failed(self, category: str | None = None):
+        super().set_progress_failed(category=category)
+        print(self.get_current_progress())
+
     def set_metrics(self, value: Dict[Any, Any], category: str):
         super().set_metrics(value, category)
         print(self.metrics_record)
@@ -150,6 +185,17 @@ class BackendLogger(BaseLogger):
         except ClientError:
             pass
 
+    def set_progress_failed(self, category: str | None = None):
+        super().set_progress_failed(category=category)
+        try:
+            progress_record = {
+                'record': self.progress_record,
+                'current_progress': self.get_current_progress(),
+            }
+            self.client.update_job(self.job_id, data={'progress_record': progress_record})
+        except ClientError:
+            pass
+
     def set_metrics(self, value: Dict[Any, Any], category: str):
         super().set_metrics(value, category)
         try: