willisapi-client 1.9.7__tar.gz → 1.9.9__tar.gz

{willisapi_client-1.9.7 → willisapi_client-1.9.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: willisapi_client
-Version: 1.9.7
+Version: 1.9.9
 Summary: A Python client for willisapi
 Home-page: https://github.com/bklynhlth/willsiapi_client
 Author: bklynhlth

{willisapi_client-1.9.7 → willisapi_client-1.9.9}/willisapi_client/__version__.py

@@ -1,7 +1,7 @@
 """Version details for willisapi_client"""
 
 __client__ = "willisapi_client"
-__latestVersion__ = "1.9.7"
+__latestVersion__ = "1.9.9"
 __url__ = "https://github.com/bklynhlth/willsiapi_client"
 __short_description__ = "A Python client for willisapi"
 __content_type__ = "text/markdown"

{willisapi_client-1.9.7 → willisapi_client-1.9.9}/willisapi_client/services/metadata/upload.py

@@ -4,10 +4,19 @@ from datetime import datetime
 import requests
 import mimetypes
 import os
+import time
+from concurrent.futures import ThreadPoolExecutor
 
-# Register extensions that Python's default mimetypes table doesn't know about.
-# The server signs presigned URLs with these types, so the client must match.
-mimetypes.add_type("application/vnd.apache.parquet", ".parquet")
+# The server signs presigned URLs with Content-Types guessed from Python's
+# built-in mimetypes table (on Linux), so the client must produce identical
+# guesses. A private MimeTypes instance holds only that built-in table —
+# the module-level mimetypes.guess_type() on Windows also reads the registry
+# (e.g. .csv -> application/vnd.ms-excel, .wav -> audio/wav), which makes the
+# PUT's Content-Type differ from the signed one and S3 reject the upload with
+# SignatureDoesNotMatch. Also register extensions the built-in table lacks,
+# mirroring the server.
+_MIME_TYPES = mimetypes.MimeTypes()
+_MIME_TYPES.add_type("application/vnd.apache.parquet", ".parquet")
 
 from willisapi_client.timer import measure
 from willisapi_client.willisapi_client import WillisapiClient
@@ -16,6 +25,7 @@ from willisapi_client.services.metadata.utils import (
     MetadataValidation,
     ProcessedMetadataValidation,
     UploadUtils,
+    build_retry_session,
     find_files_with_pattern,
     get_last_n_directories,
 )
@@ -27,6 +37,44 @@ from willisapi_client.services.metadata.archive import (
 VALID_SCORE_TYPES = ["rater", "reviewer"]
 
 
+def _put_file_to_s3(file_presigned: dict):
+    """Upload a single file to its presigned S3 URL.
+
+    Returns an error string if the upload fails, otherwise None. Presigned
+    URLs from a single recording are all signed at once and share one expiry
+    window, so callers should run these concurrently to fit within it.
+    """
+    presigned = file_presigned.get("presigned")
+    checksum = file_presigned.get("checksum")
+    recording = file_presigned.get("recording")
+    try:
+        content_type, _ = _MIME_TYPES.guess_type(recording)
+        if not content_type:
+            # Mirror the server's fallback — it signs the presigned URL with
+            # this Content-Type when the extension is unknown.
+            content_type = "application/octet-stream"
+        with open(recording, "rb") as f:
+            session = build_retry_session()
+            response = session.put(
+                presigned,
+                data=f,
+                headers={
+                    "x-amz-checksum-sha256": checksum,
+                    "x-amz-sdk-checksum-algorithm": "SHA256",
+                    "Content-Type": content_type,
+                },
+                timeout=(10, 300),
+            )
+        if response.status_code != 200:
+            return (
+                f"S3 upload failed with status code {response.status_code} "
+                f"for file {recording}: {response.text}"
+            )
+    except Exception as ex:
+        return str(ex)
+    return None
+
+
 @measure
 def upload(api_key: str, csv_path: str, **kwargs):
 
@@ -69,13 +117,14 @@ def upload(api_key: str, csv_path: str, **kwargs):
                     presigned = res.get("response", {}).get("presigned")
                     if presigned:
                         try:
-                            content_type, _ = mimetypes.guess_type(
+                            content_type, _ = _MIME_TYPES.guess_type(
                                 payload.get("filename")
                             )
                             if not content_type:
                                 content_type = "application/octet-stream"
                             with open(row.file_path, "rb") as f:
-                                response = requests.put(
+                                session = build_retry_session()
+                                response = session.put(
                                     presigned,
                                     data=f,
                                     headers={
@@ -85,13 +134,14 @@ def upload(api_key: str, csv_path: str, **kwargs):
                                         "x-amz-sdk-checksum-algorithm": "SHA256",
                                         "Content-Type": content_type,
                                     },
+                                    timeout=(10, 300),
                                 )
                             if response.status_code == 200:
                                 result_row["upload_status"] = "Success"
                             else:
                                 result_row["upload_status"] = "Failed"
                                 result_row["error"] = (
-                                    f"S3 upload failed with status code {response.status_code}"
+                                    f"S3 upload failed with status code {response.status_code}: {response.text}"
                                 )
                         except Exception as ex:
                             result_row["upload_status"] = "Failed"
@@ -161,6 +211,12 @@ def processed_upload(api_key: str, csv_path: str, output_path: str, **kwargs):
         )
 
         results = []
+        # Throttle S3 uploads: pause 3s each time the running total of files
+        # sent to S3 crosses another 100, to avoid overwhelming the connection.
+        S3_SLEEP_EVERY = 100
+        S3_SLEEP_SECONDS = 3
+        s3_uploaded_total = 0
+        next_sleep_threshold = S3_SLEEP_EVERY
         for index, row in tqdm(
             csv.transformed_df.iterrows(), total=csv.transformed_df.shape[0]
         ):
@@ -199,33 +255,26 @@ def processed_upload(api_key: str, csv_path: str, output_path: str, **kwargs):
                     result_row["upload_status"] = "Success"
                     result_row["error"] = None
 
-                    # Handle S3 upload if presigned URL is provided
+                    # Upload every file for this recording concurrently. All
+                    # presigned URLs in this response share one short expiry
+                    # window, so uploading them sequentially risks the later
+                    # files expiring (403). A thread pool sized to the CPU
+                    # count keeps them within the window (I/O-bound work).
+                    files_to_upload = res.get("response", [])
                     s3_errors = []
-                    for file_presigned in res.get("response", []):
-                        presigned = file_presigned.get("presigned")
-                        checksum = file_presigned.get("checksum")
-                        recording = file_presigned.get("recording")
-                        index = file_presigned.get("index")
-                        try:
-                            content_type, _ = mimetypes.guess_type(recording)
-                            if not content_type:
-                                content_type = "text/csv"
-                            with open(recording, "rb") as f:
-                                response = requests.put(
-                                    presigned,
-                                    data=f,
-                                    headers={
-                                        "x-amz-checksum-sha256": checksum,
-                                        "x-amz-sdk-checksum-algorithm": "SHA256",
-                                        "Content-Type": content_type,
-                                    },
-                                )
-                            if response.status_code != 200:
-                                s3_errors.append(
-                                    f"S3 upload failed with status code {response.status_code} for file {recording}"
-                                )
-                        except Exception as ex:
-                            s3_errors.append(str(ex))
+                    if files_to_upload:
+                        max_workers = min(len(files_to_upload), os.cpu_count() or 1)
+                        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+                            for error in executor.map(
+                                _put_file_to_s3, files_to_upload
+                            ):
+                                if error:
+                                    s3_errors.append(error)
+                        # Pause after every 100 files uploaded to S3.
+                        s3_uploaded_total += len(files_to_upload)
+                        while s3_uploaded_total >= next_sleep_threshold:
+                            time.sleep(S3_SLEEP_SECONDS)
+                            next_sleep_threshold += S3_SLEEP_EVERY
                     if s3_errors:
                         result_row["upload_status"] = "Failed"
                         result_row["error"] = "\n".join(s3_errors)

{willisapi_client-1.9.7 → willisapi_client-1.9.9}/willisapi_client/services/metadata/utils.py

@@ -3,9 +3,13 @@ import numpy as np
 from typing import List, Dict, Any, Tuple
 import json
 import os
+import posixpath
 import hashlib
 import base64
 import requests
+from functools import lru_cache
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
 from urllib.parse import urlsplit
 from .language_choices import (
     LANGUAGE_CHOICES,
@@ -14,6 +18,32 @@ from dateutil import parser
 
 ALLOWED_COA_NAMES = ["MADRS", "YMRS", "PHQ-9", "GAD-7", "HAM-D17", "HAMD17"]
 
+@lru_cache(maxsize=None)
+def build_retry_session(total: int = 3, backoff_factor: float = 1) -> requests.Session:
+    """Return a requests Session that retries transient network failures.
+
+    Retries connection errors (e.g. ConnectionResetError / 'Connection reset
+    by peer') and 5xx gateway responses with exponential backoff. Applied to
+    both the metadata POST and the S3 PUT uploads so a momentary blip on a
+    single record no longer fails the whole upload for that row.
+    """
+    retry = Retry(
+        total=total,
+        connect=total,
+        read=total,
+        status=total,
+        backoff_factor=backoff_factor,
+        status_forcelist=[502, 503, 504],
+        allowed_methods=["PUT", "POST"],
+        raise_on_status=False,
+    )
+    # Increased pool size to accommodate concurrent ThreadPoolExecutor uploads
+    adapter = HTTPAdapter(max_retries=retry, pool_connections=100, pool_maxsize=100)
+    session = requests.Session()
+    session.mount("https://", adapter)
+    session.mount("http://", adapter)
+    return session
+
 COA_ITEM_COUNTS = {
     "MADRS": 10,
     "YMRS": 10,
@@ -495,7 +525,10 @@ class UploadUtils:
         self, api_key: str, url: str, headers: Dict[str, str], payload: Dict[str, Any]
     ) -> Dict[str, Any]:
         try:
-            response = requests.post(url, headers=headers, json=payload)
+            session = build_retry_session()
+            response = session.post(
+                url, headers=headers, json=payload, timeout=(10, 300)
+            )
             res_json = response.json()
 
         except Exception as ex:
@@ -725,7 +758,8 @@ class ProcessedMetadataValidation:
             orig_bucket, orig_key = _get_bucket_n_key_path_from_s3url(
                 grp.iloc[0]["recording"]
             )
-            merged_key = os.path.join(os.path.dirname(orig_key), merged_name)
+            # S3 keys always use "/" — os.path.join would insert "\" on Windows.
+            merged_key = posixpath.join(posixpath.dirname(orig_key), merged_name)
 
             # Create new URI (no upload, logical merge only)
             new_uri = f"s3://{orig_bucket}/{merged_key}"
@@ -778,9 +812,11 @@ def get_last_n_directories(filepath, n=3):
     result = None
     error = None
     try:
-        parts = filepath.split(os.sep)
+        # Normalize Windows backslashes so the result is always a
+        # forward-slash S3 key regardless of the local OS.
+        parts = filepath.replace("\\", "/").split("/")
         last_parts = parts[-(n + 1) :]
-        result = os.sep.join(last_parts)
+        result = "/".join(last_parts)
     except Exception as e:
         error = "Could not extract container name and container version from file path"
     return result, error

{willisapi_client-1.9.7 → willisapi_client-1.9.9}/willisapi_client.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: willisapi-client
-Version: 1.9.7
+Version: 1.9.9
 Summary: A Python client for willisapi
 Home-page: https://github.com/bklynhlth/willsiapi_client
 Author: bklynhlth