willisapi-client 1.9.8__tar.gz → 2.0__tar.gz

{willisapi_client-1.9.8 → willisapi_client-2.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: willisapi_client
-Version: 1.9.8
+Version: 2.0
 Summary: A Python client for willisapi
 Home-page: https://github.com/bklynhlth/willsiapi_client
 Author: bklynhlth
@@ -9,6 +9,39 @@ License: Apache
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: aiohttp==3.9.0
+Requires-Dist: aiosignal==1.3.1
+Requires-Dist: async-timeout==4.0.2
+Requires-Dist: asyncio==3.4.3
+Requires-Dist: attrs==23.1.0
+Requires-Dist: certifi==2023.7.22
+Requires-Dist: charset-normalizer==3.2.0
+Requires-Dist: frozenlist==1.4.0
+Requires-Dist: idna==3.4
+Requires-Dist: multidict==6.0.4
+Requires-Dist: nest-asyncio==1.5.7
+Requires-Dist: numpy==1.25.2
+Requires-Dist: pandas==2.0.3
+Requires-Dist: pytest==7.4.0
+Requires-Dist: pytest-cov==4.1.0
+Requires-Dist: python-dateutil==2.8.2
+Requires-Dist: pytz==2023.3
+Requires-Dist: requests==2.31.0
+Requires-Dist: six==1.16.0
+Requires-Dist: tqdm==4.66.1
+Requires-Dist: tzdata==2023.3
+Requires-Dist: urllib3==2.0.7
+Requires-Dist: yarl==1.9.2
+Dynamic: author
+Dynamic: author-email
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
 
 <!-- Output copied to clipboard! -->
 

{willisapi_client-1.9.8 → willisapi_client-2.0}/willisapi_client/__version__.py

@@ -1,7 +1,7 @@
 """Version details for willisapi_client"""
 
 __client__ = "willisapi_client"
-__latestVersion__ = "1.9.8"
+__latestVersion__ = "2.0"
 __url__ = "https://github.com/bklynhlth/willsiapi_client"
 __short_description__ = "A Python client for willisapi"
 __content_type__ = "text/markdown"

{willisapi_client-1.9.8 → willisapi_client-2.0}/willisapi_client/services/metadata/upload.py

@@ -7,9 +7,16 @@ import os
 import time
 from concurrent.futures import ThreadPoolExecutor
 
-# Register extensions that Python's default mimetypes table doesn't know about.
-# The server signs presigned URLs with these types, so the client must match.
-mimetypes.add_type("application/vnd.apache.parquet", ".parquet")
+# The server signs presigned URLs with Content-Types guessed from Python's
+# built-in mimetypes table (on Linux), so the client must produce identical
+# guesses. A private MimeTypes instance holds only that built-in table —
+# the module-level mimetypes.guess_type() on Windows also reads the registry
+# (e.g. .csv -> application/vnd.ms-excel, .wav -> audio/wav), which makes the
+# PUT's Content-Type differ from the signed one and S3 reject the upload with
+# SignatureDoesNotMatch. Also register extensions the built-in table lacks,
+# mirroring the server.
+_MIME_TYPES = mimetypes.MimeTypes()
+_MIME_TYPES.add_type("application/vnd.apache.parquet", ".parquet")
 
 from willisapi_client.timer import measure
 from willisapi_client.willisapi_client import WillisapiClient
@@ -41,9 +48,11 @@ def _put_file_to_s3(file_presigned: dict):
     checksum = file_presigned.get("checksum")
     recording = file_presigned.get("recording")
     try:
-        content_type, _ = mimetypes.guess_type(recording)
+        content_type, _ = _MIME_TYPES.guess_type(recording)
         if not content_type:
-            content_type = "text/csv"
+            # Mirror the server's fallback — it signs the presigned URL with
+            # this Content-Type when the extension is unknown.
+            content_type = "application/octet-stream"
         with open(recording, "rb") as f:
             session = build_retry_session()
             response = session.put(
@@ -108,7 +117,7 @@ def upload(api_key: str, csv_path: str, **kwargs):
                     presigned = res.get("response", {}).get("presigned")
                     if presigned:
                         try:
-                            content_type, _ = mimetypes.guess_type(
+                            content_type, _ = _MIME_TYPES.guess_type(
                                 payload.get("filename")
                             )
                             if not content_type:

{willisapi_client-1.9.8 → willisapi_client-2.0}/willisapi_client/services/metadata/utils.py

@@ -3,6 +3,7 @@ import numpy as np
 from typing import List, Dict, Any, Tuple
 import json
 import os
+import posixpath
 import hashlib
 import base64
 import requests
@@ -472,15 +473,48 @@ class UploadUtils:
         except Exception as e:
             raise RuntimeError(f"Failed to calculate checksum: {e}")
 
+    def optional_value(self, column: str):
+        """Return a column's value only when the row actually supplies one.
+
+        Optional columns may be missing from the CSV entirely, or present but
+        empty on a given row -- pandas surfaces both as an absent attribute or
+        NaN. Passing that straight through would post the string "nan" to the
+        API and overwrite good data, so it is dropped instead. Callers omit
+        the key altogether, which the API treats as "leave this field alone".
+        """
+        value = getattr(self.row, column, None)
+        if value is None or pd.isna(value):
+            return None
+        text = str(value).strip()
+        return text or None
+
+    def demographic_fields(self) -> Dict[str, Any]:
+        """Optional demographics to send, omitting any the row leaves blank."""
+        supplied = {}
+        for column in ("sex", "race"):
+            value = self.optional_value(column)
+            if value is not None:
+                supplied[column] = value
+
+        age = self.optional_value("age")
+        if age is not None:
+            try:
+                # Via float first: pandas widens an int column to float as soon
+                # as any row is blank, so a whole-number age arrives as "42.0".
+                supplied["age"] = int(float(age))
+            except (TypeError, ValueError):
+                # MetadataValidation already reports a non-numeric age; dropping
+                # it here keeps a bad cell from failing the upload as well.
+                pass
+
+        return supplied
+
     def generate_payload(self) -> Dict[str, Any]:
         payload = {
             "study_id": self.row.study_id,
             "site_id": self.row.site_id,
             "rater_id": self.row.rater_id,
             "participant_id": self.row.participant_id,
-            "age": self.row.age,
-            "sex": self.row.sex,
-            "race": self.row.race,
             "language": self.row.language,
             "visit_name": self.row.visit_name,
             "visit_order": int(self.row.visit_order),
@@ -493,6 +527,7 @@ class UploadUtils:
             "is_last_recording": self.row.is_last_recording,
             # "time_collected": parser.parse(self.row.time_collected).isoformat(),
         }
+        payload.update(self.demographic_fields())
         return payload
 
     def generate_processed_payload(
@@ -518,6 +553,7 @@ class UploadUtils:
         site_country = getattr(self.row, "site_country", None)
         if site_country:
             payload["site_country"] = site_country
+        payload.update(self.demographic_fields())
         return payload
 
     def post(
@@ -757,7 +793,8 @@ class ProcessedMetadataValidation:
             orig_bucket, orig_key = _get_bucket_n_key_path_from_s3url(
                 grp.iloc[0]["recording"]
             )
-            merged_key = os.path.join(os.path.dirname(orig_key), merged_name)
+            # S3 keys always use "/" — os.path.join would insert "\" on Windows.
+            merged_key = posixpath.join(posixpath.dirname(orig_key), merged_name)
 
             # Create new URI (no upload, logical merge only)
             new_uri = f"s3://{orig_bucket}/{merged_key}"
@@ -810,9 +847,11 @@ def get_last_n_directories(filepath, n=3):
     result = None
     error = None
     try:
-        parts = filepath.split(os.sep)
+        # Normalize Windows backslashes so the result is always a
+        # forward-slash S3 key regardless of the local OS.
+        parts = filepath.replace("\\", "/").split("/")
         last_parts = parts[-(n + 1) :]
-        result = os.sep.join(last_parts)
+        result = "/".join(last_parts)
     except Exception as e:
         error = "Could not extract container name and container version from file path"
     return result, error

{willisapi_client-1.9.8 → willisapi_client-2.0}/willisapi_client.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
-Name: willisapi-client
-Version: 1.9.8
+Metadata-Version: 2.4
+Name: willisapi_client
+Version: 2.0
 Summary: A Python client for willisapi
 Home-page: https://github.com/bklynhlth/willsiapi_client
 Author: bklynhlth
@@ -9,6 +9,39 @@ License: Apache
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: aiohttp==3.9.0
+Requires-Dist: aiosignal==1.3.1
+Requires-Dist: async-timeout==4.0.2
+Requires-Dist: asyncio==3.4.3
+Requires-Dist: attrs==23.1.0
+Requires-Dist: certifi==2023.7.22
+Requires-Dist: charset-normalizer==3.2.0
+Requires-Dist: frozenlist==1.4.0
+Requires-Dist: idna==3.4
+Requires-Dist: multidict==6.0.4
+Requires-Dist: nest-asyncio==1.5.7
+Requires-Dist: numpy==1.25.2
+Requires-Dist: pandas==2.0.3
+Requires-Dist: pytest==7.4.0
+Requires-Dist: pytest-cov==4.1.0
+Requires-Dist: python-dateutil==2.8.2
+Requires-Dist: pytz==2023.3
+Requires-Dist: requests==2.31.0
+Requires-Dist: six==1.16.0
+Requires-Dist: tqdm==4.66.1
+Requires-Dist: tzdata==2023.3
+Requires-Dist: urllib3==2.0.7
+Requires-Dist: yarl==1.9.2
+Dynamic: author
+Dynamic: author-email
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
 
 <!-- Output copied to clipboard! -->