clarity-api-sdk-python 0.3.34__tar.gz → 0.3.35__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/PKG-INFO +2 -1
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/pyproject.toml +5 -1
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/clarity_api_sdk_python.egg-info/PKG-INFO +2 -1
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/clarity_api_sdk_python.egg-info/SOURCES.txt +9 -1
- clarity_api_sdk_python-0.3.35/src/clarity_api_sdk_python.egg-info/entry_points.txt +2 -0
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/clarity_api_sdk_python.egg-info/requires.txt +1 -0
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/cti/api/async_client.py +18 -14
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/cti/api/client.py +18 -14
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/cti/api/sonar_wiz_api.py +220 -4
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/cti/api/sonar_wiz_async_api.py +210 -3
- clarity_api_sdk_python-0.3.35/src/cti/cli/__init__.py +1 -0
- clarity_api_sdk_python-0.3.35/src/cti/cli/__main__.py +20 -0
- clarity_api_sdk_python-0.3.35/src/cti/cli/client.py +45 -0
- clarity_api_sdk_python-0.3.35/src/cti/cli/main.py +458 -0
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/cti/model/raw_file_device_mapping.py +12 -3
- clarity_api_sdk_python-0.3.35/tests/test_cli.py +497 -0
- clarity_api_sdk_python-0.3.35/tests/test_sdk_async_methods.py +194 -0
- clarity_api_sdk_python-0.3.35/tests/test_sdk_methods.py +242 -0
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/README.md +0 -0
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/setup.cfg +0 -0
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/clarity_api_sdk_python.egg-info/dependency_links.txt +0 -0
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/clarity_api_sdk_python.egg-info/top_level.txt +0 -0
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/cti/__init__.py +0 -0
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/cti/api/__init__.py +0 -0
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/cti/api/session.py +0 -0
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/cti/logger/__init__.py +0 -0
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/cti/logger/logger.py +0 -0
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/cti/main.py +0 -0
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/cti/main_api.py +0 -0
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/cti/model/__init__.py +0 -0
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/cti/model/altitude_source.py +0 -0
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/cti/model/attitude_source.py +0 -0
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/cti/model/deferred_object_deletion.py +0 -0
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/cti/model/depth_source.py +0 -0
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/cti/model/device.py +0 -0
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/cti/model/device_type.py +0 -0
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/cti/model/final_product.py +0 -0
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/cti/model/hierarchy.py +0 -0
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/cti/model/layback_algorithm.py +0 -0
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/cti/model/layback_source.py +0 -0
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/cti/model/layback_type.py +0 -0
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/cti/model/organization.py +0 -0
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/cti/model/platform.py +0 -0
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/cti/model/platform_type.py +0 -0
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/cti/model/position_source.py +0 -0
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/cti/model/processing_job.py +0 -0
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/cti/model/processing_log.py +0 -0
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/cti/model/project.py +0 -0
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/cti/model/projection_option.py +0 -0
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/cti/model/raw_file.py +0 -0
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/cti/model/raw_file_configuration.py +0 -0
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/cti/model/raw_file_state.py +0 -0
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/cti/model/s3.py +0 -0
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/cti/model/sidescan_ping_source.py +0 -0
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/cti/model/source.py +0 -0
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/cti/model/survey.py +0 -0
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/cti/model/target.py +0 -0
- {clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/cti/model/tow_system.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: clarity-api-sdk-python
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.35
|
|
4
4
|
Summary: A Python SDK to connect to the CTI Clarity API server.
|
|
5
5
|
Author-email: "Chesapeake Technology Inc." <support@chesapeaketech.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/chesapeake-tech/clarity-api-sdk-python
|
|
@@ -16,6 +16,7 @@ Requires-Dist: httpx_auth>=0.23.1
|
|
|
16
16
|
Requires-Dist: httpx-retries>=0.4.5
|
|
17
17
|
Requires-Dist: pydantic==2.12.3
|
|
18
18
|
Requires-Dist: structlog==25.4.0
|
|
19
|
+
Requires-Dist: typer>=0.15
|
|
19
20
|
Provides-Extra: dev
|
|
20
21
|
Requires-Dist: black==25.9.0; extra == "dev"
|
|
21
22
|
Requires-Dist: build==1.3.0; extra == "dev"
|
|
@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
|
|
|
5
5
|
|
|
6
6
|
[project]
|
|
7
7
|
name = "clarity-api-sdk-python"
|
|
8
|
-
version = "0.3.
|
|
8
|
+
version = "0.3.35"
|
|
9
9
|
authors = [
|
|
10
10
|
{ name="Chesapeake Technology Inc.", email="support@chesapeaketech.com" },
|
|
11
11
|
]
|
|
@@ -25,6 +25,7 @@ dependencies = [
|
|
|
25
25
|
"httpx-retries>=0.4.5",
|
|
26
26
|
"pydantic==2.12.3",
|
|
27
27
|
"structlog==25.4.0",
|
|
28
|
+
"typer>=0.15",
|
|
28
29
|
]
|
|
29
30
|
|
|
30
31
|
[project.optional-dependencies]
|
|
@@ -43,6 +44,9 @@ dev = [
|
|
|
43
44
|
brotli = ["httpx[brotli]>=0.28.1"]
|
|
44
45
|
http2 = ["httpx[http2]>=0.28.1"]
|
|
45
46
|
|
|
47
|
+
[project.scripts]
|
|
48
|
+
ftm = "cti.cli.__main__:main"
|
|
49
|
+
|
|
46
50
|
[project.urls]
|
|
47
51
|
"Homepage" = "https://github.com/chesapeake-tech/clarity-api-sdk-python"
|
|
48
52
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: clarity-api-sdk-python
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.35
|
|
4
4
|
Summary: A Python SDK to connect to the CTI Clarity API server.
|
|
5
5
|
Author-email: "Chesapeake Technology Inc." <support@chesapeaketech.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/chesapeake-tech/clarity-api-sdk-python
|
|
@@ -16,6 +16,7 @@ Requires-Dist: httpx_auth>=0.23.1
|
|
|
16
16
|
Requires-Dist: httpx-retries>=0.4.5
|
|
17
17
|
Requires-Dist: pydantic==2.12.3
|
|
18
18
|
Requires-Dist: structlog==25.4.0
|
|
19
|
+
Requires-Dist: typer>=0.15
|
|
19
20
|
Provides-Extra: dev
|
|
20
21
|
Requires-Dist: black==25.9.0; extra == "dev"
|
|
21
22
|
Requires-Dist: build==1.3.0; extra == "dev"
|
|
@@ -3,6 +3,7 @@ pyproject.toml
|
|
|
3
3
|
src/clarity_api_sdk_python.egg-info/PKG-INFO
|
|
4
4
|
src/clarity_api_sdk_python.egg-info/SOURCES.txt
|
|
5
5
|
src/clarity_api_sdk_python.egg-info/dependency_links.txt
|
|
6
|
+
src/clarity_api_sdk_python.egg-info/entry_points.txt
|
|
6
7
|
src/clarity_api_sdk_python.egg-info/requires.txt
|
|
7
8
|
src/clarity_api_sdk_python.egg-info/top_level.txt
|
|
8
9
|
src/cti/__init__.py
|
|
@@ -14,6 +15,10 @@ src/cti/api/client.py
|
|
|
14
15
|
src/cti/api/session.py
|
|
15
16
|
src/cti/api/sonar_wiz_api.py
|
|
16
17
|
src/cti/api/sonar_wiz_async_api.py
|
|
18
|
+
src/cti/cli/__init__.py
|
|
19
|
+
src/cti/cli/__main__.py
|
|
20
|
+
src/cti/cli/client.py
|
|
21
|
+
src/cti/cli/main.py
|
|
17
22
|
src/cti/logger/__init__.py
|
|
18
23
|
src/cti/logger/logger.py
|
|
19
24
|
src/cti/model/__init__.py
|
|
@@ -45,4 +50,7 @@ src/cti/model/sidescan_ping_source.py
|
|
|
45
50
|
src/cti/model/source.py
|
|
46
51
|
src/cti/model/survey.py
|
|
47
52
|
src/cti/model/target.py
|
|
48
|
-
src/cti/model/tow_system.py
|
|
53
|
+
src/cti/model/tow_system.py
|
|
54
|
+
tests/test_cli.py
|
|
55
|
+
tests/test_sdk_async_methods.py
|
|
56
|
+
tests/test_sdk_methods.py
|
|
@@ -25,20 +25,24 @@ class ClarityApiAsyncClient(AsyncClient):
|
|
|
25
25
|
|
|
26
26
|
def __init__(self):
|
|
27
27
|
|
|
28
|
-
#
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
28
|
+
# In local dev mode, skip OAuth2 authentication — the server
|
|
29
|
+
# accepts unauthenticated requests when CLARITY_PLATFORM=local.
|
|
30
|
+
if os.environ.get("CLARITY_PLATFORM") == "local":
|
|
31
|
+
cti_credentials = None
|
|
32
|
+
else:
|
|
33
|
+
cti_credentials = OAuth2ClientCredentials(
|
|
34
|
+
token_url=(
|
|
35
|
+
f'{os.environ.get("KEYCLOAK_SERVER_URL", "missing KEYCLOAK_SERVER_URL")}/realms/'
|
|
36
|
+
f'{os.environ.get("KEYCLOAK_REALM", "missing KEYCLOAK_REALM")}'
|
|
37
|
+
"/protocol/openid-connect/token"
|
|
38
|
+
),
|
|
39
|
+
client_id=os.environ.get(
|
|
40
|
+
"KEYCLOAK_CLIENT_ID", "missing KEYCLOAK_CLIENT_ID"
|
|
41
|
+
),
|
|
42
|
+
client_secret=os.environ.get(
|
|
43
|
+
"KEYCLOAK_CLIENT_SECRET", "missing KEYCLOAK_CLIENT_SECRET"
|
|
44
|
+
),
|
|
45
|
+
)
|
|
42
46
|
|
|
43
47
|
# retry mechanism for API requests
|
|
44
48
|
retry = Retry(total=12, backoff_factor=0.5)
|
|
@@ -22,20 +22,24 @@ class ClarityApiClient(Client):
|
|
|
22
22
|
|
|
23
23
|
def __init__(self):
|
|
24
24
|
|
|
25
|
-
#
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
25
|
+
# In local dev mode, skip OAuth2 authentication — the server
|
|
26
|
+
# accepts unauthenticated requests when CLARITY_PLATFORM=local.
|
|
27
|
+
if os.environ.get("CLARITY_PLATFORM") == "local":
|
|
28
|
+
cti_credentials = None
|
|
29
|
+
else:
|
|
30
|
+
cti_credentials = OAuth2ClientCredentials(
|
|
31
|
+
token_url=(
|
|
32
|
+
f'{os.environ.get("KEYCLOAK_SERVER_URL", "missing KEYCLOAK_SERVER_URL")}/realms/'
|
|
33
|
+
f'{os.environ.get("KEYCLOAK_REALM", "missing KEYCLOAK_REALM")}'
|
|
34
|
+
"/protocol/openid-connect/token"
|
|
35
|
+
),
|
|
36
|
+
client_id=os.environ.get(
|
|
37
|
+
"KEYCLOAK_CLIENT_ID", "missing KEYCLOAK_CLIENT_ID"
|
|
38
|
+
),
|
|
39
|
+
client_secret=os.environ.get(
|
|
40
|
+
"KEYCLOAK_CLIENT_SECRET", "missing KEYCLOAK_CLIENT_SECRET"
|
|
41
|
+
),
|
|
42
|
+
)
|
|
39
43
|
|
|
40
44
|
# retry mechanism for API requests
|
|
41
45
|
retry = Retry(total=12, backoff_factor=0.5)
|
{clarity_api_sdk_python-0.3.34 → clarity_api_sdk_python-0.3.35}/src/cti/api/sonar_wiz_api.py
RENAMED
|
@@ -4,8 +4,14 @@ Provides strongly-typed methods for interacting with the SonarWiz API,
|
|
|
4
4
|
using Pydantic models for request and response validation.
|
|
5
5
|
"""
|
|
6
6
|
|
|
7
|
+
import os
|
|
8
|
+
import time
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from urllib.parse import urlparse
|
|
7
11
|
from uuid import UUID
|
|
8
12
|
|
|
13
|
+
import httpx
|
|
14
|
+
|
|
9
15
|
from cti.model.altitude_source import AltitudeSource
|
|
10
16
|
from cti.model.altitude_source import AltitudeSourceCreate
|
|
11
17
|
from cti.model.altitude_source import AltitudeSourceUpdate
|
|
@@ -49,6 +55,7 @@ from cti.model.position_source import PositionSourceCreate
|
|
|
49
55
|
from cti.model.position_source import PositionSourceUpdate
|
|
50
56
|
from cti.model.processing_job import JobStatusCallback
|
|
51
57
|
from cti.model.processing_job import ProcessingJob
|
|
58
|
+
from cti.model.processing_job import ProcessingJobCreate
|
|
52
59
|
from cti.model.processing_job import ProcessingJobList
|
|
53
60
|
from cti.model.processing_log import ProcessingLog
|
|
54
61
|
from cti.model.processing_log import ProcessingLogCreate
|
|
@@ -110,11 +117,11 @@ class SonarWizApi:
|
|
|
110
117
|
Pydantic models for responses. HTTP errors are allowed to propagate.
|
|
111
118
|
"""
|
|
112
119
|
|
|
113
|
-
def __init__(self, client: ClarityApiClient):
|
|
114
|
-
"""Initialize API wrapper with
|
|
120
|
+
def __init__(self, client: ClarityApiClient | httpx.Client):
|
|
121
|
+
"""Initialize API wrapper with an HTTP client.
|
|
115
122
|
|
|
116
123
|
Args:
|
|
117
|
-
client: ClarityApiClient
|
|
124
|
+
client: ClarityApiClient or plain httpx.Client (for local dev).
|
|
118
125
|
"""
|
|
119
126
|
super().__init__()
|
|
120
127
|
self._client = client
|
|
@@ -446,7 +453,7 @@ class SonarWizApi:
|
|
|
446
453
|
Created project instance.
|
|
447
454
|
"""
|
|
448
455
|
response = self._client.post(
|
|
449
|
-
"/api/v1/projects
|
|
456
|
+
"/api/v1/projects", json=project.model_dump(mode="json")
|
|
450
457
|
)
|
|
451
458
|
response.raise_for_status()
|
|
452
459
|
return Project.model_validate(response.json())
|
|
@@ -1999,6 +2006,22 @@ class SonarWizApi:
|
|
|
1999
2006
|
|
|
2000
2007
|
# ── Processing Jobs ──────────────────────────────────────────────────
|
|
2001
2008
|
|
|
2009
|
+
def create_job(self, job_data: ProcessingJobCreate) -> ProcessingJob:
|
|
2010
|
+
"""Create a new processing job.
|
|
2011
|
+
|
|
2012
|
+
Args:
|
|
2013
|
+
job_data: Job creation data with survey_id and job_type.
|
|
2014
|
+
|
|
2015
|
+
Returns:
|
|
2016
|
+
Created processing job instance.
|
|
2017
|
+
"""
|
|
2018
|
+
response = self._client.post(
|
|
2019
|
+
"/api/v1/jobs",
|
|
2020
|
+
json=job_data.model_dump(mode="json"),
|
|
2021
|
+
)
|
|
2022
|
+
response.raise_for_status()
|
|
2023
|
+
return ProcessingJob.model_validate(response.json())
|
|
2024
|
+
|
|
2002
2025
|
def get_job(self, job_id: UUID | str) -> ProcessingJob:
|
|
2003
2026
|
"""Get processing job by ID.
|
|
2004
2027
|
|
|
@@ -2057,3 +2080,196 @@ class SonarWizApi:
|
|
|
2057
2080
|
)
|
|
2058
2081
|
response.raise_for_status()
|
|
2059
2082
|
return ProcessingJob.model_validate(response.json())
|
|
2083
|
+
|
|
2084
|
+
# ── High-Level Workflow Methods ──────────────────────────────────────
|
|
2085
|
+
|
|
2086
|
+
def upload_file(
|
|
2087
|
+
self,
|
|
2088
|
+
survey_id: UUID,
|
|
2089
|
+
file_path: str | Path,
|
|
2090
|
+
raw_file_configuration_id: UUID | None = None,
|
|
2091
|
+
chunk_size: int = 5 * 1024 * 1024,
|
|
2092
|
+
) -> RawFile:
|
|
2093
|
+
"""Upload a sonar file to a survey via multipart S3 upload.
|
|
2094
|
+
|
|
2095
|
+
Handles the full upload flow: create raw file record, upload parts
|
|
2096
|
+
via presigned URLs, and complete the multipart upload.
|
|
2097
|
+
|
|
2098
|
+
Args:
|
|
2099
|
+
survey_id: Survey to upload to.
|
|
2100
|
+
file_path: Local path to the sonar file (.xtf, .jsf, etc.).
|
|
2101
|
+
raw_file_configuration_id: Optional config ID. If None, uses the
|
|
2102
|
+
survey's default configuration.
|
|
2103
|
+
chunk_size: Size of each upload part in bytes (default 5MB, S3 minimum).
|
|
2104
|
+
|
|
2105
|
+
Returns:
|
|
2106
|
+
Completed RawFile with state="ready".
|
|
2107
|
+
"""
|
|
2108
|
+
file_path = Path(file_path)
|
|
2109
|
+
file_size = file_path.stat().st_size
|
|
2110
|
+
|
|
2111
|
+
# Look up default raw file configuration if not provided
|
|
2112
|
+
if raw_file_configuration_id is None:
|
|
2113
|
+
response = self._client.get(
|
|
2114
|
+
f"/api/v1/surveys/{survey_id}/rawfileconfigurations"
|
|
2115
|
+
)
|
|
2116
|
+
response.raise_for_status()
|
|
2117
|
+
configs = [RawFileConfiguration.model_validate(c) for c in response.json()]
|
|
2118
|
+
if not configs:
|
|
2119
|
+
raise ValueError(f"No raw file configuration found for survey {survey_id}")
|
|
2120
|
+
raw_file_configuration_id = configs[0].raw_file_configuration_id
|
|
2121
|
+
|
|
2122
|
+
# Initiate multipart upload
|
|
2123
|
+
raw_file_create = RawFileCreate(
|
|
2124
|
+
survey_id=survey_id,
|
|
2125
|
+
raw_file_configuration_id=raw_file_configuration_id,
|
|
2126
|
+
file_name=file_path.name,
|
|
2127
|
+
file_size=file_size,
|
|
2128
|
+
)
|
|
2129
|
+
response = self._client.post(
|
|
2130
|
+
"/api/v1/rawfiles/upload",
|
|
2131
|
+
json=raw_file_create.model_dump(mode="json"),
|
|
2132
|
+
)
|
|
2133
|
+
response.raise_for_status()
|
|
2134
|
+
raw_file = RawFileWithUpload.model_validate(response.json())
|
|
2135
|
+
|
|
2136
|
+
logger.info(
|
|
2137
|
+
"upload_started",
|
|
2138
|
+
raw_file_id=str(raw_file.raw_file_id),
|
|
2139
|
+
file_name=file_path.name,
|
|
2140
|
+
file_size=file_size,
|
|
2141
|
+
)
|
|
2142
|
+
|
|
2143
|
+
# Upload parts
|
|
2144
|
+
parts: list[dict] = []
|
|
2145
|
+
part_number = 0
|
|
2146
|
+
try:
|
|
2147
|
+
with httpx.Client(timeout=120) as s3_client, open(file_path, "rb") as f:
|
|
2148
|
+
while True:
|
|
2149
|
+
chunk = f.read(chunk_size)
|
|
2150
|
+
if not chunk:
|
|
2151
|
+
break
|
|
2152
|
+
part_number += 1
|
|
2153
|
+
|
|
2154
|
+
# Get presigned URL for this part
|
|
2155
|
+
url_response = self._client.get(
|
|
2156
|
+
f"/api/v1/rawfiles/{raw_file.raw_file_id}/upload-parts/{part_number}/url"
|
|
2157
|
+
)
|
|
2158
|
+
url_response.raise_for_status()
|
|
2159
|
+
presigned_url = PartPresignedURL.model_validate(url_response.json()).url
|
|
2160
|
+
|
|
2161
|
+
# The server generates presigned URLs using its S3_ENDPOINT_URL,
|
|
2162
|
+
# which may be a Docker-internal hostname (e.g. minio:9000).
|
|
2163
|
+
# If the client has its own S3_ENDPOINT_URL (e.g. localhost:9000),
|
|
2164
|
+
# rewrite the URL so it's reachable from the client's network.
|
|
2165
|
+
client_s3_endpoint = os.environ.get("S3_ENDPOINT_URL")
|
|
2166
|
+
if client_s3_endpoint:
|
|
2167
|
+
parsed_presigned = urlparse(presigned_url)
|
|
2168
|
+
parsed_client = urlparse(client_s3_endpoint)
|
|
2169
|
+
presigned_url = presigned_url.replace(
|
|
2170
|
+
f"{parsed_presigned.scheme}://{parsed_presigned.netloc}",
|
|
2171
|
+
f"{parsed_client.scheme}://{parsed_client.netloc}",
|
|
2172
|
+
1,
|
|
2173
|
+
)
|
|
2174
|
+
|
|
2175
|
+
# Upload directly to S3/MinIO (not through API client)
|
|
2176
|
+
put_response = s3_client.put(presigned_url, content=chunk)
|
|
2177
|
+
put_response.raise_for_status()
|
|
2178
|
+
etag = put_response.headers.get("ETag", "").strip('"')
|
|
2179
|
+
|
|
2180
|
+
parts.append({"ETag": etag, "PartNumber": part_number})
|
|
2181
|
+
logger.debug(
|
|
2182
|
+
"upload_part_complete",
|
|
2183
|
+
part=part_number,
|
|
2184
|
+
etag=etag,
|
|
2185
|
+
)
|
|
2186
|
+
except Exception:
|
|
2187
|
+
logger.error(
|
|
2188
|
+
"upload_failed",
|
|
2189
|
+
raw_file_id=str(raw_file.raw_file_id),
|
|
2190
|
+
parts_uploaded=part_number,
|
|
2191
|
+
)
|
|
2192
|
+
raise
|
|
2193
|
+
|
|
2194
|
+
# Complete multipart upload
|
|
2195
|
+
complete_body = CompleteMultipartUploadBody(
|
|
2196
|
+
parts=[Part(ETag=p["ETag"], PartNumber=p["PartNumber"]) for p in parts]
|
|
2197
|
+
)
|
|
2198
|
+
complete_response = self._client.post(
|
|
2199
|
+
f"/api/v1/rawfiles/{raw_file.raw_file_id}/upload-complete",
|
|
2200
|
+
json=complete_body.model_dump(mode="json"),
|
|
2201
|
+
)
|
|
2202
|
+
complete_response.raise_for_status()
|
|
2203
|
+
completed = RawFile.model_validate(complete_response.json())
|
|
2204
|
+
|
|
2205
|
+
logger.info(
|
|
2206
|
+
"upload_complete",
|
|
2207
|
+
raw_file_id=str(completed.raw_file_id),
|
|
2208
|
+
state=completed.state,
|
|
2209
|
+
parts=part_number,
|
|
2210
|
+
)
|
|
2211
|
+
return completed
|
|
2212
|
+
|
|
2213
|
+
def trigger_processing(self, survey_id: UUID) -> ProcessingJob:
|
|
2214
|
+
"""Trigger processing for a survey.
|
|
2215
|
+
|
|
2216
|
+
Creates a processing job and dispatches it via SQS.
|
|
2217
|
+
|
|
2218
|
+
Args:
|
|
2219
|
+
survey_id: Survey to process.
|
|
2220
|
+
|
|
2221
|
+
Returns:
|
|
2222
|
+
The created ProcessingJob.
|
|
2223
|
+
"""
|
|
2224
|
+
response = self._client.post(f"/api/v1/ingest/{survey_id}")
|
|
2225
|
+
response.raise_for_status()
|
|
2226
|
+
data = response.json()
|
|
2227
|
+
|
|
2228
|
+
# The ingest endpoint returns a dict with job_id, not a full ProcessingJob.
|
|
2229
|
+
# Fetch the full job record.
|
|
2230
|
+
if job_id := data.get("job_id"):
|
|
2231
|
+
return self.get_job(job_id)
|
|
2232
|
+
|
|
2233
|
+
return ProcessingJob.model_validate(data)
|
|
2234
|
+
|
|
2235
|
+
def wait_for_job(
|
|
2236
|
+
self,
|
|
2237
|
+
job_id: UUID | str,
|
|
2238
|
+
timeout: float = 300,
|
|
2239
|
+
poll_interval: float = 5,
|
|
2240
|
+
) -> ProcessingJob:
|
|
2241
|
+
"""Poll a processing job until it reaches a terminal state.
|
|
2242
|
+
|
|
2243
|
+
Args:
|
|
2244
|
+
job_id: Job UUID to poll.
|
|
2245
|
+
timeout: Maximum seconds to wait (default 5 minutes).
|
|
2246
|
+
poll_interval: Seconds between polls (default 5).
|
|
2247
|
+
|
|
2248
|
+
Returns:
|
|
2249
|
+
The completed or failed ProcessingJob.
|
|
2250
|
+
|
|
2251
|
+
Raises:
|
|
2252
|
+
TimeoutError: If the job doesn't reach a terminal state within timeout.
|
|
2253
|
+
"""
|
|
2254
|
+
terminal_states = {"completed", "failed", "cancelled"}
|
|
2255
|
+
deadline = time.monotonic() + timeout
|
|
2256
|
+
|
|
2257
|
+
while time.monotonic() < deadline:
|
|
2258
|
+
job = self.get_job(job_id)
|
|
2259
|
+
if job.status in terminal_states:
|
|
2260
|
+
return job
|
|
2261
|
+
|
|
2262
|
+
elapsed = timeout - (deadline - time.monotonic())
|
|
2263
|
+
logger.debug(
|
|
2264
|
+
"wait_for_job_poll",
|
|
2265
|
+
job_id=str(job_id),
|
|
2266
|
+
status=job.status,
|
|
2267
|
+
phase=job.current_phase,
|
|
2268
|
+
progress=job.progress,
|
|
2269
|
+
elapsed=round(elapsed, 1),
|
|
2270
|
+
)
|
|
2271
|
+
time.sleep(poll_interval)
|
|
2272
|
+
|
|
2273
|
+
raise TimeoutError(
|
|
2274
|
+
f"Job {job_id} did not complete within {timeout}s (last status: {job.status})"
|
|
2275
|
+
)
|