unitysvc-services 0.2.7__tar.gz → 0.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {unitysvc_services-0.2.7/src/unitysvc_services.egg-info → unitysvc_services-0.3.1}/PKG-INFO +1 -1
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/pyproject.toml +1 -1
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/src/unitysvc_services/publisher.py +259 -73
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/src/unitysvc_services/query.py +88 -15
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1/src/unitysvc_services.egg-info}/PKG-INFO +1 -1
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/CONTRIBUTING.md +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/HISTORY.md +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/LICENSE +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/MANIFEST.in +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/README.md +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/docs/api-reference.md +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/docs/cli-reference.md +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/docs/contributing.md +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/docs/data-structure.md +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/docs/development.md +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/docs/file-schemas.md +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/docs/getting-started.md +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/docs/index.md +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/docs/installation.md +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/docs/usage.md +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/docs/workflows.md +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/setup.cfg +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/src/unitysvc_services/__init__.py +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/src/unitysvc_services/cli.py +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/src/unitysvc_services/format_data.py +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/src/unitysvc_services/list.py +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/src/unitysvc_services/models/__init__.py +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/src/unitysvc_services/models/base.py +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/src/unitysvc_services/models/listing_v1.py +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/src/unitysvc_services/models/provider_v1.py +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/src/unitysvc_services/models/seller_v1.py +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/src/unitysvc_services/models/service_v1.py +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/src/unitysvc_services/populate.py +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/src/unitysvc_services/py.typed +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/src/unitysvc_services/scaffold.py +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/src/unitysvc_services/update.py +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/src/unitysvc_services/utils.py +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/src/unitysvc_services/validator.py +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/src/unitysvc_services.egg-info/SOURCES.txt +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/src/unitysvc_services.egg-info/dependency_links.txt +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/src/unitysvc_services.egg-info/entry_points.txt +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/src/unitysvc_services.egg-info/requires.txt +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/src/unitysvc_services.egg-info/top_level.txt +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/tests/__init__.py +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/tests/example_data/README.md +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/tests/example_data/provider1/README.md +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/tests/example_data/provider1/provider.toml +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/tests/example_data/provider1/services/service1/code-example.md +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/tests/example_data/provider1/services/service1/service.toml +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/tests/example_data/provider1/services/service1/svcreseller.toml +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/tests/example_data/provider1/terms-of-service.md +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/tests/example_data/provider2/README.md +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/tests/example_data/provider2/provider.json +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/tests/example_data/provider2/services/service2/code-example.md +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/tests/example_data/provider2/services/service2/service.json +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/tests/example_data/provider2/services/service2/svcreseller.json +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/tests/example_data/provider2/terms-of-service.md +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/tests/example_data/seller.json +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/tests/test_utils.py +0 -0
- {unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/tests/test_validator.py +0 -0
@@ -18,7 +18,11 @@ from .validator import DataValidator
|
|
18
18
|
|
19
19
|
|
20
20
|
class ServiceDataPublisher:
|
21
|
-
"""Publishes service data to UnitySVC backend endpoints.
|
21
|
+
"""Publishes service data to UnitySVC backend endpoints.
|
22
|
+
|
23
|
+
Uses httpx by default, with automatic fallback to curl for systems
|
24
|
+
with network restrictions (e.g., macOS with conda Python).
|
25
|
+
"""
|
22
26
|
|
23
27
|
def __init__(self) -> None:
|
24
28
|
self.base_url = os.environ.get("UNITYSVC_BASE_URL")
|
@@ -30,6 +34,7 @@ class ServiceDataPublisher:
|
|
30
34
|
raise ValueError("UNITYSVC_API_KEY environment variable not set")
|
31
35
|
|
32
36
|
self.base_url = self.base_url.rstrip("/")
|
37
|
+
self.use_curl_fallback = False
|
33
38
|
self.async_client = httpx.AsyncClient(
|
34
39
|
headers={
|
35
40
|
"X-API-Key": self.api_key,
|
@@ -97,6 +102,174 @@ class ServiceDataPublisher:
|
|
97
102
|
|
98
103
|
return result
|
99
104
|
|
105
|
+
async def _make_request_curl_async(
|
106
|
+
self, endpoint: str, method: str = "GET", data: dict[str, Any] | None = None
|
107
|
+
) -> tuple[dict[str, Any], int]:
|
108
|
+
"""Make HTTP request using curl (async version).
|
109
|
+
|
110
|
+
Args:
|
111
|
+
endpoint: API endpoint path
|
112
|
+
method: HTTP method (GET or POST)
|
113
|
+
data: JSON data for POST requests
|
114
|
+
|
115
|
+
Returns:
|
116
|
+
Tuple of (JSON response, HTTP status code)
|
117
|
+
|
118
|
+
Raises:
|
119
|
+
RuntimeError: If curl command fails
|
120
|
+
"""
|
121
|
+
url = f"{self.base_url}{endpoint}"
|
122
|
+
|
123
|
+
cmd = [
|
124
|
+
"curl",
|
125
|
+
"-s", # Silent mode
|
126
|
+
"-w",
|
127
|
+
"\n%{http_code}", # Write HTTP status code on new line
|
128
|
+
"-X",
|
129
|
+
method,
|
130
|
+
"-H",
|
131
|
+
f"X-API-Key: {self.api_key}",
|
132
|
+
"-H",
|
133
|
+
"Content-Type: application/json",
|
134
|
+
]
|
135
|
+
|
136
|
+
if data:
|
137
|
+
cmd.extend(["-d", json.dumps(data)])
|
138
|
+
|
139
|
+
cmd.append(url)
|
140
|
+
|
141
|
+
try:
|
142
|
+
proc = await asyncio.create_subprocess_exec(
|
143
|
+
*cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
|
144
|
+
)
|
145
|
+
stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=30.0)
|
146
|
+
|
147
|
+
if proc.returncode != 0:
|
148
|
+
error_msg = stderr.decode().strip() if stderr else "Unknown error"
|
149
|
+
raise RuntimeError(f"curl command failed: {error_msg}")
|
150
|
+
|
151
|
+
output = stdout.decode().strip()
|
152
|
+
lines = output.split("\n")
|
153
|
+
status_code = int(lines[-1])
|
154
|
+
body = "\n".join(lines[:-1])
|
155
|
+
|
156
|
+
response_data = json.loads(body) if body else {}
|
157
|
+
return (response_data, status_code)
|
158
|
+
|
159
|
+
except TimeoutError:
|
160
|
+
raise RuntimeError("Request timed out after 30 seconds")
|
161
|
+
except json.JSONDecodeError as e:
|
162
|
+
raise RuntimeError(f"Invalid JSON response: {e}")
|
163
|
+
|
164
|
+
async def post(self, endpoint: str, data: dict[str, Any], check_status: bool = True) -> tuple[dict[str, Any], int]:
|
165
|
+
"""Make a POST request to the backend API with automatic curl fallback.
|
166
|
+
|
167
|
+
Public utility method for making POST requests. Tries httpx first for performance,
|
168
|
+
automatically falls back to curl if network restrictions are detected (e.g., macOS
|
169
|
+
with conda Python).
|
170
|
+
|
171
|
+
Can be used by other packages that need the same fallback behavior.
|
172
|
+
|
173
|
+
Args:
|
174
|
+
endpoint: API endpoint path (e.g., "/publish/seller")
|
175
|
+
data: JSON data to post
|
176
|
+
check_status: Whether to raise on non-2xx status codes (default: True)
|
177
|
+
|
178
|
+
Returns:
|
179
|
+
Tuple of (JSON response, HTTP status code)
|
180
|
+
|
181
|
+
Raises:
|
182
|
+
RuntimeError: If both httpx and curl fail
|
183
|
+
"""
|
184
|
+
|
185
|
+
# Helper class to simulate httpx response for curl
|
186
|
+
class CurlResponse:
|
187
|
+
def __init__(self, json_data: dict[str, Any], status: int):
|
188
|
+
self._json_data = json_data
|
189
|
+
self.status_code = status
|
190
|
+
self.is_success = 200 <= status < 300
|
191
|
+
self.text = json.dumps(json_data)
|
192
|
+
|
193
|
+
def json(self) -> dict[str, Any]:
|
194
|
+
return self._json_data
|
195
|
+
|
196
|
+
def raise_for_status(self) -> None:
|
197
|
+
if not self.is_success:
|
198
|
+
raise RuntimeError(f"HTTP {self.status_code}: {self.text}")
|
199
|
+
|
200
|
+
# If we already know curl is needed, use it directly
|
201
|
+
if self.use_curl_fallback:
|
202
|
+
response_data, status_code = await self._make_request_curl_async(endpoint, method="POST", data=data)
|
203
|
+
response = CurlResponse(response_data, status_code)
|
204
|
+
else:
|
205
|
+
try:
|
206
|
+
response = await self.async_client.post(f"{self.base_url}{endpoint}", json=data)
|
207
|
+
except (httpx.ConnectError, OSError):
|
208
|
+
# Connection failed - switch to curl fallback and retry
|
209
|
+
self.use_curl_fallback = True
|
210
|
+
response_data, status_code = await self._make_request_curl_async(endpoint, method="POST", data=data)
|
211
|
+
response = CurlResponse(response_data, status_code)
|
212
|
+
|
213
|
+
if check_status:
|
214
|
+
response.raise_for_status()
|
215
|
+
|
216
|
+
return (response.json(), response.status_code)
|
217
|
+
|
218
|
+
async def check_task(self, task_id: str, poll_interval: float = 2.0, timeout: float = 300.0) -> dict[str, Any]:
|
219
|
+
"""Check and wait for task completion.
|
220
|
+
|
221
|
+
Utility function to poll a Celery task until it completes or times out.
|
222
|
+
|
223
|
+
Args:
|
224
|
+
task_id: Celery task ID to poll
|
225
|
+
poll_interval: Seconds between status checks (default: 2.0)
|
226
|
+
timeout: Maximum seconds to wait (default: 300.0)
|
227
|
+
|
228
|
+
Returns:
|
229
|
+
Task result dictionary
|
230
|
+
|
231
|
+
Raises:
|
232
|
+
ValueError: If task fails or times out
|
233
|
+
"""
|
234
|
+
import time
|
235
|
+
|
236
|
+
start_time = time.time()
|
237
|
+
|
238
|
+
while True:
|
239
|
+
elapsed = time.time() - start_time
|
240
|
+
if elapsed > timeout:
|
241
|
+
raise ValueError(f"Task {task_id} timed out after {timeout}s")
|
242
|
+
|
243
|
+
# Check task status using the get() equivalent for async
|
244
|
+
try:
|
245
|
+
if self.use_curl_fallback:
|
246
|
+
status, _ = await self._make_request_curl_async(f"/tasks/{task_id}", method="GET")
|
247
|
+
else:
|
248
|
+
try:
|
249
|
+
response = await self.async_client.get(f"{self.base_url}/tasks/{task_id}")
|
250
|
+
response.raise_for_status()
|
251
|
+
status = response.json()
|
252
|
+
except (httpx.ConnectError, OSError):
|
253
|
+
# Connection failed - switch to curl fallback
|
254
|
+
self.use_curl_fallback = True
|
255
|
+
status, _ = await self._make_request_curl_async(f"/tasks/{task_id}", method="GET")
|
256
|
+
except (httpx.HTTPError, httpx.NetworkError, httpx.TimeoutException, RuntimeError):
|
257
|
+
# Network error while checking status - retry
|
258
|
+
await asyncio.sleep(poll_interval)
|
259
|
+
continue
|
260
|
+
|
261
|
+
state = status.get("state", "PENDING")
|
262
|
+
|
263
|
+
# Check if task is complete
|
264
|
+
if status.get("status") == "completed" or state == "SUCCESS":
|
265
|
+
return status.get("result", {})
|
266
|
+
elif status.get("status") == "failed" or state == "FAILURE":
|
267
|
+
error = status.get("error", "Unknown error")
|
268
|
+
raise ValueError(f"Task {task_id} failed: {error}")
|
269
|
+
|
270
|
+
# Still processing - wait and retry
|
271
|
+
await asyncio.sleep(poll_interval)
|
272
|
+
|
100
273
|
async def _post_with_retry(
|
101
274
|
self,
|
102
275
|
endpoint: str,
|
@@ -107,7 +280,13 @@ class ServiceDataPublisher:
|
|
107
280
|
max_retries: int = 3,
|
108
281
|
) -> dict[str, Any]:
|
109
282
|
"""
|
110
|
-
Generic retry wrapper for posting data to backend API.
|
283
|
+
Generic retry wrapper for posting data to backend API with task polling.
|
284
|
+
|
285
|
+
The backend now returns HTTP 202 with a task_id. This method:
|
286
|
+
1. Submits the publish request
|
287
|
+
2. Gets the task_id from the response
|
288
|
+
3. Polls /tasks/{task_id} until completion
|
289
|
+
4. Returns the final result
|
111
290
|
|
112
291
|
Args:
|
113
292
|
endpoint: API endpoint path (e.g., "/publish/listing")
|
@@ -126,50 +305,57 @@ class ServiceDataPublisher:
|
|
126
305
|
last_exception = None
|
127
306
|
for attempt in range(max_retries):
|
128
307
|
try:
|
129
|
-
|
130
|
-
|
131
|
-
json=data,
|
132
|
-
)
|
308
|
+
# Use the public post() method with automatic curl fallback
|
309
|
+
response_json, status_code = await self.post(endpoint, data, check_status=False)
|
133
310
|
|
134
|
-
#
|
135
|
-
if
|
136
|
-
#
|
137
|
-
|
138
|
-
error_detail = "Unknown error"
|
139
|
-
try:
|
140
|
-
error_json = response.json()
|
141
|
-
error_detail = error_json.get("detail", str(error_json))
|
142
|
-
except Exception:
|
143
|
-
error_detail = response.text or f"HTTP {response.status_code}"
|
311
|
+
# Handle task-based response (HTTP 202)
|
312
|
+
if status_code == 202:
|
313
|
+
# Backend returns task_id - poll for completion
|
314
|
+
task_id = response_json.get("task_id")
|
144
315
|
|
316
|
+
if not task_id:
|
317
|
+
context_msg = f" ({context_info})" if context_info else ""
|
318
|
+
raise ValueError(f"No task_id in response for {entity_type} '{entity_name}'{context_msg}")
|
319
|
+
|
320
|
+
# Poll task status until completion using check_task utility
|
321
|
+
try:
|
322
|
+
result = await self.check_task(task_id)
|
323
|
+
return result
|
324
|
+
except ValueError as e:
|
325
|
+
# Add context to task errors
|
326
|
+
context_msg = f" ({context_info})" if context_info else ""
|
327
|
+
raise ValueError(f"Task failed for {entity_type} '{entity_name}'{context_msg}: {e}")
|
328
|
+
|
329
|
+
# Check for errors
|
330
|
+
if status_code >= 400:
|
331
|
+
# Don't retry on 4xx errors (client errors) - they won't succeed on retry
|
332
|
+
if 400 <= status_code < 500:
|
333
|
+
error_detail = response_json.get("detail", str(response_json))
|
145
334
|
context_msg = f" ({context_info})" if context_info else ""
|
146
335
|
raise ValueError(
|
147
336
|
f"Failed to publish {entity_type} '{entity_name}'{context_msg}: {error_detail}"
|
148
337
|
)
|
149
338
|
|
150
|
-
# 5xx errors
|
339
|
+
# 5xx errors - retry with exponential backoff
|
151
340
|
if attempt < max_retries - 1:
|
152
341
|
wait_time = 2**attempt # Exponential backoff: 1s, 2s, 4s
|
153
342
|
await asyncio.sleep(wait_time)
|
154
343
|
continue
|
155
344
|
else:
|
156
345
|
# Last attempt failed
|
157
|
-
error_detail = "
|
158
|
-
try:
|
159
|
-
error_json = response.json()
|
160
|
-
error_detail = error_json.get("detail", str(error_json))
|
161
|
-
except Exception:
|
162
|
-
error_detail = response.text or f"HTTP {response.status_code}"
|
163
|
-
|
346
|
+
error_detail = response_json.get("detail", str(response_json))
|
164
347
|
context_msg = f" ({context_info})" if context_info else ""
|
165
348
|
raise ValueError(
|
166
349
|
f"Failed to publish {entity_type} after {max_retries} attempts: "
|
167
350
|
f"'{entity_name}'{context_msg}: {error_detail}"
|
168
351
|
)
|
169
352
|
|
170
|
-
|
353
|
+
# Success response (2xx)
|
354
|
+
return response_json
|
171
355
|
|
172
|
-
except (httpx.NetworkError, httpx.TimeoutException) as e:
|
356
|
+
except (httpx.NetworkError, httpx.TimeoutException, RuntimeError) as e:
|
357
|
+
# Network/connection errors - the post() method should have tried curl fallback
|
358
|
+
# If we're here, both httpx and curl failed
|
173
359
|
last_exception = e
|
174
360
|
if attempt < max_retries - 1:
|
175
361
|
wait_time = 2**attempt # Exponential backoff: 1s, 2s, 4s
|
@@ -478,7 +664,7 @@ class ServiceDataPublisher:
|
|
478
664
|
console.print(f" [red]✗[/red] Failed to publish offering: [cyan]{offering_name}[/cyan] - {str(e)}")
|
479
665
|
return (offering_file, e)
|
480
666
|
|
481
|
-
def publish_all_offerings(self, data_dir: Path) -> dict[str, Any]:
|
667
|
+
async def publish_all_offerings(self, data_dir: Path) -> dict[str, Any]:
|
482
668
|
"""
|
483
669
|
Publish all service offerings found in a directory tree concurrently.
|
484
670
|
|
@@ -510,14 +696,10 @@ class ServiceDataPublisher:
|
|
510
696
|
console = Console()
|
511
697
|
|
512
698
|
# Run all offering publications concurrently with rate limiting
|
513
|
-
|
514
|
-
|
515
|
-
|
516
|
-
|
517
|
-
return await asyncio.gather(*tasks)
|
518
|
-
|
519
|
-
# Execute async tasks
|
520
|
-
task_results = asyncio.run(_publish_all())
|
699
|
+
# Create semaphore to limit concurrent requests
|
700
|
+
semaphore = asyncio.Semaphore(self.max_concurrent_requests)
|
701
|
+
tasks = [self._publish_offering_task(offering_file, console, semaphore) for offering_file in offering_files]
|
702
|
+
task_results = await asyncio.gather(*tasks)
|
521
703
|
|
522
704
|
# Process results
|
523
705
|
for offering_file, result in task_results:
|
@@ -565,7 +747,7 @@ class ServiceDataPublisher:
|
|
565
747
|
console.print(f" [red]✗[/red] Failed to publish listing: [cyan]{listing_file}[/cyan] - {str(e)}")
|
566
748
|
return (listing_file, e)
|
567
749
|
|
568
|
-
def publish_all_listings(self, data_dir: Path) -> dict[str, Any]:
|
750
|
+
async def publish_all_listings(self, data_dir: Path) -> dict[str, Any]:
|
569
751
|
"""
|
570
752
|
Publish all service listings found in a directory tree concurrently.
|
571
753
|
|
@@ -597,14 +779,10 @@ class ServiceDataPublisher:
|
|
597
779
|
console = Console()
|
598
780
|
|
599
781
|
# Run all listing publications concurrently with rate limiting
|
600
|
-
|
601
|
-
|
602
|
-
|
603
|
-
|
604
|
-
return await asyncio.gather(*tasks)
|
605
|
-
|
606
|
-
# Execute async tasks
|
607
|
-
task_results = asyncio.run(_publish_all())
|
782
|
+
# Create semaphore to limit concurrent requests
|
783
|
+
semaphore = asyncio.Semaphore(self.max_concurrent_requests)
|
784
|
+
tasks = [self._publish_listing_task(listing_file, console, semaphore) for listing_file in listing_files]
|
785
|
+
task_results = await asyncio.gather(*tasks)
|
608
786
|
|
609
787
|
# Process results
|
610
788
|
for listing_file, result in task_results:
|
@@ -647,7 +825,7 @@ class ServiceDataPublisher:
|
|
647
825
|
console.print(f" [red]✗[/red] Failed to publish provider: [cyan]{provider_name}[/cyan] - {str(e)}")
|
648
826
|
return (provider_file, e)
|
649
827
|
|
650
|
-
def publish_all_providers(self, data_dir: Path) -> dict[str, Any]:
|
828
|
+
async def publish_all_providers(self, data_dir: Path) -> dict[str, Any]:
|
651
829
|
"""
|
652
830
|
Publish all providers found in a directory tree concurrently.
|
653
831
|
|
@@ -667,14 +845,10 @@ class ServiceDataPublisher:
|
|
667
845
|
console = Console()
|
668
846
|
|
669
847
|
# Run all provider publications concurrently with rate limiting
|
670
|
-
|
671
|
-
|
672
|
-
|
673
|
-
|
674
|
-
return await asyncio.gather(*tasks)
|
675
|
-
|
676
|
-
# Execute async tasks
|
677
|
-
task_results = asyncio.run(_publish_all())
|
848
|
+
# Create semaphore to limit concurrent requests
|
849
|
+
semaphore = asyncio.Semaphore(self.max_concurrent_requests)
|
850
|
+
tasks = [self._publish_provider_task(provider_file, console, semaphore) for provider_file in provider_files]
|
851
|
+
task_results = await asyncio.gather(*tasks)
|
678
852
|
|
679
853
|
# Process results
|
680
854
|
for provider_file, result in task_results:
|
@@ -717,7 +891,7 @@ class ServiceDataPublisher:
|
|
717
891
|
console.print(f" [red]✗[/red] Failed to publish seller: [cyan]{seller_name}[/cyan] - {str(e)}")
|
718
892
|
return (seller_file, e)
|
719
893
|
|
720
|
-
def publish_all_sellers(self, data_dir: Path) -> dict[str, Any]:
|
894
|
+
async def publish_all_sellers(self, data_dir: Path) -> dict[str, Any]:
|
721
895
|
"""
|
722
896
|
Publish all sellers found in a directory tree concurrently.
|
723
897
|
|
@@ -737,14 +911,10 @@ class ServiceDataPublisher:
|
|
737
911
|
console = Console()
|
738
912
|
|
739
913
|
# Run all seller publications concurrently with rate limiting
|
740
|
-
|
741
|
-
|
742
|
-
|
743
|
-
|
744
|
-
return await asyncio.gather(*tasks)
|
745
|
-
|
746
|
-
# Execute async tasks
|
747
|
-
task_results = asyncio.run(_publish_all())
|
914
|
+
# Create semaphore to limit concurrent requests
|
915
|
+
semaphore = asyncio.Semaphore(self.max_concurrent_requests)
|
916
|
+
tasks = [self._publish_seller_task(seller_file, console, semaphore) for seller_file in seller_files]
|
917
|
+
task_results = await asyncio.gather(*tasks)
|
748
918
|
|
749
919
|
# Process results
|
750
920
|
for seller_file, result in task_results:
|
@@ -756,7 +926,7 @@ class ServiceDataPublisher:
|
|
756
926
|
|
757
927
|
return results
|
758
928
|
|
759
|
-
def publish_all_models(self, data_dir: Path) -> dict[str, Any]:
|
929
|
+
async def publish_all_models(self, data_dir: Path) -> dict[str, Any]:
|
760
930
|
"""
|
761
931
|
Publish all data types in the correct order.
|
762
932
|
|
@@ -788,7 +958,7 @@ class ServiceDataPublisher:
|
|
788
958
|
|
789
959
|
for data_type, publish_method in publish_order:
|
790
960
|
try:
|
791
|
-
results = publish_method(data_dir)
|
961
|
+
results = await publish_method(data_dir)
|
792
962
|
all_results[data_type] = results
|
793
963
|
all_results["total_success"] += results["success"]
|
794
964
|
all_results["total_failed"] += results["failed"]
|
@@ -805,9 +975,25 @@ class ServiceDataPublisher:
|
|
805
975
|
|
806
976
|
return all_results
|
807
977
|
|
978
|
+
async def aclose(self):
|
979
|
+
"""Close HTTP client asynchronously."""
|
980
|
+
await self.async_client.aclose()
|
981
|
+
|
808
982
|
def close(self):
|
809
|
-
"""Close HTTP client."""
|
810
|
-
|
983
|
+
"""Close HTTP client synchronously (best effort)."""
|
984
|
+
try:
|
985
|
+
# Try to close if there's an event loop running
|
986
|
+
loop = asyncio.get_event_loop()
|
987
|
+
if loop.is_running():
|
988
|
+
# Can't close synchronously if loop is running
|
989
|
+
# The client will be garbage collected
|
990
|
+
return
|
991
|
+
else:
|
992
|
+
# Loop exists but not running, we can use it
|
993
|
+
loop.run_until_complete(self.async_client.aclose())
|
994
|
+
except RuntimeError:
|
995
|
+
# No event loop or loop is closed - just let it be garbage collected
|
996
|
+
pass
|
811
997
|
|
812
998
|
def __enter__(self):
|
813
999
|
"""Context manager entry."""
|
@@ -870,8 +1056,8 @@ def publish_callback(
|
|
870
1056
|
|
871
1057
|
try:
|
872
1058
|
with ServiceDataPublisher() as publisher:
|
873
|
-
# Call the publish_all_models method
|
874
|
-
all_results = publisher.publish_all_models(data_path)
|
1059
|
+
# Call the publish_all_models method (now async)
|
1060
|
+
all_results = asyncio.run(publisher.publish_all_models(data_path))
|
875
1061
|
|
876
1062
|
# Display results for each data type
|
877
1063
|
data_type_display_names = {
|
@@ -965,7 +1151,7 @@ def publish_providers(
|
|
965
1151
|
else:
|
966
1152
|
console.print(f"[blue]Scanning for providers in:[/blue] {data_path}")
|
967
1153
|
console.print(f"[blue]Backend URL:[/blue] {os.getenv('UNITYSVC_BASE_URL', 'N/A')}\n")
|
968
|
-
results = publisher.publish_all_providers(data_path)
|
1154
|
+
results = asyncio.run(publisher.publish_all_providers(data_path))
|
969
1155
|
|
970
1156
|
# Display summary
|
971
1157
|
console.print("\n[bold]Publishing Summary:[/bold]")
|
@@ -1024,7 +1210,7 @@ def publish_sellers(
|
|
1024
1210
|
else:
|
1025
1211
|
console.print(f"[blue]Scanning for sellers in:[/blue] {data_path}")
|
1026
1212
|
console.print(f"[blue]Backend URL:[/blue] {os.getenv('UNITYSVC_BASE_URL', 'N/A')}\n")
|
1027
|
-
results = publisher.publish_all_sellers(data_path)
|
1213
|
+
results = asyncio.run(publisher.publish_all_sellers(data_path))
|
1028
1214
|
|
1029
1215
|
console.print("\n[bold]Publishing Summary:[/bold]")
|
1030
1216
|
console.print(f" Total found: {results['total']}")
|
@@ -1080,8 +1266,8 @@ def publish_offerings(
|
|
1080
1266
|
# Handle directory
|
1081
1267
|
else:
|
1082
1268
|
console.print(f"[blue]Scanning for service offerings in:[/blue] {data_path}")
|
1083
|
-
console.print(f"[blue]Backend URL:[/blue] {os.getenv('UNITYSVC_BASE_URL', 'N/A')}\n")
|
1084
|
-
results = publisher.publish_all_offerings(data_path)
|
1269
|
+
console.print(f"[blue]Backend URL:[/bold blue] {os.getenv('UNITYSVC_BASE_URL', 'N/A')}\n")
|
1270
|
+
results = asyncio.run(publisher.publish_all_offerings(data_path))
|
1085
1271
|
|
1086
1272
|
console.print("\n[bold]Publishing Summary:[/bold]")
|
1087
1273
|
console.print(f" Total found: {results['total']}")
|
@@ -1139,7 +1325,7 @@ def publish_listings(
|
|
1139
1325
|
else:
|
1140
1326
|
console.print(f"[blue]Scanning for service listings in:[/blue] {data_path}")
|
1141
1327
|
console.print(f"[blue]Backend URL:[/blue] {os.getenv('UNITYSVC_BASE_URL', 'N/A')}\n")
|
1142
|
-
results = publisher.publish_all_listings(data_path)
|
1328
|
+
results = asyncio.run(publisher.publish_all_listings(data_path))
|
1143
1329
|
|
1144
1330
|
console.print("\n[bold]Publishing Summary:[/bold]")
|
1145
1331
|
console.print(f" Total found: {results['total']}")
|
@@ -2,7 +2,9 @@
|
|
2
2
|
|
3
3
|
import json
|
4
4
|
import os
|
5
|
+
import subprocess
|
5
6
|
from typing import Any
|
7
|
+
from urllib.parse import urlencode
|
6
8
|
|
7
9
|
import httpx
|
8
10
|
import typer
|
@@ -14,7 +16,11 @@ console = Console()
|
|
14
16
|
|
15
17
|
|
16
18
|
class ServiceDataQuery:
|
17
|
-
"""Query service data from UnitySVC backend endpoints.
|
19
|
+
"""Query service data from UnitySVC backend endpoints.
|
20
|
+
|
21
|
+
Uses httpx by default, with automatic fallback to curl for systems
|
22
|
+
with network restrictions (e.g., macOS with conda Python).
|
23
|
+
"""
|
18
24
|
|
19
25
|
def __init__(self) -> None:
|
20
26
|
"""Initialize query client from environment variables.
|
@@ -31,6 +37,7 @@ class ServiceDataQuery:
|
|
31
37
|
raise ValueError("UNITYSVC_API_KEY environment variable not set")
|
32
38
|
|
33
39
|
self.base_url = self.base_url.rstrip("/")
|
40
|
+
self.use_curl_fallback = False
|
34
41
|
self.client = httpx.Client(
|
35
42
|
headers={
|
36
43
|
"X-API-Key": self.api_key,
|
@@ -39,6 +46,80 @@ class ServiceDataQuery:
|
|
39
46
|
timeout=30.0,
|
40
47
|
)
|
41
48
|
|
49
|
+
def _make_request_curl(self, endpoint: str, params: dict[str, Any] | None = None) -> dict[str, Any]:
|
50
|
+
"""Make HTTP GET request using curl fallback.
|
51
|
+
|
52
|
+
Args:
|
53
|
+
endpoint: API endpoint path (e.g., "/publish/sellers")
|
54
|
+
params: Query parameters
|
55
|
+
|
56
|
+
Returns:
|
57
|
+
JSON response as dictionary
|
58
|
+
|
59
|
+
Raises:
|
60
|
+
RuntimeError: If curl command fails or returns non-200 status
|
61
|
+
"""
|
62
|
+
url = f"{self.base_url}{endpoint}"
|
63
|
+
if params:
|
64
|
+
url = f"{url}?{urlencode(params)}"
|
65
|
+
|
66
|
+
cmd = [
|
67
|
+
"curl",
|
68
|
+
"-s", # Silent mode
|
69
|
+
"-f", # Fail on HTTP errors
|
70
|
+
"-H",
|
71
|
+
f"X-API-Key: {self.api_key}",
|
72
|
+
"-H",
|
73
|
+
"Accept: application/json",
|
74
|
+
url,
|
75
|
+
]
|
76
|
+
|
77
|
+
try:
|
78
|
+
result = subprocess.run(cmd, capture_output=True, text=True, check=True, timeout=30)
|
79
|
+
return json.loads(result.stdout)
|
80
|
+
except subprocess.CalledProcessError as e:
|
81
|
+
error_msg = e.stderr.strip() if e.stderr else "Unknown error"
|
82
|
+
raise RuntimeError(f"HTTP request failed: {error_msg}")
|
83
|
+
except subprocess.TimeoutExpired:
|
84
|
+
raise RuntimeError("Request timed out after 30 seconds")
|
85
|
+
except json.JSONDecodeError as e:
|
86
|
+
raise RuntimeError(f"Invalid JSON response: {e}")
|
87
|
+
|
88
|
+
def get(self, endpoint: str, params: dict[str, Any] | None = None) -> dict[str, Any]:
|
89
|
+
"""Make a GET request to the backend API with automatic curl fallback.
|
90
|
+
|
91
|
+
Public utility method for making GET requests. Tries httpx first for performance,
|
92
|
+
automatically falls back to curl if network restrictions are detected (e.g., macOS
|
93
|
+
with conda Python).
|
94
|
+
|
95
|
+
Can be used by subclasses and other packages (e.g., unitysvc_admin) that need
|
96
|
+
the same fallback behavior.
|
97
|
+
|
98
|
+
Args:
|
99
|
+
endpoint: API endpoint path (e.g., "/publish/sellers", "/admin/documents")
|
100
|
+
params: Query parameters
|
101
|
+
|
102
|
+
Returns:
|
103
|
+
JSON response as dictionary
|
104
|
+
|
105
|
+
Raises:
|
106
|
+
RuntimeError: If both httpx and curl fail
|
107
|
+
"""
|
108
|
+
# If we already know curl is needed, use it directly
|
109
|
+
if self.use_curl_fallback:
|
110
|
+
return self._make_request_curl(endpoint, params)
|
111
|
+
|
112
|
+
# Try httpx first
|
113
|
+
try:
|
114
|
+
response = self.client.get(f"{self.base_url}{endpoint}", params=params)
|
115
|
+
response.raise_for_status()
|
116
|
+
return response.json()
|
117
|
+
except (httpx.ConnectError, OSError) as e:
|
118
|
+
# Connection failed - likely network restrictions
|
119
|
+
# Fall back to curl and remember this for future requests
|
120
|
+
self.use_curl_fallback = True
|
121
|
+
return self._make_request_curl(endpoint, params)
|
122
|
+
|
42
123
|
def list_service_offerings(self, skip: int = 0, limit: int = 100) -> list[dict[str, Any]]:
|
43
124
|
"""List all service offerings from the backend.
|
44
125
|
|
@@ -46,9 +127,7 @@ class ServiceDataQuery:
|
|
46
127
|
skip: Number of records to skip (for pagination)
|
47
128
|
limit: Maximum number of records to return
|
48
129
|
"""
|
49
|
-
|
50
|
-
response.raise_for_status()
|
51
|
-
result = response.json()
|
130
|
+
result = self.get("/publish/offerings", {"skip": skip, "limit": limit})
|
52
131
|
return result.get("data", result) if isinstance(result, dict) else result
|
53
132
|
|
54
133
|
def list_service_listings(self, skip: int = 0, limit: int = 100) -> list[dict[str, Any]]:
|
@@ -58,9 +137,7 @@ class ServiceDataQuery:
|
|
58
137
|
skip: Number of records to skip (for pagination)
|
59
138
|
limit: Maximum number of records to return
|
60
139
|
"""
|
61
|
-
|
62
|
-
response.raise_for_status()
|
63
|
-
result = response.json()
|
140
|
+
result = self.get("/publish/listings", {"skip": skip, "limit": limit})
|
64
141
|
return result.get("data", result) if isinstance(result, dict) else result
|
65
142
|
|
66
143
|
def list_providers(self, skip: int = 0, limit: int = 100) -> list[dict[str, Any]]:
|
@@ -70,9 +147,7 @@ class ServiceDataQuery:
|
|
70
147
|
skip: Number of records to skip (for pagination)
|
71
148
|
limit: Maximum number of records to return
|
72
149
|
"""
|
73
|
-
|
74
|
-
response.raise_for_status()
|
75
|
-
result = response.json()
|
150
|
+
result = self.get("/publish/providers", {"skip": skip, "limit": limit})
|
76
151
|
return result.get("data", result) if isinstance(result, dict) else result
|
77
152
|
|
78
153
|
def list_sellers(self, skip: int = 0, limit: int = 100) -> list[dict[str, Any]]:
|
@@ -82,14 +157,12 @@ class ServiceDataQuery:
|
|
82
157
|
skip: Number of records to skip (for pagination)
|
83
158
|
limit: Maximum number of records to return
|
84
159
|
"""
|
85
|
-
|
86
|
-
response.raise_for_status()
|
87
|
-
result = response.json()
|
160
|
+
result = self.get("/publish/sellers", {"skip": skip, "limit": limit})
|
88
161
|
return result.get("data", result) if isinstance(result, dict) else result
|
89
162
|
|
90
163
|
def close(self):
|
91
|
-
"""Close the HTTP client."""
|
92
|
-
|
164
|
+
"""Close the HTTP client (no-op for curl-based implementation)."""
|
165
|
+
pass
|
93
166
|
|
94
167
|
def __enter__(self):
|
95
168
|
"""Context manager entry."""
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
{unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/src/unitysvc_services/models/__init__.py
RENAMED
File without changes
|
File without changes
|
{unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/src/unitysvc_services/models/listing_v1.py
RENAMED
File without changes
|
{unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/src/unitysvc_services/models/provider_v1.py
RENAMED
File without changes
|
{unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/src/unitysvc_services/models/seller_v1.py
RENAMED
File without changes
|
{unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/src/unitysvc_services/models/service_v1.py
RENAMED
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
{unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/src/unitysvc_services.egg-info/SOURCES.txt
RENAMED
File without changes
|
File without changes
|
{unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/src/unitysvc_services.egg-info/entry_points.txt
RENAMED
File without changes
|
{unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/src/unitysvc_services.egg-info/requires.txt
RENAMED
File without changes
|
{unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/src/unitysvc_services.egg-info/top_level.txt
RENAMED
File without changes
|
File without changes
|
File without changes
|
File without changes
|
{unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/tests/example_data/provider1/provider.toml
RENAMED
File without changes
|
File without changes
|
File without changes
|
File without changes
|
{unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/tests/example_data/provider1/terms-of-service.md
RENAMED
File without changes
|
File without changes
|
{unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/tests/example_data/provider2/provider.json
RENAMED
File without changes
|
File without changes
|
File without changes
|
File without changes
|
{unitysvc_services-0.2.7 → unitysvc_services-0.3.1}/tests/example_data/provider2/terms-of-service.md
RENAMED
File without changes
|
File without changes
|
File without changes
|
File without changes
|