unitysvc-services 0.1.0__py3-none-any.whl → 0.1.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- unitysvc_services/api.py +278 -0
- unitysvc_services/format_data.py +2 -7
- unitysvc_services/list.py +14 -43
- unitysvc_services/models/base.py +139 -0
- unitysvc_services/models/listing_v1.py +23 -3
- unitysvc_services/models/provider_v1.py +23 -2
- unitysvc_services/models/seller_v1.py +12 -6
- unitysvc_services/models/service_v1.py +8 -1
- unitysvc_services/populate.py +2 -6
- unitysvc_services/publisher.py +732 -467
- unitysvc_services/py.typed +0 -0
- unitysvc_services/query.py +521 -318
- unitysvc_services/update.py +10 -14
- unitysvc_services/utils.py +105 -7
- unitysvc_services/validator.py +194 -10
- {unitysvc_services-0.1.0.dist-info → unitysvc_services-0.1.4.dist-info}/METADATA +42 -39
- unitysvc_services-0.1.4.dist-info/RECORD +25 -0
- unitysvc_services-0.1.0.dist-info/RECORD +0 -23
- {unitysvc_services-0.1.0.dist-info → unitysvc_services-0.1.4.dist-info}/WHEEL +0 -0
- {unitysvc_services-0.1.0.dist-info → unitysvc_services-0.1.4.dist-info}/entry_points.txt +0 -0
- {unitysvc_services-0.1.0.dist-info → unitysvc_services-0.1.4.dist-info}/licenses/LICENSE +0 -0
- {unitysvc_services-0.1.0.dist-info → unitysvc_services-0.1.4.dist-info}/top_level.txt +0 -0
unitysvc_services/publisher.py
CHANGED
@@ -1,6 +1,9 @@
|
|
1
1
|
"""Data publisher module for posting service data to UnitySVC backend."""
|
2
2
|
|
3
|
+
import asyncio
|
4
|
+
import base64
|
3
5
|
import json
|
6
|
+
import os
|
4
7
|
import tomllib as toml
|
5
8
|
from pathlib import Path
|
6
9
|
from typing import Any
|
@@ -9,20 +12,25 @@ import httpx
|
|
9
12
|
import typer
|
10
13
|
from rich.console import Console
|
11
14
|
|
15
|
+
from .api import UnitySvcAPI
|
16
|
+
from .models.base import ProviderStatusEnum, SellerStatusEnum
|
17
|
+
from .utils import convert_convenience_fields_to_documents, find_files_by_schema
|
18
|
+
from .validator import DataValidator
|
12
19
|
|
13
|
-
class ServiceDataPublisher:
|
14
|
-
"""Publishes service data to UnitySVC backend endpoints."""
|
15
20
|
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
)
|
21
|
+
class ServiceDataPublisher(UnitySvcAPI):
|
22
|
+
"""Publishes service data to UnitySVC backend endpoints.
|
23
|
+
|
24
|
+
Inherits base HTTP client with curl fallback from UnitySvcAPI.
|
25
|
+
Extends with async operations for concurrent publishing.
|
26
|
+
"""
|
27
|
+
|
28
|
+
def __init__(self) -> None:
|
29
|
+
# Initialize base class (provides self.client as AsyncClient with curl fallback)
|
30
|
+
super().__init__()
|
31
|
+
|
32
|
+
# Semaphore to limit concurrent requests and prevent connection pool exhaustion
|
33
|
+
self.max_concurrent_requests = 15
|
26
34
|
|
27
35
|
def load_data_file(self, file_path: Path) -> dict[str, Any]:
|
28
36
|
"""Load data from JSON or TOML file."""
|
@@ -48,8 +56,6 @@ class ServiceDataPublisher:
|
|
48
56
|
return f.read()
|
49
57
|
except UnicodeDecodeError:
|
50
58
|
# If it fails, read as binary and encode as base64
|
51
|
-
import base64
|
52
|
-
|
53
59
|
with open(full_path, "rb") as f:
|
54
60
|
return base64.b64encode(f.read()).decode("ascii")
|
55
61
|
|
@@ -83,152 +89,248 @@ class ServiceDataPublisher:
|
|
83
89
|
|
84
90
|
return result
|
85
91
|
|
86
|
-
def
|
87
|
-
|
92
|
+
async def post( # type: ignore[override]
|
93
|
+
self, endpoint: str, data: dict[str, Any], check_status: bool = True
|
94
|
+
) -> tuple[dict[str, Any], int]:
|
95
|
+
"""Make a POST request to the backend API with automatic curl fallback.
|
88
96
|
|
89
|
-
|
90
|
-
|
97
|
+
Override of base class post() that returns both JSON and status code.
|
98
|
+
Uses base class client with automatic curl fallback.
|
99
|
+
|
100
|
+
Args:
|
101
|
+
endpoint: API endpoint path (e.g., "/publish/seller")
|
102
|
+
data: JSON data to post
|
103
|
+
check_status: Whether to raise on non-2xx status codes (default: True)
|
104
|
+
|
105
|
+
Returns:
|
106
|
+
Tuple of (JSON response, HTTP status code)
|
107
|
+
|
108
|
+
Raises:
|
109
|
+
RuntimeError: If both httpx and curl fail
|
91
110
|
"""
|
92
|
-
#
|
93
|
-
|
111
|
+
# Use base class client (self.client from UnitySvcQuery) with automatic curl fallback
|
112
|
+
# If we already know curl is needed, use it directly
|
113
|
+
if self.use_curl_fallback:
|
114
|
+
# Use base class curl fallback method
|
115
|
+
response_json = await super().post(endpoint, json_data=data)
|
116
|
+
# Curl POST doesn't return status code separately, assume 2xx if no exception
|
117
|
+
status_code = 200
|
118
|
+
else:
|
119
|
+
try:
|
120
|
+
response = await self.client.post(f"{self.base_url}{endpoint}", json=data)
|
121
|
+
status_code = response.status_code
|
122
|
+
|
123
|
+
if check_status:
|
124
|
+
response.raise_for_status()
|
125
|
+
|
126
|
+
response_json = response.json()
|
127
|
+
except (httpx.ConnectError, OSError):
|
128
|
+
# Connection failed - switch to curl fallback and retry
|
129
|
+
self.use_curl_fallback = True
|
130
|
+
response_json = await super().post(endpoint, json_data=data)
|
131
|
+
status_code = 200 # Assume success if curl didn't raise
|
132
|
+
|
133
|
+
return (response_json, status_code)
|
134
|
+
|
135
|
+
async def _post_with_retry(
|
136
|
+
self,
|
137
|
+
endpoint: str,
|
138
|
+
data: dict[str, Any],
|
139
|
+
entity_type: str,
|
140
|
+
entity_name: str,
|
141
|
+
context_info: str = "",
|
142
|
+
max_retries: int = 3,
|
143
|
+
) -> dict[str, Any]:
|
144
|
+
"""
|
145
|
+
Generic retry wrapper for posting data to backend API with task polling.
|
146
|
+
|
147
|
+
The backend now returns HTTP 202 with a task_id. This method:
|
148
|
+
1. Submits the publish request
|
149
|
+
2. Gets the task_id from the response
|
150
|
+
3. Polls /tasks/{task_id} until completion
|
151
|
+
4. Returns the final result
|
152
|
+
|
153
|
+
Args:
|
154
|
+
endpoint: API endpoint path (e.g., "/publish/listing")
|
155
|
+
data: JSON data to post
|
156
|
+
entity_type: Type of entity being published (for error messages)
|
157
|
+
entity_name: Name of the entity being published (for error messages)
|
158
|
+
context_info: Additional context for error messages (e.g., provider, service info)
|
159
|
+
max_retries: Maximum number of retry attempts
|
160
|
+
|
161
|
+
Returns:
|
162
|
+
Response JSON from successful API call
|
163
|
+
|
164
|
+
Raises:
|
165
|
+
ValueError: On client errors (4xx) or after exhausting retries
|
166
|
+
"""
|
167
|
+
last_exception = None
|
168
|
+
for attempt in range(max_retries):
|
169
|
+
try:
|
170
|
+
# Use the public post() method with automatic curl fallback
|
171
|
+
response_json, status_code = await self.post(endpoint, data, check_status=False)
|
94
172
|
|
95
|
-
|
96
|
-
|
97
|
-
|
173
|
+
# Handle task-based response (HTTP 202)
|
174
|
+
if status_code == 202:
|
175
|
+
# Backend returns task_id - poll for completion
|
176
|
+
task_id = response_json.get("task_id")
|
98
177
|
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
try:
|
103
|
-
services_idx = parts.index("services")
|
104
|
-
provider_name = parts[services_idx - 1]
|
105
|
-
data_with_content["provider_name"] = provider_name
|
106
|
-
except (ValueError, IndexError):
|
107
|
-
raise ValueError(
|
108
|
-
f"Cannot extract provider_name from path: {data_file}. "
|
109
|
-
f"Expected path to contain .../{{provider_name}}/services/..."
|
110
|
-
)
|
178
|
+
if not task_id:
|
179
|
+
context_msg = f" ({context_info})" if context_info else ""
|
180
|
+
raise ValueError(f"No task_id in response for {entity_type} '{entity_name}'{context_msg}")
|
111
181
|
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
182
|
+
# Poll task status until completion using check_task utility
|
183
|
+
try:
|
184
|
+
result = await self.check_task(task_id)
|
185
|
+
return result
|
186
|
+
except ValueError as e:
|
187
|
+
# Add context to task errors
|
188
|
+
context_msg = f" ({context_info})" if context_info else ""
|
189
|
+
raise ValueError(f"Task failed for {entity_type} '{entity_name}'{context_msg}: {e}")
|
190
|
+
|
191
|
+
# Check for errors
|
192
|
+
if status_code >= 400:
|
193
|
+
# Don't retry on 4xx errors (client errors) - they won't succeed on retry
|
194
|
+
if 400 <= status_code < 500:
|
195
|
+
error_detail = response_json.get("detail", str(response_json))
|
196
|
+
context_msg = f" ({context_info})" if context_info else ""
|
197
|
+
raise ValueError(
|
198
|
+
f"Failed to publish {entity_type} '{entity_name}'{context_msg}: {error_detail}"
|
199
|
+
)
|
200
|
+
|
201
|
+
# 5xx errors - retry with exponential backoff
|
202
|
+
if attempt < max_retries - 1:
|
203
|
+
wait_time = 2**attempt # Exponential backoff: 1s, 2s, 4s
|
204
|
+
await asyncio.sleep(wait_time)
|
205
|
+
continue
|
206
|
+
else:
|
207
|
+
# Last attempt failed
|
208
|
+
error_detail = response_json.get("detail", str(response_json))
|
209
|
+
context_msg = f" ({context_info})" if context_info else ""
|
210
|
+
raise ValueError(
|
211
|
+
f"Failed to publish {entity_type} after {max_retries} attempts: "
|
212
|
+
f"'{entity_name}'{context_msg}: {error_detail}"
|
213
|
+
)
|
214
|
+
|
215
|
+
# Success response (2xx)
|
216
|
+
return response_json
|
217
|
+
|
218
|
+
except (httpx.NetworkError, httpx.TimeoutException, RuntimeError) as e:
|
219
|
+
# Network/connection errors - the post() method should have tried curl fallback
|
220
|
+
# If we're here, both httpx and curl failed
|
221
|
+
last_exception = e
|
222
|
+
if attempt < max_retries - 1:
|
223
|
+
wait_time = 2**attempt # Exponential backoff: 1s, 2s, 4s
|
224
|
+
await asyncio.sleep(wait_time)
|
225
|
+
continue
|
226
|
+
else:
|
227
|
+
raise ValueError(
|
228
|
+
f"Network error after {max_retries} attempts for {entity_type} '{entity_name}': {str(e)}"
|
229
|
+
)
|
119
230
|
|
120
|
-
|
121
|
-
|
231
|
+
# Should never reach here, but just in case
|
232
|
+
if last_exception:
|
233
|
+
raise last_exception
|
234
|
+
raise ValueError("Unexpected error in retry logic")
|
122
235
|
|
123
|
-
|
124
|
-
|
125
|
-
"""
|
236
|
+
async def post_service_listing_async(self, listing_file: Path, max_retries: int = 3) -> dict[str, Any]:
|
237
|
+
"""Async version of post_service_listing for concurrent publishing with retry logic."""
|
126
238
|
# Load the listing data file
|
127
|
-
data = self.load_data_file(
|
239
|
+
data = self.load_data_file(listing_file)
|
240
|
+
|
241
|
+
# If name is not provided, use filename (without extension)
|
242
|
+
if "name" not in data or not data.get("name"):
|
243
|
+
data["name"] = listing_file.stem
|
128
244
|
|
129
245
|
# Resolve file references and include content
|
130
|
-
base_path =
|
246
|
+
base_path = listing_file.parent
|
131
247
|
data_with_content = self.resolve_file_references(data, base_path)
|
132
248
|
|
133
249
|
# Extract provider_name from directory structure
|
134
|
-
parts =
|
250
|
+
parts = listing_file.parts
|
135
251
|
try:
|
136
252
|
services_idx = parts.index("services")
|
137
253
|
provider_name = parts[services_idx - 1]
|
138
254
|
data_with_content["provider_name"] = provider_name
|
139
255
|
except (ValueError, IndexError):
|
140
256
|
raise ValueError(
|
141
|
-
f"Cannot extract provider_name from path: {
|
257
|
+
f"Cannot extract provider_name from path: {listing_file}. "
|
142
258
|
f"Expected path to contain .../{{provider_name}}/services/..."
|
143
259
|
)
|
144
260
|
|
145
261
|
# If service_name is not in listing data, find it from service files in the same directory
|
146
262
|
if "service_name" not in data_with_content or not data_with_content["service_name"]:
|
147
263
|
# Find all service files in the same directory
|
148
|
-
service_files =
|
149
|
-
for pattern in ["*.json", "*.toml"]:
|
150
|
-
for file_path in data_file.parent.glob(pattern):
|
151
|
-
try:
|
152
|
-
file_data = self.load_data_file(file_path)
|
153
|
-
if file_data.get("schema") == "service_v1":
|
154
|
-
service_files.append((file_path, file_data))
|
155
|
-
except Exception:
|
156
|
-
continue
|
264
|
+
service_files = find_files_by_schema(listing_file.parent, "service_v1")
|
157
265
|
|
158
266
|
if len(service_files) == 0:
|
159
267
|
raise ValueError(
|
160
|
-
f"Cannot find any service_v1 files in {
|
268
|
+
f"Cannot find any service_v1 files in {listing_file.parent}. "
|
161
269
|
f"Listing files must be in the same directory as a service definition."
|
162
270
|
)
|
163
271
|
elif len(service_files) > 1:
|
164
|
-
service_names = [data.get("name", "unknown") for _, data in service_files]
|
272
|
+
service_names = [data.get("name", "unknown") for _, _, data in service_files]
|
165
273
|
raise ValueError(
|
166
|
-
f"Multiple services found in {
|
167
|
-
f"Please add 'service_name' field to {
|
274
|
+
f"Multiple services found in {listing_file.parent}: {', '.join(service_names)}. "
|
275
|
+
f"Please add 'service_name' field to {listing_file.name} to specify which "
|
168
276
|
f"service this listing belongs to."
|
169
277
|
)
|
170
278
|
else:
|
171
279
|
# Exactly one service found - use it
|
172
|
-
|
280
|
+
_service_file, _format, service_data = service_files[0]
|
173
281
|
data_with_content["service_name"] = service_data.get("name")
|
174
282
|
data_with_content["service_version"] = service_data.get("version")
|
175
283
|
else:
|
176
284
|
# service_name is provided in listing data, find the matching service to get version
|
177
285
|
service_name = data_with_content["service_name"]
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
for file_path in data_file.parent.glob(pattern):
|
182
|
-
try:
|
183
|
-
file_data = self.load_data_file(file_path)
|
184
|
-
if file_data.get("schema") == "service_v1" and file_data.get("name") == service_name:
|
185
|
-
data_with_content["service_version"] = file_data.get("version")
|
186
|
-
service_found = True
|
187
|
-
break
|
188
|
-
except Exception:
|
189
|
-
continue
|
190
|
-
if service_found:
|
191
|
-
break
|
286
|
+
service_files = find_files_by_schema(
|
287
|
+
listing_file.parent, "service_v1", field_filter=(("name", service_name),)
|
288
|
+
)
|
192
289
|
|
193
|
-
if not
|
290
|
+
if not service_files:
|
194
291
|
raise ValueError(
|
195
|
-
f"Service '{service_name}' specified in {
|
292
|
+
f"Service '{service_name}' specified in {listing_file.name} not found in {listing_file.parent}."
|
196
293
|
)
|
197
294
|
|
295
|
+
# Get version from the found service
|
296
|
+
_service_file, _format, service_data = service_files[0]
|
297
|
+
data_with_content["service_version"] = service_data.get("version")
|
298
|
+
|
198
299
|
# Find seller_name from seller definition in the data directory
|
199
300
|
# Navigate up to find the data directory and look for seller file
|
200
|
-
data_dir =
|
301
|
+
data_dir = listing_file.parent
|
201
302
|
while data_dir.name != "data" and data_dir.parent != data_dir:
|
202
303
|
data_dir = data_dir.parent
|
203
304
|
|
204
305
|
if data_dir.name != "data":
|
205
306
|
raise ValueError(
|
206
|
-
f"Cannot find 'data' directory in path: {
|
307
|
+
f"Cannot find 'data' directory in path: {listing_file}. "
|
207
308
|
f"Expected path structure includes a 'data' directory."
|
208
309
|
)
|
209
310
|
|
210
|
-
# Look for seller file in the data directory
|
211
|
-
|
212
|
-
for pattern in ["seller.json", "seller.toml"]:
|
213
|
-
potential_seller = data_dir / pattern
|
214
|
-
if potential_seller.exists():
|
215
|
-
seller_file = potential_seller
|
216
|
-
break
|
311
|
+
# Look for seller file in the data directory by checking schema field
|
312
|
+
seller_files = find_files_by_schema(data_dir, "seller_v1")
|
217
313
|
|
218
|
-
if not
|
314
|
+
if not seller_files:
|
219
315
|
raise ValueError(
|
220
|
-
f"Cannot find
|
221
|
-
f"A seller definition is required in the data directory."
|
316
|
+
f"Cannot find seller_v1 file in {data_dir}. A seller definition is required in the data directory."
|
222
317
|
)
|
223
318
|
|
224
|
-
#
|
225
|
-
seller_data =
|
226
|
-
|
227
|
-
|
319
|
+
# Should only be one seller file in the data directory
|
320
|
+
_seller_file, _format, seller_data = seller_files[0]
|
321
|
+
|
322
|
+
# Check seller status - skip if incomplete
|
323
|
+
seller_status = seller_data.get("status", SellerStatusEnum.active)
|
324
|
+
if seller_status == SellerStatusEnum.incomplete:
|
325
|
+
return {
|
326
|
+
"skipped": True,
|
327
|
+
"reason": f"Seller status is '{seller_status}' - not publishing listing to backend",
|
328
|
+
"name": data.get("name", "unknown"),
|
329
|
+
}
|
228
330
|
|
229
331
|
seller_name = seller_data.get("name")
|
230
332
|
if not seller_name:
|
231
|
-
raise ValueError(
|
333
|
+
raise ValueError("Seller data missing 'name' field")
|
232
334
|
|
233
335
|
data_with_content["seller_name"] = seller_name
|
234
336
|
|
@@ -236,201 +338,201 @@ class ServiceDataPublisher:
|
|
236
338
|
if "listing_status" in data_with_content:
|
237
339
|
data_with_content["status"] = data_with_content.pop("listing_status")
|
238
340
|
|
239
|
-
# Post to the endpoint
|
240
|
-
|
241
|
-
f"{
|
242
|
-
|
341
|
+
# Post to the endpoint using retry helper
|
342
|
+
context_info = (
|
343
|
+
f"service: {data_with_content.get('service_name')}, "
|
344
|
+
f"provider: {data_with_content.get('provider_name')}, "
|
345
|
+
f"seller: {data_with_content.get('seller_name')}"
|
346
|
+
)
|
347
|
+
return await self._post_with_retry(
|
348
|
+
endpoint="/publish/listing",
|
349
|
+
data=data_with_content,
|
350
|
+
entity_type="listing",
|
351
|
+
entity_name=data.get("name", "unknown"),
|
352
|
+
context_info=context_info,
|
353
|
+
max_retries=max_retries,
|
243
354
|
)
|
244
|
-
response.raise_for_status()
|
245
|
-
return response.json()
|
246
355
|
|
247
|
-
def
|
248
|
-
"""
|
356
|
+
async def post_service_offering_async(self, data_file: Path, max_retries: int = 3) -> dict[str, Any]:
|
357
|
+
"""Async version of post_service_offering for concurrent publishing with retry logic."""
|
249
358
|
# Load the data file
|
250
359
|
data = self.load_data_file(data_file)
|
251
360
|
|
252
361
|
# Resolve file references and include content
|
253
362
|
base_path = data_file.parent
|
363
|
+
data = convert_convenience_fields_to_documents(
|
364
|
+
data, base_path, logo_field="logo", terms_field="terms_of_service"
|
365
|
+
)
|
366
|
+
|
367
|
+
# Resolve file references and include content
|
254
368
|
data_with_content = self.resolve_file_references(data, base_path)
|
255
369
|
|
256
|
-
#
|
257
|
-
|
258
|
-
|
259
|
-
|
370
|
+
# Extract provider_name from directory structure
|
371
|
+
# Find the 'services' directory and use its parent as provider_name
|
372
|
+
parts = data_file.parts
|
373
|
+
try:
|
374
|
+
services_idx = parts.index("services")
|
375
|
+
provider_name = parts[services_idx - 1]
|
376
|
+
data_with_content["provider_name"] = provider_name
|
377
|
+
|
378
|
+
# Find provider directory to check status
|
379
|
+
provider_dir = Path(*parts[:services_idx])
|
380
|
+
except (ValueError, IndexError):
|
381
|
+
raise ValueError(
|
382
|
+
f"Cannot extract provider_name from path: {data_file}. "
|
383
|
+
f"Expected path to contain .../{{provider_name}}/services/..."
|
384
|
+
)
|
385
|
+
|
386
|
+
# Check provider status - skip if incomplete
|
387
|
+
provider_files = find_files_by_schema(provider_dir, "provider_v1")
|
388
|
+
if provider_files:
|
389
|
+
# Should only be one provider file in the directory
|
390
|
+
_provider_file, _format, provider_data = provider_files[0]
|
391
|
+
provider_status = provider_data.get("status", ProviderStatusEnum.active)
|
392
|
+
if provider_status == ProviderStatusEnum.incomplete:
|
393
|
+
return {
|
394
|
+
"skipped": True,
|
395
|
+
"reason": f"Provider status is '{provider_status}' - not publishing offering to backend",
|
396
|
+
"name": data.get("name", "unknown"),
|
397
|
+
}
|
398
|
+
|
399
|
+
# Post to the endpoint using retry helper
|
400
|
+
context_info = f"provider: {data_with_content.get('provider_name')}"
|
401
|
+
return await self._post_with_retry(
|
402
|
+
endpoint="/publish/offering",
|
403
|
+
data=data_with_content,
|
404
|
+
entity_type="offering",
|
405
|
+
entity_name=data.get("name", "unknown"),
|
406
|
+
context_info=context_info,
|
407
|
+
max_retries=max_retries,
|
260
408
|
)
|
261
|
-
response.raise_for_status()
|
262
|
-
return response.json()
|
263
409
|
|
264
|
-
def
|
265
|
-
"""
|
410
|
+
async def post_provider_async(self, data_file: Path, max_retries: int = 3) -> dict[str, Any]:
|
411
|
+
"""Async version of post_provider for concurrent publishing with retry logic."""
|
266
412
|
# Load the data file
|
267
413
|
data = self.load_data_file(data_file)
|
268
414
|
|
269
|
-
#
|
415
|
+
# Check provider status - skip if incomplete
|
416
|
+
provider_status = data.get("status", ProviderStatusEnum.active)
|
417
|
+
if provider_status == ProviderStatusEnum.incomplete:
|
418
|
+
# Return success without publishing - provider is incomplete
|
419
|
+
return {
|
420
|
+
"skipped": True,
|
421
|
+
"reason": f"Provider status is '{provider_status}' - not publishing to backend",
|
422
|
+
"name": data.get("name", "unknown"),
|
423
|
+
}
|
424
|
+
|
425
|
+
# Convert convenience fields (logo, terms_of_service) to documents
|
270
426
|
base_path = data_file.parent
|
427
|
+
data = convert_convenience_fields_to_documents(
|
428
|
+
data, base_path, logo_field="logo", terms_field="terms_of_service"
|
429
|
+
)
|
430
|
+
|
431
|
+
# Resolve file references and include content
|
271
432
|
data_with_content = self.resolve_file_references(data, base_path)
|
272
433
|
|
273
|
-
# Post to the endpoint
|
274
|
-
|
275
|
-
|
276
|
-
|
434
|
+
# Post to the endpoint using retry helper
|
435
|
+
return await self._post_with_retry(
|
436
|
+
endpoint="/publish/provider",
|
437
|
+
data=data_with_content,
|
438
|
+
entity_type="provider",
|
439
|
+
entity_name=data.get("name", "unknown"),
|
440
|
+
max_retries=max_retries,
|
277
441
|
)
|
278
|
-
response.raise_for_status()
|
279
|
-
return response.json()
|
280
442
|
|
281
|
-
def
|
282
|
-
"""
|
443
|
+
async def post_seller_async(self, data_file: Path, max_retries: int = 3) -> dict[str, Any]:
|
444
|
+
"""Async version of post_seller for concurrent publishing with retry logic."""
|
445
|
+
# Load the data file
|
446
|
+
data = self.load_data_file(data_file)
|
283
447
|
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
|
288
|
-
|
289
|
-
|
290
|
-
|
291
|
-
|
292
|
-
|
293
|
-
def list_service_listings(self) -> list[dict[str, Any]]:
|
294
|
-
"""List all service listings from the backend."""
|
295
|
-
response = self.client.get(f"{self.base_url}/services/")
|
296
|
-
response.raise_for_status()
|
297
|
-
result = response.json()
|
298
|
-
# Backend returns {"data": [...], "count": N}
|
299
|
-
return result.get("data", result) if isinstance(result, dict) else result
|
300
|
-
|
301
|
-
def list_providers(self) -> list[dict[str, Any]]:
|
302
|
-
"""List all providers from the backend."""
|
303
|
-
response = self.client.get(f"{self.base_url}/providers/")
|
304
|
-
response.raise_for_status()
|
305
|
-
result = response.json()
|
306
|
-
# Backend returns {"data": [...], "count": N}
|
307
|
-
return result.get("data", result) if isinstance(result, dict) else result
|
308
|
-
|
309
|
-
def list_sellers(self) -> list[dict[str, Any]]:
|
310
|
-
"""List all sellers from the backend."""
|
311
|
-
response = self.client.get(f"{self.base_url}/sellers/")
|
312
|
-
response.raise_for_status()
|
313
|
-
result = response.json()
|
314
|
-
# Backend returns {"data": [...], "count": N}
|
315
|
-
return result.get("data", result) if isinstance(result, dict) else result
|
316
|
-
|
317
|
-
def update_service_offering_status(self, offering_id: int | str, status: str) -> dict[str, Any]:
|
318
|
-
"""
|
319
|
-
Update the status of a service offering.
|
448
|
+
# Check seller status - skip if incomplete
|
449
|
+
seller_status = data.get("status", SellerStatusEnum.active)
|
450
|
+
if seller_status == SellerStatusEnum.incomplete:
|
451
|
+
# Return success without publishing - seller is incomplete
|
452
|
+
return {
|
453
|
+
"skipped": True,
|
454
|
+
"reason": f"Seller status is '{seller_status}' - not publishing to backend",
|
455
|
+
"name": data.get("name", "unknown"),
|
456
|
+
}
|
320
457
|
|
321
|
-
|
322
|
-
|
323
|
-
|
324
|
-
- deprecated: Service is deprecated from upstream
|
325
|
-
"""
|
326
|
-
response = self.client.patch(
|
327
|
-
f"{self.base_url}/service_offering/{offering_id}/",
|
328
|
-
json={"upstream_status": status},
|
329
|
-
)
|
330
|
-
response.raise_for_status()
|
331
|
-
return response.json()
|
458
|
+
# Convert convenience fields (logo only for sellers, no terms_of_service)
|
459
|
+
base_path = data_file.parent
|
460
|
+
data = convert_convenience_fields_to_documents(data, base_path, logo_field="logo", terms_field=None)
|
332
461
|
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
|
339
|
-
|
340
|
-
|
341
|
-
|
342
|
-
|
343
|
-
- upstream_deprecated: Service is deprecated from upstream
|
344
|
-
- deprecated: Service is no longer offered to users
|
345
|
-
"""
|
346
|
-
response = self.client.patch(
|
347
|
-
f"{self.base_url}/service_listing/{listing_id}/",
|
348
|
-
json={"listing_status": status},
|
462
|
+
# Resolve file references and include content
|
463
|
+
data_with_content = self.resolve_file_references(data, base_path)
|
464
|
+
|
465
|
+
# Post to the endpoint using retry helper
|
466
|
+
return await self._post_with_retry(
|
467
|
+
endpoint="/publish/seller",
|
468
|
+
data=data_with_content,
|
469
|
+
entity_type="seller",
|
470
|
+
entity_name=data.get("name", "unknown"),
|
471
|
+
max_retries=max_retries,
|
349
472
|
)
|
350
|
-
response.raise_for_status()
|
351
|
-
return response.json()
|
352
473
|
|
353
474
|
def find_offering_files(self, data_dir: Path) -> list[Path]:
|
354
|
-
"""
|
355
|
-
|
356
|
-
|
357
|
-
Searches all JSON and TOML files and checks for schema="service_v1".
|
358
|
-
"""
|
359
|
-
offerings = []
|
360
|
-
for pattern in ["*.json", "*.toml"]:
|
361
|
-
for file_path in data_dir.rglob(pattern):
|
362
|
-
try:
|
363
|
-
data = self.load_data_file(file_path)
|
364
|
-
if data.get("schema") == "service_v1":
|
365
|
-
offerings.append(file_path)
|
366
|
-
except Exception:
|
367
|
-
# Skip files that can't be loaded or don't have schema field
|
368
|
-
pass
|
369
|
-
return sorted(offerings)
|
475
|
+
"""Find all service offering files in a directory tree."""
|
476
|
+
files = find_files_by_schema(data_dir, "service_v1")
|
477
|
+
return sorted([f[0] for f in files])
|
370
478
|
|
371
479
|
def find_listing_files(self, data_dir: Path) -> list[Path]:
|
372
|
-
"""
|
373
|
-
|
374
|
-
|
375
|
-
Searches all JSON and TOML files and checks for schema="listing_v1".
|
376
|
-
"""
|
377
|
-
listings = []
|
378
|
-
for pattern in ["*.json", "*.toml"]:
|
379
|
-
for file_path in data_dir.rglob(pattern):
|
380
|
-
try:
|
381
|
-
data = self.load_data_file(file_path)
|
382
|
-
if data.get("schema") == "listing_v1":
|
383
|
-
listings.append(file_path)
|
384
|
-
except Exception:
|
385
|
-
# Skip files that can't be loaded or don't have schema field
|
386
|
-
pass
|
387
|
-
return sorted(listings)
|
480
|
+
"""Find all service listing files in a directory tree."""
|
481
|
+
files = find_files_by_schema(data_dir, "listing_v1")
|
482
|
+
return sorted([f[0] for f in files])
|
388
483
|
|
389
484
|
def find_provider_files(self, data_dir: Path) -> list[Path]:
|
390
|
-
"""
|
391
|
-
|
392
|
-
|
393
|
-
Searches all JSON and TOML files and checks for schema="provider_v1".
|
394
|
-
"""
|
395
|
-
providers = []
|
396
|
-
for pattern in ["*.json", "*.toml"]:
|
397
|
-
for file_path in data_dir.rglob(pattern):
|
398
|
-
try:
|
399
|
-
data = self.load_data_file(file_path)
|
400
|
-
if data.get("schema") == "provider_v1":
|
401
|
-
providers.append(file_path)
|
402
|
-
except Exception:
|
403
|
-
# Skip files that can't be loaded or don't have schema field
|
404
|
-
pass
|
405
|
-
return sorted(providers)
|
485
|
+
"""Find all provider files in a directory tree."""
|
486
|
+
files = find_files_by_schema(data_dir, "provider_v1")
|
487
|
+
return sorted([f[0] for f in files])
|
406
488
|
|
407
489
|
def find_seller_files(self, data_dir: Path) -> list[Path]:
|
490
|
+
"""Find all seller files in a directory tree."""
|
491
|
+
files = find_files_by_schema(data_dir, "seller_v1")
|
492
|
+
return sorted([f[0] for f in files])
|
493
|
+
|
494
|
+
async def _publish_offering_task(
|
495
|
+
self, offering_file: Path, console: Console, semaphore: asyncio.Semaphore
|
496
|
+
) -> tuple[Path, dict[str, Any] | Exception]:
|
408
497
|
"""
|
409
|
-
|
498
|
+
Async task to publish a single offering with concurrency control.
|
410
499
|
|
411
|
-
|
500
|
+
Returns tuple of (offering_file, result_or_exception).
|
412
501
|
"""
|
413
|
-
|
414
|
-
|
415
|
-
|
416
|
-
|
417
|
-
|
418
|
-
|
419
|
-
|
420
|
-
|
421
|
-
|
422
|
-
|
423
|
-
|
424
|
-
|
425
|
-
|
502
|
+
async with semaphore: # Limit concurrent requests
|
503
|
+
try:
|
504
|
+
# Load offering data to get the name
|
505
|
+
data = self.load_data_file(offering_file)
|
506
|
+
offering_name = data.get("name", offering_file.stem)
|
507
|
+
|
508
|
+
# Publish the offering
|
509
|
+
result = await self.post_service_offering_async(offering_file)
|
510
|
+
|
511
|
+
# Print complete statement after publication
|
512
|
+
if result.get("skipped"):
|
513
|
+
reason = result.get("reason", "unknown")
|
514
|
+
console.print(f" [yellow]⊘[/yellow] Skipped offering: [cyan]{offering_name}[/cyan] - {reason}")
|
515
|
+
else:
|
516
|
+
provider_name = result.get("provider_name")
|
517
|
+
console.print(
|
518
|
+
f" [green]✓[/green] Published offering: [cyan]{offering_name}[/cyan] "
|
519
|
+
f"(provider: {provider_name})"
|
520
|
+
)
|
521
|
+
|
522
|
+
return (offering_file, result)
|
523
|
+
except Exception as e:
|
524
|
+
data = self.load_data_file(offering_file)
|
525
|
+
offering_name = data.get("name", offering_file.stem)
|
526
|
+
console.print(f" [red]✗[/red] Failed to publish offering: [cyan]{offering_name}[/cyan] - {str(e)}")
|
527
|
+
return (offering_file, e)
|
528
|
+
|
529
|
+
async def publish_all_offerings(self, data_dir: Path) -> dict[str, Any]:
|
426
530
|
"""
|
427
|
-
Publish all service offerings found in a directory tree.
|
531
|
+
Publish all service offerings found in a directory tree concurrently.
|
428
532
|
|
429
533
|
Validates data consistency before publishing.
|
430
534
|
Returns a summary of successes and failures.
|
431
535
|
"""
|
432
|
-
from .validator import DataValidator
|
433
|
-
|
434
536
|
# Validate all service directories first
|
435
537
|
validator = DataValidator(data_dir, data_dir.parent / "schema")
|
436
538
|
validation_errors = validator.validate_all_service_directories(data_dir)
|
@@ -450,25 +552,70 @@ class ServiceDataPublisher:
|
|
450
552
|
"errors": [],
|
451
553
|
}
|
452
554
|
|
453
|
-
|
454
|
-
|
455
|
-
|
456
|
-
|
457
|
-
|
555
|
+
if not offering_files:
|
556
|
+
return results
|
557
|
+
|
558
|
+
console = Console()
|
559
|
+
|
560
|
+
# Run all offering publications concurrently with rate limiting
|
561
|
+
# Create semaphore to limit concurrent requests
|
562
|
+
semaphore = asyncio.Semaphore(self.max_concurrent_requests)
|
563
|
+
tasks = [self._publish_offering_task(offering_file, console, semaphore) for offering_file in offering_files]
|
564
|
+
task_results = await asyncio.gather(*tasks)
|
565
|
+
|
566
|
+
# Process results
|
567
|
+
for offering_file, result in task_results:
|
568
|
+
if isinstance(result, Exception):
|
458
569
|
results["failed"] += 1
|
459
|
-
results["errors"].append({"file": str(offering_file), "error": str(
|
570
|
+
results["errors"].append({"file": str(offering_file), "error": str(result)})
|
571
|
+
else:
|
572
|
+
results["success"] += 1
|
460
573
|
|
461
574
|
return results
|
462
575
|
|
463
|
-
def
|
576
|
+
async def _publish_listing_task(
|
577
|
+
self, listing_file: Path, console: Console, semaphore: asyncio.Semaphore
|
578
|
+
) -> tuple[Path, dict[str, Any] | Exception]:
|
464
579
|
"""
|
465
|
-
|
580
|
+
Async task to publish a single listing with concurrency control.
|
581
|
+
|
582
|
+
Returns tuple of (listing_file, result_or_exception).
|
583
|
+
"""
|
584
|
+
async with semaphore: # Limit concurrent requests
|
585
|
+
try:
|
586
|
+
# Load listing data to get the name
|
587
|
+
data = self.load_data_file(listing_file)
|
588
|
+
listing_name = data.get("name", listing_file.stem)
|
589
|
+
|
590
|
+
# Publish the listing
|
591
|
+
result = await self.post_service_listing_async(listing_file)
|
592
|
+
|
593
|
+
# Print complete statement after publication
|
594
|
+
if result.get("skipped"):
|
595
|
+
reason = result.get("reason", "unknown")
|
596
|
+
console.print(f" [yellow]⊘[/yellow] Skipped listing: [cyan]{listing_name}[/cyan] - {reason}")
|
597
|
+
else:
|
598
|
+
service_name = result.get("service_name")
|
599
|
+
provider_name = result.get("provider_name")
|
600
|
+
console.print(
|
601
|
+
f" [green]✓[/green] Published listing: [cyan]{listing_name}[/cyan] "
|
602
|
+
f"(service: {service_name}, provider: {provider_name})"
|
603
|
+
)
|
604
|
+
|
605
|
+
return (listing_file, result)
|
606
|
+
except Exception as e:
|
607
|
+
data = self.load_data_file(listing_file)
|
608
|
+
listing_name = data.get("name", listing_file.stem)
|
609
|
+
console.print(f" [red]✗[/red] Failed to publish listing: [cyan]{listing_file}[/cyan] - {str(e)}")
|
610
|
+
return (listing_file, e)
|
611
|
+
|
612
|
+
async def publish_all_listings(self, data_dir: Path) -> dict[str, Any]:
|
613
|
+
"""
|
614
|
+
Publish all service listings found in a directory tree concurrently.
|
466
615
|
|
467
616
|
Validates data consistency before publishing.
|
468
617
|
Returns a summary of successes and failures.
|
469
618
|
"""
|
470
|
-
from .validator import DataValidator
|
471
|
-
|
472
619
|
# Validate all service directories first
|
473
620
|
validator = DataValidator(data_dir, data_dir.parent / "schema")
|
474
621
|
validation_errors = validator.validate_all_service_directories(data_dir)
|
@@ -481,21 +628,68 @@ class ServiceDataPublisher:
|
|
481
628
|
}
|
482
629
|
|
483
630
|
listing_files = self.find_listing_files(data_dir)
|
484
|
-
results: dict[str, Any] = {
|
631
|
+
results: dict[str, Any] = {
|
632
|
+
"total": len(listing_files),
|
633
|
+
"success": 0,
|
634
|
+
"failed": 0,
|
635
|
+
"errors": [],
|
636
|
+
}
|
485
637
|
|
486
|
-
|
487
|
-
|
488
|
-
|
489
|
-
|
490
|
-
|
638
|
+
if not listing_files:
|
639
|
+
return results
|
640
|
+
|
641
|
+
console = Console()
|
642
|
+
|
643
|
+
# Run all listing publications concurrently with rate limiting
|
644
|
+
# Create semaphore to limit concurrent requests
|
645
|
+
semaphore = asyncio.Semaphore(self.max_concurrent_requests)
|
646
|
+
tasks = [self._publish_listing_task(listing_file, console, semaphore) for listing_file in listing_files]
|
647
|
+
task_results = await asyncio.gather(*tasks)
|
648
|
+
|
649
|
+
# Process results
|
650
|
+
for listing_file, result in task_results:
|
651
|
+
if isinstance(result, Exception):
|
491
652
|
results["failed"] += 1
|
492
|
-
results["errors"].append({"file": str(listing_file), "error": str(
|
653
|
+
results["errors"].append({"file": str(listing_file), "error": str(result)})
|
654
|
+
else:
|
655
|
+
results["success"] += 1
|
493
656
|
|
494
657
|
return results
|
495
658
|
|
496
|
-
def
|
659
|
+
async def _publish_provider_task(
|
660
|
+
self, provider_file: Path, console: Console, semaphore: asyncio.Semaphore
|
661
|
+
) -> tuple[Path, dict[str, Any] | Exception]:
|
497
662
|
"""
|
498
|
-
|
663
|
+
Async task to publish a single provider with concurrency control.
|
664
|
+
|
665
|
+
Returns tuple of (provider_file, result_or_exception).
|
666
|
+
"""
|
667
|
+
async with semaphore: # Limit concurrent requests
|
668
|
+
try:
|
669
|
+
# Load provider data to get the name
|
670
|
+
data = self.load_data_file(provider_file)
|
671
|
+
provider_name = data.get("name", provider_file.stem)
|
672
|
+
|
673
|
+
# Publish the provider
|
674
|
+
result = await self.post_provider_async(provider_file)
|
675
|
+
|
676
|
+
# Print complete statement after publication
|
677
|
+
if result.get("skipped"):
|
678
|
+
reason = result.get("reason", "unknown")
|
679
|
+
console.print(f" [yellow]⊘[/yellow] Skipped provider: [cyan]{provider_name}[/cyan] - {reason}")
|
680
|
+
else:
|
681
|
+
console.print(f" [green]✓[/green] Published provider: [cyan]{provider_name}[/cyan]")
|
682
|
+
|
683
|
+
return (provider_file, result)
|
684
|
+
except Exception as e:
|
685
|
+
data = self.load_data_file(provider_file)
|
686
|
+
provider_name = data.get("name", provider_file.stem)
|
687
|
+
console.print(f" [red]✗[/red] Failed to publish provider: [cyan]{provider_name}[/cyan] - {str(e)}")
|
688
|
+
return (provider_file, e)
|
689
|
+
|
690
|
+
async def publish_all_providers(self, data_dir: Path) -> dict[str, Any]:
|
691
|
+
"""
|
692
|
+
Publish all providers found in a directory tree concurrently.
|
499
693
|
|
500
694
|
Returns a summary of successes and failures.
|
501
695
|
"""
|
@@ -507,19 +701,61 @@ class ServiceDataPublisher:
|
|
507
701
|
"errors": [],
|
508
702
|
}
|
509
703
|
|
510
|
-
|
511
|
-
|
512
|
-
|
513
|
-
|
514
|
-
|
704
|
+
if not provider_files:
|
705
|
+
return results
|
706
|
+
|
707
|
+
console = Console()
|
708
|
+
|
709
|
+
# Run all provider publications concurrently with rate limiting
|
710
|
+
# Create semaphore to limit concurrent requests
|
711
|
+
semaphore = asyncio.Semaphore(self.max_concurrent_requests)
|
712
|
+
tasks = [self._publish_provider_task(provider_file, console, semaphore) for provider_file in provider_files]
|
713
|
+
task_results = await asyncio.gather(*tasks)
|
714
|
+
|
715
|
+
# Process results
|
716
|
+
for provider_file, result in task_results:
|
717
|
+
if isinstance(result, Exception):
|
515
718
|
results["failed"] += 1
|
516
|
-
results["errors"].append({"file": str(provider_file), "error": str(
|
719
|
+
results["errors"].append({"file": str(provider_file), "error": str(result)})
|
720
|
+
else:
|
721
|
+
results["success"] += 1
|
517
722
|
|
518
723
|
return results
|
519
724
|
|
520
|
-
def
|
725
|
+
async def _publish_seller_task(
|
726
|
+
self, seller_file: Path, console: Console, semaphore: asyncio.Semaphore
|
727
|
+
) -> tuple[Path, dict[str, Any] | Exception]:
|
728
|
+
"""
|
729
|
+
Async task to publish a single seller with concurrency control.
|
730
|
+
|
731
|
+
Returns tuple of (seller_file, result_or_exception).
|
732
|
+
"""
|
733
|
+
async with semaphore: # Limit concurrent requests
|
734
|
+
try:
|
735
|
+
# Load seller data to get the name
|
736
|
+
data = self.load_data_file(seller_file)
|
737
|
+
seller_name = data.get("name", seller_file.stem)
|
738
|
+
|
739
|
+
# Publish the seller
|
740
|
+
result = await self.post_seller_async(seller_file)
|
741
|
+
|
742
|
+
# Print complete statement after publication
|
743
|
+
if result.get("skipped"):
|
744
|
+
reason = result.get("reason", "unknown")
|
745
|
+
console.print(f" [yellow]⊘[/yellow] Skipped seller: [cyan]{seller_name}[/cyan] - {reason}")
|
746
|
+
else:
|
747
|
+
console.print(f" [green]✓[/green] Published seller: [cyan]{seller_name}[/cyan]")
|
748
|
+
|
749
|
+
return (seller_file, result)
|
750
|
+
except Exception as e:
|
751
|
+
data = self.load_data_file(seller_file)
|
752
|
+
seller_name = data.get("name", seller_file.stem)
|
753
|
+
console.print(f" [red]✗[/red] Failed to publish seller: [cyan]{seller_name}[/cyan] - {str(e)}")
|
754
|
+
return (seller_file, e)
|
755
|
+
|
756
|
+
async def publish_all_sellers(self, data_dir: Path) -> dict[str, Any]:
|
521
757
|
"""
|
522
|
-
Publish all sellers found in a directory tree.
|
758
|
+
Publish all sellers found in a directory tree concurrently.
|
523
759
|
|
524
760
|
Returns a summary of successes and failures.
|
525
761
|
"""
|
@@ -531,27 +767,83 @@ class ServiceDataPublisher:
|
|
531
767
|
"errors": [],
|
532
768
|
}
|
533
769
|
|
534
|
-
|
535
|
-
|
536
|
-
|
537
|
-
|
538
|
-
|
770
|
+
if not seller_files:
|
771
|
+
return results
|
772
|
+
|
773
|
+
console = Console()
|
774
|
+
|
775
|
+
# Run all seller publications concurrently with rate limiting
|
776
|
+
# Create semaphore to limit concurrent requests
|
777
|
+
semaphore = asyncio.Semaphore(self.max_concurrent_requests)
|
778
|
+
tasks = [self._publish_seller_task(seller_file, console, semaphore) for seller_file in seller_files]
|
779
|
+
task_results = await asyncio.gather(*tasks)
|
780
|
+
|
781
|
+
# Process results
|
782
|
+
for seller_file, result in task_results:
|
783
|
+
if isinstance(result, Exception):
|
539
784
|
results["failed"] += 1
|
540
|
-
results["errors"].append({"file": str(seller_file), "error": str(
|
785
|
+
results["errors"].append({"file": str(seller_file), "error": str(result)})
|
786
|
+
else:
|
787
|
+
results["success"] += 1
|
541
788
|
|
542
789
|
return results
|
543
790
|
|
544
|
-
def
|
545
|
-
"""
|
546
|
-
|
791
|
+
async def publish_all_models(self, data_dir: Path) -> dict[str, Any]:
|
792
|
+
"""
|
793
|
+
Publish all data types in the correct order.
|
794
|
+
|
795
|
+
Publishing order:
|
796
|
+
1. Sellers - Must exist before listings
|
797
|
+
2. Providers - Must exist before offerings
|
798
|
+
3. Service Offerings - Must exist before listings
|
799
|
+
4. Service Listings - Depends on sellers, providers, and offerings
|
800
|
+
|
801
|
+
Returns a dict with results for each data type and overall summary.
|
802
|
+
"""
|
803
|
+
all_results: dict[str, Any] = {
|
804
|
+
"sellers": {},
|
805
|
+
"providers": {},
|
806
|
+
"offerings": {},
|
807
|
+
"listings": {},
|
808
|
+
"total_success": 0,
|
809
|
+
"total_failed": 0,
|
810
|
+
"total_found": 0,
|
811
|
+
}
|
812
|
+
|
813
|
+
# Publish in order: sellers -> providers -> offerings -> listings
|
814
|
+
publish_order = [
|
815
|
+
("sellers", self.publish_all_sellers),
|
816
|
+
("providers", self.publish_all_providers),
|
817
|
+
("offerings", self.publish_all_offerings),
|
818
|
+
("listings", self.publish_all_listings),
|
819
|
+
]
|
820
|
+
|
821
|
+
for data_type, publish_method in publish_order:
|
822
|
+
try:
|
823
|
+
results = await publish_method(data_dir)
|
824
|
+
all_results[data_type] = results
|
825
|
+
all_results["total_success"] += results["success"]
|
826
|
+
all_results["total_failed"] += results["failed"]
|
827
|
+
all_results["total_found"] += results["total"]
|
828
|
+
except Exception as e:
|
829
|
+
# If a publish method fails catastrophically, record the error
|
830
|
+
all_results[data_type] = {
|
831
|
+
"total": 0,
|
832
|
+
"success": 0,
|
833
|
+
"failed": 1,
|
834
|
+
"errors": [{"file": "N/A", "error": str(e)}],
|
835
|
+
}
|
836
|
+
all_results["total_failed"] += 1
|
837
|
+
|
838
|
+
return all_results
|
547
839
|
|
548
840
|
def __enter__(self):
|
549
|
-
"""
|
841
|
+
"""Sync context manager entry for CLI usage."""
|
550
842
|
return self
|
551
843
|
|
552
844
|
def __exit__(self, exc_type, exc_val, exc_tb):
|
553
|
-
"""
|
554
|
-
self.
|
845
|
+
"""Sync context manager exit for CLI usage."""
|
846
|
+
asyncio.run(self.aclose())
|
555
847
|
|
556
848
|
|
557
849
|
# CLI commands for publishing
|
@@ -559,35 +851,40 @@ app = typer.Typer(help="Publish data to backend")
|
|
559
851
|
console = Console()
|
560
852
|
|
561
853
|
|
562
|
-
@app.
|
563
|
-
def
|
564
|
-
|
565
|
-
|
566
|
-
help="Path to provider file or directory (default: ./data or UNITYSVC_DATA_DIR env var)",
|
567
|
-
),
|
568
|
-
backend_url: str | None = typer.Option(
|
569
|
-
None,
|
570
|
-
"--backend-url",
|
571
|
-
"-u",
|
572
|
-
help="UnitySVC backend URL (default: from UNITYSVC_BACKEND_URL env var)",
|
573
|
-
),
|
574
|
-
api_key: str | None = typer.Option(
|
854
|
+
@app.callback(invoke_without_command=True)
|
855
|
+
def publish_callback(
|
856
|
+
ctx: typer.Context,
|
857
|
+
data_path: Path | None = typer.Option(
|
575
858
|
None,
|
576
|
-
"--
|
577
|
-
"-
|
578
|
-
help="
|
859
|
+
"--data-path",
|
860
|
+
"-d",
|
861
|
+
help="Path to data directory (default: current directory)",
|
579
862
|
),
|
580
863
|
):
|
581
|
-
"""
|
582
|
-
|
583
|
-
|
864
|
+
"""
|
865
|
+
Publish data to backend.
|
866
|
+
|
867
|
+
When called without a subcommand, publishes all data types in order:
|
868
|
+
sellers → providers → offerings → listings.
|
869
|
+
|
870
|
+
Use subcommands to publish specific data types:
|
871
|
+
- providers: Publish only providers
|
872
|
+
- sellers: Publish only sellers
|
873
|
+
- offerings: Publish only service offerings
|
874
|
+
- listings: Publish only service listings
|
875
|
+
|
876
|
+
Required environment variables:
|
877
|
+
- UNITYSVC_BASE_URL: Backend API URL
|
878
|
+
- UNITYSVC_API_KEY: API key for authentication
|
879
|
+
"""
|
880
|
+
# If a subcommand was invoked, skip this callback logic
|
881
|
+
if ctx.invoked_subcommand is not None:
|
882
|
+
return
|
883
|
+
|
884
|
+
# No subcommand - publish all
|
584
885
|
# Set data path
|
585
886
|
if data_path is None:
|
586
|
-
|
587
|
-
if data_path_str:
|
588
|
-
data_path = Path(data_path_str)
|
589
|
-
else:
|
590
|
-
data_path = Path.cwd() / "data"
|
887
|
+
data_path = Path.cwd()
|
591
888
|
|
592
889
|
if not data_path.is_absolute():
|
593
890
|
data_path = Path.cwd() / data_path
|
@@ -596,38 +893,107 @@ def publish_providers(
|
|
596
893
|
console.print(f"[red]✗[/red] Path not found: {data_path}", style="bold red")
|
597
894
|
raise typer.Exit(code=1)
|
598
895
|
|
599
|
-
|
600
|
-
|
601
|
-
|
602
|
-
|
603
|
-
|
604
|
-
|
605
|
-
|
896
|
+
console.print(f"[bold blue]Publishing all data from:[/bold blue] {data_path}")
|
897
|
+
console.print(f"[bold blue]Backend URL:[/bold blue] {os.getenv('UNITYSVC_BASE_URL', 'N/A')}\n")
|
898
|
+
|
899
|
+
try:
|
900
|
+
with ServiceDataPublisher() as publisher:
|
901
|
+
# Call the publish_all_models method (now async)
|
902
|
+
all_results = asyncio.run(publisher.publish_all_models(data_path))
|
903
|
+
|
904
|
+
# Display results for each data type
|
905
|
+
data_type_display_names = {
|
906
|
+
"sellers": "Sellers",
|
907
|
+
"providers": "Providers",
|
908
|
+
"offerings": "Service Offerings",
|
909
|
+
"listings": "Service Listings",
|
910
|
+
}
|
911
|
+
|
912
|
+
for data_type in ["sellers", "providers", "offerings", "listings"]:
|
913
|
+
display_name = data_type_display_names[data_type]
|
914
|
+
results = all_results[data_type]
|
915
|
+
|
916
|
+
console.print(f"\n[bold cyan]{'=' * 60}[/bold cyan]")
|
917
|
+
console.print(f"[bold cyan]{display_name}[/bold cyan]")
|
918
|
+
console.print(f"[bold cyan]{'=' * 60}[/bold cyan]\n")
|
919
|
+
|
920
|
+
console.print(f" Total found: {results['total']}")
|
921
|
+
console.print(f" [green]✓ Success:[/green] {results['success']}")
|
922
|
+
console.print(f" [red]✗ Failed:[/red] {results['failed']}")
|
923
|
+
|
924
|
+
# Display errors if any
|
925
|
+
if results.get("errors"):
|
926
|
+
console.print(f"\n[bold red]Errors in {display_name}:[/bold red]")
|
927
|
+
for error in results["errors"]:
|
928
|
+
# Check if this is a skipped item
|
929
|
+
if isinstance(error, dict) and error.get("error", "").startswith("skipped"):
|
930
|
+
continue
|
931
|
+
console.print(f" [red]✗[/red] {error.get('file', 'unknown')}")
|
932
|
+
console.print(f" {error.get('error', 'unknown error')}")
|
933
|
+
|
934
|
+
# Final summary
|
935
|
+
console.print(f"\n[bold cyan]{'=' * 60}[/bold cyan]")
|
936
|
+
console.print("[bold]Final Publishing Summary[/bold]")
|
937
|
+
console.print(f"[bold cyan]{'=' * 60}[/bold cyan]\n")
|
938
|
+
console.print(f" Total found: {all_results['total_found']}")
|
939
|
+
console.print(f" [green]✓ Success:[/green] {all_results['total_success']}")
|
940
|
+
console.print(f" [red]✗ Failed:[/red] {all_results['total_failed']}")
|
941
|
+
|
942
|
+
if all_results["total_failed"] > 0:
|
943
|
+
console.print(
|
944
|
+
f"\n[yellow]⚠[/yellow] Completed with {all_results['total_failed']} failure(s)",
|
945
|
+
style="bold yellow",
|
946
|
+
)
|
947
|
+
raise typer.Exit(code=1)
|
948
|
+
else:
|
949
|
+
console.print(
|
950
|
+
"\n[green]✓[/green] All data published successfully!",
|
951
|
+
style="bold green",
|
952
|
+
)
|
953
|
+
|
954
|
+
except typer.Exit:
|
955
|
+
raise
|
956
|
+
except Exception as e:
|
957
|
+
console.print(f"[red]✗[/red] Failed to publish all data: {e}", style="bold red")
|
606
958
|
raise typer.Exit(code=1)
|
607
959
|
|
608
|
-
|
609
|
-
|
610
|
-
|
611
|
-
|
612
|
-
|
613
|
-
|
614
|
-
|
960
|
+
|
961
|
+
@app.command("providers")
|
962
|
+
def publish_providers(
|
963
|
+
data_path: Path | None = typer.Option(
|
964
|
+
None,
|
965
|
+
"--data-path",
|
966
|
+
"-d",
|
967
|
+
help="Path to provider file or directory (default: current directory)",
|
968
|
+
),
|
969
|
+
):
|
970
|
+
"""Publish provider(s) from a file or directory."""
|
971
|
+
|
972
|
+
# Set data path
|
973
|
+
if data_path is None:
|
974
|
+
data_path = Path.cwd()
|
975
|
+
|
976
|
+
if not data_path.is_absolute():
|
977
|
+
data_path = Path.cwd() / data_path
|
978
|
+
|
979
|
+
if not data_path.exists():
|
980
|
+
console.print(f"[red]✗[/red] Path not found: {data_path}", style="bold red")
|
615
981
|
raise typer.Exit(code=1)
|
616
982
|
|
617
983
|
try:
|
618
|
-
with ServiceDataPublisher(
|
984
|
+
with ServiceDataPublisher() as publisher:
|
619
985
|
# Handle single file
|
620
986
|
if data_path.is_file():
|
621
987
|
console.print(f"[blue]Publishing provider:[/blue] {data_path}")
|
622
|
-
console.print(f"[blue]Backend URL:[/blue] {
|
623
|
-
result = publisher.
|
988
|
+
console.print(f"[blue]Backend URL:[/blue] {os.getenv('UNITYSVC_BASE_URL', 'N/A')}\n")
|
989
|
+
result = asyncio.run(publisher.post_provider_async(data_path))
|
624
990
|
console.print("[green]✓[/green] Provider published successfully!")
|
625
991
|
console.print(f"[cyan]Response:[/cyan] {json.dumps(result, indent=2)}")
|
626
992
|
# Handle directory
|
627
993
|
else:
|
628
994
|
console.print(f"[blue]Scanning for providers in:[/blue] {data_path}")
|
629
|
-
console.print(f"[blue]Backend URL:[/blue] {
|
630
|
-
results = publisher.publish_all_providers(data_path)
|
995
|
+
console.print(f"[blue]Backend URL:[/blue] {os.getenv('UNITYSVC_BASE_URL', 'N/A')}\n")
|
996
|
+
results = asyncio.run(publisher.publish_all_providers(data_path))
|
631
997
|
|
632
998
|
# Display summary
|
633
999
|
console.print("\n[bold]Publishing Summary:[/bold]")
|
@@ -654,33 +1020,17 @@ def publish_providers(
|
|
654
1020
|
|
655
1021
|
@app.command("sellers")
|
656
1022
|
def publish_sellers(
|
657
|
-
data_path: Path | None = typer.
|
658
|
-
None,
|
659
|
-
help="Path to seller file or directory (default: ./data or UNITYSVC_DATA_DIR env var)",
|
660
|
-
),
|
661
|
-
backend_url: str | None = typer.Option(
|
1023
|
+
data_path: Path | None = typer.Option(
|
662
1024
|
None,
|
663
|
-
"--
|
664
|
-
"-
|
665
|
-
help="
|
666
|
-
),
|
667
|
-
api_key: str | None = typer.Option(
|
668
|
-
None,
|
669
|
-
"--api-key",
|
670
|
-
"-k",
|
671
|
-
help="API key for authentication (default: from UNITYSVC_API_KEY env var)",
|
1025
|
+
"--data-path",
|
1026
|
+
"-d",
|
1027
|
+
help="Path to seller file or directory (default: current directory)",
|
672
1028
|
),
|
673
1029
|
):
|
674
1030
|
"""Publish seller(s) from a file or directory."""
|
675
|
-
import os
|
676
|
-
|
677
1031
|
# Set data path
|
678
1032
|
if data_path is None:
|
679
|
-
|
680
|
-
if data_path_str:
|
681
|
-
data_path = Path(data_path_str)
|
682
|
-
else:
|
683
|
-
data_path = Path.cwd() / "data"
|
1033
|
+
data_path = Path.cwd()
|
684
1034
|
|
685
1035
|
if not data_path.is_absolute():
|
686
1036
|
data_path = Path.cwd() / data_path
|
@@ -689,38 +1039,20 @@ def publish_sellers(
|
|
689
1039
|
console.print(f"[red]✗[/red] Path not found: {data_path}", style="bold red")
|
690
1040
|
raise typer.Exit(code=1)
|
691
1041
|
|
692
|
-
# Get backend URL
|
693
|
-
backend_url = backend_url or os.getenv("UNITYSVC_BACKEND_URL")
|
694
|
-
if not backend_url:
|
695
|
-
console.print(
|
696
|
-
"[red]✗[/red] Backend URL not provided. Use --backend-url or set UNITYSVC_BACKEND_URL env var.",
|
697
|
-
style="bold red",
|
698
|
-
)
|
699
|
-
raise typer.Exit(code=1)
|
700
|
-
|
701
|
-
# Get API key
|
702
|
-
api_key = api_key or os.getenv("UNITYSVC_API_KEY")
|
703
|
-
if not api_key:
|
704
|
-
console.print(
|
705
|
-
"[red]✗[/red] API key not provided. Use --api-key or set UNITYSVC_API_KEY env var.",
|
706
|
-
style="bold red",
|
707
|
-
)
|
708
|
-
raise typer.Exit(code=1)
|
709
|
-
|
710
1042
|
try:
|
711
|
-
with ServiceDataPublisher(
|
1043
|
+
with ServiceDataPublisher() as publisher:
|
712
1044
|
# Handle single file
|
713
1045
|
if data_path.is_file():
|
714
1046
|
console.print(f"[blue]Publishing seller:[/blue] {data_path}")
|
715
|
-
console.print(f"[blue]Backend URL:[/blue] {
|
716
|
-
result = publisher.
|
1047
|
+
console.print(f"[blue]Backend URL:[/blue] {os.getenv('UNITYSVC_BASE_URL', 'N/A')}\n")
|
1048
|
+
result = asyncio.run(publisher.post_seller_async(data_path))
|
717
1049
|
console.print("[green]✓[/green] Seller published successfully!")
|
718
1050
|
console.print(f"[cyan]Response:[/cyan] {json.dumps(result, indent=2)}")
|
719
1051
|
# Handle directory
|
720
1052
|
else:
|
721
1053
|
console.print(f"[blue]Scanning for sellers in:[/blue] {data_path}")
|
722
|
-
console.print(f"[blue]Backend URL:[/blue] {
|
723
|
-
results = publisher.publish_all_sellers(data_path)
|
1054
|
+
console.print(f"[blue]Backend URL:[/blue] {os.getenv('UNITYSVC_BASE_URL', 'N/A')}\n")
|
1055
|
+
results = asyncio.run(publisher.publish_all_sellers(data_path))
|
724
1056
|
|
725
1057
|
console.print("\n[bold]Publishing Summary:[/bold]")
|
726
1058
|
console.print(f" Total found: {results['total']}")
|
@@ -745,33 +1077,17 @@ def publish_sellers(
|
|
745
1077
|
|
746
1078
|
@app.command("offerings")
|
747
1079
|
def publish_offerings(
|
748
|
-
data_path: Path | None = typer.
|
749
|
-
None,
|
750
|
-
help="Path to service offering file or directory (default: ./data or UNITYSVC_DATA_DIR env var)",
|
751
|
-
),
|
752
|
-
backend_url: str | None = typer.Option(
|
753
|
-
None,
|
754
|
-
"--backend-url",
|
755
|
-
"-u",
|
756
|
-
help="UnitySVC backend URL (default: from UNITYSVC_BACKEND_URL env var)",
|
757
|
-
),
|
758
|
-
api_key: str | None = typer.Option(
|
1080
|
+
data_path: Path | None = typer.Option(
|
759
1081
|
None,
|
760
|
-
"--
|
761
|
-
"-
|
762
|
-
help="
|
1082
|
+
"--data-path",
|
1083
|
+
"-d",
|
1084
|
+
help="Path to service offering file or directory (default: current directory)",
|
763
1085
|
),
|
764
1086
|
):
|
765
1087
|
"""Publish service offering(s) from a file or directory."""
|
766
|
-
import os
|
767
|
-
|
768
1088
|
# Set data path
|
769
1089
|
if data_path is None:
|
770
|
-
|
771
|
-
if data_path_str:
|
772
|
-
data_path = Path(data_path_str)
|
773
|
-
else:
|
774
|
-
data_path = Path.cwd() / "data"
|
1090
|
+
data_path = Path.cwd()
|
775
1091
|
|
776
1092
|
if not data_path.is_absolute():
|
777
1093
|
data_path = Path.cwd() / data_path
|
@@ -780,38 +1096,20 @@ def publish_offerings(
|
|
780
1096
|
console.print(f"[red]✗[/red] Path not found: {data_path}", style="bold red")
|
781
1097
|
raise typer.Exit(code=1)
|
782
1098
|
|
783
|
-
# Get backend URL from argument or environment
|
784
|
-
backend_url = backend_url or os.getenv("UNITYSVC_BACKEND_URL")
|
785
|
-
if not backend_url:
|
786
|
-
console.print(
|
787
|
-
"[red]✗[/red] Backend URL not provided. Use --backend-url or set UNITYSVC_BACKEND_URL env var.",
|
788
|
-
style="bold red",
|
789
|
-
)
|
790
|
-
raise typer.Exit(code=1)
|
791
|
-
|
792
|
-
# Get API key from argument or environment
|
793
|
-
api_key = api_key or os.getenv("UNITYSVC_API_KEY")
|
794
|
-
if not api_key:
|
795
|
-
console.print(
|
796
|
-
"[red]✗[/red] API key not provided. Use --api-key or set UNITYSVC_API_KEY env var.",
|
797
|
-
style="bold red",
|
798
|
-
)
|
799
|
-
raise typer.Exit(code=1)
|
800
|
-
|
801
1099
|
try:
|
802
|
-
with ServiceDataPublisher(
|
1100
|
+
with ServiceDataPublisher() as publisher:
|
803
1101
|
# Handle single file
|
804
1102
|
if data_path.is_file():
|
805
1103
|
console.print(f"[blue]Publishing service offering:[/blue] {data_path}")
|
806
|
-
console.print(f"[blue]Backend URL:[/blue] {
|
807
|
-
result = publisher.
|
1104
|
+
console.print(f"[blue]Backend URL:[/blue] {os.getenv('UNITYSVC_BASE_URL', 'N/A')}\n")
|
1105
|
+
result = asyncio.run(publisher.post_service_offering_async(data_path))
|
808
1106
|
console.print("[green]✓[/green] Service offering published successfully!")
|
809
1107
|
console.print(f"[cyan]Response:[/cyan] {json.dumps(result, indent=2)}")
|
810
1108
|
# Handle directory
|
811
1109
|
else:
|
812
1110
|
console.print(f"[blue]Scanning for service offerings in:[/blue] {data_path}")
|
813
|
-
console.print(f"[blue]Backend URL:[/blue] {
|
814
|
-
results = publisher.publish_all_offerings(data_path)
|
1111
|
+
console.print(f"[blue]Backend URL:[/bold blue] {os.getenv('UNITYSVC_BASE_URL', 'N/A')}\n")
|
1112
|
+
results = asyncio.run(publisher.publish_all_offerings(data_path))
|
815
1113
|
|
816
1114
|
console.print("\n[bold]Publishing Summary:[/bold]")
|
817
1115
|
console.print(f" Total found: {results['total']}")
|
@@ -836,33 +1134,18 @@ def publish_offerings(
|
|
836
1134
|
|
837
1135
|
@app.command("listings")
|
838
1136
|
def publish_listings(
|
839
|
-
data_path: Path | None = typer.
|
840
|
-
None,
|
841
|
-
help="Path to service listing file or directory (default: ./data or UNITYSVC_DATA_DIR env var)",
|
842
|
-
),
|
843
|
-
backend_url: str | None = typer.Option(
|
844
|
-
None,
|
845
|
-
"--backend-url",
|
846
|
-
"-u",
|
847
|
-
help="UnitySVC backend URL (default: from UNITYSVC_BACKEND_URL env var)",
|
848
|
-
),
|
849
|
-
api_key: str | None = typer.Option(
|
1137
|
+
data_path: Path | None = typer.Option(
|
850
1138
|
None,
|
851
|
-
"--
|
852
|
-
"-
|
853
|
-
help="
|
1139
|
+
"--data-path",
|
1140
|
+
"-d",
|
1141
|
+
help="Path to service listing file or directory (default: current directory)",
|
854
1142
|
),
|
855
1143
|
):
|
856
1144
|
"""Publish service listing(s) from a file or directory."""
|
857
|
-
import os
|
858
1145
|
|
859
1146
|
# Set data path
|
860
1147
|
if data_path is None:
|
861
|
-
|
862
|
-
if data_path_str:
|
863
|
-
data_path = Path(data_path_str)
|
864
|
-
else:
|
865
|
-
data_path = Path.cwd() / "data"
|
1148
|
+
data_path = Path.cwd()
|
866
1149
|
|
867
1150
|
if not data_path.is_absolute():
|
868
1151
|
data_path = Path.cwd() / data_path
|
@@ -871,38 +1154,20 @@ def publish_listings(
|
|
871
1154
|
console.print(f"[red]✗[/red] Path not found: {data_path}", style="bold red")
|
872
1155
|
raise typer.Exit(code=1)
|
873
1156
|
|
874
|
-
# Get backend URL from argument or environment
|
875
|
-
backend_url = backend_url or os.getenv("UNITYSVC_BACKEND_URL")
|
876
|
-
if not backend_url:
|
877
|
-
console.print(
|
878
|
-
"[red]✗[/red] Backend URL not provided. Use --backend-url or set UNITYSVC_BACKEND_URL env var.",
|
879
|
-
style="bold red",
|
880
|
-
)
|
881
|
-
raise typer.Exit(code=1)
|
882
|
-
|
883
|
-
# Get API key from argument or environment
|
884
|
-
api_key = api_key or os.getenv("UNITYSVC_API_KEY")
|
885
|
-
if not api_key:
|
886
|
-
console.print(
|
887
|
-
"[red]✗[/red] API key not provided. Use --api-key or set UNITYSVC_API_KEY env var.",
|
888
|
-
style="bold red",
|
889
|
-
)
|
890
|
-
raise typer.Exit(code=1)
|
891
|
-
|
892
1157
|
try:
|
893
|
-
with ServiceDataPublisher(
|
1158
|
+
with ServiceDataPublisher() as publisher:
|
894
1159
|
# Handle single file
|
895
1160
|
if data_path.is_file():
|
896
1161
|
console.print(f"[blue]Publishing service listing:[/blue] {data_path}")
|
897
|
-
console.print(f"[blue]Backend URL:[/blue] {
|
898
|
-
result = publisher.
|
1162
|
+
console.print(f"[blue]Backend URL:[/blue] {os.getenv('UNITYSVC_BASE_URL', 'N/A')}\n")
|
1163
|
+
result = asyncio.run(publisher.post_service_listing_async(data_path))
|
899
1164
|
console.print("[green]✓[/green] Service listing published successfully!")
|
900
1165
|
console.print(f"[cyan]Response:[/cyan] {json.dumps(result, indent=2)}")
|
901
1166
|
# Handle directory
|
902
1167
|
else:
|
903
1168
|
console.print(f"[blue]Scanning for service listings in:[/blue] {data_path}")
|
904
|
-
console.print(f"[blue]Backend URL:[/blue] {
|
905
|
-
results = publisher.publish_all_listings(data_path)
|
1169
|
+
console.print(f"[blue]Backend URL:[/blue] {os.getenv('UNITYSVC_BASE_URL', 'N/A')}\n")
|
1170
|
+
results = asyncio.run(publisher.publish_all_listings(data_path))
|
906
1171
|
|
907
1172
|
console.print("\n[bold]Publishing Summary:[/bold]")
|
908
1173
|
console.print(f" Total found: {results['total']}")
|