unitysvc-services 0.2.5__py3-none-any.whl → 0.2.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- unitysvc_services/models/base.py +123 -0
- unitysvc_services/models/listing_v1.py +26 -3
- unitysvc_services/models/provider_v1.py +17 -2
- unitysvc_services/models/seller_v1.py +8 -2
- unitysvc_services/models/service_v1.py +8 -1
- unitysvc_services/publisher.py +413 -137
- unitysvc_services/query.py +5 -4
- unitysvc_services/validator.py +79 -23
- {unitysvc_services-0.2.5.dist-info → unitysvc_services-0.2.7.dist-info}/METADATA +1 -1
- unitysvc_services-0.2.7.dist-info/RECORD +24 -0
- unitysvc_services-0.2.5.dist-info/RECORD +0 -24
- {unitysvc_services-0.2.5.dist-info → unitysvc_services-0.2.7.dist-info}/WHEEL +0 -0
- {unitysvc_services-0.2.5.dist-info → unitysvc_services-0.2.7.dist-info}/entry_points.txt +0 -0
- {unitysvc_services-0.2.5.dist-info → unitysvc_services-0.2.7.dist-info}/licenses/LICENSE +0 -0
- {unitysvc_services-0.2.5.dist-info → unitysvc_services-0.2.7.dist-info}/top_level.txt +0 -0
unitysvc_services/models/base.py
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
import re
|
1
2
|
from enum import StrEnum
|
2
3
|
from typing import Any
|
3
4
|
|
@@ -210,6 +211,7 @@ class ProviderStatusEnum(StrEnum):
|
|
210
211
|
"""Provider status enum."""
|
211
212
|
|
212
213
|
active = "active"
|
214
|
+
pending = "pending"
|
213
215
|
disabled = "disabled"
|
214
216
|
incomplete = "incomplete" # Provider information is incomplete
|
215
217
|
|
@@ -218,6 +220,7 @@ class SellerStatusEnum(StrEnum):
|
|
218
220
|
"""Seller status enum."""
|
219
221
|
|
220
222
|
active = "active"
|
223
|
+
pending = "pending"
|
221
224
|
disabled = "disabled"
|
222
225
|
incomplete = "incomplete" # Seller information is incomplete
|
223
226
|
|
@@ -366,3 +369,123 @@ class Pricing(BaseModel):
|
|
366
369
|
|
367
370
|
# Optional reference to upstream pricing
|
368
371
|
reference: str | None = Field(default=None, description="Reference URL to upstream pricing")
|
372
|
+
|
373
|
+
|
374
|
+
def validate_name(name: str, entity_type: str, display_name: str | None = None, *, allow_slash: bool = False) -> str:
|
375
|
+
"""
|
376
|
+
Validate that a name field uses valid identifiers.
|
377
|
+
|
378
|
+
Name format rules:
|
379
|
+
- Only letters (upper/lowercase), numbers, dots, dashes, and underscores allowed
|
380
|
+
- If allow_slash=True, slashes are also allowed for hierarchical names
|
381
|
+
- Must start and end with alphanumeric characters (not special characters)
|
382
|
+
- Cannot have consecutive slashes (when allow_slash=True)
|
383
|
+
- Cannot be empty
|
384
|
+
|
385
|
+
Args:
|
386
|
+
name: The name value to validate
|
387
|
+
entity_type: Type of entity (provider, seller, service, listing) for error messages
|
388
|
+
display_name: Optional display name to suggest a valid name from
|
389
|
+
allow_slash: Whether to allow slashes for hierarchical names (default: False)
|
390
|
+
|
391
|
+
Returns:
|
392
|
+
The validated name (unchanged if valid)
|
393
|
+
|
394
|
+
Raises:
|
395
|
+
ValueError: If the name doesn't match the required pattern
|
396
|
+
|
397
|
+
Examples:
|
398
|
+
Without slashes (providers, sellers):
|
399
|
+
- name='amazon-bedrock' or name='Amazon-Bedrock'
|
400
|
+
- name='fireworks.ai' or name='Fireworks.ai'
|
401
|
+
- name='llama-3.1' or name='Llama-3.1'
|
402
|
+
|
403
|
+
With slashes (services, listings):
|
404
|
+
- name='gpt-4' or name='GPT-4'
|
405
|
+
- name='models/gpt-4' or name='models/GPT-4'
|
406
|
+
- name='black-forest-labs/FLUX.1-dev'
|
407
|
+
- name='api/v1/completion'
|
408
|
+
"""
|
409
|
+
# Build pattern based on allow_slash parameter
|
410
|
+
if allow_slash:
|
411
|
+
# Pattern: starts with alphanumeric, can contain alphanumeric/dot/dash/underscore/slash, ends with alphanumeric
|
412
|
+
name_pattern = r"^[a-zA-Z0-9]([a-zA-Z0-9._/-]*[a-zA-Z0-9])?$"
|
413
|
+
allowed_chars = "letters, numbers, dots, dashes, underscores, and slashes"
|
414
|
+
else:
|
415
|
+
# Pattern: starts with alphanumeric, can contain alphanumeric/dot/dash/underscore, ends with alphanumeric
|
416
|
+
name_pattern = r"^[a-zA-Z0-9]([a-zA-Z0-9._-]*[a-zA-Z0-9])?$"
|
417
|
+
allowed_chars = "letters, numbers, dots, dashes, and underscores"
|
418
|
+
|
419
|
+
# Check for consecutive slashes if slashes are allowed
|
420
|
+
if allow_slash and "//" in name:
|
421
|
+
raise ValueError(f"Invalid {entity_type} name '{name}'. Name cannot contain consecutive slashes.")
|
422
|
+
|
423
|
+
if not re.match(name_pattern, name):
|
424
|
+
# Build helpful error message
|
425
|
+
error_msg = (
|
426
|
+
f"Invalid {entity_type} name '{name}'. "
|
427
|
+
f"Name must contain only {allowed_chars}. "
|
428
|
+
f"It must start and end with an alphanumeric character.\n"
|
429
|
+
)
|
430
|
+
|
431
|
+
# Suggest a valid name based on display_name if available
|
432
|
+
if display_name:
|
433
|
+
suggested_name = suggest_valid_name(display_name, allow_slash=allow_slash)
|
434
|
+
if suggested_name and suggested_name != name:
|
435
|
+
error_msg += f" Suggestion: Set name='{suggested_name}' and display_name='{display_name}'\n"
|
436
|
+
|
437
|
+
# Add appropriate examples based on allow_slash
|
438
|
+
if allow_slash:
|
439
|
+
error_msg += (
|
440
|
+
" Examples:\n"
|
441
|
+
" - name='gpt-4' or name='GPT-4'\n"
|
442
|
+
" - name='models/gpt-4' or name='models/GPT-4'\n"
|
443
|
+
" - name='black-forest-labs/FLUX.1-dev'\n"
|
444
|
+
" - name='api/v1/completion'"
|
445
|
+
)
|
446
|
+
else:
|
447
|
+
error_msg += (
|
448
|
+
" Note: Use 'display_name' field for brand names with spaces and special characters.\n"
|
449
|
+
" Examples:\n"
|
450
|
+
" - name='amazon-bedrock' or name='Amazon-Bedrock'\n"
|
451
|
+
" - name='fireworks.ai' or name='Fireworks.ai'\n"
|
452
|
+
" - name='llama-3.1' or name='Llama-3.1'"
|
453
|
+
)
|
454
|
+
|
455
|
+
raise ValueError(error_msg)
|
456
|
+
|
457
|
+
return name
|
458
|
+
|
459
|
+
|
460
|
+
def suggest_valid_name(display_name: str, *, allow_slash: bool = False) -> str:
|
461
|
+
"""
|
462
|
+
Suggest a valid name based on a display name.
|
463
|
+
|
464
|
+
Replaces invalid characters with hyphens and ensures it follows the naming rules.
|
465
|
+
Preserves the original case.
|
466
|
+
|
467
|
+
Args:
|
468
|
+
display_name: The display name to convert
|
469
|
+
allow_slash: Whether to allow slashes for hierarchical names (default: False)
|
470
|
+
|
471
|
+
Returns:
|
472
|
+
A suggested valid name
|
473
|
+
"""
|
474
|
+
if allow_slash:
|
475
|
+
# Replace characters that aren't alphanumeric, dot, dash, underscore, or slash with hyphens
|
476
|
+
suggested = re.sub(r"[^a-zA-Z0-9._/-]+", "-", display_name)
|
477
|
+
# Remove leading/trailing special characters
|
478
|
+
suggested = suggested.strip("._/-")
|
479
|
+
# Collapse multiple consecutive dashes
|
480
|
+
suggested = re.sub(r"-+", "-", suggested)
|
481
|
+
# Remove consecutive slashes
|
482
|
+
suggested = re.sub(r"/+", "/", suggested)
|
483
|
+
else:
|
484
|
+
# Replace characters that aren't alphanumeric, dot, dash, or underscore with hyphens
|
485
|
+
suggested = re.sub(r"[^a-zA-Z0-9._-]+", "-", display_name)
|
486
|
+
# Remove leading/trailing dots, dashes, or underscores
|
487
|
+
suggested = suggested.strip("._-")
|
488
|
+
# Collapse multiple consecutive dashes
|
489
|
+
suggested = re.sub(r"-+", "-", suggested)
|
490
|
+
|
491
|
+
return suggested
|
@@ -1,9 +1,15 @@
|
|
1
1
|
from datetime import datetime
|
2
2
|
from typing import Any
|
3
3
|
|
4
|
-
from pydantic import BaseModel, ConfigDict, Field
|
4
|
+
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
5
5
|
|
6
|
-
from unitysvc_services.models.base import
|
6
|
+
from unitysvc_services.models.base import (
|
7
|
+
AccessInterface,
|
8
|
+
Document,
|
9
|
+
ListingStatusEnum,
|
10
|
+
Pricing,
|
11
|
+
validate_name,
|
12
|
+
)
|
7
13
|
|
8
14
|
|
9
15
|
class ListingV1(BaseModel):
|
@@ -27,10 +33,19 @@ class ListingV1(BaseModel):
|
|
27
33
|
|
28
34
|
seller_name: str | None = Field(default=None, description="Name of the seller offering this service listing")
|
29
35
|
|
30
|
-
name: str = Field(
|
36
|
+
name: str | None = Field(
|
37
|
+
default=None,
|
31
38
|
max_length=255,
|
32
39
|
description="Name identifier for the service listing, default to filename",
|
33
40
|
)
|
41
|
+
|
42
|
+
# Display name for UI (human-readable listing name)
|
43
|
+
display_name: str | None = Field(
|
44
|
+
default=None,
|
45
|
+
max_length=200,
|
46
|
+
description="Human-readable listing name (e.g., 'Premium GPT-4 Access', 'Enterprise AI Services')",
|
47
|
+
)
|
48
|
+
|
34
49
|
# unique name for each provider, usually following upstream naming convention
|
35
50
|
# status of the service, public, deprecated etc
|
36
51
|
listing_status: ListingStatusEnum = Field(
|
@@ -67,3 +82,11 @@ class ListingV1(BaseModel):
|
|
67
82
|
user_parameters_ui_schema: dict[str, Any] | None = Field(
|
68
83
|
default=None, description="Dictionary of user parameters UI schema"
|
69
84
|
)
|
85
|
+
|
86
|
+
@field_validator("name")
|
87
|
+
@classmethod
|
88
|
+
def validate_name_format(cls, v: str | None) -> str | None:
|
89
|
+
"""Validate that listing name uses valid identifiers (allows slashes for hierarchical names)."""
|
90
|
+
if v is None:
|
91
|
+
return v
|
92
|
+
return validate_name(v, "listing", allow_slash=True)
|
@@ -1,9 +1,9 @@
|
|
1
1
|
from datetime import datetime
|
2
2
|
from typing import Any
|
3
3
|
|
4
|
-
from pydantic import BaseModel, ConfigDict, EmailStr, Field, HttpUrl
|
4
|
+
from pydantic import BaseModel, ConfigDict, EmailStr, Field, HttpUrl, field_validator
|
5
5
|
|
6
|
-
from unitysvc_services.models.base import AccessInterface, Document, ProviderStatusEnum
|
6
|
+
from unitysvc_services.models.base import AccessInterface, Document, ProviderStatusEnum, validate_name
|
7
7
|
|
8
8
|
|
9
9
|
class ProviderV1(BaseModel):
|
@@ -26,6 +26,13 @@ class ProviderV1(BaseModel):
|
|
26
26
|
# name of the provider should be the same as directory name
|
27
27
|
name: str
|
28
28
|
|
29
|
+
# Display name for UI (human-readable brand name)
|
30
|
+
display_name: str | None = Field(
|
31
|
+
default=None,
|
32
|
+
max_length=200,
|
33
|
+
description="Human-readable provider name (e.g., 'Amazon Bedrock', 'Fireworks.ai')",
|
34
|
+
)
|
35
|
+
|
29
36
|
# this field is added for convenience. It will be converted to
|
30
37
|
# documents during importing.
|
31
38
|
logo: str | HttpUrl | None = None
|
@@ -57,3 +64,11 @@ class ProviderV1(BaseModel):
|
|
57
64
|
default=ProviderStatusEnum.active,
|
58
65
|
description="Provider status: active, disabled, or incomplete",
|
59
66
|
)
|
67
|
+
|
68
|
+
@field_validator("name")
|
69
|
+
@classmethod
|
70
|
+
def validate_name_format(cls, v: str) -> str:
|
71
|
+
"""Validate that provider name uses URL-safe identifiers."""
|
72
|
+
# Note: display_name is not available in the validator context for suggesting
|
73
|
+
# Display name will be shown in error if user provides it
|
74
|
+
return validate_name(v, "provider", allow_slash=False)
|
@@ -1,8 +1,8 @@
|
|
1
1
|
from datetime import datetime
|
2
2
|
|
3
|
-
from pydantic import BaseModel, ConfigDict, EmailStr, Field, HttpUrl
|
3
|
+
from pydantic import BaseModel, ConfigDict, EmailStr, Field, HttpUrl, field_validator
|
4
4
|
|
5
|
-
from unitysvc_services.models.base import Document, SellerStatusEnum, SellerTypeEnum
|
5
|
+
from unitysvc_services.models.base import Document, SellerStatusEnum, SellerTypeEnum, validate_name
|
6
6
|
|
7
7
|
|
8
8
|
class SellerV1(BaseModel):
|
@@ -108,3 +108,9 @@ class SellerV1(BaseModel):
|
|
108
108
|
default=False,
|
109
109
|
description="Whether the seller has been verified (KYC/business verification)",
|
110
110
|
)
|
111
|
+
|
112
|
+
@field_validator("name")
|
113
|
+
@classmethod
|
114
|
+
def validate_name_format(cls, v: str) -> str:
|
115
|
+
"""Validate that seller name uses URL-safe identifiers."""
|
116
|
+
return validate_name(v, "seller", allow_slash=False)
|
@@ -1,7 +1,7 @@
|
|
1
1
|
from datetime import datetime
|
2
2
|
from typing import Any
|
3
3
|
|
4
|
-
from pydantic import BaseModel, ConfigDict, Field, HttpUrl
|
4
|
+
from pydantic import BaseModel, ConfigDict, Field, HttpUrl, field_validator
|
5
5
|
|
6
6
|
from unitysvc_services.models.base import (
|
7
7
|
AccessInterface,
|
@@ -10,6 +10,7 @@ from unitysvc_services.models.base import (
|
|
10
10
|
ServiceTypeEnum,
|
11
11
|
TagEnum,
|
12
12
|
UpstreamStatusEnum,
|
13
|
+
validate_name,
|
13
14
|
)
|
14
15
|
|
15
16
|
|
@@ -78,3 +79,9 @@ class ServiceV1(BaseModel):
|
|
78
79
|
# a list of pricing models
|
79
80
|
#
|
80
81
|
upstream_price: Pricing | None = Field(description="List of pricing information")
|
82
|
+
|
83
|
+
@field_validator("name")
|
84
|
+
@classmethod
|
85
|
+
def validate_name_format(cls, v: str) -> str:
|
86
|
+
"""Validate that service name uses valid identifiers (allows slashes for hierarchical names)."""
|
87
|
+
return validate_name(v, "service", allow_slash=True)
|
unitysvc_services/publisher.py
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
"""Data publisher module for posting service data to UnitySVC backend."""
|
2
2
|
|
3
|
+
import asyncio
|
3
4
|
import base64
|
4
5
|
import json
|
5
6
|
import os
|
@@ -29,13 +30,15 @@ class ServiceDataPublisher:
|
|
29
30
|
raise ValueError("UNITYSVC_API_KEY environment variable not set")
|
30
31
|
|
31
32
|
self.base_url = self.base_url.rstrip("/")
|
32
|
-
self.
|
33
|
+
self.async_client = httpx.AsyncClient(
|
33
34
|
headers={
|
34
35
|
"X-API-Key": self.api_key,
|
35
36
|
"Content-Type": "application/json",
|
36
37
|
},
|
37
38
|
timeout=30.0,
|
38
39
|
)
|
40
|
+
# Semaphore to limit concurrent requests and prevent connection pool exhaustion
|
41
|
+
self.max_concurrent_requests = 15
|
39
42
|
|
40
43
|
def load_data_file(self, file_path: Path) -> dict[str, Any]:
|
41
44
|
"""Load data from JSON or TOML file."""
|
@@ -94,106 +97,134 @@ class ServiceDataPublisher:
|
|
94
97
|
|
95
98
|
return result
|
96
99
|
|
97
|
-
def
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
100
|
+
async def _post_with_retry(
|
101
|
+
self,
|
102
|
+
endpoint: str,
|
103
|
+
data: dict[str, Any],
|
104
|
+
entity_type: str,
|
105
|
+
entity_name: str,
|
106
|
+
context_info: str = "",
|
107
|
+
max_retries: int = 3,
|
108
|
+
) -> dict[str, Any]:
|
102
109
|
"""
|
110
|
+
Generic retry wrapper for posting data to backend API.
|
103
111
|
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
)
|
112
|
-
|
113
|
-
# Resolve file references and include content
|
114
|
-
data_with_content = self.resolve_file_references(data, base_path)
|
115
|
-
|
116
|
-
# Extract provider_name from directory structure
|
117
|
-
# Find the 'services' directory and use its parent as provider_name
|
118
|
-
parts = data_file.parts
|
119
|
-
try:
|
120
|
-
services_idx = parts.index("services")
|
121
|
-
provider_name = parts[services_idx - 1]
|
122
|
-
data_with_content["provider_name"] = provider_name
|
112
|
+
Args:
|
113
|
+
endpoint: API endpoint path (e.g., "/publish/listing")
|
114
|
+
data: JSON data to post
|
115
|
+
entity_type: Type of entity being published (for error messages)
|
116
|
+
entity_name: Name of the entity being published (for error messages)
|
117
|
+
context_info: Additional context for error messages (e.g., provider, service info)
|
118
|
+
max_retries: Maximum number of retry attempts
|
123
119
|
|
124
|
-
|
125
|
-
|
126
|
-
except (ValueError, IndexError):
|
127
|
-
raise ValueError(
|
128
|
-
f"Cannot extract provider_name from path: {data_file}. "
|
129
|
-
f"Expected path to contain .../{{provider_name}}/services/..."
|
130
|
-
)
|
120
|
+
Returns:
|
121
|
+
Response JSON from successful API call
|
131
122
|
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
"name": data.get("name", "unknown"),
|
143
|
-
}
|
123
|
+
Raises:
|
124
|
+
ValueError: On client errors (4xx) or after exhausting retries
|
125
|
+
"""
|
126
|
+
last_exception = None
|
127
|
+
for attempt in range(max_retries):
|
128
|
+
try:
|
129
|
+
response = await self.async_client.post(
|
130
|
+
f"{self.base_url}{endpoint}",
|
131
|
+
json=data,
|
132
|
+
)
|
144
133
|
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
134
|
+
# Provide detailed error information if request fails
|
135
|
+
if not response.is_success:
|
136
|
+
# Don't retry on 4xx errors (client errors) - they won't succeed on retry
|
137
|
+
if 400 <= response.status_code < 500:
|
138
|
+
error_detail = "Unknown error"
|
139
|
+
try:
|
140
|
+
error_json = response.json()
|
141
|
+
error_detail = error_json.get("detail", str(error_json))
|
142
|
+
except Exception:
|
143
|
+
error_detail = response.text or f"HTTP {response.status_code}"
|
144
|
+
|
145
|
+
context_msg = f" ({context_info})" if context_info else ""
|
146
|
+
raise ValueError(
|
147
|
+
f"Failed to publish {entity_type} '{entity_name}'{context_msg}: {error_detail}"
|
148
|
+
)
|
149
|
+
|
150
|
+
# 5xx errors or network errors - retry with exponential backoff
|
151
|
+
if attempt < max_retries - 1:
|
152
|
+
wait_time = 2**attempt # Exponential backoff: 1s, 2s, 4s
|
153
|
+
await asyncio.sleep(wait_time)
|
154
|
+
continue
|
155
|
+
else:
|
156
|
+
# Last attempt failed
|
157
|
+
error_detail = "Unknown error"
|
158
|
+
try:
|
159
|
+
error_json = response.json()
|
160
|
+
error_detail = error_json.get("detail", str(error_json))
|
161
|
+
except Exception:
|
162
|
+
error_detail = response.text or f"HTTP {response.status_code}"
|
163
|
+
|
164
|
+
context_msg = f" ({context_info})" if context_info else ""
|
165
|
+
raise ValueError(
|
166
|
+
f"Failed to publish {entity_type} after {max_retries} attempts: "
|
167
|
+
f"'{entity_name}'{context_msg}: {error_detail}"
|
168
|
+
)
|
169
|
+
|
170
|
+
return response.json()
|
171
|
+
|
172
|
+
except (httpx.NetworkError, httpx.TimeoutException) as e:
|
173
|
+
last_exception = e
|
174
|
+
if attempt < max_retries - 1:
|
175
|
+
wait_time = 2**attempt # Exponential backoff: 1s, 2s, 4s
|
176
|
+
await asyncio.sleep(wait_time)
|
177
|
+
continue
|
178
|
+
else:
|
179
|
+
raise ValueError(
|
180
|
+
f"Network error after {max_retries} attempts for {entity_type} '{entity_name}': {str(e)}"
|
181
|
+
)
|
152
182
|
|
153
|
-
|
154
|
-
|
183
|
+
# Should never reach here, but just in case
|
184
|
+
if last_exception:
|
185
|
+
raise last_exception
|
186
|
+
raise ValueError("Unexpected error in retry logic")
|
155
187
|
|
156
|
-
|
157
|
-
|
158
|
-
"""
|
188
|
+
async def post_service_listing_async(self, listing_file: Path, max_retries: int = 3) -> dict[str, Any]:
|
189
|
+
"""Async version of post_service_listing for concurrent publishing with retry logic."""
|
159
190
|
# Load the listing data file
|
160
|
-
data = self.load_data_file(
|
191
|
+
data = self.load_data_file(listing_file)
|
161
192
|
|
162
193
|
# If name is not provided, use filename (without extension)
|
163
194
|
if "name" not in data or not data.get("name"):
|
164
|
-
data["name"] =
|
195
|
+
data["name"] = listing_file.stem
|
165
196
|
|
166
197
|
# Resolve file references and include content
|
167
|
-
base_path =
|
198
|
+
base_path = listing_file.parent
|
168
199
|
data_with_content = self.resolve_file_references(data, base_path)
|
169
200
|
|
170
201
|
# Extract provider_name from directory structure
|
171
|
-
parts =
|
202
|
+
parts = listing_file.parts
|
172
203
|
try:
|
173
204
|
services_idx = parts.index("services")
|
174
205
|
provider_name = parts[services_idx - 1]
|
175
206
|
data_with_content["provider_name"] = provider_name
|
176
207
|
except (ValueError, IndexError):
|
177
208
|
raise ValueError(
|
178
|
-
f"Cannot extract provider_name from path: {
|
209
|
+
f"Cannot extract provider_name from path: {listing_file}. "
|
179
210
|
f"Expected path to contain .../{{provider_name}}/services/..."
|
180
211
|
)
|
181
212
|
|
182
213
|
# If service_name is not in listing data, find it from service files in the same directory
|
183
214
|
if "service_name" not in data_with_content or not data_with_content["service_name"]:
|
184
215
|
# Find all service files in the same directory
|
185
|
-
service_files = find_files_by_schema(
|
216
|
+
service_files = find_files_by_schema(listing_file.parent, "service_v1")
|
186
217
|
|
187
218
|
if len(service_files) == 0:
|
188
219
|
raise ValueError(
|
189
|
-
f"Cannot find any service_v1 files in {
|
220
|
+
f"Cannot find any service_v1 files in {listing_file.parent}. "
|
190
221
|
f"Listing files must be in the same directory as a service definition."
|
191
222
|
)
|
192
223
|
elif len(service_files) > 1:
|
193
224
|
service_names = [data.get("name", "unknown") for _, _, data in service_files]
|
194
225
|
raise ValueError(
|
195
|
-
f"Multiple services found in {
|
196
|
-
f"Please add 'service_name' field to {
|
226
|
+
f"Multiple services found in {listing_file.parent}: {', '.join(service_names)}. "
|
227
|
+
f"Please add 'service_name' field to {listing_file.name} to specify which "
|
197
228
|
f"service this listing belongs to."
|
198
229
|
)
|
199
230
|
else:
|
@@ -204,11 +235,13 @@ class ServiceDataPublisher:
|
|
204
235
|
else:
|
205
236
|
# service_name is provided in listing data, find the matching service to get version
|
206
237
|
service_name = data_with_content["service_name"]
|
207
|
-
service_files = find_files_by_schema(
|
238
|
+
service_files = find_files_by_schema(
|
239
|
+
listing_file.parent, "service_v1", field_filter=(("name", service_name),)
|
240
|
+
)
|
208
241
|
|
209
242
|
if not service_files:
|
210
243
|
raise ValueError(
|
211
|
-
f"Service '{service_name}' specified in {
|
244
|
+
f"Service '{service_name}' specified in {listing_file.name} not found in {listing_file.parent}."
|
212
245
|
)
|
213
246
|
|
214
247
|
# Get version from the found service
|
@@ -217,13 +250,13 @@ class ServiceDataPublisher:
|
|
217
250
|
|
218
251
|
# Find seller_name from seller definition in the data directory
|
219
252
|
# Navigate up to find the data directory and look for seller file
|
220
|
-
data_dir =
|
253
|
+
data_dir = listing_file.parent
|
221
254
|
while data_dir.name != "data" and data_dir.parent != data_dir:
|
222
255
|
data_dir = data_dir.parent
|
223
256
|
|
224
257
|
if data_dir.name != "data":
|
225
258
|
raise ValueError(
|
226
|
-
f"Cannot find 'data' directory in path: {
|
259
|
+
f"Cannot find 'data' directory in path: {listing_file}. "
|
227
260
|
f"Expected path structure includes a 'data' directory."
|
228
261
|
)
|
229
262
|
|
@@ -257,17 +290,77 @@ class ServiceDataPublisher:
|
|
257
290
|
if "listing_status" in data_with_content:
|
258
291
|
data_with_content["status"] = data_with_content.pop("listing_status")
|
259
292
|
|
260
|
-
# Post to the endpoint
|
261
|
-
|
262
|
-
f"{
|
263
|
-
|
293
|
+
# Post to the endpoint using retry helper
|
294
|
+
context_info = (
|
295
|
+
f"service: {data_with_content.get('service_name')}, "
|
296
|
+
f"provider: {data_with_content.get('provider_name')}, "
|
297
|
+
f"seller: {data_with_content.get('seller_name')}"
|
298
|
+
)
|
299
|
+
return await self._post_with_retry(
|
300
|
+
endpoint="/publish/listing",
|
301
|
+
data=data_with_content,
|
302
|
+
entity_type="listing",
|
303
|
+
entity_name=data.get("name", "unknown"),
|
304
|
+
context_info=context_info,
|
305
|
+
max_retries=max_retries,
|
306
|
+
)
|
307
|
+
|
308
|
+
async def post_service_offering_async(self, data_file: Path, max_retries: int = 3) -> dict[str, Any]:
|
309
|
+
"""Async version of post_service_offering for concurrent publishing with retry logic."""
|
310
|
+
# Load the data file
|
311
|
+
data = self.load_data_file(data_file)
|
312
|
+
|
313
|
+
# Resolve file references and include content
|
314
|
+
base_path = data_file.parent
|
315
|
+
data = convert_convenience_fields_to_documents(
|
316
|
+
data, base_path, logo_field="logo", terms_field="terms_of_service"
|
264
317
|
)
|
265
|
-
response.raise_for_status()
|
266
|
-
return response.json()
|
267
318
|
|
268
|
-
|
269
|
-
|
319
|
+
# Resolve file references and include content
|
320
|
+
data_with_content = self.resolve_file_references(data, base_path)
|
270
321
|
|
322
|
+
# Extract provider_name from directory structure
|
323
|
+
# Find the 'services' directory and use its parent as provider_name
|
324
|
+
parts = data_file.parts
|
325
|
+
try:
|
326
|
+
services_idx = parts.index("services")
|
327
|
+
provider_name = parts[services_idx - 1]
|
328
|
+
data_with_content["provider_name"] = provider_name
|
329
|
+
|
330
|
+
# Find provider directory to check status
|
331
|
+
provider_dir = Path(*parts[:services_idx])
|
332
|
+
except (ValueError, IndexError):
|
333
|
+
raise ValueError(
|
334
|
+
f"Cannot extract provider_name from path: {data_file}. "
|
335
|
+
f"Expected path to contain .../{{provider_name}}/services/..."
|
336
|
+
)
|
337
|
+
|
338
|
+
# Check provider status - skip if incomplete
|
339
|
+
provider_files = find_files_by_schema(provider_dir, "provider_v1")
|
340
|
+
if provider_files:
|
341
|
+
# Should only be one provider file in the directory
|
342
|
+
_provider_file, _format, provider_data = provider_files[0]
|
343
|
+
provider_status = provider_data.get("status", ProviderStatusEnum.active)
|
344
|
+
if provider_status == ProviderStatusEnum.incomplete:
|
345
|
+
return {
|
346
|
+
"skipped": True,
|
347
|
+
"reason": f"Provider status is '{provider_status}' - not publishing offering to backend",
|
348
|
+
"name": data.get("name", "unknown"),
|
349
|
+
}
|
350
|
+
|
351
|
+
# Post to the endpoint using retry helper
|
352
|
+
context_info = f"provider: {data_with_content.get('provider_name')}"
|
353
|
+
return await self._post_with_retry(
|
354
|
+
endpoint="/publish/offering",
|
355
|
+
data=data_with_content,
|
356
|
+
entity_type="offering",
|
357
|
+
entity_name=data.get("name", "unknown"),
|
358
|
+
context_info=context_info,
|
359
|
+
max_retries=max_retries,
|
360
|
+
)
|
361
|
+
|
362
|
+
async def post_provider_async(self, data_file: Path, max_retries: int = 3) -> dict[str, Any]:
|
363
|
+
"""Async version of post_provider for concurrent publishing with retry logic."""
|
271
364
|
# Load the data file
|
272
365
|
data = self.load_data_file(data_file)
|
273
366
|
|
@@ -290,22 +383,17 @@ class ServiceDataPublisher:
|
|
290
383
|
# Resolve file references and include content
|
291
384
|
data_with_content = self.resolve_file_references(data, base_path)
|
292
385
|
|
293
|
-
#
|
294
|
-
|
295
|
-
|
296
|
-
|
297
|
-
|
298
|
-
|
299
|
-
|
300
|
-
f"{self.base_url}/publish/provider",
|
301
|
-
json=data_with_content,
|
386
|
+
# Post to the endpoint using retry helper
|
387
|
+
return await self._post_with_retry(
|
388
|
+
endpoint="/publish/provider",
|
389
|
+
data=data_with_content,
|
390
|
+
entity_type="provider",
|
391
|
+
entity_name=data.get("name", "unknown"),
|
392
|
+
max_retries=max_retries,
|
302
393
|
)
|
303
|
-
response.raise_for_status()
|
304
|
-
return response.json()
|
305
|
-
|
306
|
-
def post_seller(self, data_file: Path) -> dict[str, Any]:
|
307
|
-
"""Post seller data to the backend."""
|
308
394
|
|
395
|
+
async def post_seller_async(self, data_file: Path, max_retries: int = 3) -> dict[str, Any]:
|
396
|
+
"""Async version of post_seller for concurrent publishing with retry logic."""
|
309
397
|
# Load the data file
|
310
398
|
data = self.load_data_file(data_file)
|
311
399
|
|
@@ -326,18 +414,14 @@ class ServiceDataPublisher:
|
|
326
414
|
# Resolve file references and include content
|
327
415
|
data_with_content = self.resolve_file_references(data, base_path)
|
328
416
|
|
329
|
-
#
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
f"{self.base_url}/publish/seller",
|
337
|
-
json=data_with_content,
|
417
|
+
# Post to the endpoint using retry helper
|
418
|
+
return await self._post_with_retry(
|
419
|
+
endpoint="/publish/seller",
|
420
|
+
data=data_with_content,
|
421
|
+
entity_type="seller",
|
422
|
+
entity_name=data.get("name", "unknown"),
|
423
|
+
max_retries=max_retries,
|
338
424
|
)
|
339
|
-
response.raise_for_status()
|
340
|
-
return response.json()
|
341
425
|
|
342
426
|
def find_offering_files(self, data_dir: Path) -> list[Path]:
|
343
427
|
"""Find all service offering files in a directory tree."""
|
@@ -359,14 +443,48 @@ class ServiceDataPublisher:
|
|
359
443
|
files = find_files_by_schema(data_dir, "seller_v1")
|
360
444
|
return sorted([f[0] for f in files])
|
361
445
|
|
446
|
+
async def _publish_offering_task(
|
447
|
+
self, offering_file: Path, console: Console, semaphore: asyncio.Semaphore
|
448
|
+
) -> tuple[Path, dict[str, Any] | Exception]:
|
449
|
+
"""
|
450
|
+
Async task to publish a single offering with concurrency control.
|
451
|
+
|
452
|
+
Returns tuple of (offering_file, result_or_exception).
|
453
|
+
"""
|
454
|
+
async with semaphore: # Limit concurrent requests
|
455
|
+
try:
|
456
|
+
# Load offering data to get the name
|
457
|
+
data = self.load_data_file(offering_file)
|
458
|
+
offering_name = data.get("name", offering_file.stem)
|
459
|
+
|
460
|
+
# Publish the offering
|
461
|
+
result = await self.post_service_offering_async(offering_file)
|
462
|
+
|
463
|
+
# Print complete statement after publication
|
464
|
+
if result.get("skipped"):
|
465
|
+
reason = result.get("reason", "unknown")
|
466
|
+
console.print(f" [yellow]⊘[/yellow] Skipped offering: [cyan]{offering_name}[/cyan] - {reason}")
|
467
|
+
else:
|
468
|
+
provider_name = result.get("provider_name")
|
469
|
+
console.print(
|
470
|
+
f" [green]✓[/green] Published offering: [cyan]{offering_name}[/cyan] "
|
471
|
+
f"(provider: {provider_name})"
|
472
|
+
)
|
473
|
+
|
474
|
+
return (offering_file, result)
|
475
|
+
except Exception as e:
|
476
|
+
data = self.load_data_file(offering_file)
|
477
|
+
offering_name = data.get("name", offering_file.stem)
|
478
|
+
console.print(f" [red]✗[/red] Failed to publish offering: [cyan]{offering_name}[/cyan] - {str(e)}")
|
479
|
+
return (offering_file, e)
|
480
|
+
|
362
481
|
def publish_all_offerings(self, data_dir: Path) -> dict[str, Any]:
|
363
482
|
"""
|
364
|
-
Publish all service offerings found in a directory tree.
|
483
|
+
Publish all service offerings found in a directory tree concurrently.
|
365
484
|
|
366
485
|
Validates data consistency before publishing.
|
367
486
|
Returns a summary of successes and failures.
|
368
487
|
"""
|
369
|
-
|
370
488
|
# Validate all service directories first
|
371
489
|
validator = DataValidator(data_dir, data_dir.parent / "schema")
|
372
490
|
validation_errors = validator.validate_all_service_directories(data_dir)
|
@@ -386,19 +504,70 @@ class ServiceDataPublisher:
|
|
386
504
|
"errors": [],
|
387
505
|
}
|
388
506
|
|
389
|
-
|
390
|
-
|
391
|
-
|
392
|
-
|
393
|
-
|
507
|
+
if not offering_files:
|
508
|
+
return results
|
509
|
+
|
510
|
+
console = Console()
|
511
|
+
|
512
|
+
# Run all offering publications concurrently with rate limiting
|
513
|
+
async def _publish_all() -> list[tuple[Path, dict[str, Any] | Exception]]:
|
514
|
+
# Create semaphore to limit concurrent requests
|
515
|
+
semaphore = asyncio.Semaphore(self.max_concurrent_requests)
|
516
|
+
tasks = [self._publish_offering_task(offering_file, console, semaphore) for offering_file in offering_files]
|
517
|
+
return await asyncio.gather(*tasks)
|
518
|
+
|
519
|
+
# Execute async tasks
|
520
|
+
task_results = asyncio.run(_publish_all())
|
521
|
+
|
522
|
+
# Process results
|
523
|
+
for offering_file, result in task_results:
|
524
|
+
if isinstance(result, Exception):
|
394
525
|
results["failed"] += 1
|
395
|
-
results["errors"].append({"file": str(offering_file), "error": str(
|
526
|
+
results["errors"].append({"file": str(offering_file), "error": str(result)})
|
527
|
+
else:
|
528
|
+
results["success"] += 1
|
396
529
|
|
397
530
|
return results
|
398
531
|
|
532
|
+
async def _publish_listing_task(
|
533
|
+
self, listing_file: Path, console: Console, semaphore: asyncio.Semaphore
|
534
|
+
) -> tuple[Path, dict[str, Any] | Exception]:
|
535
|
+
"""
|
536
|
+
Async task to publish a single listing with concurrency control.
|
537
|
+
|
538
|
+
Returns tuple of (listing_file, result_or_exception).
|
539
|
+
"""
|
540
|
+
async with semaphore: # Limit concurrent requests
|
541
|
+
try:
|
542
|
+
# Load listing data to get the name
|
543
|
+
data = self.load_data_file(listing_file)
|
544
|
+
listing_name = data.get("name", listing_file.stem)
|
545
|
+
|
546
|
+
# Publish the listing
|
547
|
+
result = await self.post_service_listing_async(listing_file)
|
548
|
+
|
549
|
+
# Print complete statement after publication
|
550
|
+
if result.get("skipped"):
|
551
|
+
reason = result.get("reason", "unknown")
|
552
|
+
console.print(f" [yellow]⊘[/yellow] Skipped listing: [cyan]{listing_name}[/cyan] - {reason}")
|
553
|
+
else:
|
554
|
+
service_name = result.get("service_name")
|
555
|
+
provider_name = result.get("provider_name")
|
556
|
+
console.print(
|
557
|
+
f" [green]✓[/green] Published listing: [cyan]{listing_name}[/cyan] "
|
558
|
+
f"(service: {service_name}, provider: {provider_name})"
|
559
|
+
)
|
560
|
+
|
561
|
+
return (listing_file, result)
|
562
|
+
except Exception as e:
|
563
|
+
data = self.load_data_file(listing_file)
|
564
|
+
listing_name = data.get("name", listing_file.stem)
|
565
|
+
console.print(f" [red]✗[/red] Failed to publish listing: [cyan]{listing_file}[/cyan] - {str(e)}")
|
566
|
+
return (listing_file, e)
|
567
|
+
|
399
568
|
def publish_all_listings(self, data_dir: Path) -> dict[str, Any]:
|
400
569
|
"""
|
401
|
-
Publish all service listings found in a directory tree.
|
570
|
+
Publish all service listings found in a directory tree concurrently.
|
402
571
|
|
403
572
|
Validates data consistency before publishing.
|
404
573
|
Returns a summary of successes and failures.
|
@@ -422,19 +591,65 @@ class ServiceDataPublisher:
|
|
422
591
|
"errors": [],
|
423
592
|
}
|
424
593
|
|
425
|
-
|
426
|
-
|
427
|
-
|
428
|
-
|
429
|
-
|
594
|
+
if not listing_files:
|
595
|
+
return results
|
596
|
+
|
597
|
+
console = Console()
|
598
|
+
|
599
|
+
# Run all listing publications concurrently with rate limiting
|
600
|
+
async def _publish_all() -> list[tuple[Path, dict[str, Any] | Exception]]:
|
601
|
+
# Create semaphore to limit concurrent requests
|
602
|
+
semaphore = asyncio.Semaphore(self.max_concurrent_requests)
|
603
|
+
tasks = [self._publish_listing_task(listing_file, console, semaphore) for listing_file in listing_files]
|
604
|
+
return await asyncio.gather(*tasks)
|
605
|
+
|
606
|
+
# Execute async tasks
|
607
|
+
task_results = asyncio.run(_publish_all())
|
608
|
+
|
609
|
+
# Process results
|
610
|
+
for listing_file, result in task_results:
|
611
|
+
if isinstance(result, Exception):
|
430
612
|
results["failed"] += 1
|
431
|
-
results["errors"].append({"file": str(listing_file), "error": str(
|
613
|
+
results["errors"].append({"file": str(listing_file), "error": str(result)})
|
614
|
+
else:
|
615
|
+
results["success"] += 1
|
432
616
|
|
433
617
|
return results
|
434
618
|
|
619
|
+
async def _publish_provider_task(
|
620
|
+
self, provider_file: Path, console: Console, semaphore: asyncio.Semaphore
|
621
|
+
) -> tuple[Path, dict[str, Any] | Exception]:
|
622
|
+
"""
|
623
|
+
Async task to publish a single provider with concurrency control.
|
624
|
+
|
625
|
+
Returns tuple of (provider_file, result_or_exception).
|
626
|
+
"""
|
627
|
+
async with semaphore: # Limit concurrent requests
|
628
|
+
try:
|
629
|
+
# Load provider data to get the name
|
630
|
+
data = self.load_data_file(provider_file)
|
631
|
+
provider_name = data.get("name", provider_file.stem)
|
632
|
+
|
633
|
+
# Publish the provider
|
634
|
+
result = await self.post_provider_async(provider_file)
|
635
|
+
|
636
|
+
# Print complete statement after publication
|
637
|
+
if result.get("skipped"):
|
638
|
+
reason = result.get("reason", "unknown")
|
639
|
+
console.print(f" [yellow]⊘[/yellow] Skipped provider: [cyan]{provider_name}[/cyan] - {reason}")
|
640
|
+
else:
|
641
|
+
console.print(f" [green]✓[/green] Published provider: [cyan]{provider_name}[/cyan]")
|
642
|
+
|
643
|
+
return (provider_file, result)
|
644
|
+
except Exception as e:
|
645
|
+
data = self.load_data_file(provider_file)
|
646
|
+
provider_name = data.get("name", provider_file.stem)
|
647
|
+
console.print(f" [red]✗[/red] Failed to publish provider: [cyan]{provider_name}[/cyan] - {str(e)}")
|
648
|
+
return (provider_file, e)
|
649
|
+
|
435
650
|
def publish_all_providers(self, data_dir: Path) -> dict[str, Any]:
|
436
651
|
"""
|
437
|
-
Publish all providers found in a directory tree.
|
652
|
+
Publish all providers found in a directory tree concurrently.
|
438
653
|
|
439
654
|
Returns a summary of successes and failures.
|
440
655
|
"""
|
@@ -446,19 +661,65 @@ class ServiceDataPublisher:
|
|
446
661
|
"errors": [],
|
447
662
|
}
|
448
663
|
|
449
|
-
|
450
|
-
|
451
|
-
|
452
|
-
|
453
|
-
|
664
|
+
if not provider_files:
|
665
|
+
return results
|
666
|
+
|
667
|
+
console = Console()
|
668
|
+
|
669
|
+
# Run all provider publications concurrently with rate limiting
|
670
|
+
async def _publish_all() -> list[tuple[Path, dict[str, Any] | Exception]]:
|
671
|
+
# Create semaphore to limit concurrent requests
|
672
|
+
semaphore = asyncio.Semaphore(self.max_concurrent_requests)
|
673
|
+
tasks = [self._publish_provider_task(provider_file, console, semaphore) for provider_file in provider_files]
|
674
|
+
return await asyncio.gather(*tasks)
|
675
|
+
|
676
|
+
# Execute async tasks
|
677
|
+
task_results = asyncio.run(_publish_all())
|
678
|
+
|
679
|
+
# Process results
|
680
|
+
for provider_file, result in task_results:
|
681
|
+
if isinstance(result, Exception):
|
454
682
|
results["failed"] += 1
|
455
|
-
results["errors"].append({"file": str(provider_file), "error": str(
|
683
|
+
results["errors"].append({"file": str(provider_file), "error": str(result)})
|
684
|
+
else:
|
685
|
+
results["success"] += 1
|
456
686
|
|
457
687
|
return results
|
458
688
|
|
689
|
+
async def _publish_seller_task(
|
690
|
+
self, seller_file: Path, console: Console, semaphore: asyncio.Semaphore
|
691
|
+
) -> tuple[Path, dict[str, Any] | Exception]:
|
692
|
+
"""
|
693
|
+
Async task to publish a single seller with concurrency control.
|
694
|
+
|
695
|
+
Returns tuple of (seller_file, result_or_exception).
|
696
|
+
"""
|
697
|
+
async with semaphore: # Limit concurrent requests
|
698
|
+
try:
|
699
|
+
# Load seller data to get the name
|
700
|
+
data = self.load_data_file(seller_file)
|
701
|
+
seller_name = data.get("name", seller_file.stem)
|
702
|
+
|
703
|
+
# Publish the seller
|
704
|
+
result = await self.post_seller_async(seller_file)
|
705
|
+
|
706
|
+
# Print complete statement after publication
|
707
|
+
if result.get("skipped"):
|
708
|
+
reason = result.get("reason", "unknown")
|
709
|
+
console.print(f" [yellow]⊘[/yellow] Skipped seller: [cyan]{seller_name}[/cyan] - {reason}")
|
710
|
+
else:
|
711
|
+
console.print(f" [green]✓[/green] Published seller: [cyan]{seller_name}[/cyan]")
|
712
|
+
|
713
|
+
return (seller_file, result)
|
714
|
+
except Exception as e:
|
715
|
+
data = self.load_data_file(seller_file)
|
716
|
+
seller_name = data.get("name", seller_file.stem)
|
717
|
+
console.print(f" [red]✗[/red] Failed to publish seller: [cyan]{seller_name}[/cyan] - {str(e)}")
|
718
|
+
return (seller_file, e)
|
719
|
+
|
459
720
|
def publish_all_sellers(self, data_dir: Path) -> dict[str, Any]:
|
460
721
|
"""
|
461
|
-
Publish all sellers found in a directory tree.
|
722
|
+
Publish all sellers found in a directory tree concurrently.
|
462
723
|
|
463
724
|
Returns a summary of successes and failures.
|
464
725
|
"""
|
@@ -470,13 +731,28 @@ class ServiceDataPublisher:
|
|
470
731
|
"errors": [],
|
471
732
|
}
|
472
733
|
|
473
|
-
|
474
|
-
|
475
|
-
|
476
|
-
|
477
|
-
|
734
|
+
if not seller_files:
|
735
|
+
return results
|
736
|
+
|
737
|
+
console = Console()
|
738
|
+
|
739
|
+
# Run all seller publications concurrently with rate limiting
|
740
|
+
async def _publish_all() -> list[tuple[Path, dict[str, Any] | Exception]]:
|
741
|
+
# Create semaphore to limit concurrent requests
|
742
|
+
semaphore = asyncio.Semaphore(self.max_concurrent_requests)
|
743
|
+
tasks = [self._publish_seller_task(seller_file, console, semaphore) for seller_file in seller_files]
|
744
|
+
return await asyncio.gather(*tasks)
|
745
|
+
|
746
|
+
# Execute async tasks
|
747
|
+
task_results = asyncio.run(_publish_all())
|
748
|
+
|
749
|
+
# Process results
|
750
|
+
for seller_file, result in task_results:
|
751
|
+
if isinstance(result, Exception):
|
478
752
|
results["failed"] += 1
|
479
|
-
results["errors"].append({"file": str(seller_file), "error": str(
|
753
|
+
results["errors"].append({"file": str(seller_file), "error": str(result)})
|
754
|
+
else:
|
755
|
+
results["success"] += 1
|
480
756
|
|
481
757
|
return results
|
482
758
|
|
@@ -530,8 +806,8 @@ class ServiceDataPublisher:
|
|
530
806
|
return all_results
|
531
807
|
|
532
808
|
def close(self):
|
533
|
-
"""Close
|
534
|
-
self.
|
809
|
+
"""Close HTTP client."""
|
810
|
+
asyncio.run(self.async_client.aclose())
|
535
811
|
|
536
812
|
def __enter__(self):
|
537
813
|
"""Context manager entry."""
|
@@ -682,7 +958,7 @@ def publish_providers(
|
|
682
958
|
if data_path.is_file():
|
683
959
|
console.print(f"[blue]Publishing provider:[/blue] {data_path}")
|
684
960
|
console.print(f"[blue]Backend URL:[/blue] {os.getenv('UNITYSVC_BASE_URL', 'N/A')}\n")
|
685
|
-
result = publisher.
|
961
|
+
result = asyncio.run(publisher.post_provider_async(data_path))
|
686
962
|
console.print("[green]✓[/green] Provider published successfully!")
|
687
963
|
console.print(f"[cyan]Response:[/cyan] {json.dumps(result, indent=2)}")
|
688
964
|
# Handle directory
|
@@ -741,7 +1017,7 @@ def publish_sellers(
|
|
741
1017
|
if data_path.is_file():
|
742
1018
|
console.print(f"[blue]Publishing seller:[/blue] {data_path}")
|
743
1019
|
console.print(f"[blue]Backend URL:[/blue] {os.getenv('UNITYSVC_BASE_URL', 'N/A')}\n")
|
744
|
-
result = publisher.
|
1020
|
+
result = asyncio.run(publisher.post_seller_async(data_path))
|
745
1021
|
console.print("[green]✓[/green] Seller published successfully!")
|
746
1022
|
console.print(f"[cyan]Response:[/cyan] {json.dumps(result, indent=2)}")
|
747
1023
|
# Handle directory
|
@@ -798,7 +1074,7 @@ def publish_offerings(
|
|
798
1074
|
if data_path.is_file():
|
799
1075
|
console.print(f"[blue]Publishing service offering:[/blue] {data_path}")
|
800
1076
|
console.print(f"[blue]Backend URL:[/blue] {os.getenv('UNITYSVC_BASE_URL', 'N/A')}\n")
|
801
|
-
result = publisher.
|
1077
|
+
result = asyncio.run(publisher.post_service_offering_async(data_path))
|
802
1078
|
console.print("[green]✓[/green] Service offering published successfully!")
|
803
1079
|
console.print(f"[cyan]Response:[/cyan] {json.dumps(result, indent=2)}")
|
804
1080
|
# Handle directory
|
@@ -856,7 +1132,7 @@ def publish_listings(
|
|
856
1132
|
if data_path.is_file():
|
857
1133
|
console.print(f"[blue]Publishing service listing:[/blue] {data_path}")
|
858
1134
|
console.print(f"[blue]Backend URL:[/blue] {os.getenv('UNITYSVC_BASE_URL', 'N/A')}\n")
|
859
|
-
result = publisher.
|
1135
|
+
result = asyncio.run(publisher.post_service_listing_async(data_path))
|
860
1136
|
console.print("[green]✓[/green] Service listing published successfully!")
|
861
1137
|
console.print(f"[cyan]Response:[/cyan] {json.dumps(result, indent=2)}")
|
862
1138
|
# Handle directory
|
unitysvc_services/query.py
CHANGED
@@ -421,7 +421,7 @@ def query_offerings(
|
|
421
421
|
help="Output format: table, json",
|
422
422
|
),
|
423
423
|
fields: str = typer.Option(
|
424
|
-
"id,
|
424
|
+
"id,name,service_type,provider_name,status",
|
425
425
|
"--fields",
|
426
426
|
help=(
|
427
427
|
"Comma-separated list of fields to display. Available fields: "
|
@@ -447,7 +447,7 @@ def query_offerings(
|
|
447
447
|
unitysvc_services query offerings
|
448
448
|
|
449
449
|
# Show only specific fields
|
450
|
-
unitysvc_services query offerings --fields id,
|
450
|
+
unitysvc_services query offerings --fields id,name,status
|
451
451
|
|
452
452
|
# Retrieve more than 100 records
|
453
453
|
unitysvc_services query offerings --limit 500
|
@@ -469,7 +469,7 @@ def query_offerings(
|
|
469
469
|
"provider_id",
|
470
470
|
"status",
|
471
471
|
"price",
|
472
|
-
"
|
472
|
+
"name",
|
473
473
|
"service_type",
|
474
474
|
"provider_name",
|
475
475
|
}
|
@@ -578,7 +578,7 @@ def query_listings(
|
|
578
578
|
|
579
579
|
# Show all available fields
|
580
580
|
unitysvc_services query listings --fields \\
|
581
|
-
id,service_name,service_type,seller_name,listing_type,status,provider_name
|
581
|
+
id,name,service_name,service_type,seller_name,listing_type,status,provider_name
|
582
582
|
"""
|
583
583
|
# Parse fields list
|
584
584
|
field_list = [f.strip() for f in fields.split(",")]
|
@@ -586,6 +586,7 @@ def query_listings(
|
|
586
586
|
# Define allowed fields from ServiceListingPublic model
|
587
587
|
allowed_fields = {
|
588
588
|
"id",
|
589
|
+
"name",
|
589
590
|
"offering_id",
|
590
591
|
"offering_status",
|
591
592
|
"seller_id",
|
unitysvc_services/validator.py
CHANGED
@@ -205,6 +205,54 @@ class DataValidator:
|
|
205
205
|
normalized = normalized.strip("-")
|
206
206
|
return normalized
|
207
207
|
|
208
|
+
def validate_with_pydantic_model(self, data: dict[str, Any], schema_name: str) -> list[str]:
|
209
|
+
"""
|
210
|
+
Validate data using Pydantic models for additional validation rules.
|
211
|
+
|
212
|
+
This complements JSON schema validation with Pydantic field validators
|
213
|
+
like name format validation.
|
214
|
+
|
215
|
+
Args:
|
216
|
+
data: The data to validate
|
217
|
+
schema_name: The schema name (e.g., 'provider_v1', 'seller_v1')
|
218
|
+
|
219
|
+
Returns:
|
220
|
+
List of validation error messages
|
221
|
+
"""
|
222
|
+
from pydantic import BaseModel
|
223
|
+
|
224
|
+
from unitysvc_services.models import ListingV1, ProviderV1, SellerV1, ServiceV1
|
225
|
+
|
226
|
+
errors: list[str] = []
|
227
|
+
|
228
|
+
# Map schema names to Pydantic model classes
|
229
|
+
model_map: dict[str, type[BaseModel]] = {
|
230
|
+
"provider_v1": ProviderV1,
|
231
|
+
"seller_v1": SellerV1,
|
232
|
+
"service_v1": ServiceV1,
|
233
|
+
"listing_v1": ListingV1,
|
234
|
+
}
|
235
|
+
|
236
|
+
if schema_name not in model_map:
|
237
|
+
return errors # No Pydantic model for this schema
|
238
|
+
|
239
|
+
model_class = model_map[schema_name]
|
240
|
+
|
241
|
+
try:
|
242
|
+
# Validate using the Pydantic model
|
243
|
+
model_class.model_validate(data)
|
244
|
+
|
245
|
+
except Exception as e:
|
246
|
+
# Extract meaningful error message from Pydantic ValidationError
|
247
|
+
error_msg = str(e)
|
248
|
+
# Pydantic errors can be verbose, try to extract just the relevant part
|
249
|
+
if "validation error" in error_msg.lower():
|
250
|
+
errors.append(f"Pydantic validation error: {error_msg}")
|
251
|
+
else:
|
252
|
+
errors.append(error_msg)
|
253
|
+
|
254
|
+
return errors
|
255
|
+
|
208
256
|
def load_data_file(self, file_path: Path) -> tuple[dict[str, Any] | None, list[str]]:
|
209
257
|
"""Load data from JSON or TOML file."""
|
210
258
|
errors: list[str] = []
|
@@ -259,6 +307,10 @@ class DataValidator:
|
|
259
307
|
except Exception as e:
|
260
308
|
errors.append(f"Validation error: {e}")
|
261
309
|
|
310
|
+
# Also validate using Pydantic models for additional validation rules
|
311
|
+
pydantic_errors = self.validate_with_pydantic_model(data, schema_name)
|
312
|
+
errors.extend(pydantic_errors)
|
313
|
+
|
262
314
|
# Find Union[str, HttpUrl] fields and validate file references
|
263
315
|
union_fields = self.find_union_fields(schema)
|
264
316
|
file_ref_errors = self.validate_file_references(data, file_path, union_fields)
|
@@ -308,6 +360,10 @@ class DataValidator:
|
|
308
360
|
|
309
361
|
# Find all data files with seller_v1 schema
|
310
362
|
for file_path in self.data_dir.rglob("*"):
|
363
|
+
# Skip hidden directories (those starting with .)
|
364
|
+
if any(part.startswith(".") for part in file_path.parts):
|
365
|
+
continue
|
366
|
+
|
311
367
|
if file_path.is_file() and file_path.suffix in [".json", ".toml"]:
|
312
368
|
try:
|
313
369
|
data, load_errors = self.load_data_file(file_path)
|
@@ -341,20 +397,17 @@ class DataValidator:
|
|
341
397
|
|
342
398
|
warnings: list[str] = []
|
343
399
|
|
344
|
-
# Find all provider files
|
345
|
-
provider_files =
|
400
|
+
# Find all provider files (skip hidden directories)
|
401
|
+
provider_files = [
|
402
|
+
f for f in self.data_dir.glob("*/provider.*") if not any(part.startswith(".") for part in f.parts)
|
403
|
+
]
|
346
404
|
|
347
405
|
for provider_file in provider_files:
|
348
406
|
try:
|
349
|
-
# Load provider data
|
350
|
-
data =
|
351
|
-
if
|
352
|
-
|
353
|
-
data = json.load(f)
|
354
|
-
elif provider_file.suffix == ".toml":
|
355
|
-
with open(provider_file, "rb") as f:
|
356
|
-
data = toml.load(f)
|
357
|
-
else:
|
407
|
+
# Load provider data using existing helper method
|
408
|
+
data, load_errors = self.load_data_file(provider_file)
|
409
|
+
if load_errors or data is None:
|
410
|
+
warnings.append(f"Failed to load provider file {provider_file}: {load_errors}")
|
358
411
|
continue
|
359
412
|
|
360
413
|
# Parse as ProviderV1
|
@@ -391,20 +444,15 @@ class DataValidator:
|
|
391
444
|
|
392
445
|
warnings: list[str] = []
|
393
446
|
|
394
|
-
# Find all seller files
|
395
|
-
seller_files =
|
447
|
+
# Find all seller files (skip hidden files)
|
448
|
+
seller_files = [f for f in self.data_dir.glob("seller.*") if not f.name.startswith(".")]
|
396
449
|
|
397
450
|
for seller_file in seller_files:
|
398
451
|
try:
|
399
|
-
# Load seller data
|
400
|
-
data =
|
401
|
-
if
|
402
|
-
|
403
|
-
data = json.load(f)
|
404
|
-
elif seller_file.suffix == ".toml":
|
405
|
-
with open(seller_file, "rb") as f:
|
406
|
-
data = toml.load(f)
|
407
|
-
else:
|
452
|
+
# Load seller data using existing helper method
|
453
|
+
data, load_errors = self.load_data_file(seller_file)
|
454
|
+
if load_errors or data is None:
|
455
|
+
warnings.append(f"Failed to load seller file {seller_file}: {load_errors}")
|
408
456
|
continue
|
409
457
|
|
410
458
|
# Parse as SellerV1
|
@@ -448,8 +496,12 @@ class DataValidator:
|
|
448
496
|
provider_warnings,
|
449
497
|
) # Warnings, not errors
|
450
498
|
|
451
|
-
# Find all data and MD files recursively
|
499
|
+
# Find all data and MD files recursively, skipping hidden directories
|
452
500
|
for file_path in self.data_dir.rglob("*"):
|
501
|
+
# Skip hidden directories (those starting with .)
|
502
|
+
if any(part.startswith(".") for part in file_path.parts):
|
503
|
+
continue
|
504
|
+
|
453
505
|
if file_path.is_file() and file_path.suffix in [".json", ".toml", ".md"]:
|
454
506
|
relative_path = file_path.relative_to(self.data_dir)
|
455
507
|
|
@@ -560,6 +612,10 @@ class DataValidator:
|
|
560
612
|
|
561
613
|
for pattern in ["*.json", "*.toml"]:
|
562
614
|
for file_path in data_dir.rglob(pattern):
|
615
|
+
# Skip hidden directories (those starting with .)
|
616
|
+
if any(part.startswith(".") for part in file_path.parts):
|
617
|
+
continue
|
618
|
+
|
563
619
|
try:
|
564
620
|
data, load_errors = self.load_data_file(file_path)
|
565
621
|
if load_errors or data is None:
|
@@ -0,0 +1,24 @@
|
|
1
|
+
unitysvc_services/__init__.py,sha256=J6F3RlZCJUVjhZoprfbrYCxe3l9ynQQbGO7pf7FyqlM,110
|
2
|
+
unitysvc_services/cli.py,sha256=OK0IZyAckxP15jRWU_W49hl3t7XcNRtd8BoDMyRKqNM,682
|
3
|
+
unitysvc_services/format_data.py,sha256=Jl9Vj3fRX852fHSUa5DzO-oiFQwuQHC3WMCDNIlo1Lc,5460
|
4
|
+
unitysvc_services/list.py,sha256=QDp9BByaoeFeJxXJN9RQ-jU99mH9Guq9ampfXCbpZmI,7033
|
5
|
+
unitysvc_services/populate.py,sha256=zkcjIy8BWuQSO7JwiRNHKgGoxQvc3ujluUQdYixdBvY,6626
|
6
|
+
unitysvc_services/publisher.py,sha256=xQqIajb3JRDX9Qg6N94hqtT_mc0NBYbUYKKMm4zsKyE,48686
|
7
|
+
unitysvc_services/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
8
|
+
unitysvc_services/query.py,sha256=x2VUnfva21-mVd-JgtChajNBgXG1AQJ6c3umCw2FNWU,24089
|
9
|
+
unitysvc_services/scaffold.py,sha256=Y73IX8vskImxSvxDgR0mvEFuAMYnBKfttn3bjcz3jmQ,40331
|
10
|
+
unitysvc_services/update.py,sha256=K9swocTUnqqiSgARo6GmuzTzUySSpyqqPPW4xF7ZU-g,9659
|
11
|
+
unitysvc_services/utils.py,sha256=GN0gkVTU8fOx2G0EbqnWmx8w9eFsoPfRprPjwCyPYkE,11371
|
12
|
+
unitysvc_services/validator.py,sha256=VAII5mu_Jdyr96v4nwXzihsoAj7DJiXN6LjhL8lGGUo,29054
|
13
|
+
unitysvc_services/models/__init__.py,sha256=hJCc2KSZmIHlKWKE6GpLGdeVB6LIpyVUKiOKnwmKvCs,200
|
14
|
+
unitysvc_services/models/base.py,sha256=3FdlR-_tBOFC2JbVNFNQA4-D1Lhlo5UZQh1QDgKnS_I,18293
|
15
|
+
unitysvc_services/models/listing_v1.py,sha256=PPb9hIdWQp80AWKLxFXYBDcWXzNcDrO4v6rqt5_i2qo,3083
|
16
|
+
unitysvc_services/models/provider_v1.py,sha256=76EK1i0hVtdx_awb00-ZMtSj4Oc9Zp4xZ-DeXmG3iTY,2701
|
17
|
+
unitysvc_services/models/seller_v1.py,sha256=oll2ZZBPBDX8wslHrbsCKf_jIqHNte2VEj5RJ9bawR4,3520
|
18
|
+
unitysvc_services/models/service_v1.py,sha256=Xpk-K-95M1LRqYM8nNJcll8t-lsW9Xdi2_bVbYNs8-M,3019
|
19
|
+
unitysvc_services-0.2.7.dist-info/licenses/LICENSE,sha256=_p8V6A8OMPu2HIztn3O01v0-urZFwk0Dd3Yk_PTIlL8,1065
|
20
|
+
unitysvc_services-0.2.7.dist-info/METADATA,sha256=8KqcRPrJwkYb9zCgT3rytQTOrmIBGm3hHctp-7VYM3A,6628
|
21
|
+
unitysvc_services-0.2.7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
22
|
+
unitysvc_services-0.2.7.dist-info/entry_points.txt,sha256=-vodnbPmo7QQmFu8jdG6sCyGRVM727w9Nhwp4Vwau_k,64
|
23
|
+
unitysvc_services-0.2.7.dist-info/top_level.txt,sha256=GIotQj-Ro2ruR7eupM1r58PWqIHTAq647ORL7E2kneo,18
|
24
|
+
unitysvc_services-0.2.7.dist-info/RECORD,,
|
@@ -1,24 +0,0 @@
|
|
1
|
-
unitysvc_services/__init__.py,sha256=J6F3RlZCJUVjhZoprfbrYCxe3l9ynQQbGO7pf7FyqlM,110
|
2
|
-
unitysvc_services/cli.py,sha256=OK0IZyAckxP15jRWU_W49hl3t7XcNRtd8BoDMyRKqNM,682
|
3
|
-
unitysvc_services/format_data.py,sha256=Jl9Vj3fRX852fHSUa5DzO-oiFQwuQHC3WMCDNIlo1Lc,5460
|
4
|
-
unitysvc_services/list.py,sha256=QDp9BByaoeFeJxXJN9RQ-jU99mH9Guq9ampfXCbpZmI,7033
|
5
|
-
unitysvc_services/populate.py,sha256=zkcjIy8BWuQSO7JwiRNHKgGoxQvc3ujluUQdYixdBvY,6626
|
6
|
-
unitysvc_services/publisher.py,sha256=s3px0i1ov6FisnnYG-gkkMHwJhnGbf-Ug225vosfxxM,35733
|
7
|
-
unitysvc_services/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
8
|
-
unitysvc_services/query.py,sha256=M8-zklRYvIaLZSzmDVroZBf48OyGUa1nvhUaLzZXeuU,24092
|
9
|
-
unitysvc_services/scaffold.py,sha256=Y73IX8vskImxSvxDgR0mvEFuAMYnBKfttn3bjcz3jmQ,40331
|
10
|
-
unitysvc_services/update.py,sha256=K9swocTUnqqiSgARo6GmuzTzUySSpyqqPPW4xF7ZU-g,9659
|
11
|
-
unitysvc_services/utils.py,sha256=GN0gkVTU8fOx2G0EbqnWmx8w9eFsoPfRprPjwCyPYkE,11371
|
12
|
-
unitysvc_services/validator.py,sha256=zuFA44ezKlfUTtdJ8M2Xd7nk1Eot4HxbBksEUaIIpZs,26790
|
13
|
-
unitysvc_services/models/__init__.py,sha256=hJCc2KSZmIHlKWKE6GpLGdeVB6LIpyVUKiOKnwmKvCs,200
|
14
|
-
unitysvc_services/models/base.py,sha256=gm3xlcC35QNRST5ikJPhdk-dTTXoY9D_5Jxkyt8SBCU,13173
|
15
|
-
unitysvc_services/models/listing_v1.py,sha256=8cSd91weXGpP5tX9aQg0iagi1H3gSd5WJddVhOJji3c,2421
|
16
|
-
unitysvc_services/models/provider_v1.py,sha256=cYK5kDDmzQEnLvUC2C8dKz-ZXci7hVn3fjNrJkaSr10,2050
|
17
|
-
unitysvc_services/models/seller_v1.py,sha256=SU4rqYAh9hE4EeUrEkqaVrLwusenV7MotPF77VcsRKo,3263
|
18
|
-
unitysvc_services/models/service_v1.py,sha256=u16zqM3khrJoTw_v0d45tMcKXjko5k_v3w8xwUtZ6nM,2720
|
19
|
-
unitysvc_services-0.2.5.dist-info/licenses/LICENSE,sha256=_p8V6A8OMPu2HIztn3O01v0-urZFwk0Dd3Yk_PTIlL8,1065
|
20
|
-
unitysvc_services-0.2.5.dist-info/METADATA,sha256=G6wUb0qoOpZtYWLYnp-MfOgv3tMZWHqh-K78vAm1POI,6628
|
21
|
-
unitysvc_services-0.2.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
22
|
-
unitysvc_services-0.2.5.dist-info/entry_points.txt,sha256=-vodnbPmo7QQmFu8jdG6sCyGRVM727w9Nhwp4Vwau_k,64
|
23
|
-
unitysvc_services-0.2.5.dist-info/top_level.txt,sha256=GIotQj-Ro2ruR7eupM1r58PWqIHTAq647ORL7E2kneo,18
|
24
|
-
unitysvc_services-0.2.5.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|