xcpcio 0.67.0__py3-none-any.whl → 0.74.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of xcpcio might be problematic. Click here for more details.
- xcpcio/__version__.py +1 -1
- xcpcio/api/__init__.py +2 -0
- xcpcio/api/client.py +2 -2
- xcpcio/api/models.py +14 -3
- xcpcio/app/clics_uploader.py +103 -0
- xcpcio/clics/clics_api_client.py +36 -0
- xcpcio/clics/contest_archiver.py +1 -0
- xcpcio/clics/contest_uploader.py +304 -0
- xcpcio/clics/reader/contest_package_reader.py +32 -20
- xcpcio/types.py +21 -1
- {xcpcio-0.67.0.dist-info → xcpcio-0.74.0.dist-info}/METADATA +2 -2
- {xcpcio-0.67.0.dist-info → xcpcio-0.74.0.dist-info}/RECORD +14 -12
- {xcpcio-0.67.0.dist-info → xcpcio-0.74.0.dist-info}/entry_points.txt +1 -0
- {xcpcio-0.67.0.dist-info → xcpcio-0.74.0.dist-info}/WHEEL +0 -0
xcpcio/__version__.py
CHANGED
xcpcio/api/__init__.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
from .client import ApiClient
|
|
2
2
|
from .models import (
|
|
3
|
+
FileData,
|
|
3
4
|
HTTPValidationError,
|
|
4
5
|
UploadBoardDataReq,
|
|
5
6
|
UploadBoardDataResp,
|
|
@@ -8,6 +9,7 @@ from .models import (
|
|
|
8
9
|
|
|
9
10
|
__all__ = [
|
|
10
11
|
ApiClient,
|
|
12
|
+
FileData,
|
|
11
13
|
HTTPValidationError,
|
|
12
14
|
UploadBoardDataReq,
|
|
13
15
|
UploadBoardDataResp,
|
xcpcio/api/client.py
CHANGED
|
@@ -2,7 +2,7 @@ from typing import Any, Dict, Optional
|
|
|
2
2
|
|
|
3
3
|
import aiohttp
|
|
4
4
|
|
|
5
|
-
from .models import UploadBoardDataReq, UploadBoardDataResp
|
|
5
|
+
from .models import FileData, UploadBoardDataReq, UploadBoardDataResp
|
|
6
6
|
|
|
7
7
|
|
|
8
8
|
class ApiClient:
|
|
@@ -51,7 +51,7 @@ class ApiClient:
|
|
|
51
51
|
config: Optional[str] = None,
|
|
52
52
|
teams: Optional[str] = None,
|
|
53
53
|
submissions: Optional[str] = None,
|
|
54
|
-
extra_files: Optional[Dict[str,
|
|
54
|
+
extra_files: Optional[Dict[str, FileData]] = None,
|
|
55
55
|
) -> UploadBoardDataResp:
|
|
56
56
|
await self._ensure_session()
|
|
57
57
|
await self._ensure_token()
|
xcpcio/api/models.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from typing import Dict, List, Optional, Union
|
|
1
|
+
from typing import Dict, List, Literal, Optional, Union
|
|
2
2
|
|
|
3
3
|
from pydantic import BaseModel, Field
|
|
4
4
|
|
|
@@ -13,13 +13,24 @@ class HTTPValidationError(BaseModel):
|
|
|
13
13
|
detail: Optional[List[ValidationError]] = Field(None, title="Detail")
|
|
14
14
|
|
|
15
15
|
|
|
16
|
+
class FileData(BaseModel):
|
|
17
|
+
content: str = Field(..., title="Content")
|
|
18
|
+
encoding: Optional[Literal["base64"]] = Field(None, title="Encoding")
|
|
19
|
+
checksum: Optional[str] = Field(None, title="Checksum")
|
|
20
|
+
|
|
21
|
+
|
|
16
22
|
class UploadBoardDataReq(BaseModel):
|
|
17
23
|
token: str = Field(..., title="Token")
|
|
18
24
|
config: Optional[str] = Field(None, title="Config")
|
|
19
25
|
teams: Optional[str] = Field(None, title="Teams")
|
|
20
26
|
submissions: Optional[str] = Field(None, title="Submissions")
|
|
21
|
-
extra_files: Optional[Dict[str,
|
|
27
|
+
extra_files: Optional[Dict[str, FileData]] = Field(None, title="Extra Files")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class FileValidationResult(BaseModel):
|
|
31
|
+
checksum_valid: bool = Field(..., title="Checksum Valid")
|
|
32
|
+
message: Optional[str] = Field(None, title="Message")
|
|
22
33
|
|
|
23
34
|
|
|
24
35
|
class UploadBoardDataResp(BaseModel):
|
|
25
|
-
|
|
36
|
+
file_validations: Dict[str, FileValidationResult] = Field(default_factory=dict, title="File Validations")
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import logging
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Optional
|
|
5
|
+
|
|
6
|
+
import click
|
|
7
|
+
|
|
8
|
+
from xcpcio import __version__
|
|
9
|
+
from xcpcio.clics.clics_api_client import APICredentials
|
|
10
|
+
from xcpcio.clics.contest_uploader import ContestUploader, UploaderConfig
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def setup_logging(level: str = "INFO"):
|
|
14
|
+
logging.basicConfig(
|
|
15
|
+
level=getattr(logging, level.upper()),
|
|
16
|
+
format="%(asctime)s [%(name)s] %(filename)s:%(lineno)d %(levelname)s: %(message)s",
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@click.command()
|
|
21
|
+
@click.version_option(__version__)
|
|
22
|
+
@click.option("--base-url", required=True, help="Base URL of the CLICS API (e.g., https://domjudge/api)")
|
|
23
|
+
@click.option("--contest-id", required=True, help="Contest ID to fetch")
|
|
24
|
+
@click.option("--username", "-u", required=True, help="HTTP Basic Auth username for CLICS API")
|
|
25
|
+
@click.option("--password", "-p", required=True, help="HTTP Basic Auth password for CLICS API")
|
|
26
|
+
@click.option("--xcpcio-api-url", default="https://api.xcpcio.com", help="XCPCIO API URL")
|
|
27
|
+
@click.option("--xcpcio-api-token", required=True, help="XCPCIO API token")
|
|
28
|
+
@click.option(
|
|
29
|
+
"--cache-dir", type=click.Path(path_type=Path), help="Directory for checksum cache files (default: ~/.xcpcio/)"
|
|
30
|
+
)
|
|
31
|
+
@click.option("--timeout", default=30, type=int, help="Request timeout in seconds")
|
|
32
|
+
@click.option("--max-concurrent", default=10, type=int, help="Max concurrent requests for CLICS API")
|
|
33
|
+
@click.option("--interval", default=5, type=int, help="Polling interval in seconds")
|
|
34
|
+
@click.option(
|
|
35
|
+
"--log-level",
|
|
36
|
+
default="INFO",
|
|
37
|
+
type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR"], case_sensitive=False),
|
|
38
|
+
help="Log level",
|
|
39
|
+
)
|
|
40
|
+
@click.option("--verbose", "-v", is_flag=True, help="Enable verbose logging (same as --log-level DEBUG)")
|
|
41
|
+
def main(
|
|
42
|
+
base_url: str,
|
|
43
|
+
contest_id: str,
|
|
44
|
+
username: str,
|
|
45
|
+
password: str,
|
|
46
|
+
xcpcio_api_url: str,
|
|
47
|
+
xcpcio_api_token: str,
|
|
48
|
+
cache_dir: Optional[Path],
|
|
49
|
+
timeout: int,
|
|
50
|
+
max_concurrent: int,
|
|
51
|
+
interval: int,
|
|
52
|
+
log_level: str,
|
|
53
|
+
verbose: bool,
|
|
54
|
+
):
|
|
55
|
+
"""
|
|
56
|
+
Upload CLICS Contest API data to XCPCIO with polling support.
|
|
57
|
+
|
|
58
|
+
This tool fetches contest data from a CLICS API and uploads raw JSON data to the xcpcio board-admin service.
|
|
59
|
+
It continuously polls every 5 seconds and uses SHA256 checksums to avoid uploading unchanged files.
|
|
60
|
+
|
|
61
|
+
Examples:
|
|
62
|
+
|
|
63
|
+
clics-uploader --base-url https://domjudge/api --contest-id contest123 -u admin -p secret --xcpcio-api-token YOUR_TOKEN
|
|
64
|
+
"""
|
|
65
|
+
|
|
66
|
+
if verbose:
|
|
67
|
+
log_level = "DEBUG"
|
|
68
|
+
|
|
69
|
+
setup_logging(log_level)
|
|
70
|
+
|
|
71
|
+
credentials = APICredentials(username=username, password=password)
|
|
72
|
+
config = UploaderConfig(
|
|
73
|
+
clics_base_url=base_url.rstrip("/"),
|
|
74
|
+
contest_id=contest_id,
|
|
75
|
+
clics_credentials=credentials,
|
|
76
|
+
xcpcio_api_url=xcpcio_api_url,
|
|
77
|
+
xcpcio_api_token=xcpcio_api_token,
|
|
78
|
+
cache_dir=cache_dir,
|
|
79
|
+
timeout=timeout,
|
|
80
|
+
max_concurrent=max_concurrent,
|
|
81
|
+
poll_interval=interval,
|
|
82
|
+
version=__version__,
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
click.echo(f"Fetching contest '{contest_id}' from {base_url}")
|
|
86
|
+
click.echo(f"Uploading to XCPCIO: {xcpcio_api_url}")
|
|
87
|
+
|
|
88
|
+
async def run():
|
|
89
|
+
async with ContestUploader(config) as uploader:
|
|
90
|
+
await uploader.run_loop()
|
|
91
|
+
|
|
92
|
+
try:
|
|
93
|
+
asyncio.run(run())
|
|
94
|
+
except KeyboardInterrupt:
|
|
95
|
+
click.echo(click.style("\nClics Uploader stopped by user", fg="yellow"), err=True)
|
|
96
|
+
raise click.Abort()
|
|
97
|
+
except Exception as e:
|
|
98
|
+
click.echo(click.style(f"Error: {e}", fg="red"), err=True)
|
|
99
|
+
raise click.ClickException(str(e))
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
if __name__ == "__main__":
|
|
103
|
+
main()
|
xcpcio/clics/clics_api_client.py
CHANGED
|
@@ -168,6 +168,42 @@ class ClicsApiClient:
|
|
|
168
168
|
logger.error(f"Failed to download {file_url} after retries: {e}")
|
|
169
169
|
return False
|
|
170
170
|
|
|
171
|
+
@retry(
|
|
172
|
+
stop=stop_after_attempt(3),
|
|
173
|
+
wait=wait_exponential(multiplier=1, min=1, max=10),
|
|
174
|
+
retry=retry_if_exception_type((asyncio.TimeoutError, aiohttp.ClientError)),
|
|
175
|
+
before_sleep=before_sleep_log(logger, logging.WARNING),
|
|
176
|
+
reraise=True,
|
|
177
|
+
)
|
|
178
|
+
async def _fetch_file_content_internal(self, file_url: str, override_timeout: Optional[int] = None) -> bytes:
|
|
179
|
+
"""Internal method to fetch file content as bytes with retry logic"""
|
|
180
|
+
logger.info(f"Fetching file content from {file_url}")
|
|
181
|
+
timeout = aiohttp.ClientTimeout(total=override_timeout or self._config.timeout)
|
|
182
|
+
async with self._session.get(file_url, timeout=timeout) as response:
|
|
183
|
+
if response.status != 200:
|
|
184
|
+
raise aiohttp.ClientResponseError(
|
|
185
|
+
request_info=response.request_info, history=response.history, status=response.status
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
content = await response.read()
|
|
189
|
+
logger.debug(f"Fetched {len(content)} bytes from {file_url}")
|
|
190
|
+
return content
|
|
191
|
+
|
|
192
|
+
async def fetch_file_content(self, file_url: Optional[str], override_timeout: Optional[int] = None) -> Optional[bytes]:
|
|
193
|
+
"""Fetch file content as bytes from URL"""
|
|
194
|
+
if not file_url:
|
|
195
|
+
return None
|
|
196
|
+
|
|
197
|
+
if not file_url.startswith(("http://", "https://")):
|
|
198
|
+
file_url = self._build_url(file_url)
|
|
199
|
+
|
|
200
|
+
async with self._semaphore:
|
|
201
|
+
try:
|
|
202
|
+
return await self._fetch_file_content_internal(file_url, override_timeout)
|
|
203
|
+
except Exception as e:
|
|
204
|
+
logger.error(f"Failed to fetch file content from {file_url} after retries: {e}")
|
|
205
|
+
return None
|
|
206
|
+
|
|
171
207
|
async def fetch_api_info(self) -> Optional[Dict[str, Any]]:
|
|
172
208
|
"""Fetch API root endpoint information and parse provider details"""
|
|
173
209
|
data = await self.fetch_json("/")
|
xcpcio/clics/contest_archiver.py
CHANGED
|
@@ -0,0 +1,304 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import hashlib
|
|
3
|
+
import json
|
|
4
|
+
import logging
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from datetime import datetime
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Dict, Optional
|
|
9
|
+
|
|
10
|
+
from pydantic import BaseModel
|
|
11
|
+
|
|
12
|
+
from xcpcio.api.client import ApiClient
|
|
13
|
+
from xcpcio.api.models import FileData, UploadBoardDataResp
|
|
14
|
+
from xcpcio.clics.clics_api_client import APICredentials, ClicsApiClient, ClicsApiConfig
|
|
15
|
+
|
|
16
|
+
logger = logging.getLogger(__name__)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class FileChecksumEntry(BaseModel):
|
|
20
|
+
checksum: str
|
|
21
|
+
updated_at: str
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class FileChecksumCacheData(BaseModel):
|
|
25
|
+
checksums: Dict[str, FileChecksumEntry]
|
|
26
|
+
updated_at: str
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class FileChecksumCache:
|
|
30
|
+
def __init__(
|
|
31
|
+
self, cache_dir: Optional[Path] = None, contest_id: Optional[str] = None, version: Optional[str] = None
|
|
32
|
+
):
|
|
33
|
+
if cache_dir:
|
|
34
|
+
cache_dir = Path(cache_dir)
|
|
35
|
+
else:
|
|
36
|
+
cache_dir = Path.home() / ".xcpcio"
|
|
37
|
+
|
|
38
|
+
cache_dir.mkdir(parents=True, exist_ok=True)
|
|
39
|
+
|
|
40
|
+
if contest_id:
|
|
41
|
+
self.cache_file = cache_dir / f"clics_uploader_{contest_id}.json"
|
|
42
|
+
else:
|
|
43
|
+
self.cache_file = cache_dir / "clics_uploader_cache.json"
|
|
44
|
+
|
|
45
|
+
self.version_file = cache_dir / "version"
|
|
46
|
+
self.version = version
|
|
47
|
+
self.checksums: Dict[str, FileChecksumEntry] = {}
|
|
48
|
+
self._load_cache()
|
|
49
|
+
|
|
50
|
+
def _load_version(self) -> Optional[str]:
|
|
51
|
+
if self.version_file.exists():
|
|
52
|
+
try:
|
|
53
|
+
with open(self.version_file, "r") as f:
|
|
54
|
+
return f.read().strip()
|
|
55
|
+
except Exception as e:
|
|
56
|
+
logger.warning(f"Failed to read version file: {e}")
|
|
57
|
+
return None
|
|
58
|
+
|
|
59
|
+
def _save_version(self):
|
|
60
|
+
if not self.version:
|
|
61
|
+
return
|
|
62
|
+
try:
|
|
63
|
+
with open(self.version_file, "w") as f:
|
|
64
|
+
f.write(self.version)
|
|
65
|
+
except Exception as e:
|
|
66
|
+
logger.warning(f"Failed to save version file: {e}")
|
|
67
|
+
|
|
68
|
+
def _load_cache(self):
|
|
69
|
+
cached_version = self._load_version()
|
|
70
|
+
if cached_version != self.version:
|
|
71
|
+
logger.info(f"Cache version mismatch (cached: {cached_version}, current: {self.version}), clearing cache")
|
|
72
|
+
self.checksums = {}
|
|
73
|
+
return
|
|
74
|
+
|
|
75
|
+
if self.cache_file.exists():
|
|
76
|
+
try:
|
|
77
|
+
with open(self.cache_file, "r") as f:
|
|
78
|
+
data = json.load(f)
|
|
79
|
+
|
|
80
|
+
cache_data = FileChecksumCacheData.model_validate(data)
|
|
81
|
+
self.checksums = cache_data.checksums
|
|
82
|
+
logger.debug(f"Loaded cache with {len(self.checksums)} entries")
|
|
83
|
+
except Exception as e:
|
|
84
|
+
logger.warning(f"Failed to load cache: {e}")
|
|
85
|
+
self.checksums = {}
|
|
86
|
+
|
|
87
|
+
def _save_cache(self):
|
|
88
|
+
try:
|
|
89
|
+
self.cache_file.parent.mkdir(parents=True, exist_ok=True)
|
|
90
|
+
cache_data = FileChecksumCacheData(
|
|
91
|
+
checksums=self.checksums,
|
|
92
|
+
updated_at=datetime.now().isoformat(),
|
|
93
|
+
)
|
|
94
|
+
with open(self.cache_file, "w") as f:
|
|
95
|
+
json.dump(cache_data.model_dump(), f, indent=2)
|
|
96
|
+
self._save_version()
|
|
97
|
+
logger.debug(f"Saved cache with {len(self.checksums)} entries")
|
|
98
|
+
except Exception as e:
|
|
99
|
+
logger.warning(f"Failed to save cache: {e}")
|
|
100
|
+
|
|
101
|
+
def calculate_checksum(self, content: str) -> str:
|
|
102
|
+
return hashlib.sha256(content.encode("utf-8")).hexdigest()
|
|
103
|
+
|
|
104
|
+
def has_changed(self, key: str, content: str) -> bool:
|
|
105
|
+
checksum = self.calculate_checksum(content)
|
|
106
|
+
old_entry = self.checksums.get(key)
|
|
107
|
+
return not old_entry or old_entry.checksum != checksum
|
|
108
|
+
|
|
109
|
+
def update_checksum(self, key: str, content: str):
|
|
110
|
+
checksum = self.calculate_checksum(content)
|
|
111
|
+
self.checksums[key] = FileChecksumEntry(
|
|
112
|
+
checksum=checksum,
|
|
113
|
+
updated_at=datetime.now().isoformat(),
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
def save(self):
|
|
117
|
+
self._save_cache()
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
@dataclass
|
|
121
|
+
class UploaderConfig:
|
|
122
|
+
clics_base_url: str
|
|
123
|
+
contest_id: str
|
|
124
|
+
clics_credentials: APICredentials
|
|
125
|
+
xcpcio_api_url: str
|
|
126
|
+
xcpcio_api_token: str
|
|
127
|
+
cache_dir: Optional[Path] = None
|
|
128
|
+
timeout: int = 30
|
|
129
|
+
max_concurrent: int = 10
|
|
130
|
+
poll_interval: int = 5
|
|
131
|
+
version: Optional[str] = None
|
|
132
|
+
|
|
133
|
+
def to_clics_api_config(self) -> ClicsApiConfig:
|
|
134
|
+
return ClicsApiConfig(
|
|
135
|
+
base_url=self.clics_base_url,
|
|
136
|
+
credentials=self.clics_credentials,
|
|
137
|
+
timeout=self.timeout,
|
|
138
|
+
max_concurrent=self.max_concurrent,
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
class ContestUploader:
|
|
143
|
+
ENDPOINTS_TO_FETCH = (
|
|
144
|
+
"access",
|
|
145
|
+
"accounts",
|
|
146
|
+
"awards",
|
|
147
|
+
"groups",
|
|
148
|
+
"judgement-types",
|
|
149
|
+
"judgements",
|
|
150
|
+
"languages",
|
|
151
|
+
"organizations",
|
|
152
|
+
"problems",
|
|
153
|
+
"state",
|
|
154
|
+
"submissions",
|
|
155
|
+
"teams",
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
def __init__(self, config: UploaderConfig):
|
|
159
|
+
self._config = config
|
|
160
|
+
self._clics_client = ClicsApiClient(config.to_clics_api_config())
|
|
161
|
+
self._api_client = ApiClient(
|
|
162
|
+
base_url=config.xcpcio_api_url,
|
|
163
|
+
token=config.xcpcio_api_token,
|
|
164
|
+
timeout=config.timeout,
|
|
165
|
+
)
|
|
166
|
+
self._cache = FileChecksumCache(
|
|
167
|
+
cache_dir=config.cache_dir,
|
|
168
|
+
contest_id=config.contest_id,
|
|
169
|
+
version=config.version,
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
async def __aenter__(self):
|
|
173
|
+
await self._clics_client.__aenter__()
|
|
174
|
+
await self._api_client.__aenter__()
|
|
175
|
+
return self
|
|
176
|
+
|
|
177
|
+
async def __aexit__(self, exc_type, exc_val, exc_tb):
|
|
178
|
+
await self._clics_client.__aexit__(exc_type, exc_val, exc_tb)
|
|
179
|
+
await self._api_client.__aexit__(exc_type, exc_val, exc_tb)
|
|
180
|
+
|
|
181
|
+
def _process_file_validations(
|
|
182
|
+
self, file_map: Dict[str, FileData], file_keys: list[str], response: UploadBoardDataResp
|
|
183
|
+
):
|
|
184
|
+
for file_key in file_keys:
|
|
185
|
+
if file_key in file_map:
|
|
186
|
+
validation = response.file_validations.get(file_key)
|
|
187
|
+
if validation:
|
|
188
|
+
if validation.checksum_valid:
|
|
189
|
+
self._cache.update_checksum(file_key, file_map[file_key].content)
|
|
190
|
+
else:
|
|
191
|
+
logger.warning(f"File checksum validation failed for {file_key}: {validation.message}")
|
|
192
|
+
else:
|
|
193
|
+
logger.warning(f"No validation result for {file_key}, will retry on next poll")
|
|
194
|
+
|
|
195
|
+
async def _fetch_api_data(
|
|
196
|
+
self,
|
|
197
|
+
api_files: Dict[str, FileData],
|
|
198
|
+
changed_api_files: list[str],
|
|
199
|
+
unchanged_files: list[str],
|
|
200
|
+
):
|
|
201
|
+
api_data = await self._clics_client.fetch_api_info()
|
|
202
|
+
if api_data:
|
|
203
|
+
filename = "api.json"
|
|
204
|
+
content = json.dumps(api_data, ensure_ascii=False)
|
|
205
|
+
|
|
206
|
+
if self._cache.has_changed(filename, content):
|
|
207
|
+
checksum = self._cache.calculate_checksum(content)
|
|
208
|
+
api_files[filename] = FileData(content=content, checksum=checksum)
|
|
209
|
+
changed_api_files.append(filename)
|
|
210
|
+
else:
|
|
211
|
+
unchanged_files.append(filename)
|
|
212
|
+
|
|
213
|
+
async def _fetch_contest_data(
|
|
214
|
+
self,
|
|
215
|
+
) -> tuple[Dict[str, FileData], list[str], list[str]]:
|
|
216
|
+
api_files = {}
|
|
217
|
+
changed_api_files = []
|
|
218
|
+
unchanged_files = []
|
|
219
|
+
|
|
220
|
+
contest_data = await self._clics_client.fetch_json(f"contests/{self._config.contest_id}")
|
|
221
|
+
if contest_data:
|
|
222
|
+
filename = "contest.json"
|
|
223
|
+
content = json.dumps(contest_data, ensure_ascii=False)
|
|
224
|
+
|
|
225
|
+
if self._cache.has_changed(filename, content):
|
|
226
|
+
checksum = self._cache.calculate_checksum(content)
|
|
227
|
+
api_files[filename] = FileData(content=content, checksum=checksum)
|
|
228
|
+
changed_api_files.append(filename)
|
|
229
|
+
else:
|
|
230
|
+
unchanged_files.append(filename)
|
|
231
|
+
|
|
232
|
+
return api_files, changed_api_files, unchanged_files
|
|
233
|
+
|
|
234
|
+
async def _fetch_endpoint_data(
|
|
235
|
+
self,
|
|
236
|
+
api_files: Dict[str, FileData],
|
|
237
|
+
changed_api_files: list[str],
|
|
238
|
+
unchanged_files: list[str],
|
|
239
|
+
):
|
|
240
|
+
for endpoint in self.ENDPOINTS_TO_FETCH:
|
|
241
|
+
data = await self._clics_client.fetch_json(f"contests/{self._config.contest_id}/{endpoint}")
|
|
242
|
+
if data is not None:
|
|
243
|
+
filename = f"{endpoint}.json"
|
|
244
|
+
content = json.dumps(data, ensure_ascii=False)
|
|
245
|
+
|
|
246
|
+
if self._cache.has_changed(filename, content):
|
|
247
|
+
checksum = self._cache.calculate_checksum(content)
|
|
248
|
+
api_files[filename] = FileData(content=content, checksum=checksum)
|
|
249
|
+
changed_api_files.append(filename)
|
|
250
|
+
else:
|
|
251
|
+
unchanged_files.append(filename)
|
|
252
|
+
|
|
253
|
+
async def _upload_api_files(
|
|
254
|
+
self, api_files: Dict[str, FileData], changed_api_files: list[str]
|
|
255
|
+
) -> Optional[UploadBoardDataResp]:
|
|
256
|
+
if not api_files:
|
|
257
|
+
return None
|
|
258
|
+
|
|
259
|
+
logger.info(f"Uploading {len(api_files)} API file(s) to XCPCIO...")
|
|
260
|
+
response = await self._api_client.upload_board_data(extra_files=api_files)
|
|
261
|
+
logger.info(f"API upload successful! ({len(api_files)} files)")
|
|
262
|
+
|
|
263
|
+
self._process_file_validations(api_files, changed_api_files, response)
|
|
264
|
+
self._cache.save()
|
|
265
|
+
return response
|
|
266
|
+
|
|
267
|
+
async def fetch_and_upload(self) -> Optional[Dict]:
|
|
268
|
+
api_files, changed_api_files, unchanged_files = ({}, [], [])
|
|
269
|
+
|
|
270
|
+
await self._fetch_api_data(api_files, changed_api_files, unchanged_files)
|
|
271
|
+
|
|
272
|
+
contest_files, contest_changed, contest_unchanged = await self._fetch_contest_data()
|
|
273
|
+
api_files.update(contest_files)
|
|
274
|
+
changed_api_files.extend(contest_changed)
|
|
275
|
+
unchanged_files.extend(contest_unchanged)
|
|
276
|
+
|
|
277
|
+
await self._fetch_endpoint_data(api_files, changed_api_files, unchanged_files)
|
|
278
|
+
|
|
279
|
+
if changed_api_files:
|
|
280
|
+
logger.info(f"Changed API files ({len(changed_api_files)}): {', '.join(changed_api_files)}")
|
|
281
|
+
if unchanged_files:
|
|
282
|
+
logger.debug(f"Unchanged files ({len(unchanged_files)}): {', '.join(unchanged_files)}")
|
|
283
|
+
|
|
284
|
+
if not api_files:
|
|
285
|
+
logger.info("No changed files to upload")
|
|
286
|
+
return None
|
|
287
|
+
|
|
288
|
+
response = await self._upload_api_files(api_files, changed_api_files)
|
|
289
|
+
|
|
290
|
+
logger.info(f"All uploads completed! ({len(api_files)} files)")
|
|
291
|
+
return response.model_dump() if response else None
|
|
292
|
+
|
|
293
|
+
async def run_loop(self):
|
|
294
|
+
iteration = 0
|
|
295
|
+
while True:
|
|
296
|
+
iteration += 1
|
|
297
|
+
try:
|
|
298
|
+
logger.info(f"[Iteration #{iteration}] Starting fetch and upload...")
|
|
299
|
+
await self.fetch_and_upload()
|
|
300
|
+
except Exception as e:
|
|
301
|
+
logger.exception(f"Error during fetch and upload: {e}", exc_info=True)
|
|
302
|
+
|
|
303
|
+
logger.info(f"Waiting {self._config.poll_interval} seconds until next poll...")
|
|
304
|
+
await asyncio.sleep(self._config.poll_interval)
|
|
@@ -27,17 +27,26 @@ class ContestPackageReader(BaseContestReader):
|
|
|
27
27
|
res[item[id_name]].append(item)
|
|
28
28
|
return res
|
|
29
29
|
|
|
30
|
-
def _load_json_file(
|
|
30
|
+
def _load_json_file(
|
|
31
|
+
self,
|
|
32
|
+
filepath: str,
|
|
33
|
+
default_value: Optional[Union[Dict[str, Any], List[Any]]] = None,
|
|
34
|
+
) -> Union[Dict[str, Any], List[Any]]:
|
|
31
35
|
full_path = self.contest_package_dir / filepath
|
|
32
36
|
try:
|
|
33
37
|
with open(full_path, "r", encoding="utf-8") as f:
|
|
34
38
|
return json.load(f)
|
|
35
39
|
except FileNotFoundError:
|
|
40
|
+
if default_value is not None:
|
|
41
|
+
logger.warning(f"File not found, will load default value. [full_path={full_path}]")
|
|
42
|
+
return default_value
|
|
36
43
|
raise HTTPException(status_code=404, detail=f"File not found: {filepath}")
|
|
37
44
|
except json.JSONDecodeError as e:
|
|
38
45
|
raise HTTPException(status_code=500, detail=f"Invalid JSON in file {filepath}: {e}")
|
|
39
46
|
|
|
40
|
-
def _load_ndjson_file(
|
|
47
|
+
def _load_ndjson_file(
|
|
48
|
+
self, filepath: str, default_value: Optional[List[Dict[str, Any]]] = None
|
|
49
|
+
) -> List[Dict[str, Any]]:
|
|
41
50
|
full_path = self.contest_package_dir / filepath
|
|
42
51
|
try:
|
|
43
52
|
data = list()
|
|
@@ -46,60 +55,63 @@ class ContestPackageReader(BaseContestReader):
|
|
|
46
55
|
data.append(json.loads(line))
|
|
47
56
|
return data
|
|
48
57
|
except FileNotFoundError:
|
|
58
|
+
if default_value is not None:
|
|
59
|
+
logger.warning(f"File not found, will load default value. [full_path={full_path}]")
|
|
60
|
+
return default_value
|
|
49
61
|
raise HTTPException(status_code=404, detail=f"File not found: {filepath}")
|
|
50
62
|
except json.JSONDecodeError as e:
|
|
51
63
|
raise HTTPException(status_code=500, detail=f"Invalid JSON in file {filepath}: {e}")
|
|
52
64
|
|
|
53
65
|
def _load_indexes(self) -> None:
|
|
54
|
-
self.access = self._load_json_file("access.json")
|
|
66
|
+
self.access = self._load_json_file("access.json", default_value={})
|
|
55
67
|
|
|
56
|
-
self.accounts = self._load_json_file("accounts.json")
|
|
68
|
+
self.accounts = self._load_json_file("accounts.json", default_value=[])
|
|
57
69
|
self.accounts_by_id = {account["id"]: account for account in self.accounts}
|
|
58
70
|
|
|
59
|
-
self.api_info = self._load_json_file("api.json")
|
|
71
|
+
self.api_info = self._load_json_file("api.json", default_value={})
|
|
60
72
|
|
|
61
|
-
self.awards = self._load_json_file("awards.json")
|
|
73
|
+
self.awards = self._load_json_file("awards.json", default_value=[])
|
|
62
74
|
self.awards_by_id = {award["id"]: award for award in self.awards}
|
|
63
75
|
|
|
64
|
-
self.clarifications = self._load_json_file("clarifications.json")
|
|
76
|
+
self.clarifications = self._load_json_file("clarifications.json", default_value=[])
|
|
65
77
|
self.clarifications_by_id = {clarification["id"]: clarification for clarification in self.clarifications}
|
|
66
78
|
|
|
67
|
-
self.contest = self._load_json_file("contest.json")
|
|
68
|
-
self.contest_state = self._load_json_file("state.json")
|
|
79
|
+
self.contest = self._load_json_file("contest.json", default_value={})
|
|
80
|
+
self.contest_state = self._load_json_file("state.json", default_value={})
|
|
69
81
|
|
|
70
|
-
self.groups = self._load_json_file("groups.json")
|
|
82
|
+
self.groups = self._load_json_file("groups.json", default_value=[])
|
|
71
83
|
self.groups_by_id = {group["id"]: group for group in self.groups}
|
|
72
84
|
|
|
73
|
-
self.judgement_types = self._load_json_file("judgement-types.json")
|
|
85
|
+
self.judgement_types = self._load_json_file("judgement-types.json", default_value=[])
|
|
74
86
|
self.judgement_types_by_id = {judgement_type["id"]: judgement_type for judgement_type in self.judgement_types}
|
|
75
87
|
|
|
76
|
-
self.judgements = self._load_json_file("judgements.json")
|
|
88
|
+
self.judgements = self._load_json_file("judgements.json", default_value=[])
|
|
77
89
|
self.judgements_by_id = {judgement["id"]: judgement for judgement in self.judgements}
|
|
78
90
|
self.judgements_by_submission_id = self._create_index_by_id(self.judgements, "submission_id")
|
|
79
91
|
|
|
80
|
-
self.languages = self._load_json_file("languages.json")
|
|
92
|
+
self.languages = self._load_json_file("languages.json", default_value=[])
|
|
81
93
|
self.languages_by_id = {language["id"]: language for language in self.languages}
|
|
82
94
|
|
|
83
|
-
self.organizations = self._load_json_file("organizations.json")
|
|
95
|
+
self.organizations = self._load_json_file("organizations.json", default_value=[])
|
|
84
96
|
self.organizations_by_id = {org["id"]: org for org in self.organizations}
|
|
85
97
|
|
|
86
|
-
self.problems = self._load_json_file("problems.json")
|
|
98
|
+
self.problems = self._load_json_file("problems.json", default_value=[])
|
|
87
99
|
self.problems_by_id = {problem["id"]: problem for problem in self.problems}
|
|
88
100
|
|
|
89
|
-
self.runs = self._load_json_file("runs.json")
|
|
101
|
+
self.runs = self._load_json_file("runs.json", default_value=[])
|
|
90
102
|
self.runs_by_id = {run["id"]: run for run in self.runs}
|
|
91
103
|
self.runs_by_judgement_id = self._create_index_by_id(self.runs, "judgement_id")
|
|
92
104
|
|
|
93
|
-
self.submissions = self._load_json_file("submissions.json")
|
|
105
|
+
self.submissions = self._load_json_file("submissions.json", default_value=[])
|
|
94
106
|
self.submissions_by_id = {submission["id"]: submission for submission in self.submissions}
|
|
95
107
|
|
|
96
|
-
self.teams = self._load_json_file("teams.json")
|
|
108
|
+
self.teams = self._load_json_file("teams.json", default_value=[])
|
|
97
109
|
self.teams_by_id = {team["id"]: team for team in self.teams}
|
|
98
110
|
|
|
99
|
-
self.event_feed = self._load_ndjson_file("event-feed.ndjson")
|
|
111
|
+
self.event_feed = self._load_ndjson_file("event-feed.ndjson", default_value=[])
|
|
100
112
|
self.event_feed_tokens = [event["token"] for event in self.event_feed]
|
|
101
113
|
|
|
102
|
-
self.contest_id = self.contest
|
|
114
|
+
self.contest_id = self.contest.get("id", "")
|
|
103
115
|
|
|
104
116
|
def _get_file_attr(self, expected_href: str, base_path: Path, files: List[Dict]) -> FileAttr:
|
|
105
117
|
for file in files:
|
xcpcio/types.py
CHANGED
|
@@ -63,6 +63,7 @@ MedalPreset = Literal["ccpc", "icpc"]
|
|
|
63
63
|
BannerMode = Literal["ONLY_BANNER", "ALL"]
|
|
64
64
|
Lang = Literal["en", "zh-CN"]
|
|
65
65
|
DateTimeISO8601String = str
|
|
66
|
+
UrlString = str
|
|
66
67
|
|
|
67
68
|
|
|
68
69
|
class I18NStringSet(BaseModel):
|
|
@@ -87,6 +88,21 @@ class Image(BaseModel):
|
|
|
87
88
|
height: Optional[int] = None
|
|
88
89
|
|
|
89
90
|
|
|
91
|
+
class DataItem(BaseModel):
|
|
92
|
+
url: UrlString
|
|
93
|
+
version: Optional[str] = None
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
class Organization(BaseModel):
|
|
97
|
+
id: str
|
|
98
|
+
name: Text
|
|
99
|
+
|
|
100
|
+
logo: Optional[Image] = None
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
Organizations = List[Organization]
|
|
104
|
+
|
|
105
|
+
|
|
90
106
|
class BalloonColor(BaseModel):
|
|
91
107
|
color: str
|
|
92
108
|
background_color: str
|
|
@@ -146,6 +162,8 @@ class Team(BaseModel):
|
|
|
146
162
|
name: Text = ""
|
|
147
163
|
|
|
148
164
|
organization: str = ""
|
|
165
|
+
organization_id: Optional[str] = None
|
|
166
|
+
|
|
149
167
|
group: List[str] = Field(default_factory=list)
|
|
150
168
|
tag: Optional[List[str]] = None
|
|
151
169
|
|
|
@@ -219,7 +237,9 @@ class Contest(BaseModel):
|
|
|
219
237
|
|
|
220
238
|
options: Optional[ContestOptions] = None
|
|
221
239
|
|
|
222
|
-
|
|
240
|
+
organizations: Optional[Union[DataItem, Organizations]] = None
|
|
241
|
+
|
|
242
|
+
thaw_time: int = Field(default=0x3F3F3F3F3F3F3F3F, exclude=True)
|
|
223
243
|
|
|
224
244
|
def append_balloon_color(self, color: BalloonColor):
|
|
225
245
|
if self.balloon_color is None:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: xcpcio
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.74.0
|
|
4
4
|
Summary: xcpcio python lib
|
|
5
5
|
Project-URL: homepage, https://github.com/xcpcio/xcpcio
|
|
6
6
|
Project-URL: documentation, https://github.com/xcpcio/xcpcio
|
|
@@ -79,7 +79,7 @@ uv run pytest tests/test_types.py
|
|
|
79
79
|
|
|
80
80
|
For detailed documentation, visit:
|
|
81
81
|
|
|
82
|
-
- [
|
|
82
|
+
- [Clics Utility Guide](https://xcpcio.com/guide/clics-utility)
|
|
83
83
|
|
|
84
84
|
## License
|
|
85
85
|
|
|
@@ -1,15 +1,17 @@
|
|
|
1
1
|
xcpcio/__init__.py,sha256=NB6wpVr5JUrOx2vLIQSVtYuCz0d7kNFE39TSQlvoENk,125
|
|
2
|
-
xcpcio/__version__.py,sha256=
|
|
2
|
+
xcpcio/__version__.py,sha256=bqW3xNfcKZLCjzET76nCOrvRVs1ULNTmgQNmTC__WlI,172
|
|
3
3
|
xcpcio/constants.py,sha256=MjpAgNXiBlUsx1S09m7JNT-nekNDR-aE6ggvGL3fg0I,2297
|
|
4
|
-
xcpcio/types.py,sha256=
|
|
5
|
-
xcpcio/api/__init__.py,sha256=
|
|
6
|
-
xcpcio/api/client.py,sha256=
|
|
7
|
-
xcpcio/api/models.py,sha256=
|
|
4
|
+
xcpcio/types.py,sha256=CLhRp_HaVbK9YK8yNOHDNDIis8XOAo5BWz5lZS-qbvw,7927
|
|
5
|
+
xcpcio/api/__init__.py,sha256=fFRUU0d_cUG-uzG2rLgirqRng8Z3UoV5ZCkBiYuXdu0,302
|
|
6
|
+
xcpcio/api/client.py,sha256=TEcf3tjLu537HHhyBBVHlQ3fFL7_abjG3IrQtp39oks,2389
|
|
7
|
+
xcpcio/api/models.py,sha256=DBS9TjtPSX32TpwO3gYnnGo7RFHLHrxYpRIAhxCHmLs,1258
|
|
8
8
|
xcpcio/app/clics_archiver.py,sha256=wd7zkbq2oyWfkS09w0f3KBr32F2yCcTMx36H5HFMgRg,7995
|
|
9
9
|
xcpcio/app/clics_server.py,sha256=-zUixNf08yICT2sry23h72ZrEm6NPb30bH5AHCXZlBM,4623
|
|
10
|
+
xcpcio/app/clics_uploader.py,sha256=Hw2buDX7lO8kghcxmLLv3P-F4uO73BDBPctmVTb7GEw,3592
|
|
10
11
|
xcpcio/clics/__init__.py,sha256=coTZiqxzXesn2SYmI2ZCsDZW6XaFi_6p-PFozZ4dfl4,150
|
|
11
|
-
xcpcio/clics/clics_api_client.py,sha256=
|
|
12
|
-
xcpcio/clics/contest_archiver.py,sha256=
|
|
12
|
+
xcpcio/clics/clics_api_client.py,sha256=N6mYlou6eTrQYiUKxE4fdwCClCt9DrmVyKQmpc53iaY,9670
|
|
13
|
+
xcpcio/clics/contest_archiver.py,sha256=9q3qB4hOPfd4ttaG78cQ7sg7vWkumO_KUZpqwYlHDlM,9933
|
|
14
|
+
xcpcio/clics/contest_uploader.py,sha256=eGvPnFkxpCGVl43hoi2_6fBXR3c4jj3Td-R4i9wHt1M,10849
|
|
13
15
|
xcpcio/clics/api_server/__init__.py,sha256=ASvVJ_ibGkXFDSNmy05eb9gESXRS8hjYHCrBecSnaS0,174
|
|
14
16
|
xcpcio/clics/api_server/app.py,sha256=dYvkxaR8PjkDIHbSX8YyU3TvPud93Dg29k6midylakE,979
|
|
15
17
|
xcpcio/clics/api_server/dependencies.py,sha256=2Zhom6vUnAOikr9bh-_kXYCc3g6JGXReVuQX7Ok90r4,1444
|
|
@@ -38,9 +40,9 @@ xcpcio/clics/model/__init__.py,sha256=cZE1q5JY-iHDEKZpsx0UZaMhH-23H4oAHaYOkW7dZ5
|
|
|
38
40
|
xcpcio/clics/model/model_2023_06/__init__.py,sha256=VzBaFcAwYw9G18p0Lh7rNPrvchyaYx_jgw6YE4W1yNg,168
|
|
39
41
|
xcpcio/clics/model/model_2023_06/model.py,sha256=bVMDWpJTwPSpz1fHPxWrWerxCBIboH3LKVZpIZGQ2pY,15287
|
|
40
42
|
xcpcio/clics/reader/__init__.py,sha256=Nfi78X8J1tJPh7WeSRPLMRUprlS2JYelYJHW4DfyJ7U,162
|
|
41
|
-
xcpcio/clics/reader/contest_package_reader.py,sha256=
|
|
43
|
+
xcpcio/clics/reader/contest_package_reader.py,sha256=W9pNAl6Za1LXG_OjDYj-szVPXczeHBUhL-ajOApL7BU,15089
|
|
42
44
|
xcpcio/clics/reader/interface.py,sha256=lK2JXU1n8GJ4PecXnfFBijMaCVLYk404e4QwV_Ti2Hk,3918
|
|
43
|
-
xcpcio-0.
|
|
44
|
-
xcpcio-0.
|
|
45
|
-
xcpcio-0.
|
|
46
|
-
xcpcio-0.
|
|
45
|
+
xcpcio-0.74.0.dist-info/METADATA,sha256=OTxJWftLDzlvdKzbx0TAISCLxyxoUs0RcF0a1s5g3js,2237
|
|
46
|
+
xcpcio-0.74.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
47
|
+
xcpcio-0.74.0.dist-info/entry_points.txt,sha256=KxviS5TVXlZ9KqS9UNMx3nI15xhEGvkTq86RaPhUhs4,158
|
|
48
|
+
xcpcio-0.74.0.dist-info/RECORD,,
|
|
File without changes
|