xcpcio 0.68.0__tar.gz → 0.69.0__tar.gz

{xcpcio-0.68.0 → xcpcio-0.69.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xcpcio
-Version: 0.68.0
+Version: 0.69.0
 Summary: xcpcio python lib
 Project-URL: homepage, https://github.com/xcpcio/xcpcio
 Project-URL: documentation, https://github.com/xcpcio/xcpcio

{xcpcio-0.68.0 → xcpcio-0.69.0}/pyproject.toml

@@ -40,6 +40,7 @@ repository = "https://github.com/xcpcio/xcpcio"
 [project.scripts]
 clics-archiver = "xcpcio.app.clics_archiver:main"
 clics-server = "xcpcio.app.clics_server:main"
+clics-uploader = "xcpcio.app.clics_uploader:main"
 
 [tool.ruff]
 line-length = 120

{xcpcio-0.68.0 → xcpcio-0.69.0}/xcpcio/__version__.py

@@ -1,4 +1,4 @@
 # This file is auto-generated by Hatchling. As such, do not:
 #   - modify
 #   - track in version control e.g. be sure to add to .gitignore
-__version__ = VERSION = '0.68.0'
+__version__ = VERSION = '0.69.0'

{xcpcio-0.68.0 → xcpcio-0.69.0}/xcpcio/api/__init__.py

@@ -1,5 +1,6 @@
 from .client import ApiClient
 from .models import (
+    FileData,
     HTTPValidationError,
     UploadBoardDataReq,
     UploadBoardDataResp,
@@ -8,6 +9,7 @@ from .models import (
 
 __all__ = [
     ApiClient,
+    FileData,
     HTTPValidationError,
     UploadBoardDataReq,
     UploadBoardDataResp,

{xcpcio-0.68.0 → xcpcio-0.69.0}/xcpcio/api/client.py

@@ -2,7 +2,7 @@ from typing import Any, Dict, Optional
 
 import aiohttp
 
-from .models import UploadBoardDataReq, UploadBoardDataResp
+from .models import FileData, UploadBoardDataReq, UploadBoardDataResp
 
 
 class ApiClient:
@@ -51,7 +51,7 @@ class ApiClient:
         config: Optional[str] = None,
         teams: Optional[str] = None,
         submissions: Optional[str] = None,
-        extra_files: Optional[Dict[str, str]] = None,
+        extra_files: Optional[Dict[str, FileData]] = None,
     ) -> UploadBoardDataResp:
         await self._ensure_session()
         await self._ensure_token()

{xcpcio-0.68.0 → xcpcio-0.69.0}/xcpcio/api/models.py

@@ -1,4 +1,4 @@
-from typing import Dict, List, Optional, Union
+from typing import Dict, List, Literal, Optional, Union
 
 from pydantic import BaseModel, Field
 
@@ -13,13 +13,24 @@ class HTTPValidationError(BaseModel):
     detail: Optional[List[ValidationError]] = Field(None, title="Detail")
 
 
+class FileData(BaseModel):
+    content: str = Field(..., title="Content")
+    encoding: Optional[Literal["base64"]] = Field(None, title="Encoding")
+    checksum: Optional[str] = Field(None, title="Checksum")
+
+
 class UploadBoardDataReq(BaseModel):
     token: str = Field(..., title="Token")
     config: Optional[str] = Field(None, title="Config")
     teams: Optional[str] = Field(None, title="Teams")
     submissions: Optional[str] = Field(None, title="Submissions")
-    extra_files: Optional[Dict[str, str]] = Field(None, title="Extra Files")
+    extra_files: Optional[Dict[str, FileData]] = Field(None, title="Extra Files")
+
+
+class FileValidationResult(BaseModel):
+    checksum_valid: bool = Field(..., title="Checksum Valid")
+    message: Optional[str] = Field(None, title="Message")
 
 
 class UploadBoardDataResp(BaseModel):
-    pass
+    file_validations: Dict[str, FileValidationResult] = Field(default_factory=dict, title="File Validations")

xcpcio-0.69.0/xcpcio/app/clics_uploader.py

@@ -0,0 +1,100 @@
+import asyncio
+import logging
+from pathlib import Path
+from typing import Optional
+
+import click
+
+from xcpcio import __version__
+from xcpcio.clics.clics_api_client import APICredentials
+from xcpcio.clics.contest_uploader import ContestUploader, UploaderConfig
+
+
+def setup_logging(level: str = "INFO"):
+    logging.basicConfig(
+        level=getattr(logging, level.upper()),
+        format="%(asctime)s [%(name)s] %(filename)s:%(lineno)d %(levelname)s: %(message)s",
+    )
+
+
+@click.command()
+@click.version_option(__version__)
+@click.option("--base-url", required=True, help="Base URL of the CLICS API (e.g., https://domjudge/api)")
+@click.option("--contest-id", required=True, help="Contest ID to fetch")
+@click.option("--username", "-u", required=True, help="HTTP Basic Auth username for CLICS API")
+@click.option("--password", "-p", required=True, help="HTTP Basic Auth password for CLICS API")
+@click.option("--xcpcio-api-url", default="https://api.xcpcio.com", help="XCPCIO API URL")
+@click.option("--xcpcio-api-token", required=True, help="XCPCIO API token")
+@click.option(
+    "--cache-dir", type=click.Path(path_type=Path), help="Directory for checksum cache files (default: ~/.xcpcio/)"
+)
+@click.option("--timeout", default=30, type=int, help="Request timeout in seconds")
+@click.option("--max-concurrent", default=10, type=int, help="Max concurrent requests for CLICS API")
+@click.option(
+    "--log-level",
+    default="INFO",
+    type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR"], case_sensitive=False),
+    help="Log level",
+)
+@click.option("--verbose", "-v", is_flag=True, help="Enable verbose logging (same as --log-level DEBUG)")
+def main(
+    base_url: str,
+    contest_id: str,
+    username: str,
+    password: str,
+    xcpcio_api_url: str,
+    xcpcio_api_token: str,
+    cache_dir: Optional[Path],
+    timeout: int,
+    max_concurrent: int,
+    log_level: str,
+    verbose: bool,
+):
+    """
+    Upload CLICS Contest API data to Board Admin with polling support.
+
+    This tool fetches contest data from a CLICS API and uploads raw JSON data to the xcpcio board-admin service.
+    It continuously polls every 5 seconds and uses SHA256 checksums to avoid uploading unchanged files.
+
+    Examples:
+
+        clics-uploader --base-url https://domjudge/api --contest-id contest123 -u admin -p secret --xcpcio-api-token YOUR_TOKEN
+    """
+
+    if verbose:
+        log_level = "DEBUG"
+
+    setup_logging(log_level)
+
+    credentials = APICredentials(username=username, password=password)
+    config = UploaderConfig(
+        clics_base_url=base_url.rstrip("/"),
+        contest_id=contest_id,
+        clics_credentials=credentials,
+        xcpcio_api_url=xcpcio_api_url,
+        xcpcio_api_token=xcpcio_api_token,
+        cache_dir=cache_dir,
+        timeout=timeout,
+        max_concurrent=max_concurrent,
+        version=__version__,
+    )
+
+    click.echo(f"Fetching contest '{contest_id}' from {base_url}")
+    click.echo(f"Uploading to XCPCIO: {xcpcio_api_url}")
+
+    async def run():
+        async with ContestUploader(config) as uploader:
+            await uploader.run_loop()
+
+    try:
+        asyncio.run(run())
+    except KeyboardInterrupt:
+        click.echo(click.style("\nClics Uploader stopped by user", fg="yellow"), err=True)
+        raise click.Abort()
+    except Exception as e:
+        click.echo(click.style(f"Error: {e}", fg="red"), err=True)
+        raise click.ClickException(str(e))
+
+
+if __name__ == "__main__":
+    main()

{xcpcio-0.68.0 → xcpcio-0.69.0}/xcpcio/clics/clics_api_client.py

@@ -168,6 +168,42 @@ class ClicsApiClient:
                 logger.error(f"Failed to download {file_url} after retries: {e}")
                 return False
 
+    @retry(
+        stop=stop_after_attempt(3),
+        wait=wait_exponential(multiplier=1, min=1, max=10),
+        retry=retry_if_exception_type((asyncio.TimeoutError, aiohttp.ClientError)),
+        before_sleep=before_sleep_log(logger, logging.WARNING),
+        reraise=True,
+    )
+    async def _fetch_file_content_internal(self, file_url: str, override_timeout: Optional[int] = None) -> bytes:
+        """Internal method to fetch file content as bytes with retry logic"""
+        logger.info(f"Fetching file content from {file_url}")
+        timeout = aiohttp.ClientTimeout(total=override_timeout or self._config.timeout)
+        async with self._session.get(file_url, timeout=timeout) as response:
+            if response.status != 200:
+                raise aiohttp.ClientResponseError(
+                    request_info=response.request_info, history=response.history, status=response.status
+                )
+
+            content = await response.read()
+            logger.debug(f"Fetched {len(content)} bytes from {file_url}")
+            return content
+
+    async def fetch_file_content(self, file_url: Optional[str], override_timeout: Optional[int] = None) -> Optional[bytes]:
+        """Fetch file content as bytes from URL"""
+        if not file_url:
+            return None
+
+        if not file_url.startswith(("http://", "https://")):
+            file_url = self._build_url(file_url)
+
+        async with self._semaphore:
+            try:
+                return await self._fetch_file_content_internal(file_url, override_timeout)
+            except Exception as e:
+                logger.error(f"Failed to fetch file content from {file_url} after retries: {e}")
+                return None
+
     async def fetch_api_info(self) -> Optional[Dict[str, Any]]:
         """Fetch API root endpoint information and parse provider details"""
         data = await self.fetch_json("/")

{xcpcio-0.68.0 → xcpcio-0.69.0}/xcpcio/clics/contest_archiver.py

@@ -91,6 +91,7 @@ class ContestArchiver:
         "runs",
         "clarifications",
         "awards",
+        "scoreboard",
     ]
 
     def __init__(self, config: ArchiveConfig):

xcpcio-0.69.0/xcpcio/clics/contest_uploader.py

@@ -0,0 +1,281 @@
+import asyncio
+import hashlib
+import json
+import logging
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+from typing import Dict, Optional
+
+from pydantic import BaseModel
+
+from xcpcio.api.client import ApiClient
+from xcpcio.api.models import FileData, UploadBoardDataResp
+from xcpcio.clics.clics_api_client import APICredentials, ClicsApiClient, ClicsApiConfig
+
+logger = logging.getLogger(__name__)
+
+POLL_INTERVAL = 5
+
+
+class FileChecksumEntry(BaseModel):
+    checksum: str
+    updated_at: str
+
+
+class FileChecksumCacheData(BaseModel):
+    checksums: Dict[str, FileChecksumEntry]
+    updated_at: str
+
+
+class FileChecksumCache:
+    def __init__(
+        self, cache_dir: Optional[Path] = None, contest_id: Optional[str] = None, version: Optional[str] = None
+    ):
+        if cache_dir:
+            cache_dir = Path(cache_dir)
+        else:
+            cache_dir = Path.home() / ".xcpcio"
+
+        cache_dir.mkdir(parents=True, exist_ok=True)
+
+        if contest_id:
+            self.cache_file = cache_dir / f"clics_uploader_{contest_id}.json"
+        else:
+            self.cache_file = cache_dir / "clics_uploader_cache.json"
+
+        self.version_file = cache_dir / "version"
+        self.version = version
+        self.checksums: Dict[str, FileChecksumEntry] = {}
+        self._load_cache()
+
+    def _load_version(self) -> Optional[str]:
+        if self.version_file.exists():
+            try:
+                with open(self.version_file, "r") as f:
+                    return f.read().strip()
+            except Exception as e:
+                logger.warning(f"Failed to read version file: {e}")
+        return None
+
+    def _save_version(self):
+        if not self.version:
+            return
+        try:
+            with open(self.version_file, "w") as f:
+                f.write(self.version)
+        except Exception as e:
+            logger.warning(f"Failed to save version file: {e}")
+
+    def _load_cache(self):
+        cached_version = self._load_version()
+        if cached_version != self.version:
+            logger.info(f"Cache version mismatch (cached: {cached_version}, current: {self.version}), clearing cache")
+            self.checksums = {}
+            return
+
+        if self.cache_file.exists():
+            try:
+                with open(self.cache_file, "r") as f:
+                    data = json.load(f)
+
+                cache_data = FileChecksumCacheData.model_validate(data)
+                self.checksums = cache_data.checksums
+                logger.debug(f"Loaded cache with {len(self.checksums)} entries")
+            except Exception as e:
+                logger.warning(f"Failed to load cache: {e}")
+                self.checksums = {}
+
+    def _save_cache(self):
+        try:
+            self.cache_file.parent.mkdir(parents=True, exist_ok=True)
+            cache_data = FileChecksumCacheData(
+                checksums=self.checksums,
+                updated_at=datetime.now().isoformat(),
+            )
+            with open(self.cache_file, "w") as f:
+                json.dump(cache_data.model_dump(), f, indent=2)
+            self._save_version()
+            logger.debug(f"Saved cache with {len(self.checksums)} entries")
+        except Exception as e:
+            logger.warning(f"Failed to save cache: {e}")
+
+    def calculate_checksum(self, content: str) -> str:
+        return hashlib.sha256(content.encode("utf-8")).hexdigest()
+
+    def has_changed(self, key: str, content: str) -> bool:
+        checksum = self.calculate_checksum(content)
+        old_entry = self.checksums.get(key)
+        return not old_entry or old_entry.checksum != checksum
+
+    def update_checksum(self, key: str, content: str):
+        checksum = self.calculate_checksum(content)
+        self.checksums[key] = FileChecksumEntry(
+            checksum=checksum,
+            updated_at=datetime.now().isoformat(),
+        )
+
+    def save(self):
+        self._save_cache()
+
+
+@dataclass
+class UploaderConfig:
+    clics_base_url: str
+    contest_id: str
+    clics_credentials: APICredentials
+    xcpcio_api_url: str
+    xcpcio_api_token: str
+    cache_dir: Optional[Path] = None
+    timeout: int = 30
+    max_concurrent: int = 10
+    version: Optional[str] = None
+
+    def to_clics_api_config(self) -> ClicsApiConfig:
+        return ClicsApiConfig(
+            base_url=self.clics_base_url,
+            credentials=self.clics_credentials,
+            timeout=self.timeout,
+            max_concurrent=self.max_concurrent,
+        )
+
+
+class ContestUploader:
+    ENDPOINTS_TO_FETCH = (
+        "access",
+        "accounts",
+        "awards",
+        "groups",
+        "judgement-types",
+        "judgements",
+        "languages",
+        "organizations",
+        "problems",
+        "state",
+        "submissions",
+        "teams",
+    )
+
+    def __init__(self, config: UploaderConfig):
+        self._config = config
+        self._clics_client = ClicsApiClient(config.to_clics_api_config())
+        self._api_client = ApiClient(
+            base_url=config.xcpcio_api_url,
+            token=config.xcpcio_api_token,
+            timeout=config.timeout,
+        )
+        self._cache = FileChecksumCache(
+            cache_dir=config.cache_dir,
+            contest_id=config.contest_id,
+            version=config.version,
+        )
+
+    async def __aenter__(self):
+        await self._clics_client.__aenter__()
+        await self._api_client.__aenter__()
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        await self._clics_client.__aexit__(exc_type, exc_val, exc_tb)
+        await self._api_client.__aexit__(exc_type, exc_val, exc_tb)
+
+    def _process_file_validations(
+        self, file_map: Dict[str, FileData], file_keys: list[str], response: UploadBoardDataResp
+    ):
+        for file_key in file_keys:
+            if file_key in file_map:
+                validation = response.file_validations.get(file_key)
+                if validation:
+                    if validation.checksum_valid:
+                        self._cache.update_checksum(file_key, file_map[file_key].content)
+                    else:
+                        logger.warning(f"File checksum validation failed for {file_key}: {validation.message}")
+                else:
+                    logger.warning(f"No validation result for {file_key}, will retry on next poll")
+
+    async def _fetch_contest_data(
+        self,
+    ) -> tuple[Dict[str, FileData], list[str], list[str]]:
+        api_files = {}
+        changed_api_files = []
+        unchanged_files = []
+
+        contest_data = await self._clics_client.fetch_json(f"contests/{self._config.contest_id}")
+        if contest_data:
+            filename = "contest.json"
+            content = json.dumps(contest_data, ensure_ascii=False)
+
+            if self._cache.has_changed(filename, content):
+                checksum = self._cache.calculate_checksum(content)
+                api_files[filename] = FileData(content=content, checksum=checksum)
+                changed_api_files.append(filename)
+            else:
+                unchanged_files.append(filename)
+
+        return api_files, changed_api_files, unchanged_files
+
+    async def _fetch_endpoint_data(
+        self,
+        api_files: Dict[str, FileData],
+        changed_api_files: list[str],
+        unchanged_files: list[str],
+    ):
+        for endpoint in self.ENDPOINTS_TO_FETCH:
+            data = await self._clics_client.fetch_json(f"contests/{self._config.contest_id}/{endpoint}")
+            if data is not None:
+                filename = f"{endpoint}.json"
+                content = json.dumps(data, ensure_ascii=False)
+
+                if self._cache.has_changed(filename, content):
+                    checksum = self._cache.calculate_checksum(content)
+                    api_files[filename] = FileData(content=content, checksum=checksum)
+                    changed_api_files.append(filename)
+                else:
+                    unchanged_files.append(filename)
+
+    async def _upload_api_files(
+        self, api_files: Dict[str, FileData], changed_api_files: list[str]
+    ) -> Optional[UploadBoardDataResp]:
+        if not api_files:
+            return None
+
+        logger.info(f"Uploading {len(api_files)} API file(s) to Board Admin...")
+        response = await self._api_client.upload_board_data(extra_files=api_files)
+        logger.info(f"API upload successful! ({len(api_files)} files)")
+
+        self._process_file_validations(api_files, changed_api_files, response)
+        self._cache.save()
+        return response
+
+    async def fetch_and_upload(self) -> Optional[Dict]:
+        await self._clics_client.fetch_api_info()
+
+        api_files, changed_api_files, unchanged_files = await self._fetch_contest_data()
+        await self._fetch_endpoint_data(api_files, changed_api_files, unchanged_files)
+
+        if changed_api_files:
+            logger.info(f"Changed API files ({len(changed_api_files)}): {', '.join(changed_api_files)}")
+        if unchanged_files:
+            logger.debug(f"Unchanged files ({len(unchanged_files)}): {', '.join(unchanged_files)}")
+
+        if not api_files:
+            logger.info("No changed files to upload")
+            return None
+
+        response = await self._upload_api_files(api_files, changed_api_files)
+
+        logger.info(f"All uploads completed! ({len(api_files)} files)")
+        return response.model_dump() if response else None
+
+    async def run_loop(self, poll_interval: int = POLL_INTERVAL):
+        iteration = 0
+        while True:
+            iteration += 1
+            try:
+                logger.info(f"[Iteration #{iteration}] Starting fetch and upload...")
+                await self.fetch_and_upload()
+            except Exception as e:
+                logger.exception(f"Error during fetch and upload: {e}", exc_info=True)
+
+            logger.info(f"Waiting {poll_interval} seconds until next poll...")
+            await asyncio.sleep(poll_interval)