cafs-cache-cdn-client 1.0.5__py3-none-any.whl

cafs_cache_cdn_client/client.py

@@ -0,0 +1,142 @@
+import asyncio
+import functools
+from collections.abc import Awaitable, Callable
+from os.path import normpath
+from pathlib import Path
+from typing import Any, Self, TypeVar
+
+import aiofiles.os as aio_os
+
+from cafs_cache_cdn_client.cafs import CAFSClient
+from cafs_cache_cdn_client.file_utils import (
+    LocalFile,
+    compare_file_lists,
+    set_file_stat,
+    walk,
+)
+from cafs_cache_cdn_client.repo import RepoClient
+
+__all__ = ('CacheCdnClient',)
+
+
+CAFS_SERVER_ROOT = '/cache'
+
+
+T = TypeVar('T')
+
+
+def needs_cafs_client(func: Callable[..., Awaitable[T]]) -> Callable[..., Awaitable[T]]:
+    @functools.wraps(func)
+    async def wrapper(self: 'CacheCdnClient', *args: Any, **kwargs: Any) -> T:
+        await self._init_cafs_client()
+        return await func(self, *args, **kwargs)
+
+    return wrapper
+
+
+class CacheCdnClient:
+    _cafs_client: CAFSClient | None = None
+    _repo_client: RepoClient
+
+    __connection_per_cafs_server: int
+    __cafs_client_lock = asyncio.Lock()
+
+    def __init__(self, server: str, connection_per_cafs_server: int = 1) -> None:
+        self._repo_client = RepoClient(server)
+        self.__connection_per_cafs_server = connection_per_cafs_server
+
+    async def _init_cafs_client(self) -> None:
+        async with self.__cafs_client_lock:
+            if self._cafs_client:
+                return
+            blob_urls = await self._repo_client.get_blob_urls()
+            self._cafs_client = await CAFSClient(
+                CAFS_SERVER_ROOT,
+                blob_urls,
+                connection_per_server=self.__connection_per_cafs_server,
+            ).__aenter__()
+
+    @needs_cafs_client
+    async def push(
+        self,
+        repo: str,
+        ref: str,
+        directory: Path | str,
+        ttl_hours: int = 0,
+        comment: str | None = None,
+    ) -> None:
+        if isinstance(directory, str):
+            directory = Path(directory)
+        if not directory.is_dir():
+            raise ValueError(f'{directory} is not a directory')
+        files = walk(directory)
+        hashes = await self._cafs_client.stream_batch(
+            [directory / file.path for file in files]
+        )
+        await self._repo_client.post_ref_info(
+            repo,
+            ref,
+            {
+                'archive': False,
+                'ttl': ttl_hours * 60 * 60 * 10**9,
+                'comment': comment,
+                'files': [
+                    {
+                        'blob': blob,
+                        'path': file.path.as_posix(),
+                        'mtime': file.mtime,
+                        'mode': file.mode,
+                    }
+                    for blob, file in zip(hashes, files)
+                ],
+            },
+        )
+
+    async def check(self, repo: str, ref: str) -> bool:
+        return await self._repo_client.is_ref_exist(repo, ref)
+
+    async def delete(self, repo: str, ref: str) -> None:
+        await self._repo_client.delete_ref(repo, ref)
+
+    async def attach(self, repo: str, ref: str, file_path: Path) -> None:
+        await self._repo_client.attach_file(repo, ref, file_path)
+
+    @needs_cafs_client
+    async def pull(self, repo: str, ref: str, directory: Path | str) -> None:
+        if isinstance(directory, str):
+            directory = Path(directory)
+        await aio_os.makedirs(directory, exist_ok=True)
+        ref_info = await self._repo_client.get_ref_info(repo, ref)
+        remote_files = [
+            LocalFile(
+                path=Path(normpath(file['path'])),
+                mtime=file['mtime'],
+                mode=file['mode'],
+                blob=file['blob'],
+            )
+            for file in ref_info['files']
+        ]
+        local_files = walk(directory)
+        to_remove, to_add, to_update = await compare_file_lists(
+            local_files, remote_files, directory
+        )
+        for file in to_remove:
+            await aio_os.unlink(directory / file.path)
+        if to_add:
+            await self._cafs_client.pull_batch(
+                [(file.blob, directory / file.path) for file in to_add]
+            )
+        for file in to_add + to_update:
+            set_file_stat(file, directory)
+
+    async def tag(self, repo: str, ref: str, tag: str) -> None:
+        await self._repo_client.tag_ref(repo, ref, tag)
+
+    async def __aenter__(self) -> Self:
+        return self
+
+    async def __aexit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
+        async with self.__cafs_client_lock:
+            if not self._cafs_client:
+                return
+            await self._cafs_client.__aexit__(exc_type, exc_val, exc_tb)

cafs_cache_cdn_client/file_utils.py

@@ -0,0 +1,96 @@
+import os
+import sys
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+from cafs_cache_cdn_client.cafs.blob.hash_ import calc_hash_file
+
+__all__ = (
+    'LocalFile',
+    'walk',
+    'compare_file_lists',
+    'set_file_stat',
+)
+
+
+if sys.platform == 'win32':
+
+    def is_same_mtime(
+        t1: int,
+        t2: int,
+    ) -> bool:
+        return t1 // 100 == t2 // 100
+else:
+
+    def is_same_mtime(
+        t1: int,
+        t2: int,
+    ) -> bool:
+        return t1 == t2
+
+
+@dataclass
+class LocalFile:
+    path: Path
+    mtime: int
+    mode: int
+    blob: str | None = None
+
+
+def walk(directory: Path) -> list[LocalFile]:
+    results = []
+    for root, _, files in directory.walk():
+        for file in files:
+            file_ = root / file
+            file_stat = file_.stat()
+            results.append(
+                LocalFile(
+                    path=file_.relative_to(directory),
+                    mtime=file_stat.st_mtime_ns,
+                    mode=file_stat.st_mode & 0o777,
+                )
+            )
+    return results
+
+
+async def compare_file_lists(
+    src_files: list[LocalFile],
+    dst_files: list[LocalFile],
+    directory: Path,
+) -> tuple[list[LocalFile], list[LocalFile], list[LocalFile]]:
+    src_files_dict = {file.path: file for file in src_files}
+    dst_files_dict = {file.path: file for file in dst_files}
+    to_remove = src_files_dict.keys() - dst_files_dict.keys()
+    to_add = dst_files_dict.keys() - src_files_dict.keys()
+    to_update = set()
+    for same_file in src_files_dict.keys() & dst_files_dict.keys():
+        if not dst_files_dict[same_file].blob:
+            to_remove.add(same_file)
+            to_add.add(same_file)
+            continue
+        if not src_files_dict[same_file].blob:
+            src_files_dict[same_file].blob = await calc_hash_file(directory / same_file)
+        if src_files_dict[same_file].blob != dst_files_dict[same_file].blob:
+            to_remove.add(same_file)
+            to_add.add(same_file)
+            continue
+        if (
+            not is_same_mtime(
+                src_files_dict[same_file].mtime, dst_files_dict[same_file].mtime
+            )
+            or src_files_dict[same_file].mode != dst_files_dict[same_file].mode
+        ):
+            to_update.add(same_file)
+
+    return (
+        [src_files_dict[file] for file in to_remove],
+        [dst_files_dict[file] for file in to_add],
+        [src_files_dict[file] for file in to_update],
+    )
+
+
+def set_file_stat(file: LocalFile, directory: Path) -> None:
+    file_ = directory / file.path
+    file_.chmod(file.mode)
+    os.utime(file_, ns=(time.time_ns(), file.mtime))

cafs_cache_cdn_client/repo/__init__.py

@@ -0,0 +1,3 @@
+from .client import RepoClient
+
+__all__ = ('RepoClient',)

cafs_cache_cdn_client/repo/client.py

@@ -0,0 +1,102 @@
+from collections.abc import Iterable
+from http import HTTPMethod
+from pathlib import Path
+from typing import Any, cast
+from urllib.parse import quote, urljoin
+
+import aiofiles
+import aiohttp
+import yarl
+
+import cafs_cache_cdn_client.repo.datatypes as dt
+
+__all__ = ('RepoClient',)
+
+
+class RepoClient:
+    server_base_url: str
+
+    def __init__(self, server: str) -> None:
+        self.server_base_url = server
+
+    async def _request(
+        self,
+        endpoint: str,
+        params: Iterable[tuple[str, str]] | None = None,
+        method: HTTPMethod = HTTPMethod.GET,
+        data: dict | bytes | aiohttp.FormData | None = None,
+        headers: dict[str, str] | None = None,
+        json_request: bool = True,
+    ) -> Any:
+        if params:
+            endpoint += '?' + '&'.join(f'{k}={quote(v)}' for k, v in params)
+        headers = headers.copy() if headers else {}
+        if json_request:
+            headers['Content-Type'] = 'application/json'
+        url_ = yarl.URL(urljoin(self.server_base_url, endpoint), encoded=True)
+        if json_request:
+            data_arg = {'json': data}
+        else:
+            data_arg = {'data': data}
+        async with aiohttp.ClientSession(
+            headers=headers, requote_redirect_url=False
+        ) as session:
+            async with session.request(
+                method, url_, **data_arg, raise_for_status=True
+            ) as resp:
+                if resp.headers.get('Content-Type') == 'application/json':
+                    return await resp.json()
+                return await resp.read()
+
+    async def get_settings(self) -> dt.SettingsResponse:
+        return cast(
+            dt.SettingsResponse, await self._request('/settings', method=HTTPMethod.GET)
+        )
+
+    async def get_blob_urls(self) -> list[str]:
+        settings = await self.get_settings()
+        return settings['blob_urls']
+
+    async def is_ref_exist(self, repo: str, ref: str) -> bool:
+        try:
+            await self._request(f'/repository/{repo}/{ref}', method=HTTPMethod.HEAD)
+            return True
+        except aiohttp.ClientResponseError as e:
+            if e.status == 404:
+                return False
+            raise
+
+    async def delete_ref(self, repo: str, ref: str) -> None:
+        try:
+            await self._request(f'/repository/{repo}/{ref}', method=HTTPMethod.DELETE)
+        except aiohttp.ClientResponseError as e:
+            if e.status == 404:
+                return
+            raise
+
+    async def get_ref_info(self, repo: str, ref: str) -> dt.RefInfoResponse:
+        return cast(
+            dt.RefInfoResponse,
+            await self._request(f'/repository/{repo}/{ref}', method=HTTPMethod.GET),
+        )
+
+    async def tag_ref(self, repo: str, ref: str, tag: str) -> None:
+        await self._request(
+            f'/repository/{repo}/{ref}/tag', method=HTTPMethod.PUT, data={'tag': tag}
+        )
+
+    async def post_ref_info(self, repo: str, ref: str, data: dt.RefInfoBody) -> None:
+        await self._request(
+            f'/repository/{repo}/{ref}', method=HTTPMethod.POST, data=data
+        )
+
+    async def attach_file(self, repo: str, ref: str, file_path: Path) -> None:
+        async with aiofiles.open(file_path, 'rb') as f:
+            form_data = aiohttp.FormData()
+            form_data.add_field('file', f, filename=file_path.name)
+            await self._request(
+                f'/repository/{repo}/{ref}/attach',
+                method=HTTPMethod.POST,
+                data=form_data,
+                json_request=False,
+            )

cafs_cache_cdn_client/repo/datatypes.py

@@ -0,0 +1,34 @@
+from typing import TypedDict
+
+__all__ = (
+    'SettingsResponse',
+    'RefInfoResponse',
+    'RefInfoBody',
+    'FileMetadata',
+)
+
+
+class FileMetadata(TypedDict):
+    blob: str
+    path: str
+    mtime: int
+    mode: int
+
+
+class SettingsResponse(TypedDict):
+    blob_urls: list[str]
+
+
+class RefInfoResponse(TypedDict):
+    revision: int
+    archive: bool
+    ttl: int
+    updated: int
+    files: list[FileMetadata]
+
+
+class RefInfoBody(TypedDict):
+    archive: bool
+    ttl: int
+    comment: str | None
+    files: list[FileMetadata]

cafs_cache_cdn_client-1.0.5.dist-info/METADATA

@@ -0,0 +1,99 @@
+Metadata-Version: 2.3
+Name: cafs-cache-cdn-client
+Version: 1.0.5
+Summary: Async Cache CDN client implementation
+Keywords: cafs,cache
+Author: Konstantin Belov
+Author-email: k.belov@gaijin.team
+Requires-Python: >=3.11,<4.0
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Dist: aiofiles
+Requires-Dist: aiohttp
+Requires-Dist: blake3
+Requires-Dist: zstandard
+Description-Content-Type: text/markdown
+
+# Cache CDN Client
+A Python client library for interacting with the Cache CDN service based on CAFS, allowing efficient pushing and pulling of cached content.
+## Installation
+``` bash
+pip install cafs-cache-cdn-client
+```
+## Features
+- Asynchronous API for high-performance operations
+- Push local directories to cache
+- Pull cached content to local directories
+- Check existence of cached references
+- Tag references for easier access
+- Attach additional files to existing references
+- Delete references when no longer needed
+
+## Usage Example
+```python
+import asyncio
+import logging
+from pathlib import Path
+from cafs_cache_cdn_client import CacheCdnClient
+
+# Configure logging to see detailed operation information
+logging.basicConfig(level=logging.DEBUG)
+
+
+async def main():
+    # Initialize the client with the server URL
+    # The connection_per_cafs_server parameter controls concurrency
+    client = CacheCdnClient(
+        'http://cache-server.example.com:8300',
+        connection_per_cafs_server=10
+    )
+
+    # Use as an async context manager to ensure proper resource cleanup
+    async with client:
+        # Push a local directory to cache with a 2-hour TTL
+        await client.push('project_name', 'build_artifacts',
+                          '/path/to/build/output', ttl_hours=2,
+                          comment='Build artifacts from CI run #123')
+
+        # Check if a reference exists
+        exists = await client.check('project_name', 'build_artifacts')
+        print(f"Reference exists: {exists}")
+
+        # Pull cached content to a local directory
+        await client.pull('project_name', 'build_artifacts',
+                          '/path/to/destination')
+
+        # Tag a reference for easier access later
+        await client.tag('project_name', 'build_artifacts', 'latest_stable')
+
+        # Attach an additional file to an existing reference
+        await client.attach('project_name', 'build_artifacts',
+                            Path('/path/to/metadata.json'))
+
+        # Delete a reference when no longer needed
+        await client.delete('project_name', 'old_artifacts')
+
+
+# Run the example
+if __name__ == '__main__':
+    asyncio.run(main())
+```
+
+## API Reference
+### `CacheCdnClient`
+- **Constructor**: `CacheCdnClient(server: str, connection_per_cafs_server: int = 1)`
+    - `server`: URL of the cache server
+    - `connection_per_cafs_server`: Number of concurrent connections per CAFS server
+
+- **Methods**:
+    - `push(repo: str, ref: str, directory: Path | str, ttl_hours: int = 0, comment: str | None = None)` - Push a local directory to cache
+    - `pull(repo: str, ref: str, directory: Path | str)` - Pull cached content to a local directory
+    - `check(repo: str, ref: str) -> bool` - Check if a reference exists
+    - `tag(repo: str, ref: str, tag: str)` - Create a tag for a reference
+    - `attach(repo: str, ref: str, file_path: Path)` - Attach a file to an existing reference
+    - `delete(repo: str, ref: str)` - Delete a reference
+

cafs_cache_cdn_client-1.0.5.dist-info/RECORD

@@ -0,0 +1,18 @@
+cafs_cache_cdn_client/__init__.py,sha256=N-Gpb6OMWWaEruRwbPjkrZ0Uta8gsQFmq1yFhGcvtcc,35
+cafs_cache_cdn_client/cafs/README.md,sha256=bcn9CEX4O74jY4kRi2oJb37oi55aneyQxDji76SU4Aw,2554
+cafs_cache_cdn_client/cafs/__init__.py,sha256=Ae2WHVqgI9dEuWG5cX-8nbuUH_wpW-5dACt6JGM3FcE,123
+cafs_cache_cdn_client/cafs/blob/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cafs_cache_cdn_client/cafs/blob/hash_.py,sha256=tI4qenyHnEEtKnNUOq4a25L1GnlKjoieyPGQDoz14rA,736
+cafs_cache_cdn_client/cafs/blob/package.py,sha256=pOMF5Dw42APHqIlRfcs1YPscqjcbO32Z35aEG0SqoHo,5865
+cafs_cache_cdn_client/cafs/blob/utils.py,sha256=tqAOvPWIL-sOwXwPiH9Kbsqn5VekJtlf6UO8V9TTtwM,1021
+cafs_cache_cdn_client/cafs/client.py,sha256=TJxwc_PFFmSeCxfTjybh--NrCDzuzuxh3Si42Lwm4Tc,18232
+cafs_cache_cdn_client/cafs/exceptions.py,sha256=I4E3lFrQ_ysR8gDIh3E0Gz4OMihfLD6vLj_KwKzxnTU,654
+cafs_cache_cdn_client/cafs/types.py,sha256=q7zqDQ8yKRn-HRSsXDuzawmikAEcyoYQ-_p2WmuDFy4,324
+cafs_cache_cdn_client/client.py,sha256=d7Dw2U9h8dPYRDPu0mbnXc62SlLy4ruvQk2Alj3cVk8,4584
+cafs_cache_cdn_client/file_utils.py,sha256=kDaak0n3emLIn0DXB02yWFkUhYxuLyTmH93WPszKEUQ,2642
+cafs_cache_cdn_client/repo/__init__.py,sha256=JzCxKznnLV7WElJuznG_jL2s9tYPecJL5uJRnPZsW5M,58
+cafs_cache_cdn_client/repo/client.py,sha256=FEiLefNmvJKwA2gz6dj2gcvt3uKRzLqBARGy4dvXP98,3545
+cafs_cache_cdn_client/repo/datatypes.py,sha256=aTlzFoEibfAOQTqcsg-RmFjnwenS_rBCEMKtN6glOhY,531
+cafs_cache_cdn_client-1.0.5.dist-info/METADATA,sha256=m-dhAfuTdQz97XW280baoDd2vZDxEQW0dtJF-eLqbFI,3679
+cafs_cache_cdn_client-1.0.5.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
+cafs_cache_cdn_client-1.0.5.dist-info/RECORD,,

cafs_cache_cdn_client-1.0.5.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 2.1.2
+Root-Is-Purelib: true
+Tag: py3-none-any