cafs-cache-cdn-client 1.0.5__tar.gz

cafs_cache_cdn_client-1.0.5/PKG-INFO

@@ -0,0 +1,99 @@
+Metadata-Version: 2.3
+Name: cafs-cache-cdn-client
+Version: 1.0.5
+Summary: Async Cache CDN client implementation
+Keywords: cafs,cache
+Author: Konstantin Belov
+Author-email: k.belov@gaijin.team
+Requires-Python: >=3.11,<4.0
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Dist: aiofiles
+Requires-Dist: aiohttp
+Requires-Dist: blake3
+Requires-Dist: zstandard
+Description-Content-Type: text/markdown
+
+# Cache CDN Client
+A Python client library for interacting with the Cache CDN service based on CAFS, allowing efficient pushing and pulling of cached content.
+## Installation
+``` bash
+pip install cafs-cache-cdn-client
+```
+## Features
+- Asynchronous API for high-performance operations
+- Push local directories to cache
+- Pull cached content to local directories
+- Check existence of cached references
+- Tag references for easier access
+- Attach additional files to existing references
+- Delete references when no longer needed
+
+## Usage Example
+```python
+import asyncio
+import logging
+from pathlib import Path
+from cafs_cache_cdn_client import CacheCdnClient
+
+# Configure logging to see detailed operation information
+logging.basicConfig(level=logging.DEBUG)
+
+
+async def main():
+    # Initialize the client with the server URL
+    # The connection_per_cafs_server parameter controls concurrency
+    client = CacheCdnClient(
+        'http://cache-server.example.com:8300',
+        connection_per_cafs_server=10
+    )
+
+    # Use as an async context manager to ensure proper resource cleanup
+    async with client:
+        # Push a local directory to cache with a 2-hour TTL
+        await client.push('project_name', 'build_artifacts',
+                          '/path/to/build/output', ttl_hours=2,
+                          comment='Build artifacts from CI run #123')
+
+        # Check if a reference exists
+        exists = await client.check('project_name', 'build_artifacts')
+        print(f"Reference exists: {exists}")
+
+        # Pull cached content to a local directory
+        await client.pull('project_name', 'build_artifacts',
+                          '/path/to/destination')
+
+        # Tag a reference for easier access later
+        await client.tag('project_name', 'build_artifacts', 'latest_stable')
+
+        # Attach an additional file to an existing reference
+        await client.attach('project_name', 'build_artifacts',
+                            Path('/path/to/metadata.json'))
+
+        # Delete a reference when no longer needed
+        await client.delete('project_name', 'old_artifacts')
+
+
+# Run the example
+if __name__ == '__main__':
+    asyncio.run(main())
+```
+
+## API Reference
+### `CacheCdnClient`
+- **Constructor**: `CacheCdnClient(server: str, connection_per_cafs_server: int = 1)`
+    - `server`: URL of the cache server
+    - `connection_per_cafs_server`: Number of concurrent connections per CAFS server
+
+- **Methods**:
+    - `push(repo: str, ref: str, directory: Path | str, ttl_hours: int = 0, comment: str | None = None)` - Push a local directory to cache
+    - `pull(repo: str, ref: str, directory: Path | str)` - Pull cached content to a local directory
+    - `check(repo: str, ref: str) -> bool` - Check if a reference exists
+    - `tag(repo: str, ref: str, tag: str)` - Create a tag for a reference
+    - `attach(repo: str, ref: str, file_path: Path)` - Attach a file to an existing reference
+    - `delete(repo: str, ref: str)` - Delete a reference
+

cafs_cache_cdn_client-1.0.5/README.md

@@ -0,0 +1,78 @@
+# Cache CDN Client
+A Python client library for interacting with the Cache CDN service based on CAFS, allowing efficient pushing and pulling of cached content.
+## Installation
+``` bash
+pip install cafs-cache-cdn-client
+```
+## Features
+- Asynchronous API for high-performance operations
+- Push local directories to cache
+- Pull cached content to local directories
+- Check existence of cached references
+- Tag references for easier access
+- Attach additional files to existing references
+- Delete references when no longer needed
+
+## Usage Example
+```python
+import asyncio
+import logging
+from pathlib import Path
+from cafs_cache_cdn_client import CacheCdnClient
+
+# Configure logging to see detailed operation information
+logging.basicConfig(level=logging.DEBUG)
+
+
+async def main():
+    # Initialize the client with the server URL
+    # The connection_per_cafs_server parameter controls concurrency
+    client = CacheCdnClient(
+        'http://cache-server.example.com:8300',
+        connection_per_cafs_server=10
+    )
+
+    # Use as an async context manager to ensure proper resource cleanup
+    async with client:
+        # Push a local directory to cache with a 2-hour TTL
+        await client.push('project_name', 'build_artifacts',
+                          '/path/to/build/output', ttl_hours=2,
+                          comment='Build artifacts from CI run #123')
+
+        # Check if a reference exists
+        exists = await client.check('project_name', 'build_artifacts')
+        print(f"Reference exists: {exists}")
+
+        # Pull cached content to a local directory
+        await client.pull('project_name', 'build_artifacts',
+                          '/path/to/destination')
+
+        # Tag a reference for easier access later
+        await client.tag('project_name', 'build_artifacts', 'latest_stable')
+
+        # Attach an additional file to an existing reference
+        await client.attach('project_name', 'build_artifacts',
+                            Path('/path/to/metadata.json'))
+
+        # Delete a reference when no longer needed
+        await client.delete('project_name', 'old_artifacts')
+
+
+# Run the example
+if __name__ == '__main__':
+    asyncio.run(main())
+```
+
+## API Reference
+### `CacheCdnClient`
+- **Constructor**: `CacheCdnClient(server: str, connection_per_cafs_server: int = 1)`
+    - `server`: URL of the cache server
+    - `connection_per_cafs_server`: Number of concurrent connections per CAFS server
+
+- **Methods**:
+    - `push(repo: str, ref: str, directory: Path | str, ttl_hours: int = 0, comment: str | None = None)` - Push a local directory to cache
+    - `pull(repo: str, ref: str, directory: Path | str)` - Pull cached content to a local directory
+    - `check(repo: str, ref: str) -> bool` - Check if a reference exists
+    - `tag(repo: str, ref: str, tag: str)` - Create a tag for a reference
+    - `attach(repo: str, ref: str, file_path: Path)` - Attach a file to an existing reference
+    - `delete(repo: str, ref: str)` - Delete a reference

cafs_cache_cdn_client-1.0.5/cafs_cache_cdn_client/__init__.py

@@ -0,0 +1 @@
+from .client import CacheCdnClient

cafs_cache_cdn_client-1.0.5/cafs_cache_cdn_client/cafs/README.md

@@ -0,0 +1,63 @@
+# CAFS Client
+
+CAFS Client is a Python library that provides an asynchronous interface for interacting with CAFS servers.
+
+More information about CAFS protocol can be found in the
+[G-CVSNT](https://github.com/GaijinEntertainment/G-CVSNT/tree/master/cvsnt/cvsnt-2.5.05.3744/keyValueServer) repository.
+
+## Usage Example
+
+Below is a complete example demonstrating all major functionality of the CAFSClient:
+
+```python
+import asyncio
+from pathlib import Path
+from cafs_cache_cdn_client.cafs import CAFSClient, CompressionT
+
+
+async def cafs_client_demo():
+    
+    client = CAFSClient(
+        server_root='/data',
+        servers=['localhost', 'example.com:2403'],
+        connection_per_server=2,
+        connect_timeout=5.0
+    )
+    
+    async with client:
+        # 1. Upload a file (stream operation)
+        source_file = Path('./sample.txt')
+        blob_hash = await client.stream(
+            path=source_file,
+            compression=CompressionT.ZSTD,
+        )
+        print(f'File uploaded with hash: {blob_hash}')
+        
+        # 2. Check if the file exists on the server
+        exists = await client.check(blob_hash)
+        print(f'File exists: {exists}')
+        
+        # 3. Get the file size
+        size = await client.size(blob_hash)
+        print(f'File size: {size} bytes')
+        
+        # 4. Download the file (pull operation)
+        download_path = Path('./downloaded_sample.txt')
+        await client.pull(blob_hash, download_path)
+
+if __name__ == '__main__':
+    asyncio.run(cafs_client_demo())
+```
+
+## Retry Mechanism
+
+The CAFSClient implements a robust retry mechanism. This feature ensures that operations attempt to complete even if some servers or connections are unavailable:
+
+- When `retry=True` is specified (default for most operations), the client will automatically retry the operation across all available connections in the pool.
+- The client will iterate through all available connections until either:
+  1. The operation succeeds
+  2. All connections in the pool have been exhausted without success
+
+This behavior makes the client resilient to temporary network issues or server unavailability when multiple servers are configured. For critical operations, always use the default `retry=True` setting to maximize the chances of operation success in distributed environments.
+
+If a specific operation needs to fail immediately without attempting other connections, you can disable this behavior by setting `retry=False` when calling methods like `pull()`, `check()`, `size()`, and `stream()` of the client.

cafs_cache_cdn_client-1.0.5/cafs_cache_cdn_client/cafs/__init__.py

@@ -0,0 +1,7 @@
+from .blob.package import CompressionT
+from .client import CAFSClient
+
+__all__ = (
+    'CAFSClient',
+    'CompressionT',
+)

cafs_cache_cdn_client-1.0.5/cafs_cache_cdn_client/cafs/blob/hash_.py

@@ -0,0 +1,34 @@
+from pathlib import Path
+
+import aiofiles
+from blake3 import blake3
+
+from cafs_cache_cdn_client.cafs.types import AsyncReader
+
+__all__ = (
+    'calc_hash',
+    'calc_hash_file',
+)
+
+DEFAULT_BUFFER_SIZE = 4 * 1024 * 1024
+
+
+async def calc_hash(
+    reader: 'AsyncReader', buffer_size: int = DEFAULT_BUFFER_SIZE
+) -> str:
+    hasher = blake3()  # pylint: disable=not-callable
+
+    while True:
+        buffer = await reader.read(buffer_size)
+        if not buffer:
+            break
+        hasher.update(buffer)
+
+    return hasher.hexdigest()
+
+
+async def calc_hash_file(
+    file_path: Path, buffer_size: int = DEFAULT_BUFFER_SIZE
+) -> str:
+    async with aiofiles.open(file_path, 'rb') as f:
+        return await calc_hash(f, buffer_size)

cafs_cache_cdn_client-1.0.5/cafs_cache_cdn_client/cafs/blob/package.py

@@ -0,0 +1,198 @@
+import zlib
+from enum import Enum
+from logging import Logger, LoggerAdapter, getLogger
+from typing import Protocol
+
+try:
+    import zstandard as zstd
+except ImportError:
+    zstd = None  # type: ignore[assignment]
+
+from cafs_cache_cdn_client.cafs.types import AsyncReader, AsyncWriter
+
+__all__ = (
+    'CompressionT',
+    'Packer',
+    'Unpacker',
+)
+
+
+module_logger = getLogger(__name__)
+
+
+class CompressionT(bytes, Enum):
+    ZSTD = b'ZSTD'
+    ZLIB = b'ZLIB'
+    NONE = b'NONE'
+
+    def __str__(self) -> str:
+        return self.decode('utf-8')
+
+
+FULL_HEADER_SIZE = 16
+COMPRESSION_HEADER_SIZE = 4
+DEFAULT_CHUNK_SIZE = 16 * 1024 * 1024
+
+
+class Compressor(Protocol):
+    def compress(self, data: bytes) -> bytes:
+        pass
+
+    def flush(self) -> bytes:
+        pass
+
+
+class Decompressor(Protocol):
+    def decompress(self, data: bytes) -> bytes:
+        pass
+
+    def flush(self) -> bytes:
+        pass
+
+
+class Packer:
+    logger: Logger | LoggerAdapter
+    chunk_size: int
+
+    _reader: 'AsyncReader'
+    _eof_reached: bool
+    _buffer: bytearray
+    _compressor: Compressor | None
+
+    def __init__(
+        self,
+        reader: 'AsyncReader',
+        compression: CompressionT = CompressionT.NONE,
+        chunk_size: int = DEFAULT_CHUNK_SIZE,
+        logger: Logger | LoggerAdapter | None = None,
+    ) -> None:
+        self._reader = reader
+        self._eof_reached = False
+        self.chunk_size = chunk_size
+
+        self._compressor = None
+        if compression == CompressionT.ZLIB:
+            self._compressor = zlib.compressobj()
+        elif compression == CompressionT.ZSTD:
+            if not zstd:
+                raise RuntimeError(
+                    'ZSTD compression is not available, please install zstandard'
+                )
+            self._compressor = zstd.ZstdCompressor().compressobj()
+
+        self._buffer = bytearray(
+            compression + b'\x00' * (FULL_HEADER_SIZE - COMPRESSION_HEADER_SIZE)
+        )
+        self.logger = logger or module_logger
+        self.logger.debug('Initialized packer with compression: %s', compression)
+
+    async def read(self, size: int = -1) -> bytes:
+        if size == 0:
+            return b''
+
+        while (size > 0 and len(self._buffer) < size) and not self._eof_reached:
+            await self._fill_buffer()
+
+        if size < 0 or len(self._buffer) <= size:
+            result = bytes(self._buffer)
+            self._buffer.clear()
+            return result
+
+        result = bytes(self._buffer[:size])
+        self._buffer = self._buffer[size:]
+        return result
+
+    async def _fill_buffer(self) -> None:
+        chunk = await self._reader.read(self.chunk_size)
+        self.logger.debug('Filling buffer with chunk of %d bytes', len(chunk))
+
+        if not chunk:
+            self._eof_reached = True
+            self.logger.debug('EOF reached')
+            if self._compressor:
+                data = self._compressor.flush()
+                self.logger.debug('Flushing compressor: %d bytes', len(data))
+                self._buffer.extend(data)
+            return
+
+        if not self._compressor:
+            self._buffer.extend(chunk)
+            return
+
+        data = self._compressor.compress(chunk)
+        self.logger.debug('Got %d bytes from compressor', len(data))
+        self._buffer.extend(data)
+
+
+class Unpacker:
+    logger: Logger | LoggerAdapter
+    chunk_size: int
+
+    _writer: 'AsyncWriter'
+    _header: bytearray
+    _buffer: bytearray
+    _decompressor: Decompressor | None
+
+    def __init__(
+        self,
+        writer: 'AsyncWriter',
+        chunk_size: int = DEFAULT_CHUNK_SIZE,
+        logger: Logger | LoggerAdapter | None = None,
+    ) -> None:
+        self._writer = writer
+        self._buffer = bytearray()
+        self._decompressor = None
+        self._header = bytearray()
+        self.chunk_size = chunk_size
+        self.logger = logger or module_logger
+
+    async def write(self, data: bytes, /) -> None:
+        if not data:
+            return
+        await self._fill_buffer(data)
+        if len(self._buffer) >= self.chunk_size:
+            await self._writer.write(self._buffer)
+            self._buffer.clear()
+
+    async def flush(self) -> None:
+        if self._decompressor:
+            data = self._decompressor.flush()
+            self.logger.debug('Flushing decompressor: %d bytes', len(data))
+            self._buffer.extend(data)
+        if self._buffer:
+            await self._writer.write(self._buffer)
+            self._buffer.clear()
+            await self._writer.flush()
+
+    async def _fill_buffer(self, data: bytes) -> None:
+        self.logger.debug('Filling buffer with chunk of %d bytes', len(data))
+        if len(self._header) < FULL_HEADER_SIZE:
+            header_offset = FULL_HEADER_SIZE - len(self._header)
+            self._header.extend(data[:header_offset])
+            data = data[header_offset:]
+            if len(self._header) < FULL_HEADER_SIZE:
+                return
+
+            compression_type = CompressionT(self._header[:COMPRESSION_HEADER_SIZE])
+            self.logger.debug('Extracted compression type: %s', compression_type)
+
+            if compression_type == CompressionT.NONE:
+                self._decompressor = None
+            elif compression_type == CompressionT.ZLIB:
+                d = zlib.decompressobj()
+                self._decompressor = d
+            elif compression_type == CompressionT.ZSTD:
+                if not zstd:
+                    raise RuntimeError('zstandard is not available')
+                self._decompressor = zstd.ZstdDecompressor().decompressobj()
+
+        if not data:
+            return
+
+        if not self._decompressor:
+            self._buffer.extend(data)
+            return
+
+        data = self._decompressor.decompress(data)
+        self.logger.debug('Got %d bytes from decompressor', len(data))
+        self._buffer.extend(data)

cafs_cache_cdn_client-1.0.5/cafs_cache_cdn_client/cafs/blob/utils.py

@@ -0,0 +1,37 @@
+from pathlib import Path
+
+from .package import CompressionT
+
+__all__ = ('choose_compression',)
+
+MAGIC_HEADER_SIZE = 4
+MINIMAL_COMPRESSION_SIZE = 1024
+
+
+# Magic header prefixes for various compression formats
+MAGIC_HEADER_PREFIXES = [
+    bytes([0x1F, 0x8B]),  # gzip
+    bytes([0x42, 0x5A, 0x68]),  # bzip2
+    bytes([0x50, 0x4B, 0x03]),  # zip
+    bytes([0x28, 0xB5, 0x2F, 0xFD]),  # zstd
+    bytes([0x78, 0x01]),  # default compression level
+]
+
+
+def is_file_already_compressed(file_path: Path) -> bool:
+    with open(file_path, 'rb') as file:
+        magic_header_buff = file.read(MAGIC_HEADER_SIZE)
+
+    return any(magic_header_buff.startswith(prefix) for prefix in MAGIC_HEADER_PREFIXES)
+
+
+def choose_compression(
+    file_path: Path, preferred_compression: CompressionT = CompressionT.NONE
+) -> CompressionT:
+    if file_path.stat().st_size < MINIMAL_COMPRESSION_SIZE:
+        return CompressionT.NONE
+
+    if is_file_already_compressed(file_path):
+        return CompressionT.NONE
+
+    return preferred_compression