cafs-cache-cdn-client 1.0.8__tar.gz → 1.0.9__tar.gz

{cafs_cache_cdn_client-1.0.8 → cafs_cache_cdn_client-1.0.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: cafs-cache-cdn-client
-Version: 1.0.8
+Version: 1.0.9
 Summary: Async Cache CDN client implementation
 Keywords: cafs,cache
 Author: Konstantin Belov
@@ -49,7 +49,8 @@ async def main():
     # The connection_per_cafs_server parameter controls concurrency
     client = CacheCdnClient(
         'http://cache-server.example.com:8300',
-        connection_per_cafs_server=10
+        connection_per_cafs_server=10,
+        verbose_debug=True,
     )
 
     # Use as an async context manager to ensure proper resource cleanup
@@ -89,6 +90,8 @@ if __name__ == '__main__':
 - **Constructor**: `CacheCdnClient(server: str, connection_per_cafs_server: int = 1)`
     - `server`: URL of the cache server
     - `connection_per_cafs_server`: Number of concurrent connections per CAFS server
+    - `logger`: Optional logger for custom logging
+    - `verbose_debug`: Enable verbose debug logging (default: `False`)
 
 - **Methods**:
     - `push(repo: str, ref: str, directory: Path | str, ttl_hours: int = 0, comment: str | None = None, compression: CompressionT = CompressionT.NONE)` - Push a local directory to cache

{cafs_cache_cdn_client-1.0.8 → cafs_cache_cdn_client-1.0.9}/README.md

@@ -29,7 +29,8 @@ async def main():
     # The connection_per_cafs_server parameter controls concurrency
     client = CacheCdnClient(
         'http://cache-server.example.com:8300',
-        connection_per_cafs_server=10
+        connection_per_cafs_server=10,
+        verbose_debug=True,
     )
 
     # Use as an async context manager to ensure proper resource cleanup
@@ -69,6 +70,8 @@ if __name__ == '__main__':
 - **Constructor**: `CacheCdnClient(server: str, connection_per_cafs_server: int = 1)`
     - `server`: URL of the cache server
     - `connection_per_cafs_server`: Number of concurrent connections per CAFS server
+    - `logger`: Optional logger for custom logging
+    - `verbose_debug`: Enable verbose debug logging (default: `False`)
 
 - **Methods**:
     - `push(repo: str, ref: str, directory: Path | str, ttl_hours: int = 0, comment: str | None = None, compression: CompressionT = CompressionT.NONE)` - Push a local directory to cache

{cafs_cache_cdn_client-1.0.8 → cafs_cache_cdn_client-1.0.9}/cafs_cache_cdn_client/cafs/README.md

@@ -11,17 +11,22 @@ Below is a complete example demonstrating all major functionality of the CAFSCli
 
 ```python
 import asyncio
+import logging
 from pathlib import Path
 from cafs_cache_cdn_client.cafs import CAFSClient, CompressionT
 
 
+# Configure logging to see detailed operation information
+logging.basicConfig(level=logging.DEBUG)
+
 async def cafs_client_demo():
     
     client = CAFSClient(
         server_root='/data',
         servers=['localhost', 'example.com:2403'],
         connection_per_server=2,
-        connect_timeout=5.0
+        connect_timeout=5.0,
+        verbose_debug=True,  # Enable verbose debug logging
     )
     
     async with client:

{cafs_cache_cdn_client-1.0.8 → cafs_cache_cdn_client-1.0.9}/cafs_cache_cdn_client/cafs/blob/package.py

@@ -53,6 +53,7 @@ class Decompressor(Protocol):
 class Packer:
     logger: Logger | LoggerAdapter
     chunk_size: int
+    verbose_debug: bool
 
     _reader: 'AsyncReader'
     _eof_reached: bool
@@ -65,10 +66,12 @@ class Packer:
         compression: CompressionT = CompressionT.NONE,
         chunk_size: int = DEFAULT_CHUNK_SIZE,
         logger: Logger | LoggerAdapter | None = None,
+        verbose_debug: bool = False,
     ) -> None:
         self._reader = reader
         self._eof_reached = False
         self.chunk_size = chunk_size
+        self.verbose_debug = verbose_debug
 
         self._compressor = None
         if compression == CompressionT.ZLIB:
@@ -104,14 +107,16 @@ class Packer:
 
     async def _fill_buffer(self) -> None:
         chunk = await self._reader.read(self.chunk_size)
-        self.logger.debug('Filling buffer with chunk of %d bytes', len(chunk))
+        if self.verbose_debug:
+            self.logger.debug('Filling buffer with chunk of %d bytes', len(chunk))
 
         if not chunk:
             self._eof_reached = True
             self.logger.debug('EOF reached')
             if self._compressor:
                 data = self._compressor.flush()
-                self.logger.debug('Flushing compressor: %d bytes', len(data))
+                if self.verbose_debug:
+                    self.logger.debug('Flushing compressor: %d bytes', len(data))
                 self._buffer.extend(data)
             return
 
@@ -120,13 +125,15 @@ class Packer:
             return
 
         data = self._compressor.compress(chunk)
-        self.logger.debug('Got %d bytes from compressor', len(data))
+        if self.verbose_debug:
+            self.logger.debug('Got %d bytes from compressor', len(data))
         self._buffer.extend(data)
 
 
 class Unpacker:
     logger: Logger | LoggerAdapter
     chunk_size: int
+    verbose_debug: bool
 
     _writer: 'AsyncWriter'
     _header: bytearray
@@ -138,6 +145,7 @@ class Unpacker:
         writer: 'AsyncWriter',
         chunk_size: int = DEFAULT_CHUNK_SIZE,
         logger: Logger | LoggerAdapter | None = None,
+        verbose_debug: bool = False,
     ) -> None:
         self._writer = writer
         self._buffer = bytearray()
@@ -145,6 +153,7 @@ class Unpacker:
         self._header = bytearray()
         self.chunk_size = chunk_size
         self.logger = logger or module_logger
+        self.verbose_debug = verbose_debug
 
     async def write(self, data: bytes, /) -> None:
         if not data:
@@ -157,7 +166,8 @@ class Unpacker:
     async def flush(self) -> None:
         if self._decompressor:
             data = self._decompressor.flush()
-            self.logger.debug('Flushing decompressor: %d bytes', len(data))
+            if self.verbose_debug:
+                self.logger.debug('Flushing decompressor: %d bytes', len(data))
             self._buffer.extend(data)
         if self._buffer:
             await self._writer.write(self._buffer)
@@ -165,7 +175,8 @@ class Unpacker:
             await self._writer.flush()
 
     async def _fill_buffer(self, data: bytes) -> None:
-        self.logger.debug('Filling buffer with chunk of %d bytes', len(data))
+        if self.verbose_debug:
+            self.logger.debug('Filling buffer with chunk of %d bytes', len(data))
         if len(self._header) < FULL_HEADER_SIZE:
             header_offset = FULL_HEADER_SIZE - len(self._header)
             self._header.extend(data[:header_offset])
@@ -194,5 +205,6 @@ class Unpacker:
             return
 
         data = self._decompressor.decompress(data)
-        self.logger.debug('Got %d bytes from decompressor', len(data))
+        if self.verbose_debug:
+            self.logger.debug('Got %d bytes from decompressor', len(data))
         self._buffer.extend(data)

{cafs_cache_cdn_client-1.0.8 → cafs_cache_cdn_client-1.0.9}/cafs_cache_cdn_client/cafs/client.py

@@ -8,7 +8,7 @@ from collections.abc import (
 )
 from contextlib import asynccontextmanager
 from enum import Enum
-from logging import LoggerAdapter, getLogger
+from logging import Logger, LoggerAdapter, getLogger
 from pathlib import Path
 from typing import Any, Self, TypeVar
 
@@ -80,7 +80,8 @@ class CAFSConnection:
     port: int
     timeout: float
     server_root: bytes
-    module_logger: ConnectionLoggerAdapter
+    logger: ConnectionLoggerAdapter
+    verbose_debug: bool
 
     _reader: asyncio.StreamReader | None = None
     _writer: asyncio.StreamWriter | None = None
@@ -91,6 +92,8 @@ class CAFSConnection:
         host: str,
         port: int = CAFS_DEFAULT_PORT,
         timeout: float = DEFAULT_CONNECT_TIMEOUT,
+        logger: Logger | LoggerAdapter | None = None,
+        verbose_debug: bool = False,
     ) -> None:
         self.server_root = server_root.encode('utf-8')
         self.host = host
@@ -98,8 +101,10 @@ class CAFSConnection:
         self.timeout = timeout
         self.is_connected = False
         self.logger = ConnectionLoggerAdapter(
-            module_logger, {'host': host, 'port': port, 'connection_id': id(self)}
+            logger or module_logger,
+            {'host': host, 'port': port, 'connection_id': id(self)},
         )
+        self.verbose_debug = verbose_debug
 
     async def connect(self) -> None:
         if self.is_connected:
@@ -219,9 +224,10 @@ class CAFSConnection:
         chunk = await reader.read(STREAM_MAX_CHUNK_SIZE)
         while chunk:
             size_header = len(chunk).to_bytes(2, 'little')
-            self.logger.debug(
-                'Streaming chunk of size: %d (%s)', len(chunk), size_header.hex()
-            )
+            if self.verbose_debug:
+                self.logger.debug(
+                    'Streaming chunk of size: %d (%s)', len(chunk), size_header.hex()
+                )
             await self._send(size_header)
             await self._send(chunk)
             chunk = await reader.read(STREAM_MAX_CHUNK_SIZE)
@@ -262,10 +268,12 @@ class CAFSConnection:
 
         while received < blob_size:
             chunk_size = min(STREAM_MAX_CHUNK_SIZE, blob_size - received)
-            self.logger.debug('Pulling chunk of size: %d', chunk_size)
+            if self.verbose_debug:
+                self.logger.debug('Pulling chunk of size: %d', chunk_size)
             chunk = await self._receive(chunk_size)
             received += chunk_size
-            self.logger.debug('Received %d bytes', len(chunk))
+            if self.verbose_debug:
+                self.logger.debug('Received %d bytes', len(chunk))
             await writer.write(chunk)
 
         await writer.flush()
@@ -278,6 +286,8 @@ class ConnectionPool:
     server_root: str
     servers: set[tuple[str, int]]
     connection_per_server: int
+    logger: Logger | LoggerAdapter
+    verbose_debug: bool
 
     _lock: asyncio.Lock
     _connections: set[CAFSConnection]
@@ -290,6 +300,8 @@ class ConnectionPool:
         servers: Collection[tuple[str, int]],
         connection_per_server: int = 1,
         connect_timeout: float = DEFAULT_CONNECT_TIMEOUT,
+        logger: Logger | LoggerAdapter | None = None,
+        verbose_debug: bool = False,
     ) -> None:
         self.server_root = server_root
         self.connect_timeout = connect_timeout
@@ -300,26 +312,40 @@ class ConnectionPool:
         self._connection_queue = asyncio.Queue()
         self._lock = asyncio.Lock()
         self._close_event = asyncio.Event()
+        self.logger = logger or module_logger
+        self.verbose_debug = verbose_debug
 
     async def get_connection_count(self) -> int:
         async with self._lock:
             return len(self._connections)
 
     async def initialize(self) -> None:
+        self.logger.debug(
+            'Initializing connection pool with %d servers (%d connections each)',
+            len(self.servers),
+            self.connection_per_server,
+        )
         for server in self.servers:
             host, port = server
             for _ in range(self.connection_per_server):
                 conn = CAFSConnection(
-                    self.server_root, host, port, timeout=self.connect_timeout
+                    self.server_root,
+                    host,
+                    port,
+                    timeout=self.connect_timeout,
+                    logger=self.logger,
+                    verbose_debug=self.verbose_debug,
                 )
                 self._connections.add(conn)
                 await self._connection_queue.put(conn)
+        self.logger.debug('Connection pool initialized')
 
     async def _get_connection(self) -> CAFSConnection:
         if self._close_event.is_set():
             raise EmptyConnectionPoolError()
         get_task = asyncio.create_task(self._connection_queue.get())
         close_task = asyncio.create_task(self._close_event.wait())
+        self.logger.debug('Waiting for connection')
         _, pending = await asyncio.wait(
             [get_task, close_task], return_when=asyncio.FIRST_COMPLETED
         )
@@ -330,19 +356,25 @@ class ConnectionPool:
         if get_task in pending:
             raise EmptyConnectionPoolError()
 
-        return get_task.result()
+        conn = get_task.result()
+        self.logger.debug('Got connection %s', id(conn))
+        return conn
 
     async def _release_connection(self, conn: CAFSConnection) -> None:
+        self.logger.debug('Releasing connection %s', id(conn))
         await self._connection_queue.put(conn)
 
     async def _delete_connection(self, conn: CAFSConnection) -> None:
+        self.logger.debug('Deleting connection %s', id(conn))
         await conn.disconnect()
         async with self._lock:
             self._connections.remove(conn)
             if not self._connections:
+                self.logger.debug('Connection pool is empty, closing')
                 self._close_event.set()
 
     async def close(self) -> None:
+        self.logger.debug('Closing connection pool')
         async with self._lock:
             self._close_event.set()
             for conn in self._connections:
@@ -387,6 +419,9 @@ async def _until_pool_empty_wrapper(
 
 
 class CAFSClient:
+    logger: Logger | LoggerAdapter
+    verbose_debug: bool
+
     _connection_pool: ConnectionPool
 
     def __init__(
@@ -395,19 +430,31 @@ class CAFSClient:
         servers: Collection[str],
         connection_per_server: int = 1,
         connect_timeout: float = DEFAULT_CONNECT_TIMEOUT,
+        logger: Logger | LoggerAdapter | None = None,
+        verbose_debug: bool = False,
     ) -> None:
+        self.verbose_debug = verbose_debug
         servers_ = {self.parse_server_uri(server) for server in servers}
+        self.logger = logger or module_logger
         self._connection_pool = ConnectionPool(
-            server_root, servers_, connection_per_server, connect_timeout
+            server_root,
+            servers_,
+            connection_per_server,
+            connect_timeout,
+            logger=self.logger,
+            verbose_debug=self.verbose_debug,
         )
 
     async def pull(self, blob_hash: str, path: Path, retry: bool = True) -> None:
+        self.logger.info('Pulling %s to %s', blob_hash, path)
         await aio_os.makedirs(path.parent, exist_ok=True)
 
         async def _pull() -> None:
             async with aiofiles.open(path, 'wb') as file:
                 async with self._connection_pool.connection() as conn:
-                    unpacker = Unpacker(file, logger=conn.logger)
+                    unpacker = Unpacker(
+                        file, logger=conn.logger, verbose_debug=self.verbose_debug
+                    )
                     await conn.pull(blob_hash, unpacker)
 
         await _until_pool_empty_wrapper(_pull, retry=retry)
@@ -416,10 +463,16 @@ class CAFSClient:
         self,
         blobs: list[tuple[str, Path]],
         retry: bool = True,
+        max_concurrent: int | None = None,
     ) -> None:
         if not blobs:
             return
 
+        max_concurrent = min(
+            max_concurrent or await self._connection_pool.get_connection_count(),
+            len(blobs),
+        )
+
         files_queue: asyncio.Queue[tuple[str, Path]] = asyncio.Queue()
         for blob_hash, blob_path in blobs:
             files_queue.put_nowait((blob_hash, blob_path))
@@ -439,7 +492,10 @@ class CAFSClient:
                     files_queue.task_done()
 
         stop_event = asyncio.Event()
-        workers = [asyncio.create_task(worker(stop_event)) for _ in range(len(blobs))]
+        self.logger.debug('Initializing %d workers', max_concurrent)
+        workers = [
+            asyncio.create_task(worker(stop_event)) for _ in range(max_concurrent)
+        ]
         errors = await asyncio.gather(*workers, return_exceptions=True)
 
         for err in errors:
@@ -468,11 +524,19 @@ class CAFSClient:
     ) -> str:
         blob_hash: str = await calc_hash_file(path)
         compression = choose_compression(path, preferred_compression=compression)
+        self.logger.info(
+            'Streaming %s to %s with %s compression', path, blob_hash, compression
+        )
 
         async def _stream() -> str:
             async with aiofiles.open(path, 'rb') as file:
                 async with self._connection_pool.connection() as conn:
-                    packer = Packer(file, compression=compression, logger=conn.logger)
+                    packer = Packer(
+                        file,
+                        compression=compression,
+                        logger=conn.logger,
+                        verbose_debug=self.verbose_debug,
+                    )
                     await conn.stream(blob_hash, packer)
             return blob_hash
 
@@ -516,6 +580,7 @@ class CAFSClient:
                     files_queue.task_done()
 
         stop_event = asyncio.Event()
+        self.logger.debug('Initializing %d workers', max_concurrent)
         workers = [
             asyncio.create_task(worker(stop_event)) for _ in range(max_concurrent)
         ]

{cafs_cache_cdn_client-1.0.8 → cafs_cache_cdn_client-1.0.9}/cafs_cache_cdn_client/client.py

@@ -1,6 +1,7 @@
 import asyncio
 import functools
 from collections.abc import Awaitable, Callable
+from logging import Logger, LoggerAdapter, getLogger
 from os.path import normpath
 from pathlib import Path
 from typing import Any, Self, TypeVar
@@ -19,6 +20,9 @@ from cafs_cache_cdn_client.repo import RepoClient
 __all__ = ('CacheCdnClient',)
 
 
+package_logger = getLogger(__name__.split('.')[0])
+
+
 CAFS_SERVER_ROOT = '/cache'
 
 
@@ -35,25 +39,44 @@ def needs_cafs_client(func: Callable[..., Awaitable[T]]) -> Callable[..., Awaita
 
 
 class CacheCdnClient:
+    verbose_debug: bool
+
     _cafs_client: CAFSClient | None = None
     _repo_client: RepoClient
 
     __connection_per_cafs_server: int
     __cafs_client_lock = asyncio.Lock()
 
-    def __init__(self, server: str, connection_per_cafs_server: int = 1) -> None:
-        self._repo_client = RepoClient(server)
+    def __init__(
+        self,
+        server: str,
+        connection_per_cafs_server: int = 1,
+        logger: Logger | LoggerAdapter | None = None,
+        verbose_debug: bool = False,
+    ) -> None:
+        self.logger = logger or package_logger
+        self._repo_client = RepoClient(server, logger=self.logger)
         self.__connection_per_cafs_server = connection_per_cafs_server
+        self.verbose_debug = verbose_debug
 
     async def _init_cafs_client(self) -> None:
+        self.logger.debug(
+            'Initializing CAFS client with %d connections per server',
+            self.__connection_per_cafs_server,
+        )
         async with self.__cafs_client_lock:
             if self._cafs_client:
+                self.logger.debug('CAFS client already initialized')
                 return
+            self.logger.debug('Fetching blob URLs from the server')
             blob_urls = await self._repo_client.get_blob_urls()
+            self.logger.debug('Blob URLs: %s', blob_urls)
             self._cafs_client = await CAFSClient(
                 CAFS_SERVER_ROOT,
                 blob_urls,
                 connection_per_server=self.__connection_per_cafs_server,
+                logger=self.logger,
+                verbose_debug=self.verbose_debug,
             ).__aenter__()
 
     @needs_cafs_client
@@ -66,15 +89,25 @@ class CacheCdnClient:
         comment: str | None = None,
         compression: CompressionT = CompressionT.NONE,
     ) -> None:
+        self.logger.info(
+            'Pushing %s to %s/%s with ttl=%d hours, compression=%s',
+            directory,
+            repo,
+            ref,
+            ttl_hours,
+            compression,
+        )
         if isinstance(directory, str):
             directory = Path(directory)
         if not directory.is_dir():
             raise ValueError(f'{directory} is not a directory')
         files = walk(directory)
+        self.logger.debug('Uploading %d files to CAFS server')
         hashes = await self._cafs_client.stream_batch(
             [directory / file.path for file in files],
             compression=compression,
         )
+        self.logger.debug('CAFS upload complete, uploading metadata to the server')
         await self._repo_client.post_ref_info(
             repo,
             ref,
@@ -93,21 +126,34 @@ class CacheCdnClient:
                 ],
             },
         )
+        self.logger.info('Pushed %d files to %s/%s successfully', len(files), repo, ref)
 
     async def check(self, repo: str, ref: str) -> bool:
-        return await self._repo_client.is_ref_exist(repo, ref)
+        self.logger.info('Checking %s/%s', repo, ref)
+        res = await self._repo_client.is_ref_exist(repo, ref)
+        if res:
+            self.logger.info('Ref %s/%s exists', repo, ref)
+        else:
+            self.logger.info('Ref %s/%s does not exist', repo, ref)
+        return res
 
     async def delete(self, repo: str, ref: str) -> None:
+        self.logger.info('Deleting %s/%s', repo, ref)
         await self._repo_client.delete_ref(repo, ref)
+        self.logger.info('Deleted %s/%s successfully', repo, ref)
 
     async def attach(self, repo: str, ref: str, file_path: Path) -> None:
+        self.logger.info('Attaching %s to %s/%s', file_path, repo, ref)
         await self._repo_client.attach_file(repo, ref, file_path)
+        self.logger.info('Attached %s to %s/%s successfully', file_path, repo, ref)
 
     @needs_cafs_client
     async def pull(self, repo: str, ref: str, directory: Path | str) -> None:
+        self.logger.info('Pulling %s/%s to %s', repo, ref, directory)
         if isinstance(directory, str):
             directory = Path(directory)
         await aio_os.makedirs(directory, exist_ok=True)
+        self.logger.debug('Fetching info about %s/%s from the server', repo, ref)
         ref_info = await self._repo_client.get_ref_info(repo, ref)
         remote_files = [
             LocalFile(
@@ -118,21 +164,41 @@ class CacheCdnClient:
             )
             for file in ref_info['files']
         ]
+        self.logger.debug('%d files on the server', len(remote_files))
         local_files = walk(directory)
+        self.logger.debug('%d files locally', len(local_files))
         to_remove, to_add, to_update = await compare_file_lists(
             local_files, remote_files, directory
         )
+        self.logger.debug(
+            'Files to remove: %d, files to add: %d, files to update: %d',
+            len(to_remove),
+            len(to_add),
+            len(to_update),
+        )
         for file in to_remove:
             await aio_os.unlink(directory / file.path)
         if to_add:
+            self.logger.debug('Downloading %d files from CAFS server', len(to_add))
             await self._cafs_client.pull_batch(
                 [(file.blob, directory / file.path) for file in to_add]
             )
+            self.logger.debug('CAFS download complete')
         for file in to_add + to_update:
             set_file_stat(file, directory)
+        self.logger.info(
+            'Pulled %d files from %s/%s successfully, updated %d files, removed %d files',
+            len(to_add),
+            repo,
+            ref,
+            len(to_update),
+            len(to_remove),
+        )
 
     async def tag(self, repo: str, ref: str, tag: str) -> None:
+        self.logger.info('Tagging %s/%s to %s', repo, ref, tag)
         await self._repo_client.tag_ref(repo, ref, tag)
+        self.logger.info('Tagged %s/%s to %s successfully', repo, ref, tag)
 
     async def __aenter__(self) -> Self:
         return self

{cafs_cache_cdn_client-1.0.8 → cafs_cache_cdn_client-1.0.9}/cafs_cache_cdn_client/repo/client.py

@@ -1,5 +1,6 @@
 from collections.abc import Iterable
 from http import HTTPMethod
+from logging import Logger, LoggerAdapter, getLogger
 from pathlib import Path
 from typing import Any, cast
 from urllib.parse import quote, urljoin
@@ -13,11 +14,18 @@ import cafs_cache_cdn_client.repo.datatypes as dt
 __all__ = ('RepoClient',)
 
 
+module_logger = getLogger(__name__)
+
+
 class RepoClient:
     server_base_url: str
+    logger: Logger | LoggerAdapter
 
-    def __init__(self, server: str) -> None:
+    def __init__(
+        self, server: str, logger: Logger | LoggerAdapter | None = None
+    ) -> None:
         self.server_base_url = server
+        self.logger = logger or module_logger
 
     async def _request(
         self,
@@ -38,6 +46,7 @@ class RepoClient:
             data_arg = {'json': data}
         else:
             data_arg = {'data': data}
+        self.logger.debug('Requesting %s', url_)
         async with aiohttp.ClientSession(
             headers=headers, requote_redirect_url=False
         ) as session:

{cafs_cache_cdn_client-1.0.8 → cafs_cache_cdn_client-1.0.9}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cafs-cache-cdn-client"
-version = "1.0.8"
+version = "1.0.9"
 description = "Async Cache CDN client implementation"
 authors = [
     { name = "Konstantin Belov", "email" = "k.belov@gaijin.team" },