investify-utils 2.0.0a3__tar.gz → 2.0.0a5__tar.gz

{investify_utils-2.0.0a3 → investify_utils-2.0.0a5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: investify-utils
-Version: 2.0.0a3
+Version: 2.0.0a5
 Summary: Shared utilities for Investify services
 Author-Email: Investify <dev@investify.vn>
 License: MIT
@@ -20,8 +20,10 @@ Provides-Extra: postgres-async
 Requires-Dist: pandas>=2.0; extra == "postgres-async"
 Requires-Dist: sqlalchemy>=2.0; extra == "postgres-async"
 Requires-Dist: asyncpg>=0.29; extra == "postgres-async"
-Provides-Extra: postgres-all
-Requires-Dist: investify-utils[postgres,postgres-async]; extra == "postgres-all"
+Provides-Extra: kafka
+Requires-Dist: confluent-kafka>=2.0; extra == "kafka"
+Provides-Extra: s3
+Requires-Dist: boto3>=1.34; extra == "s3"
 Provides-Extra: dev
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: pytest-asyncio; extra == "dev"
@@ -40,9 +42,6 @@ pip install investify-utils[postgres]
 
 # Async PostgreSQL client (asyncpg + SQLAlchemy)
 pip install investify-utils[postgres-async]
-
-# Both clients
-pip install investify-utils[postgres-all]
 ```
 
 ## PostgreSQL Clients

{investify_utils-2.0.0a3 → investify_utils-2.0.0a5}/README.md

@@ -10,9 +10,6 @@ pip install investify-utils[postgres]
 
 # Async PostgreSQL client (asyncpg + SQLAlchemy)
 pip install investify-utils[postgres-async]
-
-# Both clients
-pip install investify-utils[postgres-all]
 ```
 
 ## PostgreSQL Clients

investify_utils-2.0.0a5/investify_utils/kafka/__init__.py

@@ -0,0 +1,43 @@
+"""
+Kafka Avro producer and consumer clients.
+
+Sync (for Celery workers, scripts):
+    from investify_utils.kafka import AvroProducer, AvroConsumer
+
+Async (for LangGraph, FastAPI):
+    from investify_utils.kafka import AsyncAvroProducer, AsyncAvroConsumer
+"""
+
+
+def __getattr__(name: str):
+    """Lazy import to avoid loading confluent-kafka if not needed."""
+    if name == "AvroProducer":
+        from investify_utils.kafka.sync_producer import AvroProducer
+
+        return AvroProducer
+    if name == "AvroConsumer":
+        from investify_utils.kafka.sync_consumer import AvroConsumer
+
+        return AvroConsumer
+    if name == "OffsetTracker":
+        from investify_utils.kafka.sync_consumer import OffsetTracker
+
+        return OffsetTracker
+    if name == "AsyncAvroProducer":
+        from investify_utils.kafka.async_producer import AsyncAvroProducer
+
+        return AsyncAvroProducer
+    if name == "AsyncAvroConsumer":
+        from investify_utils.kafka.async_consumer import AsyncAvroConsumer
+
+        return AsyncAvroConsumer
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
+__all__ = [
+    "AvroProducer",
+    "AvroConsumer",
+    "OffsetTracker",
+    "AsyncAvroProducer",
+    "AsyncAvroConsumer",
+]

investify_utils-2.0.0a5/investify_utils/kafka/async_consumer.py

@@ -0,0 +1,210 @@
+"""
+Asynchronous Avro consumer using confluent-kafka with asyncio.
+
+Features:
+- Non-blocking poll with async/await
+- Suitable for FastAPI websocket streaming
+- Background message fetching with queue
+
+Usage:
+    from investify_utils.kafka import AsyncAvroConsumer
+
+    consumer = AsyncAvroConsumer(
+        topic="my-topic",
+        subject="my-topic-value",
+        schema_registry_url="http://localhost:8081",
+        bootstrap_servers="localhost:9092",
+        group_id="my-consumer-group",
+    )
+
+    # In FastAPI websocket
+    @app.websocket("/ws")
+    async def websocket_endpoint(websocket: WebSocket):
+        await websocket.accept()
+        async for key, value in consumer:
+            await websocket.send_json(value)
+"""
+
+import asyncio
+import logging
+from typing import AsyncIterator
+
+from confluent_kafka import DeserializingConsumer, TopicPartition
+from confluent_kafka.schema_registry import SchemaRegistryClient
+from confluent_kafka.schema_registry.avro import AvroDeserializer
+from confluent_kafka.serialization import StringDeserializer
+
+logger = logging.getLogger(__name__)
+
+
+class AsyncAvroConsumer:
+    """
+    Asynchronous Avro consumer for async frameworks.
+
+    Uses a background thread for polling (confluent-kafka is not async-native)
+    and an asyncio queue to bridge to async code.
+
+    Args:
+        topic: Kafka topic name or list of topics
+        subject: Schema Registry subject name
+        schema_registry_url: Schema Registry URL
+        bootstrap_servers: Kafka bootstrap servers
+        group_id: Consumer group ID
+        seek_to_end: Start from latest offset (default: False)
+        queue_size: Max messages to buffer (default: 100)
+        **kwargs: Additional Kafka consumer config
+    """
+
+    def __init__(
+        self,
+        topic: str | list[str],
+        subject: str,
+        schema_registry_url: str,
+        bootstrap_servers: str,
+        group_id: str,
+        seek_to_end: bool = False,
+        queue_size: int = 100,
+        **kwargs,
+    ):
+        self._schema_registry_url = schema_registry_url
+        self._bootstrap_servers = bootstrap_servers
+        self._subject = subject
+        self._group_id = group_id
+        self._topic = topic
+        self._seek_to_end = seek_to_end
+        self._queue_size = queue_size
+        self._kwargs = kwargs
+
+        self._consumer: DeserializingConsumer | None = None
+        self._queue: asyncio.Queue | None = None
+        self._poll_task: asyncio.Task | None = None
+        self._running = False
+
+    def _init_consumer(self) -> DeserializingConsumer:
+        """Initialize the consumer."""
+        schema_registry_client = SchemaRegistryClient({"url": self._schema_registry_url})
+        registered_schema = schema_registry_client.get_latest_version(self._subject)
+        schema_str = registered_schema.schema.schema_str
+
+        avro_deserializer = AvroDeserializer(schema_registry_client, schema_str)
+
+        consumer_config = {
+            "bootstrap.servers": self._bootstrap_servers,
+            "group.id": self._group_id,
+            "key.deserializer": StringDeserializer("utf_8"),
+            "value.deserializer": avro_deserializer,
+            **self._kwargs,
+        }
+        consumer = DeserializingConsumer(consumer_config)
+
+        topic_list = self._topic if isinstance(self._topic, list) else [self._topic]
+
+        if self._seek_to_end:
+
+            def seek_to_end_assign(c, partitions):
+                for p in partitions:
+                    high_offset = c.get_watermark_offsets(p)[1]
+                    p.offset = high_offset
+                c.assign(partitions)
+
+            consumer.subscribe(topic_list, on_assign=seek_to_end_assign)
+        else:
+            consumer.subscribe(topic_list)
+
+        return consumer
+
+    async def start(self) -> None:
+        """Start consuming messages in background."""
+        if self._running:
+            return
+
+        self._consumer = self._init_consumer()
+        self._queue = asyncio.Queue(maxsize=self._queue_size)
+        self._running = True
+        self._poll_task = asyncio.create_task(self._poll_loop())
+
+    async def _poll_loop(self) -> None:
+        """Background task for polling messages."""
+        loop = asyncio.get_running_loop()
+
+        while self._running:
+            # Run blocking poll in thread executor
+            msg = await loop.run_in_executor(None, self._consumer.poll, 0.1)
+
+            if msg is None:
+                continue
+
+            if msg.error():
+                logger.error(f"Consumer error: {msg.error()}")
+                continue
+
+            # Put message in queue (blocks if full)
+            await self._queue.put((msg.key(), msg.value(), msg))
+
+    async def poll(self, timeout: float = 1.0) -> tuple[str | None, dict | None, object] | None:
+        """
+        Poll for a single message.
+
+        Args:
+            timeout: Maximum time to wait in seconds
+
+        Returns:
+            Tuple of (key, value, raw_msg) or None if timeout
+        """
+        if not self._running:
+            await self.start()
+
+        try:
+            return await asyncio.wait_for(self._queue.get(), timeout=timeout)
+        except asyncio.TimeoutError:
+            return None
+
+    async def __aiter__(self) -> AsyncIterator[tuple[str | None, dict | None]]:
+        """
+        Async iterator for consuming messages.
+
+        Yields:
+            Tuple of (key, value) for each message
+        """
+        if not self._running:
+            await self.start()
+
+        while self._running:
+            try:
+                key, value, _ = await asyncio.wait_for(self._queue.get(), timeout=1.0)
+                yield key, value
+            except asyncio.TimeoutError:
+                continue
+
+    def commit(self) -> None:
+        """Commit current offsets."""
+        if self._consumer:
+            self._consumer.commit()
+
+    def commit_offsets(self, offsets: list[TopicPartition]) -> None:
+        """Commit specific offsets."""
+        if self._consumer:
+            self._consumer.commit(offsets=offsets)
+
+    async def close(self) -> None:
+        """Stop consuming and close the consumer."""
+        self._running = False
+
+        if self._poll_task:
+            self._poll_task.cancel()
+            try:
+                await self._poll_task
+            except asyncio.CancelledError:
+                pass
+            self._poll_task = None
+
+        if self._consumer:
+            self._consumer.close()
+            self._consumer = None
+
+    async def __aenter__(self):
+        await self.start()
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        await self.close()

investify_utils-2.0.0a5/investify_utils/kafka/async_producer.py

@@ -0,0 +1,151 @@
+"""
+Asynchronous Avro producer using confluent-kafka with asyncio.
+
+Features:
+- Non-blocking produce with async/await
+- Background asyncio task for polling
+- Suitable for async frameworks (LangGraph, FastAPI)
+
+Usage:
+    from investify_utils.kafka import AsyncAvroProducer
+
+    producer = AsyncAvroProducer(
+        topic="my-topic",
+        subject="my-topic-value",
+        schema_registry_url="http://localhost:8081",
+        bootstrap_servers="localhost:9092",
+    )
+
+    # In async context
+    await producer.produce(key="key1", value={"field": "value"})
+    producer.close()
+"""
+
+import asyncio
+import logging
+
+from confluent_kafka import KafkaException, SerializingProducer
+from confluent_kafka.schema_registry import SchemaRegistryClient, record_subject_name_strategy
+from confluent_kafka.schema_registry.avro import AvroSerializer
+from confluent_kafka.serialization import StringSerializer
+
+logger = logging.getLogger(__name__)
+
+
+class AsyncAvroProducer:
+    """
+    Asynchronous Avro producer for async frameworks.
+
+    Args:
+        topic: Kafka topic name
+        subject: Schema Registry subject name
+        schema_registry_url: Schema Registry URL
+        bootstrap_servers: Kafka bootstrap servers
+        **kwargs: Additional Kafka producer config
+    """
+
+    def __init__(
+        self,
+        topic: str,
+        subject: str,
+        schema_registry_url: str,
+        bootstrap_servers: str,
+        **kwargs,
+    ):
+        self.topic = topic
+        self._schema_registry_url = schema_registry_url
+        self._bootstrap_servers = bootstrap_servers
+        self._subject = subject
+        self._kwargs = kwargs
+        self._producer: SerializingProducer | None = None
+        self._poll_task: asyncio.Task | None = None
+
+    @property
+    def producer(self) -> SerializingProducer:
+        """Lazy producer initialization."""
+        if self._producer is None:
+            schema_registry_client = SchemaRegistryClient({"url": self._schema_registry_url})
+            registered_schema = schema_registry_client.get_latest_version(self._subject)
+            schema_str = registered_schema.schema.schema_str
+
+            avro_serializer = AvroSerializer(
+                schema_registry_client,
+                schema_str,
+                conf={
+                    "auto.register.schemas": False,
+                    "subject.name.strategy": record_subject_name_strategy,
+                },
+            )
+
+            producer_config = {
+                "bootstrap.servers": self._bootstrap_servers,
+                "key.serializer": StringSerializer("utf_8"),
+                "value.serializer": avro_serializer,
+                **self._kwargs,
+            }
+            self._producer = SerializingProducer(producer_config)
+
+            # Start background polling task
+            self._poll_task = asyncio.create_task(self._poll_loop())
+
+        return self._producer
+
+    async def _poll_loop(self) -> None:
+        """Background task for polling delivery callbacks."""
+        while True:
+            self._producer.poll(0.1)
+            await asyncio.sleep(0.1)
+
+    async def produce(
+        self,
+        value: dict,
+        key: str | None = None,
+        on_delivery: callable | None = None,
+    ) -> asyncio.Future:
+        """
+        Produce a message asynchronously.
+
+        Args:
+            value: Message value (dict matching Avro schema)
+            key: Optional message key
+            on_delivery: Optional callback(err, msg) for delivery confirmation
+
+        Returns:
+            Future that resolves to the delivered message
+        """
+        loop = asyncio.get_running_loop()
+        result = loop.create_future()
+
+        def ack(err, msg):
+            if err:
+                loop.call_soon_threadsafe(result.set_exception, KafkaException(err))
+            else:
+                loop.call_soon_threadsafe(result.set_result, msg)
+            if on_delivery:
+                loop.call_soon_threadsafe(on_delivery, err, msg)
+
+        self.producer.produce(self.topic, key=key, value=value, on_delivery=ack)
+        return await result
+
+    def flush(self, timeout: float = 10.0) -> int:
+        """
+        Wait for all messages to be delivered.
+
+        Args:
+            timeout: Maximum time to wait in seconds
+
+        Returns:
+            Number of messages still in queue
+        """
+        if self._producer:
+            return self._producer.flush(timeout)
+        return 0
+
+    def close(self) -> None:
+        """Cancel polling task and flush pending messages."""
+        if self._poll_task:
+            self._poll_task.cancel()
+            self._poll_task = None
+        if self._producer:
+            self._producer.flush()
+            self._producer = None

investify_utils-2.0.0a5/investify_utils/kafka/sync_consumer.py

@@ -0,0 +1,201 @@
+"""
+Synchronous Avro consumer using confluent-kafka.
+
+Features:
+- Avro deserialization with Schema Registry
+- Offset tracking for reliable commits
+- Seek to end option for real-time streaming
+
+Usage:
+    from investify_utils.kafka import AvroConsumer
+
+    consumer = AvroConsumer(
+        topic="my-topic",
+        subject="my-topic-value",
+        schema_registry_url="http://localhost:8081",
+        bootstrap_servers="localhost:9092",
+        group_id="my-consumer-group",
+    )
+
+    try:
+        while True:
+            msg = consumer.poll(timeout=1.0)
+            if msg is None:
+                continue
+            print(msg.key(), msg.value())
+            consumer.commit()
+    finally:
+        consumer.close()
+"""
+
+import logging
+from collections import defaultdict
+
+from confluent_kafka import DeserializingConsumer, TopicPartition
+from confluent_kafka.schema_registry import SchemaRegistryClient
+from confluent_kafka.schema_registry.avro import AvroDeserializer
+from confluent_kafka.serialization import StringDeserializer
+
+logger = logging.getLogger(__name__)
+
+
+class OffsetTracker:
+    """
+    Track message offsets for reliable commits with out-of-order processing.
+
+    Only commits offsets where all previous offsets have been marked done,
+    preventing data loss when processing messages in parallel.
+    """
+
+    def __init__(self):
+        self._tracker = defaultdict(lambda: defaultdict(lambda: defaultdict(bool)))
+
+    def log(self, msg, done: bool = False) -> None:
+        """
+        Log a message offset.
+
+        Args:
+            msg: Kafka message
+            done: Whether processing is complete
+        """
+        topic = msg.topic()
+        partition = msg.partition()
+        offset = msg.offset()
+        self._tracker[topic][partition][offset] = done
+
+    def prepare_commit_offsets(self) -> list[TopicPartition]:
+        """
+        Get offsets safe to commit.
+
+        Returns:
+            List of TopicPartition with offsets to commit
+        """
+        new_tracker = defaultdict(lambda: defaultdict(lambda: defaultdict(bool)))
+        to_commit_offsets = []
+
+        for topic, partitions in self._tracker.items():
+            for partition, offsets in partitions.items():
+                sorted_offsets = sorted(offsets.items())
+                to_commit_offset = None
+                update_to_commit_offset = True
+
+                for offset, done in sorted_offsets:
+                    update_to_commit_offset = update_to_commit_offset and done
+                    if update_to_commit_offset:
+                        to_commit_offset = offset + 1
+                    else:
+                        new_tracker[topic][partition][offset] = done
+
+                if to_commit_offset is not None:
+                    to_commit_offsets.append(TopicPartition(topic, partition, to_commit_offset))
+
+        self._tracker = new_tracker
+        return to_commit_offsets
+
+
+class AvroConsumer:
+    """
+    Synchronous Avro consumer.
+
+    Args:
+        topic: Kafka topic name or list of topics
+        subject: Schema Registry subject name
+        schema_registry_url: Schema Registry URL
+        bootstrap_servers: Kafka bootstrap servers
+        group_id: Consumer group ID
+        seek_to_end: Start from latest offset (default: False)
+        **kwargs: Additional Kafka consumer config
+    """
+
+    def __init__(
+        self,
+        topic: str | list[str],
+        subject: str,
+        schema_registry_url: str,
+        bootstrap_servers: str,
+        group_id: str,
+        seek_to_end: bool = False,
+        **kwargs,
+    ):
+        self._schema_registry_url = schema_registry_url
+        self._bootstrap_servers = bootstrap_servers
+        self._subject = subject
+        self._group_id = group_id
+        self._topic = topic
+        self._seek_to_end = seek_to_end
+        self._kwargs = kwargs
+        self._consumer: DeserializingConsumer | None = None
+
+    @property
+    def consumer(self) -> DeserializingConsumer:
+        """Lazy consumer initialization."""
+        if self._consumer is None:
+            schema_registry_client = SchemaRegistryClient({"url": self._schema_registry_url})
+            registered_schema = schema_registry_client.get_latest_version(self._subject)
+            schema_str = registered_schema.schema.schema_str
+
+            avro_deserializer = AvroDeserializer(schema_registry_client, schema_str)
+
+            consumer_config = {
+                "bootstrap.servers": self._bootstrap_servers,
+                "group.id": self._group_id,
+                "key.deserializer": StringDeserializer("utf_8"),
+                "value.deserializer": avro_deserializer,
+                **self._kwargs,
+            }
+            self._consumer = DeserializingConsumer(consumer_config)
+
+            topic_list = self._topic if isinstance(self._topic, list) else [self._topic]
+
+            if self._seek_to_end:
+
+                def seek_to_end_assign(consumer, partitions):
+                    for p in partitions:
+                        high_offset = consumer.get_watermark_offsets(p)[1]
+                        p.offset = high_offset
+                    consumer.assign(partitions)
+
+                self._consumer.subscribe(topic_list, on_assign=seek_to_end_assign)
+            else:
+                self._consumer.subscribe(topic_list)
+
+        return self._consumer
+
+    def poll(self, timeout: float = 1.0):
+        """
+        Poll for a message.
+
+        Args:
+            timeout: Maximum time to wait in seconds
+
+        Returns:
+            Message object or None if no message available
+        """
+        msg = self.consumer.poll(timeout)
+        if msg is None:
+            return None
+
+        if msg.error():
+            logger.error(f"Consumer error: {msg.error()}")
+            return None
+
+        return msg
+
+    def commit(self) -> None:
+        """Commit current offsets."""
+        self.consumer.commit()
+
+    def commit_offsets(self, offsets: list[TopicPartition]) -> None:
+        """
+        Commit specific offsets.
+
+        Args:
+            offsets: List of TopicPartition with offsets to commit
+        """
+        self.consumer.commit(offsets=offsets)
+
+    def close(self) -> None:
+        """Close the consumer."""
+        if self._consumer:
+            self._consumer.close()
+            self._consumer = None

investify_utils-2.0.0a5/investify_utils/kafka/sync_producer.py

@@ -0,0 +1,140 @@
+"""
+Synchronous Avro producer using confluent-kafka.
+
+Features:
+- Avro serialization with Schema Registry
+- Background polling thread for delivery callbacks
+- Thread-safe produce operations
+
+Usage:
+    from investify_utils.kafka import AvroProducer
+
+    producer = AvroProducer(
+        topic="my-topic",
+        subject="my-topic-value",
+        schema_registry_url="http://localhost:8081",
+        bootstrap_servers="localhost:9092",
+    )
+
+    producer.produce(key="key1", value={"field": "value"})
+    producer.flush()
+    producer.close()
+"""
+
+import logging
+import threading
+
+from confluent_kafka import SerializingProducer
+from confluent_kafka.schema_registry import SchemaRegistryClient, record_subject_name_strategy
+from confluent_kafka.schema_registry.avro import AvroSerializer
+from confluent_kafka.serialization import StringSerializer
+
+logger = logging.getLogger(__name__)
+
+
+class AvroProducer:
+    """
+    Synchronous Avro producer with background polling.
+
+    Args:
+        topic: Kafka topic name
+        subject: Schema Registry subject name (usually "{topic}-value")
+        schema_registry_url: Schema Registry URL
+        bootstrap_servers: Kafka bootstrap servers
+        **kwargs: Additional Kafka producer config
+    """
+
+    def __init__(
+        self,
+        topic: str,
+        subject: str,
+        schema_registry_url: str,
+        bootstrap_servers: str,
+        **kwargs,
+    ):
+        self.topic = topic
+        self._schema_registry_url = schema_registry_url
+        self._bootstrap_servers = bootstrap_servers
+        self._subject = subject
+        self._kwargs = kwargs
+        self._producer: SerializingProducer | None = None
+        self._shutdown_event: threading.Event | None = None
+        self._poll_thread: threading.Thread | None = None
+
+    @property
+    def producer(self) -> SerializingProducer:
+        """Lazy producer initialization."""
+        if self._producer is None:
+            schema_registry_client = SchemaRegistryClient({"url": self._schema_registry_url})
+            registered_schema = schema_registry_client.get_latest_version(self._subject)
+            schema_str = registered_schema.schema.schema_str
+
+            avro_serializer = AvroSerializer(
+                schema_registry_client,
+                schema_str,
+                conf={
+                    "auto.register.schemas": False,
+                    "subject.name.strategy": record_subject_name_strategy,
+                },
+            )
+
+            producer_config = {
+                "bootstrap.servers": self._bootstrap_servers,
+                "key.serializer": StringSerializer("utf_8"),
+                "value.serializer": avro_serializer,
+                **self._kwargs,
+            }
+            self._producer = SerializingProducer(producer_config)
+
+            # Start background polling thread
+            self._shutdown_event = threading.Event()
+            self._poll_thread = threading.Thread(target=self._poll_loop, daemon=True)
+            self._poll_thread.start()
+
+        return self._producer
+
+    def _poll_loop(self) -> None:
+        """Background thread for polling delivery callbacks."""
+        while not self._shutdown_event.is_set():
+            self._producer.poll(0.1)
+            self._shutdown_event.wait(0.1)
+
+    def produce(
+        self,
+        value: dict,
+        key: str | None = None,
+        on_delivery: callable | None = None,
+    ) -> None:
+        """
+        Produce a message to Kafka.
+
+        Args:
+            value: Message value (dict matching Avro schema)
+            key: Optional message key
+            on_delivery: Optional callback(err, msg) for delivery confirmation
+        """
+        try:
+            self.producer.produce(self.topic, key=key, value=value, on_delivery=on_delivery)
+        except Exception as e:
+            logger.error(f"Failed to produce message: {e!r}")
+            raise
+
+    def flush(self, timeout: float = 10.0) -> int:
+        """
+        Wait for all messages to be delivered.
+
+        Args:
+            timeout: Maximum time to wait in seconds
+
+        Returns:
+            Number of messages still in queue (0 if all delivered)
+        """
+        return self.producer.flush(timeout)
+
+    def close(self) -> None:
+        """Flush pending messages and stop background polling."""
+        if self._shutdown_event:
+            self._shutdown_event.set()
+        if self._producer:
+            self._producer.flush()
+            self._producer = None

investify_utils-2.0.0a5/investify_utils/s3/__init__.py

@@ -0,0 +1,18 @@
+"""
+S3-compatible object storage client.
+
+Usage:
+    from investify_utils.s3 import S3Client
+"""
+
+
+def __getattr__(name: str):
+    """Lazy import to avoid loading boto3 if not needed."""
+    if name == "S3Client":
+        from investify_utils.s3.sync_client import S3Client
+
+        return S3Client
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
+__all__ = ["S3Client"]

investify_utils-2.0.0a5/investify_utils/s3/sync_client.py

@@ -0,0 +1,226 @@
+"""
+S3-compatible object storage client using boto3.
+
+Features:
+- Works with AWS S3, Ceph RGW, MinIO, and other S3-compatible services
+- Lazy client initialization (safe for Celery prefork)
+- Common operations: upload, download, get, put, delete, list
+
+Usage:
+    from investify_utils.s3 import S3Client
+
+    client = S3Client(
+        endpoint_url="https://s3.example.com",
+        access_key="access_key",
+        secret_key="secret_key",
+    )
+
+    # Upload file
+    client.upload_file("local.pdf", bucket="my-bucket", key="remote.pdf")
+
+    # Get object as bytes
+    data = client.get_object("my-bucket", "remote.pdf")
+
+    # Put object from bytes/string
+    client.put_object("my-bucket", "file.txt", b"content", content_type="text/plain")
+"""
+
+import io
+import os
+from typing import IO
+
+import boto3
+from botocore.config import Config
+from botocore.exceptions import ClientError
+
+
+class S3Client:
+    """
+    S3-compatible object storage client with lazy initialization.
+
+    Args:
+        endpoint_url: S3 endpoint URL (e.g., https://s3.amazonaws.com)
+        access_key: AWS access key ID
+        secret_key: AWS secret access key
+        region: AWS region (default: None)
+        **kwargs: Additional boto3 client options
+    """
+
+    def __init__(
+        self,
+        endpoint_url: str,
+        access_key: str,
+        secret_key: str,
+        region: str | None = None,
+        **kwargs,
+    ):
+        self._endpoint_url = endpoint_url
+        self._access_key = access_key
+        self._secret_key = secret_key
+        self._region = region
+        self._kwargs = kwargs
+        self._client = None
+
+    @property
+    def client(self):
+        """Lazy client initialization - created on first access."""
+        if self._client is None:
+            self._client = boto3.client(
+                "s3",
+                endpoint_url=self._endpoint_url,
+                aws_access_key_id=self._access_key,
+                aws_secret_access_key=self._secret_key,
+                region_name=self._region,
+                config=Config(signature_version="s3v4"),
+                **self._kwargs,
+            )
+        return self._client
+
+    def list_buckets(self) -> list[str]:
+        """List all buckets."""
+        response = self.client.list_buckets()
+        return [bucket["Name"] for bucket in response["Buckets"]]
+
+    def list_objects(
+        self,
+        bucket: str,
+        prefix: str = "",
+        max_keys: int | None = None,
+    ) -> list[dict]:
+        """
+        List objects in a bucket with optional prefix filter.
+
+        Args:
+            bucket: Bucket name
+            prefix: Filter objects by prefix (e.g., "folder/")
+            max_keys: Maximum number of objects to return (None = all)
+
+        Returns:
+            List of object metadata dicts with keys: Key, Size, LastModified
+        """
+        objects = []
+        paginator = self.client.get_paginator("list_objects_v2")
+
+        for page in paginator.paginate(Bucket=bucket, Prefix=prefix):
+            for obj in page.get("Contents", []):
+                objects.append({
+                    "Key": obj["Key"],
+                    "Size": obj["Size"],
+                    "LastModified": obj["LastModified"],
+                })
+                if max_keys and len(objects) >= max_keys:
+                    return objects
+
+        return objects
+
+    def upload_file(self, file_path: str, bucket: str, key: str | None = None) -> None:
+        """
+        Upload a local file to S3.
+
+        Args:
+            file_path: Local file path
+            bucket: Bucket name
+            key: Object key (default: basename of file_path)
+        """
+        if key is None:
+            key = os.path.basename(file_path)
+        self.client.upload_file(file_path, bucket, key)
+
+    def download_file(self, bucket: str, key: str, file_path: str) -> None:
+        """
+        Download an object to a local file.
+
+        Args:
+            bucket: Bucket name
+            key: Object key
+            file_path: Local file path to save to
+        """
+        self.client.download_file(bucket, key, file_path)
+
+    def get_object(self, bucket: str, key: str) -> IO[bytes]:
+        """
+        Get object content as a file-like BytesIO object.
+
+        Args:
+            bucket: Bucket name
+            key: Object key
+
+        Returns:
+            BytesIO object with object content
+        """
+        response = self.client.get_object(Bucket=bucket, Key=key)
+        return io.BytesIO(response["Body"].read())
+
+    def put_object(
+        self,
+        bucket: str,
+        key: str,
+        data: str | bytes | IO[bytes],
+        content_type: str | None = None,
+        content_disposition: str | None = None,
+    ) -> None:
+        """
+        Upload data directly to S3.
+
+        Args:
+            bucket: Bucket name
+            key: Object key
+            data: Content as string, bytes, or file-like object
+            content_type: MIME type (e.g., "application/pdf")
+            content_disposition: Content-Disposition header value
+        """
+        params = {"Bucket": bucket, "Key": key, "Body": data}
+        if content_type:
+            params["ContentType"] = content_type
+        if content_disposition:
+            params["ContentDisposition"] = content_disposition
+        self.client.put_object(**params)
+
+    def delete_object(self, bucket: str, key: str) -> None:
+        """Delete a single object."""
+        self.client.delete_object(Bucket=bucket, Key=key)
+
+    def delete_prefix(self, bucket: str, prefix: str) -> int:
+        """
+        Delete all objects with a given prefix.
+
+        Args:
+            bucket: Bucket name
+            prefix: Prefix to match (e.g., "folder/" deletes all in folder)
+
+        Returns:
+            Number of objects deleted
+        """
+        deleted_count = 0
+        paginator = self.client.get_paginator("list_objects_v2")
+
+        for page in paginator.paginate(Bucket=bucket, Prefix=prefix):
+            contents = page.get("Contents", [])
+            if not contents:
+                continue
+
+            delete_keys = [{"Key": obj["Key"]} for obj in contents]
+            self.client.delete_objects(Bucket=bucket, Delete={"Objects": delete_keys})
+            deleted_count += len(delete_keys)
+
+        return deleted_count
+
+    def exists(self, bucket: str, key: str) -> bool:
+        """
+        Check if an object exists.
+
+        Args:
+            bucket: Bucket name
+            key: Object key
+
+        Returns:
+            True if object exists, False otherwise
+        """
+        try:
+            self.client.head_object(Bucket=bucket, Key=key)
+            return True
+        except ClientError as e:
+            if e.response["Error"]["Code"] == "404":
+                return False
+            raise
+

{investify_utils-2.0.0a3 → investify_utils-2.0.0a5}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "pdm.backend"
 
 [project]
 name = "investify-utils"
-version = "2.0.0a3"
+version = "2.0.0a5"
 description = "Shared utilities for Investify services"
 readme = "README.md"
 requires-python = ">=3.12"
@@ -40,8 +40,11 @@ postgres-async = [
     "sqlalchemy>=2.0",
     "asyncpg>=0.29",
 ]
-postgres-all = [
-    "investify-utils[postgres,postgres-async]",
+kafka = [
+    "confluent-kafka>=2.0",
+]
+s3 = [
+    "boto3>=1.34",
 ]
 dev = [
     "pytest",