minix 0.1.0__tar.gz

minix-0.1.0/PKG-INFO

@@ -0,0 +1,21 @@
+Metadata-Version: 2.3
+Name: minix
+Version: 0.1.0
+Summary: 
+Author: AmirHossein Advari
+Author-email: amiradvari@gmail.com
+Requires-Python: >=3.12
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: aiokafka (>=0.12.0,<0.13.0)
+Requires-Dist: boto3 (>=1.39.14,<2.0.0)
+Requires-Dist: celery (>=5.5.3,<6.0.0)
+Requires-Dist: dotenv (>=0.9.9,<0.10.0)
+Requires-Dist: fastapi (>=0.116.1,<0.117.0)
+Requires-Dist: pymysql (>=1.1.1,<2.0.0)
+Requires-Dist: sqlalchemy (>=2.0.41,<3.0.0)
+Requires-Dist: uvicorn (>=0.35.0,<0.36.0)
+Description-Content-Type: text/markdown
+
+Simple Backend framework for data applications.

minix-0.1.0/README.md

@@ -0,0 +1 @@
+Simple Backend framework for data applications.

minix-0.1.0/pyproject.toml

@@ -0,0 +1,28 @@
+[project]
+name = "minix"
+version = "0.1.0"
+description = ""
+authors = [
+    {name = "AmirHossein Advari",email = "amiradvari@gmail.com"}
+]
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "fastapi (>=0.116.1,<0.117.0)",
+    "uvicorn (>=0.35.0,<0.36.0)",
+    "celery (>=5.5.3,<6.0.0)",
+    "boto3 (>=1.39.14,<2.0.0)",
+    "aiokafka (>=0.12.0,<0.13.0)",
+    "sqlalchemy (>=2.0.41,<3.0.0)",
+    "dotenv (>=0.9.9,<0.10.0)",
+    "pymysql (>=1.1.1,<2.0.0)"
+]
+
+
+[build-system]
+requires = ["poetry-core>=2.0.0,<3.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+
+[tool.poetry.extras]
+ai = ["torch", "mlflow"]

minix-0.1.0/src/minix/core/bootstrap/__init__.py

@@ -0,0 +1 @@
+from .bootstrap import bootstrap

minix-0.1.0/src/minix/core/bootstrap/bootstrap.py

@@ -0,0 +1,58 @@
+import pymysql
+
+from src.minix.core.connectors import Connector
+from src.minix.core.module import Module
+from fastapi import FastAPI
+from src.minix.core.registry import Registry
+from src.minix.core.scheduler import SchedulerConfig, Scheduler
+pymysql.install_as_MySQLdb()
+import os
+import dotenv
+dotenv.load_dotenv()
+
+
+def register_connectors(connectors: list[Connector]):
+    for connector in connectors:
+        Registry().register(connector.__class__, connector)
+def register_scheduler():
+    Registry().register(Scheduler, Scheduler(
+        SchedulerConfig()
+        .set_broker_url(os.getenv('CELERY_BROKER_URL'))
+        .set_result_backend(os.getenv('CELERY_RESULT_BACKEND'))
+        .set_task_serializer('json')
+        .set_result_serializer('json')
+        .set_accept_content(['json'])
+        .set_timezone('GMT')
+    ))
+def register_fast_api():
+    Registry().register(FastAPI, FastAPI())
+def register_modules(modules: list[Module]):
+    fast_api = False
+    scheduler = False
+    for module in modules:
+        if module.controllers is not None and len(module.controllers) > 0:
+            fast_api = True
+        if module.periodic_tasks is not None and len(module.periodic_tasks) > 0:
+            scheduler = True
+        if module.tasks is not None and len(module.tasks) > 0:
+            scheduler = True
+
+    if fast_api:
+        register_fast_api()
+    if scheduler:
+        register_scheduler()
+    for module in modules:
+        module.install()
+
+
+def bootstrap(
+        modules: list[Module] = None,
+        connectors: list[Connector] = None
+):
+
+    if modules:
+        register_modules(modules)
+    if connectors:
+        register_connectors(connectors)
+
+

minix-0.1.0/src/minix/core/connectors/__init__.py

@@ -0,0 +1 @@
+from .connector import Connector

minix-0.1.0/src/minix/core/connectors/connector.py

@@ -0,0 +1,4 @@
+
+
+class Connector:
+    pass

minix-0.1.0/src/minix/core/connectors/object_storage_connector/__init__.py

@@ -0,0 +1,4 @@
+from .connector import ObjectStorageConnector
+from .config import ObjectStorageConfig
+
+__all__ = ['ObjectStorageConnector', 'ObjectStorageConfig'] 

minix-0.1.0/src/minix/core/connectors/object_storage_connector/config.py

@@ -0,0 +1,11 @@
+from pydantic import BaseModel
+
+class ObjectStorageConfig(BaseModel):
+    """Configuration for object storage connection."""
+    endpoint_url: str
+    access_key: str
+    secret_key: str
+    bucket_name: str
+    use_ssl: bool = True
+    verify_ssl: bool = True
+

minix-0.1.0/src/minix/core/connectors/object_storage_connector/connector.py

@@ -0,0 +1,121 @@
+import boto3
+from botocore.exceptions import ClientError
+from typing import Optional, BinaryIO, Dict, Any
+import logging
+from .config import ObjectStorageConfig
+from .. import Connector
+
+logger = logging.getLogger(__name__)
+
+class ObjectStorageConnector(Connector):
+    """Connector for object storage operations."""
+    
+    def __init__(self, config: ObjectStorageConfig):
+        self.config = config
+        self.client = self._create_client()
+        self.bucket_name = config.bucket_name
+        
+    def _create_client(self) -> boto3.client:
+        """Create and return an S3 client."""
+        return boto3.client(
+            's3',
+            endpoint_url=self.config.endpoint_url,
+            aws_access_key_id=self.config.access_key,
+            aws_secret_access_key=self.config.secret_key,
+            use_ssl=self.config.use_ssl,
+            verify=self.config.verify_ssl
+        )
+    
+    async def upload_file(self, file_obj: BinaryIO, object_key: str, metadata: Optional[Dict[str, str]] = None) -> bool:
+        """Upload a file to object storage."""
+        try:
+            extra_args = {'Metadata': metadata} if metadata else {}
+            self.client.upload_fileobj(
+                file_obj,
+                self.bucket_name,
+                object_key,
+                ExtraArgs=extra_args
+            )
+            return True
+        except ClientError as e:
+            logger.error(f"Error uploading file {object_key}: {str(e)}")
+            return False
+            
+    async def download_file(self, object_key: str, file_obj: BinaryIO) -> bool:
+        """Download a file from object storage."""
+        try:
+            self.client.download_fileobj(
+                self.bucket_name,
+                object_key,
+                file_obj
+            )
+            return True
+        except ClientError as e:
+            logger.error(f"Error downloading file {object_key}: {str(e)}")
+            return False
+
+    async def download_to_dir(self, object_key: str, directory: str) -> bool:
+        """Download a file to a specified directory."""
+        try:
+            file_path = f"{directory}/{object_key}"
+            with open(file_path, 'wb') as file_obj:
+                return await self.download_file(object_key, file_obj)
+        except Exception as e:
+            logger.error(f"Error downloading file {object_key} to {directory}: {str(e)}")
+            return False
+
+    async def delete_file(self, object_key: str) -> bool:
+        """Delete a file from object storage."""
+        try:
+            self.client.delete_object(
+                Bucket=self.bucket_name,
+                Key=object_key
+            )
+            return True
+        except ClientError as e:
+            logger.error(f"Error deleting file {object_key}: {str(e)}")
+            return False
+            
+    async def get_file_metadata(self, object_key: str) -> Optional[Dict[str, Any]]:
+        """Get metadata for a file."""
+        try:
+            response = self.client.head_object(
+                Bucket=self.bucket_name,
+                Key=object_key
+            )
+            return {
+                'content_type': response.get('ContentType'),
+                'content_length': response.get('ContentLength'),
+                'last_modified': response.get('LastModified'),
+                'metadata': response.get('Metadata', {})
+            }
+        except ClientError as e:
+            logger.error(f"Error getting metadata for file {object_key}: {str(e)}")
+            return None
+            
+    async def list_files(self, prefix: str = "") -> list:
+        """List files in the bucket with optional prefix."""
+        try:
+            response = self.client.list_objects_v2(
+                Bucket=self.bucket_name,
+                Prefix=prefix
+            )
+            return [item['Key'] for item in response.get('Contents', [])]
+        except ClientError as e:
+            logger.error(f"Error listing files with prefix {prefix}: {str(e)}")
+            return []
+            
+    async def generate_presigned_url(self, object_key: str, expiration: int = 3600) -> Optional[str]:
+        """Generate a presigned URL for temporary access to a file."""
+        try:
+            return self.client.generate_presigned_url(
+                'get_object',
+                Params={
+                    'Bucket': self.bucket_name,
+                    'Key': object_key
+                },
+                ExpiresIn=expiration
+            )
+        except ClientError as e:
+            logger.error(f"Error generating presigned URL for {object_key}: {str(e)}")
+            return None 

minix-0.1.0/src/minix/core/connectors/qdrant_connector/__init__.py

@@ -0,0 +1 @@
+from .connector import QdrantConnector

minix-0.1.0/src/minix/core/connectors/qdrant_connector/connector.py

@@ -0,0 +1,20 @@
+from qdrant_client import AsyncQdrantClient
+
+from src.core.connectors import Connector
+
+
+class QdrantConnector(Connector):
+    def __init__(self, url: str, api_key: str = None):
+        self.url = url
+        self.api_key = api_key
+        self.client = None
+
+    async def connect(self) -> None:
+        self.client = AsyncQdrantClient(url=self.url, api_key=self.api_key)
+
+    async def disconnect(self) -> None:
+        # qdrant-client doesn't need explicit disconnect, but close can be added for compatibility
+        self.client = None
+
+    async def is_connected(self) -> bool:
+        return self.client is not None

minix-0.1.0/src/minix/core/connectors/sql_connector/__init__.py

@@ -0,0 +1 @@
+from .sql_connector import SqlConnectorConfig, SqlConnector

minix-0.1.0/src/minix/core/connectors/sql_connector/sql_connector.py

@@ -0,0 +1,96 @@
+from sqlalchemy.orm import sessionmaker, scoped_session
+from sqlalchemy import create_engine
+
+from src.minix.core.connectors import Connector
+
+
+class SqlConnectorConfig:
+    def __init__(
+            self,
+            username: str = None,
+            password: str = None,
+            host: str = None,
+            port: int = None,
+            database: str = None,
+            driver: str = None,
+            config_dict: dict = None,
+            read_from_dict: bool = False
+    ):
+        if read_from_dict:
+            self.read_from_dict(config_dict)
+
+        else:
+            self.username = username
+            self.password = password
+            self.host = host
+            self.port = port
+            self.database = database
+            self.driver = driver
+
+    def read_from_dict(self, config_dict: dict):
+        self.username = config_dict.get('username')
+        self.password = config_dict.get('password')
+        self.host = config_dict.get('host')
+        self.port = config_dict.get('port')
+        self.database = config_dict.get('database')
+        self.driver = config_dict.get('driver')
+
+    def to_dict(self) -> dict:
+        return {
+            'username': self.username,
+            'password': self.password,
+            'host': self.host,
+            'port': self.port,
+            'database': self.database,
+            'driver': self.driver
+        }
+
+    def __str__(self):
+        return f'username: {self.username}, password: {self.password}, host: {self.host}, port: {self.port}, database: {self.database}, driver: {self.driver}'
+
+
+class SqlConnector(Connector):
+    def __init__(
+            self,
+            sql_connector_config: SqlConnectorConfig
+    ):
+        self.username = sql_connector_config.username
+        self.password = sql_connector_config.password
+        self.host = sql_connector_config.host
+        self.port = sql_connector_config.port
+        self.database = sql_connector_config.database
+        self.driver = sql_connector_config.driver
+        print(self.get_connection_string(self.driver))
+        self.engine = create_engine(
+            self.get_connection_string(self.driver),
+            echo=False,
+            pool_pre_ping=True,
+            pool_recycle=3600  # Optional: recycle after 1 hour (seconds)
+        )
+        self.Session = scoped_session(
+            sessionmaker(
+                bind=self.engine,
+                autocommit=False,
+                autoflush=False
+            )
+        )
+
+    def get_session(self):
+        return self.Session()
+
+    def get_engine(self):
+        return self.engine
+
+    def get_connection_string(self, driver: str) -> str:
+        if driver == 'mysql':
+            return self.get_mysql_connection_string()
+        elif driver == 'clickhouse':
+            return self.clickhouse_connection_string()
+        else:
+            raise Exception('Driver not supported')
+
+    def get_mysql_connection_string(self) -> str:
+        return f'mysql+pymysql://{self.username}:{self.password}@{self.host}:{self.port}/{self.database}'
+
+    def clickhouse_connection_string(self) -> str:
+        return f'clickhouse://{self.username}:{self.password}@{self.host}:{self.port}/{self.database}'

minix-0.1.0/src/minix/core/consumer/__init__.py

@@ -0,0 +1 @@
+from .async_consumer import AsyncConsumer, AsyncConsumerConfig

minix-0.1.0/src/minix/core/consumer/async_consumer.py

@@ -0,0 +1,83 @@
+import asyncio
+import threading
+import json
+from abc import ABC, abstractmethod
+from typing import List
+from aiokafka import AIOKafkaConsumer
+from pydantic import BaseModel
+
+
+class AsyncConsumerConfig(BaseModel):
+    topics: List[str]
+    group_id: str
+    bootstrap_servers: List[str] | None = None
+    name: str
+
+class AsyncConsumer(ABC):
+    def __init__(self):
+        self._thread = None
+        self._loop = None
+        self._task = None
+        self._consumer = None
+        self._running = False
+
+
+
+    @abstractmethod
+    def get_config(self) -> AsyncConsumerConfig:
+        """Return Kafka config including topic, bootstrap_servers, group_id"""
+        pass
+
+    @abstractmethod
+    async def run(self, message: dict):
+        """Process a single Kafka message."""
+        pass
+
+    async def _consume(self):
+        config = self.get_config()
+        topic = config.pop("topic")
+
+        self._consumer = AIOKafkaConsumer(
+            topic,
+            **config.model_dump(),
+            value_deserializer=lambda m: json.loads(m.decode("utf-8"))
+        )
+
+        await self._consumer.start()
+        self._running = True
+        print(f"[{self.__class__.__name__}] Started on topic '{topic}'")
+
+        try:
+            async for msg in self._consumer:
+                await self.run(msg.value)
+        except asyncio.CancelledError:
+            print(f"[{self.__class__.__name__}] Cancelled")
+        except Exception as e:
+            print(f"[{self.__class__.__name__}] Error: {e}")
+        finally:
+            await self._consumer.stop()
+            self._running = False
+            print(f"[{self.__class__.__name__}] Stopped")
+
+    def start_in_thread(self):
+        def thread_target():
+            try:
+                self._loop = asyncio.new_event_loop()
+                asyncio.set_event_loop(self._loop)
+                self._task = self._loop.create_task(self._consume())
+                self._loop.run_until_complete(self._task)
+            except Exception as e:
+                print(f"[{self.__class__.__name__}] Fatal thread error: {e}")
+            finally:
+                self._loop.close()
+
+        self._thread = threading.Thread(target=thread_target, daemon=True)
+        self._thread.start()
+
+    def stop(self):
+        if self._loop and self._task and not self._task.done():
+            def shutdown():
+                self._task.cancel()
+
+            self._loop.call_soon_threadsafe(shutdown)
+            self._thread.join(timeout=10)

minix-0.1.0/src/minix/core/controller/__init__.py

@@ -0,0 +1 @@
+from .controller import Controller

minix-0.1.0/src/minix/core/controller/controller.py

@@ -0,0 +1,32 @@
+from abc import abstractmethod
+from typing import List
+from fastapi import APIRouter
+
+
+class Controller:
+    def __init__(self, tags: List[str] = None):
+        self.router = APIRouter(prefix= self.get_prefix(), tags=tags)
+        self.define_routes()
+
+
+    @property
+    def get_router(self):
+        return self.router
+
+
+    @abstractmethod
+    def get_prefix(self):
+        """
+        Returns the prefix for the controller's routes.
+        This method should be implemented by subclasses.
+        """
+        raise NotImplementedError("Subclasses must implement this method.")
+
+
+    @abstractmethod
+    def define_routes(self):
+        """
+        Defines the routes for the controller.
+        This method should be implemented by subclasses.
+        """
+        raise NotImplementedError("Subclasses must implement this method.")

minix-0.1.0/src/minix/core/entity/__init__.py

@@ -0,0 +1,4 @@
+from .entity import Entity
+from .sql_entity import SqlEntity
+from .redis_entity import RedisEntity
+from .qdrant_entity import QdrantEntity

minix-0.1.0/src/minix/core/entity/entity.py

@@ -0,0 +1,4 @@
+
+
+class Entity:
+    pass

minix-0.1.0/src/minix/core/entity/qdrant_entity.py

@@ -0,0 +1,26 @@
+from abc import abstractmethod
+from datetime import datetime
+from typing import List, Dict, Any
+
+from pydantic import BaseModel
+from src.minix.core.entity import Entity
+
+
+class QdrantEntity(Entity, BaseModel):
+    id: str
+    created_at: datetime
+    vector: List[float]
+
+
+
+    @property
+    def payload(self) -> Dict[str, Any]:
+        return vars(self)
+
+    @staticmethod
+    @abstractmethod
+    def collection()-> str:
+        """
+        Returns the name of the Qdrant collection this entity belongs to.
+        """
+        pass

minix-0.1.0/src/minix/core/entity/redis_entity.py

@@ -0,0 +1,4 @@
+from src.minix.core.entity import Entity
+
+class RedisEntity(Entity):
+    pass

minix-0.1.0/src/minix/core/entity/sql_entity.py

@@ -0,0 +1,25 @@
+from datetime import datetime
+
+from sqlalchemy import Integer, DateTime, func
+from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, MappedAsDataclass
+from src.minix.core.entity import Entity
+
+class Base(DeclarativeBase):
+    pass
+
+
+class SqlEntity(Base, Entity):
+    __abstract__ = True
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    created_at : Mapped[datetime] = mapped_column(DateTime, nullable=False, default=func.now(), index=True)
+    updated_at : Mapped[datetime] = mapped_column(DateTime, nullable=False, default=func.now(), onupdate=func.now(), index=True)
+
+    def __repr__(self):
+        return f'<{self.__class__.__name__} {self.id}>'
+
+
+
+
+
+
+

minix-0.1.0/src/minix/core/install/__init__.py

@@ -0,0 +1 @@
+from .installable import Installable

minix-0.1.0/src/minix/core/install/installable.py

@@ -0,0 +1,11 @@
+from abc import ABC, abstractmethod
+
+
+class Installable(ABC):
+
+    @abstractmethod
+    def install(self):
+        pass
+
+
+

minix-0.1.0/src/minix/core/model/__init__.py

@@ -0,0 +1,4 @@
+from .model import Model
+from .mlflow_model import MlflowModel
+from .model_registry import ModelRegistry
+from .embedding_model import EmbeddingModel

minix-0.1.0/src/minix/core/model/embedding_model.py

@@ -0,0 +1,20 @@
+from abc import ABC, abstractmethod
+import numpy as np
+from src.minix.core.model import Model
+
+
+class EmbeddingModel(Model, ABC):
+
+    def predict(self, text: str):
+        """Predict using the model"""
+        return self.embed(text)
+
+
+    @abstractmethod
+    def embed(self, text: str) -> np.ndarray:
+        raise NotImplementedError("The embed method must be implemented by subclasses.")
+
+    @abstractmethod
+    def embed_batch(self, texts: list[str]) -> np.ndarray:
+        """Embed a batch of texts"""
+        raise NotImplementedError("The embed_batch method must be implemented by subclasses.")