muscles-data-sql 2.0.0__py3-none-any.whl

muscles/data/sql/__init__.py

@@ -0,0 +1,114 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from .native_migrations import NativeSqlMigration, NativeSqlMigrationBackend, NativeSqlMigrationSourceBackend
+    from .drivers import SqlDriverCatalog
+    from .native import SqlDatabase, SqlTransaction
+    from .model_mapper import ModelMapper
+    from .config import DatabaseConfig
+    from .config import SqlConnectionConfig
+    from .connections import SqlConnectionRegistry, UnknownSqlConnection
+    from .engine import EngineManager
+    from .mapping import map_model
+    from .providers import SqlResourceGeneratorProvider
+    from .query import FilterClause, JoinClause, QuerySpec
+    from .repository import SqlRepository
+    from .uow import UnitOfWork
+    from .sqlalchemy_compat import text
+
+__all__ = [
+    "NativeSqlMigration",
+    "NativeSqlMigrationBackend",
+    "NativeSqlMigrationSourceBackend",
+    "SqlDriverCatalog",
+    "SqlDatabase",
+    "SqlTransaction",
+    "ModelMapper",
+    "DatabaseConfig",
+    "SqlConnectionConfig",
+    "SqlConnectionRegistry",
+    "UnknownSqlConnection",
+    "EngineManager",
+    "map_model",
+    "SqlRepository",
+    "UnitOfWork",
+    "text",
+    "QuerySpec",
+    "FilterClause",
+    "JoinClause",
+    "SqlResourceGeneratorProvider",
+]
+
+
+def __getattr__(name: str):
+    if name in {"NativeSqlMigration", "NativeSqlMigrationBackend", "NativeSqlMigrationSourceBackend"}:
+        from . import native_migrations
+
+        return getattr(native_migrations, name)
+    if name == "SqlDriverCatalog":
+        from .drivers import SqlDriverCatalog
+
+        return SqlDriverCatalog
+    if name == "ModelMapper":
+        from .model_mapper import ModelMapper
+
+        return ModelMapper
+    if name in {"SqlDatabase", "SqlTransaction"}:
+        from .native import SqlDatabase, SqlTransaction
+
+        return {"SqlDatabase": SqlDatabase, "SqlTransaction": SqlTransaction}[name]
+    if name == "DatabaseConfig":
+        from .config import DatabaseConfig
+
+        return DatabaseConfig
+    if name == "SqlConnectionConfig":
+        from .config import SqlConnectionConfig
+
+        return SqlConnectionConfig
+    if name == "SqlConnectionRegistry":
+        from .connections import SqlConnectionRegistry
+
+        return SqlConnectionRegistry
+    if name == "UnknownSqlConnection":
+        from .connections import UnknownSqlConnection
+
+        return UnknownSqlConnection
+    if name == "EngineManager":
+        from .engine import EngineManager
+
+        return EngineManager
+    if name == "map_model":
+        from .mapping import map_model
+
+        return map_model
+    if name == "SqlRepository":
+        from .repository import SqlRepository
+
+        return SqlRepository
+    if name == "UnitOfWork":
+        from .uow import UnitOfWork
+
+        return UnitOfWork
+    if name == "text":
+        from .sqlalchemy_compat import text
+
+        return text
+    if name == "QuerySpec":
+        from .query import QuerySpec
+
+        return QuerySpec
+    if name == "FilterClause":
+        from .query import FilterClause
+
+        return FilterClause
+    if name == "JoinClause":
+        from .query import JoinClause
+
+        return JoinClause
+    if name == "SqlResourceGeneratorProvider":
+        from .providers import SqlResourceGeneratorProvider
+
+        return SqlResourceGeneratorProvider
+    raise AttributeError(name)

muscles/data/sql/cli.py

@@ -0,0 +1,76 @@
+import json
+from pathlib import Path
+
+import click
+
+from .config import DatabaseConfig
+from .connections import SqlConnectionRegistry, load_connection_registry
+from .providers import SqlGenerationRequest, SqlResourceGeneratorProvider
+
+
+@click.group()
+def main():
+    pass
+
+
+def _registry_from_options(url: str | None, config_path: str | None) -> SqlConnectionRegistry:
+    if config_path:
+        return load_connection_registry(config_path)
+    if url:
+        return SqlConnectionRegistry.from_database_config(DatabaseConfig(url=url))
+    raise click.UsageError("Use --url or --config.")
+
+
+@main.command("doctor")
+@click.option("--url", required=False)
+@click.option("--config", "config_path", type=click.Path(exists=True, dir_okay=False), required=False)
+@click.option("--connection", default="default", show_default=True)
+@click.option("--all", "all_connections", is_flag=True, default=False)
+def doctor(url: str | None, config_path: str | None, connection: str, all_connections: bool):
+    registry = _registry_from_options(url, config_path)
+    report = registry.inspect_all() if all_connections else registry.inspect(connection)
+    click.echo(json.dumps(report, ensure_ascii=False))
+
+
+@main.command("inspect")
+@click.option("--url", required=False)
+@click.option("--config", "config_path", type=click.Path(exists=True, dir_okay=False), required=False)
+@click.option("--connection", default="default", show_default=True)
+def inspect_cmd(url: str | None, config_path: str | None, connection: str):
+    registry = _registry_from_options(url, config_path)
+    click.echo(json.dumps(registry.inspect(connection), ensure_ascii=False))
+
+
+@main.group("generate")
+def generate():
+    pass
+
+
+@generate.command("resource")
+@click.argument("name")
+@click.option("--sql", is_flag=True, default=False)
+def generate_resource(name: str, sql: bool):
+    if sql:
+        provider = SqlResourceGeneratorProvider()
+        generated = provider.generate(
+            project_root=Path.cwd(),
+            request=SqlGenerationRequest(
+                generator_type="sql-resource",
+                name=name,
+                force=False,
+                with_tests=False,
+            ),
+        )
+        click.echo(generated[0])
+        return
+
+    root = Path("generated")
+    root.mkdir(exist_ok=True)
+    target = root / f"{name}.py"
+    content = f"class {name.title().replace('_', '')}Resource:\n    sql_enabled = {sql}\n"
+    target.write_text(content, encoding="utf-8")
+    click.echo(str(target))
+
+
+if __name__ == "__main__":
+    main()

muscles/data/sql/config.py

@@ -0,0 +1,50 @@
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass(slots=True)
+class DatabaseConfig:
+    url: str
+    echo: bool = False
+    pool_size: int = 5
+    max_overflow: int = 10
+    future: bool = True
+
+
+@dataclass(slots=True)
+class SqlConnectionConfig:
+    name: str
+    url: str
+    echo: bool = False
+    pool_size: int = 5
+    max_overflow: int = 10
+    future: bool = True
+    role: str | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def safe_url(self) -> str:
+        from sqlalchemy.engine import make_url
+
+        parsed = make_url(self.url)
+        rendered = parsed.render_as_string(hide_password=True)
+        if parsed.get_backend_name() == "sqlite" and parsed.database == ":memory:":
+            rendered = rendered.replace("%3Amemory%3A", ":memory:")
+        return rendered
+
+    def to_database_config(self) -> DatabaseConfig:
+        return DatabaseConfig(
+            url=self.url,
+            echo=self.echo,
+            pool_size=self.pool_size,
+            max_overflow=self.max_overflow,
+            future=self.future,
+        )
+
+    def to_contract(self) -> dict[str, Any]:
+        return {
+            "name": self.name,
+            "url": self.safe_url,
+            "role": self.role,
+            "metadata": dict(self.metadata),
+        }

muscles/data/sql/connections.py

@@ -0,0 +1,105 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Iterable, Mapping
+
+from .config import DatabaseConfig, SqlConnectionConfig
+from .engine import EngineManager
+
+
+class UnknownSqlConnection(KeyError):
+    def __init__(self, name: str):
+        super().__init__(f"Unknown SQL connection: {name}")
+        self.name = name
+
+
+class SqlConnectionRegistry:
+    def __init__(self, configs: Iterable[SqlConnectionConfig] | None = None):
+        self._configs: dict[str, SqlConnectionConfig] = {}
+        self._managers: dict[str, EngineManager] = {}
+        for config in configs or ():
+            self.register(config)
+
+    @classmethod
+    def from_database_config(cls, config: DatabaseConfig, *, name: str = "default") -> "SqlConnectionRegistry":
+        return cls(
+            [
+                SqlConnectionConfig(
+                    name=name,
+                    url=config.url,
+                    echo=config.echo,
+                    pool_size=config.pool_size,
+                    max_overflow=config.max_overflow,
+                    future=config.future,
+                )
+            ]
+        )
+
+    def register(self, config: SqlConnectionConfig) -> None:
+        self._configs[config.name] = config
+        self._managers.pop(config.name, None)
+
+    def names(self) -> list[str]:
+        return sorted(self._configs)
+
+    def config(self, name: str = "default") -> SqlConnectionConfig:
+        try:
+            return self._configs[name]
+        except KeyError as exc:
+            raise UnknownSqlConnection(name) from exc
+
+    def manager(self, name: str = "default") -> EngineManager:
+        if name not in self._managers:
+            self._managers[name] = EngineManager(self.config(name).to_database_config())
+        return self._managers[name]
+
+    def session_factory(self, name: str = "default"):
+        return self.manager(name).session_factory
+
+    def session(self, name: str = "default"):
+        return self.manager(name).session()
+
+    def inspect(self, name: str = "default") -> dict[str, Any]:
+        from muscles.data.sql.inspect import inspect_sql_layer
+
+        config = self.config(name)
+        report = inspect_sql_layer(self.manager(name))
+        report["connection"] = config.to_contract()
+        return report
+
+    def inspect_all(self) -> dict[str, Any]:
+        reports = {name: self.inspect(name) for name in self.names()}
+        status = "ok" if all(report.get("status") == "ok" for report in reports.values()) else "error"
+        return {"status": status, "connections": reports}
+
+
+def connection_config_from_mapping(name: str, payload: str | Mapping[str, Any]) -> SqlConnectionConfig:
+    if isinstance(payload, str):
+        return SqlConnectionConfig(name=name, url=payload)
+    return SqlConnectionConfig(
+        name=str(payload.get("name", name)),
+        url=str(payload["url"]),
+        echo=bool(payload.get("echo", False)),
+        pool_size=int(payload.get("pool_size", 5)),
+        max_overflow=int(payload.get("max_overflow", 10)),
+        future=bool(payload.get("future", True)),
+        role=payload.get("role"),
+        metadata=dict(payload.get("metadata") or {}),
+    )
+
+
+def connection_registry_from_mapping(payload: Mapping[str, Any]) -> SqlConnectionRegistry:
+    raw_connections = payload.get("connections", payload)
+    if not isinstance(raw_connections, Mapping):
+        raise ValueError("SQL connection config must contain a mapping of connections.")
+    configs = [connection_config_from_mapping(str(name), value) for name, value in raw_connections.items()]
+    return SqlConnectionRegistry(configs)
+
+
+def load_connection_registry(path: str | Path) -> SqlConnectionRegistry:
+    import json
+
+    payload = json.loads(Path(path).read_text(encoding="utf-8"))
+    if not isinstance(payload, Mapping):
+        raise ValueError("SQL connection config must be a JSON object.")
+    return connection_registry_from_mapping(payload)

muscles/data/sql/data_adapter.py

@@ -0,0 +1,71 @@
+"""Optional integration with muscles-data; the SQL core does not import it."""
+from __future__ import annotations
+
+from threading import Lock
+from typing import Mapping
+
+from muscles.data.errors import DataConfigurationError
+from muscles.data.models import DataCapability, HealthResult
+
+from .drivers import SqlDriverCatalog, SqlDriverConfigError, SqlDriverFactory, SqlDriverNotFoundError
+
+
+class NativeSqlAdapter:
+    resource_type = "native_sql"
+
+    def __init__(self, config, drivers: SqlDriverCatalog):
+        self.config = config
+        self._drivers = drivers
+        self._db = None
+        self._lock = Lock()
+        self._closed = False
+
+    def _database(self):
+        with self._lock:
+            if self._closed:
+                raise RuntimeError("SQL resource is closed")
+            if self._db is None:
+                options = self.config.resolved_options()
+                driver = options.get("driver")
+                try:
+                    self._db = self._drivers.open(driver, options)
+                except (SqlDriverNotFoundError, SqlDriverConfigError) as exc:
+                    raise DataConfigurationError(str(exc)) from exc
+            return self._db
+
+    def transaction(self):
+        return self._database().transaction()
+
+    def inspect(self):
+        return {"status": "ok", "details": {"driver": self.config.options.get("driver")}}
+
+    def health(self):
+        try:
+            self._database().check_health()
+        except Exception:
+            return HealthResult(status="failed", message="SQL health check failed")
+        return HealthResult(status="ok")
+
+    def close(self):
+        with self._lock:
+            if self._db is not None:
+                self._db.close()
+            self._closed = True
+
+
+class NativeSqlFactory:
+    resource_type = "native_sql"
+
+    def __init__(self, *, drivers: SqlDriverCatalog | Mapping[str, SqlDriverFactory] | None = None):
+        if isinstance(drivers, SqlDriverCatalog):
+            self._drivers = drivers
+        else:
+            self._drivers = SqlDriverCatalog.with_defaults()
+            for name, factory in (drivers or {}).items():
+                self._drivers.register(name, factory)
+
+    def capabilities(self, config):
+        return {DataCapability.SQL_TRANSACTION, DataCapability.HEALTHCHECK}
+
+    def create(self, config):
+        return NativeSqlAdapter(config, self._drivers)

muscles/data/sql/drivers.py

@@ -0,0 +1,52 @@
+"""Registration of independent SQL connection factories."""
+from __future__ import annotations
+
+from typing import Any, Callable, Mapping
+
+from .native import SqlDatabase
+
+SqlDriverFactory = Callable[[Mapping[str, Any]], SqlDatabase]
+
+
+class SqlDriverNotFoundError(LookupError):
+    """The requested driver has not been registered."""
+
+
+class SqlDriverConfigError(ValueError):
+    """A driver's required configuration is missing."""
+
+
+def _sqlite_driver(options: Mapping[str, Any]) -> SqlDatabase:
+    if not options.get("path"):
+        raise SqlDriverConfigError("sqlite driver requires path")
+    return SqlDatabase.sqlite(options["path"])
+
+
+def _postgres_driver(options: Mapping[str, Any]) -> SqlDatabase:
+    if not options.get("url"):
+        raise SqlDriverConfigError("postgres driver requires url or url_env")
+    return SqlDatabase.postgres(options["url"])
+
+
+class SqlDriverCatalog:
+    def __init__(self) -> None:
+        self._drivers: dict[str, SqlDriverFactory] = {}
+
+    @classmethod
+    def with_defaults(cls) -> SqlDriverCatalog:
+        catalog = cls()
+        catalog.register("sqlite", _sqlite_driver)
+        catalog.register("postgres", _postgres_driver)
+        return catalog
+
+    def register(self, name: str, factory: SqlDriverFactory) -> None:
+        if not isinstance(name, str) or not name.strip() or not callable(factory):
+            raise ValueError("SQL drivers require a nonempty name and callable factory")
+        if name in self._drivers:
+            raise ValueError(f"SQL driver '{name}' is already registered")
+        self._drivers[name] = factory
+
+    def open(self, name: str, options: Mapping[str, Any]) -> SqlDatabase:
+        if not isinstance(name, str) or name not in self._drivers:
+            raise SqlDriverNotFoundError("SQL driver is not registered")
+        return self._drivers[name](options)

muscles/data/sql/engine.py

@@ -0,0 +1,19 @@
+from typing import Any
+
+from .config import DatabaseConfig
+
+
+class EngineManager:
+    def __init__(self, config: DatabaseConfig):
+        from sqlalchemy import create_engine
+        from sqlalchemy.orm import sessionmaker
+
+        kwargs: dict[str, Any] = {"echo": config.echo, "future": config.future}
+        if not config.url.startswith("sqlite"):
+            kwargs["pool_size"] = config.pool_size
+            kwargs["max_overflow"] = config.max_overflow
+        self.engine = create_engine(config.url, **kwargs)
+        self.session_factory = sessionmaker(bind=self.engine, autoflush=False, autocommit=False, future=True)
+
+    def session(self):
+        return self.session_factory()

muscles/data/sql/inspect.py

@@ -0,0 +1,26 @@
+def inspect_sql_layer(engine_manager) -> dict:
+    from time import perf_counter
+    from sqlalchemy import inspect, text
+
+    started = perf_counter()
+    with engine_manager.engine.connect() as conn:
+        conn.execute(text("SELECT 1"))
+    latency_ms = round((perf_counter() - started) * 1000, 3)
+    inspector = inspect(engine_manager.engine)
+    tables = []
+    for table_name in inspector.get_table_names():
+        tables.append(
+            {
+                "name": table_name,
+                "columns": [column["name"] for column in inspector.get_columns(table_name)],
+                "indexes": [index["name"] for index in inspector.get_indexes(table_name)],
+                "pk": inspector.get_pk_constraint(table_name).get("constrained_columns", []),
+            }
+        )
+    return {
+        "status": "ok",
+        "dialect": engine_manager.engine.dialect.name,
+        "driver": engine_manager.engine.dialect.driver,
+        "latency_ms": latency_ms,
+        "tables": tables,
+    }

muscles/data/sql/mapping.py

@@ -0,0 +1,71 @@
+from typing import Any
+
+TYPE_MAP: dict[str, str] = {
+    "String": "String",
+    "Text": "Text",
+    "Integer": "Integer",
+    "Numeric": "Numeric",
+    "Float": "Float",
+    "Boolean": "Boolean",
+    "Date": "Date",
+    "DateTime": "DateTime",
+}
+
+
+def _sa():
+    from sqlalchemy import Boolean, Column, Date, DateTime, Float, Integer, MetaData, Numeric, String, Table, Text
+
+    return {
+        "Boolean": Boolean,
+        "Column": Column,
+        "Date": Date,
+        "DateTime": DateTime,
+        "Float": Float,
+        "Integer": Integer,
+        "MetaData": MetaData,
+        "Numeric": Numeric,
+        "String": String,
+        "Table": Table,
+        "Text": Text,
+    }
+
+
+def _resolve_type(column_type: Any):
+    sa = _sa()
+    if isinstance(column_type, type):
+        name = column_type.__name__
+    else:
+        name = getattr(column_type, "__class__", type(column_type)).__name__
+    return sa.get(TYPE_MAP.get(name, "String"), sa["String"])
+
+
+def map_model(model_cls: type, table_name: str | None = None):
+    sa = _sa()
+    metadata = sa["MetaData"]()
+    columns = []
+    for field_name, field_def in model_cls.__dict__.items():
+        if field_name.startswith("_"):
+            continue
+        field_type = getattr(field_def, "field_type", None)
+        if field_type is None:
+            continue
+        sql_type = _resolve_type(field_type)
+        is_pk = bool(getattr(field_def, "primary_key", False))
+        nullable = bool(getattr(field_def, "nullable", not is_pk))
+        autoincrement = bool(
+            getattr(field_def, "autoincrement", False)
+            or (is_pk and sql_type is sa["Integer"])
+        )
+        columns.append(
+            sa["Column"](
+                field_name,
+                sql_type,
+                primary_key=is_pk,
+                nullable=nullable,
+                autoincrement=autoincrement,
+            )
+        )
+
+    if not columns:
+        raise ValueError(f"No mappable columns found in model {model_cls.__name__}")
+    return sa["Table"](table_name or model_cls.__name__.lower(), metadata, *columns, extend_existing=True)

muscles/data/sql/migration_lock.py

@@ -0,0 +1,53 @@
+"""Optional SQL-backed coordination lock for the migration registry."""
+from contextlib import contextmanager
+from pathlib import Path
+
+
+class SqlMigrationExecutionLock:
+    """All runners must use the same database URL and this lock implementation."""
+
+    def __init__(self, url: str):
+        self.url = url
+
+    @contextmanager
+    def __call__(self):
+        from sqlalchemy.engine import make_url
+
+        sqlalchemy_url = self.url.replace("postgresql+psycopg2://", "postgresql+psycopg://", 1)
+        sqlalchemy_url = sqlalchemy_url.replace("postgresql://", "postgresql+psycopg://", 1)
+        url = make_url(sqlalchemy_url)
+        dialect = url.get_backend_name()
+        if dialect == "sqlite":
+            import fcntl
+
+            database = url.database
+            if not database or database == ":memory:" or database.startswith("file:"):
+                raise ValueError("SQLite migration coordinator requires a shared file, not memory or a URI")
+            lock_path = Path(f"{Path(database).resolve()}.migrations.lock")
+            with lock_path.open("a+b") as handle:
+                try:
+                    fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+                except BlockingIOError as exc:
+                    raise RuntimeError("Migration execution is locked") from exc
+                try:
+                    yield
+                finally:
+                    fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
+        elif dialect == "postgresql":
+            from sqlalchemy import create_engine, text
+            from sqlalchemy.pool import NullPool
+
+            engine = create_engine(url, poolclass=NullPool)
+            try:
+                with engine.connect() as connection:
+                    acquired = connection.execute(text("SELECT pg_try_advisory_lock(:key)"), {"key": 778212474}).scalar_one()
+                    if not acquired:
+                        raise RuntimeError("Migration execution is locked")
+                    try:
+                        yield
+                    finally:
+                        connection.execute(text("SELECT pg_advisory_unlock(:key)"), {"key": 778212474})
+            finally:
+                engine.dispose()
+        else:
+            raise ValueError("Migration coordinator supports only SQLite and PostgreSQL")

muscles/data/sql/model_mapper.py

@@ -0,0 +1,30 @@
+"""Explicit mapping between Muscles models and SQL result records."""
+from __future__ import annotations
+
+from typing import Any, Generic, Iterable, Mapping, TypeVar
+
+T = TypeVar("T")
+
+
+class ModelMapper(Generic[T]):
+    def __init__(self, model_type: type[T], fields: Mapping[str, str]):
+        if not fields or len(set(fields.values())) != len(fields):
+            raise ValueError("fields must contain unique record names")
+        self.model_type = model_type
+        self.fields = dict(fields)
+
+    def from_record(self, record: Mapping[str, Any]) -> T:
+        model = self.model_type(**{field: record[column] for field, column in self.fields.items()})
+        unknown = self.fields.keys() - set(model.fields())
+        if unknown:
+            raise ValueError(f"Unknown model fields: {', '.join(sorted(unknown))}")
+        return model
+
+    def from_records(self, records: Iterable[Mapping[str, Any]]) -> list[T]:
+        return [self.from_record(record) for record in records]
+
+    def to_record(self, model: T) -> dict[str, Any]:
+        if not isinstance(model, self.model_type):
+            raise TypeError("model has an unexpected type")
+        values = model.as_dict()
+        return {column: values[field] for field, column in self.fields.items()}