framework-py 0.1.0.dev0__py3-none-any.whl

framework/config_home.py

@@ -0,0 +1,194 @@
+"""The global config folder and syncing it with the R package's folder.
+
+Python and the ``fw`` CLI share one folder. The R package keeps its own,
+because CRAN does not allow a package to write outside
+``tools::R_user_dir()``. ``config_sync()`` copies settings and saved logins
+from R's folder when someone uses both languages.
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+import sys
+from dataclasses import dataclass, field
+from datetime import date
+from pathlib import Path
+from typing import Any
+
+import pandas as pd
+
+from framework import _yaml
+
+# Files config_sync() copies. Token files hold a secret and are written 0600.
+_SYNC_FILES = ("settings.yml", "cloud.yml")
+_TOKEN_PREFIX = "cloud-token"
+
+
+def config_dir() -> Path:
+    """The global Framework config folder for Python and ``fw``.
+
+    ``FW_CONFIG_HOME`` when set, otherwise ``$XDG_CONFIG_HOME/framework``,
+    otherwise ``~/.config/framework`` (on Windows too). The folder is not
+    created until Framework writes to it.
+    """
+    custom = os.environ.get("FW_CONFIG_HOME")
+    if custom:
+        return Path(custom).expanduser()
+    xdg = os.environ.get("XDG_CONFIG_HOME")
+    base = Path(xdg).expanduser() if xdg else Path.home() / ".config"
+    return base / "framework"
+
+
+def r_config_dir() -> Path:
+    """The R package's config folder, ``tools::R_user_dir("framework", "config")``.
+
+    This ignores ``FW_CONFIG_HOME``, which points both tools at the same
+    folder when it is set.
+    """
+    custom = os.environ.get("R_USER_CONFIG_DIR") or os.environ.get("XDG_CONFIG_HOME")
+    if custom:
+        base = Path(custom).expanduser()
+    elif sys.platform == "win32":
+        base = Path(os.environ.get("APPDATA", str(Path.home()))) / "R" / "config"
+    elif sys.platform == "darwin":
+        base = Path.home() / "Library" / "Preferences" / "org.R-project.R"
+    else:
+        base = Path.home() / ".config"
+    return base / "R" / "framework"
+
+
+def ensure_config_dir() -> Path:
+    """Create the config folder if needed and return it."""
+    folder = config_dir()
+    folder.mkdir(parents=True, exist_ok=True)
+    return folder
+
+
+@dataclass(frozen=True)
+class SyncResult:
+    """What ``config_sync()`` copied, or would copy with ``dry_run=True``."""
+
+    source: Path
+    target: Path
+    copied: list[str] = field(default_factory=list)
+    skipped: list[str] = field(default_factory=list)
+    dry_run: bool = False
+
+    def __str__(self) -> str:
+        verb = "Would copy" if self.dry_run else "Copied"
+        if not self.copied:
+            return f"Nothing to copy from {self.source} (Python's folder is up to date)."
+        return f"{verb} {', '.join(self.copied)} from {self.source} to {self.target}."
+
+
+def _syncable(path: Path) -> bool:
+    name = path.name
+    if name.startswith(".") or not path.is_file():
+        return False
+    return name in _SYNC_FILES or name.startswith(_TOKEN_PREFIX)
+
+
+def config_sync(from_dir: str | Path | None = None, *, dry_run: bool = False) -> SyncResult:
+    """Copy settings and saved logins from the R package's config folder.
+
+    A file is copied when Python's folder does not have it or R's copy is
+    newer. The files are ``settings.yml``, ``cloud.yml``, and the
+    ``cloud-token*`` files used when no system keyring is available.
+    Modification times are kept, token files are made readable only by you,
+    and nothing is written to R's folder.
+
+    Args:
+        from_dir: The folder to copy from. Default: ``r_config_dir()``.
+        dry_run: Report what would be copied without copying.
+    """
+    source = Path(from_dir).expanduser() if from_dir is not None else r_config_dir()
+    target = config_dir()
+    copied: list[str] = []
+    skipped: list[str] = []
+
+    if source.resolve() == target.resolve() or not source.is_dir():
+        return SyncResult(source, target, copied, skipped, dry_run)
+
+    for path in sorted(source.iterdir()):
+        if not _syncable(path):
+            continue
+        destination = target / path.name
+        if destination.exists() and destination.stat().st_mtime >= path.stat().st_mtime:
+            skipped.append(path.name)
+            continue
+        copied.append(path.name)
+        if dry_run:
+            continue
+        target.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(path, destination)
+        if path.name.startswith(_TOKEN_PREFIX):
+            destination.chmod(0o600)
+
+    return SyncResult(source, target, copied, skipped, dry_run)
+
+
+# The project registry -----------------------------------------------------------
+
+
+def _global_settings_path() -> Path:
+    return config_dir() / "settings.yml"
+
+
+def global_settings() -> dict[str, Any]:
+    """The user's global settings (``settings.yml`` in the config folder), or ``{}``."""
+    path = _global_settings_path()
+    if not path.is_file():
+        return {}
+    data = _yaml.read(path)
+    return data if isinstance(data, dict) else {}
+
+
+def _save_global_settings(settings: dict[str, Any]) -> None:
+    ensure_config_dir()
+    _yaml.write(_global_settings_path(), settings)
+
+
+def project_register(path: str | Path) -> int:
+    """Add a project to the registry in the global settings. Returns its id."""
+    settings = global_settings()
+    projects = settings.get("projects") or []
+    resolved = str(Path(path).expanduser().resolve())
+    for project in projects:
+        if project.get("path") == resolved:
+            return int(project.get("id", 0))
+    next_id = max((int(p.get("id", 0)) for p in projects), default=0) + 1
+    projects.append({"id": next_id, "path": resolved, "created": f"{date.today():%Y-%m-%d}"})
+    settings["projects"] = projects
+    _save_global_settings(settings)
+    return next_id
+
+
+def project_unregister(project: int | str | Path) -> bool:
+    """Remove a project from the registry by id or path. Returns whether it was there."""
+    settings = global_settings()
+    projects = settings.get("projects") or []
+    if isinstance(project, int):
+        kept = [p for p in projects if int(p.get("id", 0)) != project]
+    else:
+        resolved = str(Path(project).expanduser().resolve())
+        kept = [p for p in projects if p.get("path") != resolved]
+    if len(kept) == len(projects):
+        return False
+    settings["projects"] = kept
+    _save_global_settings(settings)
+    return True
+
+
+def project_list() -> pd.DataFrame:
+    """Registered projects, with columns id, path, and created."""
+    projects = global_settings().get("projects") or []
+    rows = [
+        {
+            "id": int(p.get("id", 0)),
+            "path": str(p.get("path", "")),
+            "created": str(p.get("created", "")),
+        }
+        for p in projects
+    ]
+    return pd.DataFrame(rows, columns=["id", "path", "created"])

framework/connections.py

@@ -0,0 +1,442 @@
+"""Database connections defined in the project settings.
+
+Connections live under ``connections`` in the settings (usually
+``settings/connections.yml``)::
+
+    connections:
+      default_database: framework
+      databases:
+        framework: {driver: sqlite, database: framework.db}
+        warehouse:
+          driver: postgres
+          host: env("POSTGRES_HOST")
+          port: 5432
+          database: analytics
+          user: env("POSTGRES_USER")
+          password: env("POSTGRES_PASSWORD")
+
+Credentials come from ``.env`` through ``env()``, so they never sit in the
+settings. Drivers: sqlite (built in), postgres (``postgres`` extra), mysql or
+mariadb (``mysql``), duckdb (``duckdb``), and sqlserver (``sqlserver``).
+"""
+
+from __future__ import annotations
+
+import re
+import sqlite3
+from collections.abc import Iterator, Mapping, Sequence
+from contextlib import contextmanager
+from importlib import import_module, metadata
+from types import TracebackType
+from typing import Any
+
+import pandas as pd
+
+from framework._paths import in_root
+from framework.errors import FrameworkError
+from framework.project import project_root
+from framework.settings import settings_read
+
+_HOST = re.compile(r"^[a-zA-Z0-9.-]+$")
+
+# Driver name -> (canonical driver, Python module, extra)
+DRIVERS: dict[str, tuple[str, str, str | None]] = {
+    "sqlite": ("sqlite", "sqlite3", None),
+    "postgres": ("postgres", "psycopg", "postgres"),
+    "postgresql": ("postgres", "psycopg", "postgres"),
+    "mysql": ("mysql", "pymysql", "mysql"),
+    "mariadb": ("mysql", "pymysql", "mysql"),
+    "duckdb": ("duckdb", "duckdb", "duckdb"),
+    "sqlserver": ("sqlserver", "pyodbc", "sqlserver"),
+    "mssql": ("sqlserver", "pyodbc", "sqlserver"),
+}
+_DISTRIBUTIONS = {
+    "sqlite3": None,
+    "psycopg": "psycopg",
+    "pymysql": "PyMySQL",
+    "duckdb": "duckdb",
+    "pyodbc": "pyodbc",
+}
+
+
+# Finding connections ------------------------------------------------------------
+
+
+def _section(config: Mapping[str, Any], key: str) -> dict[str, Any]:
+    value = config.get(key)
+    return value if isinstance(value, dict) else {}
+
+
+def database_configs(config: Mapping[str, Any] | None = None) -> dict[str, dict[str, Any]]:
+    """Every database connection in the settings, by name.
+
+    Looked up in ``connections.databases``, then in ``connections`` itself
+    (the older flat form), then in a top-level ``databases`` section, where
+    projects created by R before 2.0 have it.
+    """
+    config = config if config is not None else settings_read()
+    connections = _section(config, "connections")
+    found: dict[str, dict[str, Any]] = {}
+    for source in (
+        _section(config, "databases"),
+        {k: v for k, v in connections.items() if isinstance(v, dict) and "driver" in v},
+        _section(connections, "databases"),
+    ):
+        found.update(
+            {name: dict(entry) for name, entry in source.items() if isinstance(entry, dict)}
+        )
+    return found
+
+
+def default_database(config: Mapping[str, Any] | None = None) -> str | None:
+    config = config if config is not None else settings_read()
+    value = _section(config, "connections").get("default_database") or config.get(
+        "default_database"
+    )
+    return str(value) if value else None
+
+
+def _database_config(name: str) -> dict[str, Any]:
+    configs = database_configs()
+    if name not in configs:
+        available = ", ".join(configs) or "none"
+        raise FrameworkError(
+            f"No connection configuration found for '{name}' (available: {available})"
+        )
+    entry = configs[name]
+    if not entry.get("driver"):
+        raise FrameworkError(f"No driver specified for connection '{name}'")
+    return entry
+
+
+# Connecting -----------------------------------------------------------------------
+
+
+def _require(module: str, extra: str | None, what: str) -> Any:
+    try:
+        return import_module(module)
+    except ImportError as error:
+        hint = f" Install it with: pip install framework-py[{extra}]" if extra else ""
+        raise FrameworkError(f"{what} connections need the {module} package.{hint}") from error
+
+
+def _missing(config: Mapping[str, Any], fields: Sequence[str], what: str) -> None:
+    missing = [field for field in fields if config.get(field) in (None, "")]
+    if missing:
+        raise FrameworkError(f"{what} configuration missing required fields: {', '.join(missing)}")
+
+
+def _port(value: Any, default: int | None = None) -> int:
+    if value in (None, "") and default is not None:
+        return default
+    try:
+        port = int(str(value).strip())
+    except ValueError:
+        port = 0
+    if not 1 <= port <= 65535:
+        raise FrameworkError(f"Invalid port number: {value}")
+    return port
+
+
+def _host(value: Any) -> str:
+    host = str(value)
+    if not _HOST.match(host):
+        raise FrameworkError(f"Invalid host name: {host}")
+    return host
+
+
+def _file_path(config: Mapping[str, Any], what: str) -> str:
+    if not config.get("database"):
+        raise FrameworkError(f"{what} configuration must include 'database' path")
+    path = in_root(str(config["database"]), project_root())
+    if not path.parent.is_dir():
+        raise FrameworkError(f"Database directory does not exist: {path.parent}")
+    return str(path)
+
+
+def _connect_sqlite(config: Mapping[str, Any]) -> Any:
+    return sqlite3.connect(_file_path(config, "SQLite"))
+
+
+def _connect_postgres(config: Mapping[str, Any]) -> Any:
+    psycopg = _require("psycopg", "postgres", "PostgreSQL")
+    _missing(config, ("host", "port", "database", "user", "password"), "PostgreSQL")
+    host, port = _host(config["host"]), _port(config["port"])
+    connection = psycopg.connect(
+        host=host,
+        port=port,
+        dbname=config["database"],
+        user=config["user"],
+        password=config["password"],
+        connect_timeout=10,
+    )
+    schema = config.get("schema") or "public"
+    sql = import_module("psycopg.sql")
+    with connection.cursor() as cursor:
+        cursor.execute(sql.SQL("SET search_path TO {}").format(sql.Identifier(schema)))
+    connection.commit()
+    return connection
+
+
+def _connect_mysql(config: Mapping[str, Any]) -> Any:
+    pymysql = _require("pymysql", "mysql", "MySQL/MariaDB")
+    _missing(config, ("host", "port", "database", "user", "password"), "MySQL/MariaDB")
+    host, port = _host(config["host"]), _port(config["port"])
+    return pymysql.connect(
+        host=host,
+        port=port,
+        database=config["database"],
+        user=config["user"],
+        password=config["password"],
+        connect_timeout=10,
+    )
+
+
+def _connect_duckdb(config: Mapping[str, Any]) -> Any:
+    duckdb = _require("duckdb", "duckdb", "DuckDB")
+    options = {
+        key: config[key] for key in ("memory_limit", "threads") if config.get(key) is not None
+    }
+    return duckdb.connect(
+        _file_path(config, "DuckDB"), read_only=bool(config.get("read_only")), config=options
+    )
+
+
+def _odbc_value(value: Any) -> str:
+    """An ODBC connection string value, braced so ';' and '}' cannot break out of it."""
+    return "{" + str(value).replace("}", "}}") + "}"
+
+
+def _connect_sqlserver(config: Mapping[str, Any]) -> Any:
+    pyodbc = _require("pyodbc", "sqlserver", "SQL Server")
+    _missing(config, ("host", "database", "user", "password"), "SQL Server")
+    port = _port(config.get("port"), default=1433)
+    host = _host(config["host"])
+    server = host if port == 1433 else f"{host},{port}"
+    parts = {
+        "DRIVER": config.get("odbc_driver") or "ODBC Driver 17 for SQL Server",
+        "SERVER": server,
+        "DATABASE": config["database"],
+        "UID": config["user"],
+        "PWD": config["password"],
+        # As in R. ODBC Driver 18 otherwise refuses servers with self-signed
+        # certificates, which local and test servers have.
+        "TrustServerCertificate": "yes"
+        if config.get("trust_server_certificate", True) is not False
+        else "no",
+    }
+    text = ";".join(f"{key}={_odbc_value(value)}" for key, value in parts.items())
+    return pyodbc.connect(text, timeout=10)
+
+
+_CONNECTORS = {
+    "sqlite": _connect_sqlite,
+    "postgres": _connect_postgres,
+    "mysql": _connect_mysql,
+    "duckdb": _connect_duckdb,
+    "sqlserver": _connect_sqlserver,
+}
+
+
+class Connection:
+    """A DB-API connection from ``db_connect()``.
+
+    It behaves like the driver's own connection (``cursor()``, ``commit()``,
+    and so on) and closes when a ``with`` block ends::
+
+        with fw.db_connect("warehouse") as conn:
+            cursor = conn.cursor()
+
+    Attributes:
+        name: The connection name in the settings.
+        driver: sqlite, postgres, mysql, duckdb, or sqlserver.
+        raw: The driver's connection object.
+    """
+
+    def __init__(self, name: str, driver: str, raw: Any) -> None:
+        self.name = name
+        self.driver = driver
+        self.raw = raw
+        self._in_transaction = False
+
+    def __getattr__(self, attribute: str) -> Any:
+        return getattr(self.raw, attribute)
+
+    def __enter__(self) -> Connection:
+        return self
+
+    def __exit__(
+        self,
+        kind: type[BaseException] | None,
+        error: BaseException | None,
+        traceback: TracebackType | None,
+    ) -> None:
+        self.raw.close()
+
+    def __repr__(self) -> str:
+        return f"<framework Connection {self.name!r} ({self.driver})>"
+
+
+def db_connect(name: str) -> Connection:
+    """Open a database connection defined in the settings.
+
+    Raises:
+        FrameworkError: The connection is not configured, its driver package
+            is not installed, or connecting failed.
+    """
+    entry = _database_config(name)
+    driver_name = str(entry["driver"]).lower()
+    if driver_name not in DRIVERS:
+        raise FrameworkError(f"Unsupported database driver for '{name}': {entry['driver']}")
+    driver = DRIVERS[driver_name][0]
+    try:
+        raw = _CONNECTORS[driver](entry)
+    except FrameworkError:
+        raise
+    except Exception as error:
+        raise FrameworkError(f"Failed to connect to '{name}': {error}") from error
+    return Connection(name, driver, raw)
+
+
+# Queries ------------------------------------------------------------------------------
+
+
+def _frame(cursor: Any) -> pd.DataFrame:
+    columns = [column[0] for column in cursor.description or []]
+    return pd.DataFrame.from_records([tuple(row) for row in cursor.fetchall()], columns=columns)
+
+
+def db_query(
+    query: str, connection_name: str, params: Sequence[Any] | Mapping[str, Any] | None = None
+) -> pd.DataFrame:
+    """Run a query and return the rows as a DataFrame.
+
+    Pass values in ``params``, never by formatting them into the SQL. The
+    placeholder style is the driver's: ``?`` for sqlite, duckdb, and
+    sqlserver, and ``%s`` for postgres and mysql.
+    """
+    with db_connect(connection_name) as connection:
+        cursor = connection.cursor()
+        try:
+            cursor.execute(query, params) if params is not None else cursor.execute(query)
+            return _frame(cursor)
+        except Exception as error:
+            raise FrameworkError(
+                f"Failed to execute query on connection '{connection_name}': {error}"
+            ) from error
+        finally:
+            cursor.close()
+
+
+def db_execute(
+    query: str, connection_name: str, params: Sequence[Any] | Mapping[str, Any] | None = None
+) -> int:
+    """Run a statement, commit it, and return the number of rows affected."""
+    with db_connect(connection_name) as connection:
+        cursor = connection.cursor()
+        try:
+            cursor.execute(query, params) if params is not None else cursor.execute(query)
+            count = cursor.rowcount
+            connection.commit()
+            return int(count if count is not None else -1)
+        except Exception as error:
+            raise FrameworkError(
+                f"Failed to execute statement on connection '{connection_name}': {error}"
+            ) from error
+        finally:
+            cursor.close()
+
+
+@contextmanager
+def db_transaction(connection: Connection) -> Iterator[Connection]:
+    """Commit the block's statements together, or roll all of them back on an error.
+
+    A transaction inside another reuses the outer one::
+
+        with fw.db_connect("warehouse") as conn, fw.db_transaction(conn):
+            conn.cursor().execute("UPDATE ...")
+    """
+    if connection._in_transaction:
+        yield connection
+        return
+    connection._in_transaction = True
+    try:
+        yield connection
+        connection.commit()
+    except Exception as error:
+        try:
+            connection.rollback()
+        except Exception as rollback_error:  # pragma: no cover (driver failure)
+            raise FrameworkError(f"Failed to roll back transaction: {rollback_error}") from error
+        raise FrameworkError(f"Transaction failed and was rolled back: {error}") from error
+    finally:
+        connection._in_transaction = False
+
+
+# Listing ------------------------------------------------------------------------------
+
+
+def db_list() -> pd.DataFrame:
+    """Database connections in the settings: name, driver, host, port, and database."""
+    rows = [
+        {
+            "name": name,
+            "driver": entry.get("driver"),
+            "host": entry.get("host"),
+            "port": entry.get("port"),
+            "database": entry.get("database") or entry.get("dbname"),
+        }
+        for name, entry in database_configs().items()
+    ]
+    return pd.DataFrame(rows, columns=["name", "driver", "host", "port", "database"])
+
+
+def connections_list() -> pd.DataFrame:
+    """Every connection: databases and storage buckets, with the defaults marked."""
+    from framework.storage import bucket_configs, default_bucket
+
+    config = settings_read()
+    rows = [
+        {
+            "kind": "database",
+            "name": name,
+            "driver": entry.get("driver"),
+            "location": entry.get("host") or entry.get("database"),
+            "default": name == default_database(config),
+        }
+        for name, entry in database_configs(config).items()
+    ]
+    rows += [
+        {
+            "kind": "storage",
+            "name": name,
+            "driver": entry.get("provider") or entry.get("driver") or "s3",
+            "location": entry.get("bucket"),
+            "default": name == default_bucket(config),
+        }
+        for name, entry in bucket_configs(config).items()
+    ]
+    return pd.DataFrame(rows, columns=["kind", "name", "driver", "location", "default"])
+
+
+def db_drivers_status() -> pd.DataFrame:
+    """Which database drivers are installed, and the extra that installs each one."""
+    rows = []
+    for module in dict.fromkeys(entry[1] for entry in DRIVERS.values()):
+        drivers = sorted({name for name, entry in DRIVERS.items() if entry[1] == module})
+        extra = next(entry[2] for entry in DRIVERS.values() if entry[1] == module)
+        distribution = _DISTRIBUTIONS[module]
+        try:
+            version = metadata.version(distribution) if distribution else sqlite3.sqlite_version
+        except metadata.PackageNotFoundError:
+            version = None
+        rows.append(
+            {
+                "drivers": ", ".join(drivers),
+                "package": distribution or "sqlite3 (built in)",
+                "installed": version is not None,
+                "version": version,
+                "install": f"pip install framework-py[{extra}]" if extra else "",
+            }
+        )
+    return pd.DataFrame(rows, columns=["drivers", "package", "installed", "version", "install"])

framework/console.py

@@ -0,0 +1,41 @@
+"""Capturing printed output."""
+
+from __future__ import annotations
+
+import io
+from collections.abc import Iterator
+from contextlib import contextmanager, redirect_stderr, redirect_stdout
+from dataclasses import dataclass, field
+
+
+@dataclass
+class CapturedOutput:
+    """Text printed inside a ``capture_output()`` block, stdout and stderr together."""
+
+    output: str = ""
+    _buffer: io.StringIO = field(default_factory=io.StringIO, repr=False)
+
+    @property
+    def lines(self) -> list[str]:
+        return self.output.splitlines()
+
+
+@contextmanager
+def capture_output() -> Iterator[CapturedOutput]:
+    """Capture everything printed to stdout and stderr inside the block.
+
+    ::
+
+        with fw.capture_output() as captured:
+            model.summary()
+        captured.lines
+
+    Exceptions are not caught: the output printed before one is still
+    available in ``captured.output``.
+    """
+    captured = CapturedOutput()
+    try:
+        with redirect_stdout(captured._buffer), redirect_stderr(captured._buffer):
+            yield captured
+    finally:
+        captured.output = captured._buffer.getvalue()