framework-py 0.1.0.dev0__py3-none-any.whl

framework/db.py

@@ -0,0 +1,222 @@
+"""The project database, ``framework.db``.
+
+The schema comes from the R package's ``init.sql``, so R and Python read and
+write the same tables. Values follow what R stores:
+
+- Timestamps in the ``data``, ``cache``, and ``meta`` tables are seconds since
+  the Unix epoch, stored as REAL. RSQLite writes R's date-times that way.
+- Booleans are stored as 0 and 1.
+
+Readers accept both epoch seconds and date strings, because older rows and
+the ``results`` table hold strings.
+"""
+
+from __future__ import annotations
+
+import re
+import sqlite3
+import time
+from collections.abc import Iterator
+from contextlib import contextmanager
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from importlib import resources
+from pathlib import Path
+
+from framework.project import find_settings_file
+
+DB_NAME = "framework.db"
+
+
+def _schema_statements() -> dict[str, str]:
+    """CREATE TABLE statements from init.sql, keyed by table name."""
+    sql = resources.files("framework").joinpath("_assets/init.sql").read_text(encoding="utf-8")
+    statements: dict[str, str] = {}
+    for statement in sql.split(";"):
+        statement = statement.strip()
+        match = re.match(r"CREATE TABLE\s+(\w+)", statement, re.IGNORECASE)
+        if match:
+            statements[match.group(1)] = re.sub(
+                r"^CREATE TABLE\s+", "CREATE TABLE IF NOT EXISTS ", statement, flags=re.IGNORECASE
+            )
+    return statements
+
+
+def db_path(root: str | Path | None = None) -> Path:
+    """Where framework.db is: the project root, or the current folder outside a project."""
+    if root is not None:
+        return Path(root) / DB_NAME
+    found = find_settings_file()
+    return (found.parent if found else Path.cwd()) / DB_NAME
+
+
+def ensure_db(root: str | Path | None = None) -> Path:
+    """Create framework.db, or add any tables it is missing. Returns its path."""
+    path = db_path(root)
+    connection = sqlite3.connect(path)
+    try:
+        existing = {
+            row[0]
+            for row in connection.execute("SELECT name FROM sqlite_master WHERE type='table'")
+        }
+        for table, statement in _schema_statements().items():
+            if table not in existing:
+                connection.execute(statement)
+        connection.commit()
+    finally:
+        connection.close()
+    return path
+
+
+@contextmanager
+def connect(root: str | Path | None = None) -> Iterator[sqlite3.Connection]:
+    """A connection to framework.db that commits on success and always closes."""
+    path = ensure_db(root)
+    connection = sqlite3.connect(path)
+    connection.row_factory = sqlite3.Row
+    try:
+        with connection:
+            yield connection
+    finally:
+        connection.close()
+
+
+def timestamp() -> float:
+    """The current time as R stores it: seconds since the Unix epoch."""
+    return time.time()
+
+
+def parse_timestamp(value: object) -> datetime | None:
+    """A stored timestamp as an aware datetime in local time, or None.
+
+    Accepts epoch seconds and the date strings R and Python have written.
+    """
+    if value is None or value == "":
+        return None
+    if isinstance(value, int | float):
+        return datetime.fromtimestamp(value, tz=UTC).astimezone()
+    text = str(value)
+    try:
+        return datetime.fromtimestamp(float(text), tz=UTC).astimezone()
+    except ValueError:
+        pass
+    for pattern in ("%Y-%m-%dT%H:%M:%S%z", "%Y-%m-%d %H:%M:%S%z"):
+        try:
+            return datetime.strptime(text, pattern).astimezone()
+        except ValueError:
+            continue
+    try:
+        # A string without an offset is local time, as R writes it.
+        return datetime.fromisoformat(text).astimezone()
+    except ValueError:
+        return None
+
+
+# The data table -------------------------------------------------------------
+
+
+@dataclass(frozen=True)
+class DataRecord:
+    """A row of the ``data`` table: what Framework knows about one data file."""
+
+    name: str
+    path: str | None
+    type: str | None
+    delimiter: str | None
+    locked: bool
+    hash: str | None
+    last_read_at: datetime | None
+    created_at: datetime | None
+    updated_at: datetime | None
+
+
+def get_data_record(name: str, root: str | Path | None = None) -> DataRecord | None:
+    with connect(root) as connection:
+        row = connection.execute(
+            "SELECT name, path, type, delimiter, locked, hash, last_read_at, created_at, "
+            "updated_at FROM data WHERE name = ?",
+            (name,),
+        ).fetchone()
+    if row is None:
+        return None
+    return DataRecord(
+        name=row["name"],
+        path=row["path"],
+        type=row["type"],
+        delimiter=row["delimiter"],
+        locked=bool(row["locked"]),
+        hash=row["hash"],
+        last_read_at=parse_timestamp(row["last_read_at"]),
+        created_at=parse_timestamp(row["created_at"]),
+        updated_at=parse_timestamp(row["updated_at"]),
+    )
+
+
+def set_data_record(
+    name: str,
+    *,
+    path: str | None,
+    type: str | None,
+    delimiter: str | None,
+    locked: bool,
+    hash: str | None,
+    root: str | Path | None = None,
+) -> None:
+    """Insert or update a ``data`` row, as R's ``.set_data()`` does."""
+    now = timestamp()
+    with connect(root) as connection:
+        exists = connection.execute("SELECT 1 FROM data WHERE name = ?", (name,)).fetchone()
+        if exists:
+            connection.execute(
+                "UPDATE data SET path = ?, type = ?, delimiter = ?, locked = ?, hash = ?, "
+                "last_read_at = ?, updated_at = ? WHERE name = ?",
+                (path, type, delimiter, int(locked), hash, now, now, name),
+            )
+        else:
+            connection.execute(
+                "INSERT INTO data (name, path, type, delimiter, locked, hash, last_read_at, "
+                "created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
+                (name, path, type, delimiter, int(locked), hash, now, now, now),
+            )
+
+
+def update_data_hash(name: str, hash: str, root: str | Path | None = None) -> None:
+    """Record a new hash for a data file, as R's ``.update_data_with_hash()`` does."""
+    now = timestamp()
+    with connect(root) as connection:
+        exists = connection.execute("SELECT 1 FROM data WHERE name = ?", (name,)).fetchone()
+        if exists:
+            connection.execute(
+                "UPDATE data SET hash = ?, last_read_at = ?, updated_at = ? WHERE name = ?",
+                (hash, now, now, name),
+            )
+        else:
+            connection.execute(
+                "INSERT INTO data (name, hash, last_read_at, created_at, updated_at) "
+                "VALUES (?, ?, ?, ?, ?)",
+                (name, hash, now, now, now),
+            )
+
+
+# The meta table -------------------------------------------------------------
+
+
+def get_meta(key: str, root: str | Path | None = None) -> str | None:
+    with connect(root) as connection:
+        row = connection.execute("SELECT value FROM meta WHERE key = ?", (key,)).fetchone()
+    return None if row is None else row["value"]
+
+
+def set_meta(key: str, value: str, root: str | Path | None = None) -> None:
+    now = timestamp()
+    with connect(root) as connection:
+        exists = connection.execute("SELECT 1 FROM meta WHERE key = ?", (key,)).fetchone()
+        if exists:
+            connection.execute(
+                "UPDATE meta SET value = ?, updated_at = ? WHERE key = ?", (value, now, key)
+            )
+        else:
+            connection.execute(
+                "INSERT INTO meta (key, value, created_at, updated_at) VALUES (?, ?, ?, ?)",
+                (key, value, now, now),
+            )

framework/dotenv.py

@@ -0,0 +1,79 @@
+"""Reading and writing ``.env`` files with the same rules as R's dotenv package.
+
+The rules, from dotenv 1.0:
+
+- Lines that start with ``#`` and blank lines are skipped. A ``#`` later in a
+  line is part of the value.
+- Each other line is ``[export ]KEY=VALUE``. The key is everything before the
+  first ``=``, kept exactly as written.
+- A value wrapped in matching single or double quotes loses the quotes.
+  Nothing inside the quotes is unescaped.
+- A line that does not match is an error.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+from collections.abc import Mapping
+from pathlib import Path
+
+from framework.errors import SettingsError
+
+_LINE = re.compile(r"^\s*(?:export\s+)?(?P<key>[^=]+)=(?P<q>['\"]?)(?P<value>.*)(?P=q)\s*$")
+
+
+def parse(text: str) -> dict[str, str]:
+    """Parse ``.env`` text into a dict, in file order. Later keys win."""
+    values: dict[str, str] = {}
+    for line in text.splitlines():
+        if line.startswith("#") or not line.strip():
+            continue
+        match = _LINE.match(line)
+        if match is None:
+            raise SettingsError(f"Cannot parse .env line: {line[:40]}")
+        values[match["key"]] = match["value"]
+    return values
+
+
+def read(path: str | Path) -> dict[str, str]:
+    """Read a ``.env`` file. A missing file reads as empty."""
+    file = Path(path)
+    if not file.is_file():
+        return {}
+    return parse(file.read_text(encoding="utf-8"))
+
+
+def load(path: str | Path) -> dict[str, str]:
+    """Read a ``.env`` file and set its values in ``os.environ``.
+
+    Values replace variables that are already set, as R's ``load_dot_env()``
+    does, so a project behaves the same from R and Python.
+    """
+    values = read(path)
+    os.environ.update(values)
+    return values
+
+
+def value(path: str | Path, key: str) -> str | None:
+    """One value from a ``.env`` file, or None when it is missing or empty."""
+    found = read(path).get(key, "")
+    return found or None
+
+
+def add_missing(path: str | Path, values: Mapping[str, str]) -> list[str]:
+    """Append ``KEY=value`` lines for keys the file does not already define.
+
+    Returns the keys that were added. Existing lines are never changed.
+    """
+    file = Path(path)
+    text = file.read_text(encoding="utf-8") if file.exists() else ""
+    present = parse(text)
+    added = [key for key in values if key not in present]
+    if not added:
+        return []
+    if text and not text.endswith("\n"):
+        text += "\n"
+    text += "".join(f"{key}={values[key]}\n" for key in added)
+    file.write_text(text, encoding="utf-8")
+    return added

framework/errors.py

@@ -0,0 +1,39 @@
+"""Exceptions raised by Framework."""
+
+from __future__ import annotations
+
+
+class FrameworkError(Exception):
+    """Base class for every error Framework raises."""
+
+
+class ProjectNotFoundError(FrameworkError):
+    """No Framework project (a folder with settings.yml or config.yml) was found."""
+
+
+class SettingsError(FrameworkError):
+    """A settings file is missing, cannot be parsed, or holds an invalid value."""
+
+
+class DataNotFoundError(FrameworkError):
+    """A data catalog name or file path does not resolve to a data file."""
+
+
+class UnsupportedFormatError(FrameworkError):
+    """A file format Framework cannot read or write from Python."""
+
+
+class IntegrityError(FrameworkError):
+    """A locked data file no longer matches the hash recorded in framework.db."""
+
+
+class FrameworkWarning(UserWarning):
+    """Base class for warnings Framework issues."""
+
+
+class DataChangedWarning(FrameworkWarning):
+    """An unlocked data file changed since Framework last recorded its hash."""
+
+
+class CacheWarning(FrameworkWarning):
+    """A cache entry could not be used, so it was treated as a miss."""

framework/global_config.py

@@ -0,0 +1,238 @@
+"""Global settings, the settings catalog, and user templates.
+
+The global settings live in ``settings.yml`` in ``config_dir()``. Python
+keeps its own config folder, separate from R's; ``config_sync()`` copies R's
+settings over.
+"""
+
+from __future__ import annotations
+
+import copy
+from importlib.resources import files
+from pathlib import Path
+from typing import Any
+
+from framework import _yaml
+from framework._log import inform, show_messages
+from framework.config_home import (
+    _global_settings_path,
+    _save_global_settings,
+    config_dir,
+    global_settings,
+)
+from framework.errors import FrameworkError, SettingsError
+from framework.settings import modify_list
+from framework.spec.check import secret_env_values
+from framework.spec.resolve import TYPES
+
+# Settings ---------------------------------------------------------------------
+
+# Sections a new value replaces instead of merging into, as in R.
+_REPLACED = (("defaults", "packages"), ("defaults", "connections"), ("defaults", "env"))
+_TYPE_REPLACED = ("directories", "extra_directories", "directories_enabled")
+
+
+def _validate(settings: dict[str, Any]) -> None:
+    author = settings.get("author")
+    if author is not None:
+        if not isinstance(author, dict):
+            raise SettingsError("author must be a mapping of name, email, and affiliation")
+        for key in ("name", "email", "affiliation", "orcid", "url"):
+            if author.get(key) is not None and not isinstance(author[key], str):
+                raise SettingsError(f"author.{key} must be text")
+    for key in ("defaults", "project_types", "git", "privacy", "global"):
+        if settings.get(key) is not None and not isinstance(settings[key], dict):
+            raise SettingsError(f"{key} must be a mapping")
+    for root in (
+        settings.get("projects_root"),
+        (settings.get("global") or {}).get("projects_root"),
+    ):
+        if root is not None and not isinstance(root, str):
+            raise SettingsError("projects_root must be a folder path")
+    types = settings.get("project_types") or {}
+    unknown = sorted(set(types) - set(TYPES))
+    if unknown:
+        raise SettingsError(f"Unknown project types: {', '.join(unknown)}")
+    # R stores defaults.env as .env text, or as {raw, variables}. A plain
+    # mapping of names to values is read as the variables.
+    env = (settings.get("defaults") or {}).get("env")
+    raw: Any
+    variables: Any
+    if isinstance(env, str):
+        raw, variables = env, None
+    elif isinstance(env, dict) and env and set(env) <= {"raw", "variables"}:
+        raw, variables = env.get("raw"), env.get("variables")
+    elif isinstance(env, dict) or env is None:
+        raw, variables = None, env
+    else:
+        raise SettingsError("defaults.env must be .env text or a mapping")
+    secrets = secret_env_values(raw if isinstance(raw, str) else None, variables)
+    if secrets:
+        raise SettingsError(
+            f"defaults.env: secrets must not be stored in settings ({', '.join(secrets)}). "
+            "Leave the values empty and put them in each project's .env file."
+        )
+
+
+def _tilde(path: str) -> str:
+    expanded = Path(path).expanduser()
+    home = Path.home()
+    try:
+        return "~/" + expanded.relative_to(home).as_posix()
+    except ValueError:
+        return path
+
+
+def configure_global(
+    settings: dict[str, Any] | None = None, *, validate: bool = True
+) -> dict[str, Any]:
+    """Read or update the global settings.
+
+    Args:
+        settings: Values to merge into the global settings, such as
+            ``{"author": {"name": "Ada Lovelace"}}``. Nested mappings merge,
+            except ``defaults.packages``, ``defaults.connections``,
+            ``defaults.env``, and each project type's directories, which are
+            replaced. Without ``settings``, nothing changes.
+        validate: Check the values before saving. Secrets in
+            ``defaults.env`` are refused.
+
+    Returns:
+        The global settings after the update.
+    """
+    current = global_settings()
+    if settings is None:
+        return current
+    if not isinstance(settings, dict):
+        raise TypeError("settings must be a dict")
+    updated: dict[str, Any] = modify_list(copy.deepcopy(current), copy.deepcopy(settings))
+    for section, key in _REPLACED:
+        value = (settings.get(section) or {}).get(key)
+        if value is not None:
+            updated.setdefault(section, {})[key] = copy.deepcopy(value)
+    for type_name, type_settings in (settings.get("project_types") or {}).items():
+        for key in _TYPE_REPLACED:
+            value = (type_settings or {}).get(key)
+            if value is None:
+                continue
+            if key == "directories" and isinstance(value, dict):
+                value = {k: v for k, v in value.items() if v}
+            updated["project_types"][type_name][key] = copy.deepcopy(value)
+    if validate:
+        _validate(updated)
+    for holder in (updated, updated.get("global") or {}):
+        if "projects_root" in holder:
+            if holder["projects_root"]:
+                holder["projects_root"] = _tilde(holder["projects_root"])
+            else:
+                del holder["projects_root"]
+    _save_global_settings(updated)
+    show_messages()
+    inform(f"✓ Global settings updated in {_global_settings_path()}")
+    return updated
+
+
+# The settings catalog ---------------------------------------------------------
+
+_GITIGNORE = {
+    "project": "project",
+    "project_sensitive": "sensitive",
+    "course": "course",
+    "presentation": "presentation",
+}
+
+
+def load_settings_catalog(include_user: bool = True, validate: bool = True) -> dict[str, Any]:
+    """The defaults new Python projects start from.
+
+    The catalog has ``defaults`` (Framework's package defaults, layer 0 of
+    ``spec_resolve()``) and ``project_types``, each with its ``.gitignore``
+    text. With ``include_user``, ``settings-catalog.yml`` in ``config_dir()``
+    is merged over it.
+    """
+    shipped = files("framework").joinpath("templates")
+    defaults = _yaml.load(shipped.joinpath("defaults.yml").read_text(encoding="utf-8")) or {}
+    catalog: dict[str, Any] = {
+        "meta": {"version": 2, "language": "python"},
+        "defaults": defaults,
+        "project_types": {
+            name: {
+                "gitignore": shipped.joinpath(f"gitignore/{_GITIGNORE[name]}").read_text(
+                    encoding="utf-8"
+                )
+            }
+            for name in _GITIGNORE
+        },
+    }
+    user_path = config_dir() / "settings-catalog.yml"
+    if include_user and user_path.is_file():
+        user = _yaml.read(user_path)
+        if isinstance(user, dict):
+            catalog = modify_list(catalog, user)
+    if validate:
+        for key in ("defaults", "project_types"):
+            if not isinstance(catalog.get(key), dict):
+                raise SettingsError(f"Settings catalog is missing '{key}'")
+    return catalog
+
+
+# User templates ---------------------------------------------------------------
+
+# Template names and the stub files they replace. make_notebook(),
+# make_script(), and make_presentation() use a user template when there is
+# no project stub of the same name.
+TEMPLATES = {
+    "notebook": "notebook-default.qmd",
+    "script": "script-default.py",
+    "presentation": "notebook-revealjs.qmd",
+}
+
+
+def templates_dir() -> Path:
+    """Where your templates are kept: ``templates/`` in ``config_dir()``."""
+    return config_dir() / "templates"
+
+
+def _template_file(name: str) -> str:
+    if name not in TEMPLATES:
+        hint = (
+            " .gitignore content comes from the project type and from layers; "
+            "add rules with a facet."
+            if name.startswith("gitignore")
+            else ""
+        )
+        raise FrameworkError(
+            f"Unknown template: {name}. Templates are {', '.join(TEMPLATES)}.{hint}"
+        )
+    return TEMPLATES[name]
+
+
+def framework_template_path(name: str) -> Path:
+    """The file your version of a template is saved in (it may not exist yet)."""
+    return templates_dir() / _template_file(name)
+
+
+def read_framework_template(name: str) -> str:
+    """A template's text: yours when you saved one, otherwise Framework's."""
+    path = framework_template_path(name)
+    if path.is_file():
+        return path.read_text(encoding="utf-8")
+    shipped = files("framework").joinpath(f"templates/stubs/{_template_file(name)}")
+    return shipped.read_text(encoding="utf-8")
+
+
+def write_framework_template(name: str, contents: str) -> Path:
+    """Save your version of a template. Returns its path."""
+    if not isinstance(contents, str):
+        raise TypeError("contents must be text")
+    path = framework_template_path(name)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(contents if contents.endswith("\n") else contents + "\n", encoding="utf-8")
+    return path
+
+
+def reset_framework_template(name: str) -> Path:
+    """Remove your version of a template, so Framework's is used again."""
+    path = framework_template_path(name)
+    path.unlink(missing_ok=True)
+    return path