framework-py 0.1.0.dev0__py3-none-any.whl → 0.1.0.dev2__py3-none-any.whl

framework/connections.py

@@ -280,6 +280,13 @@ class Connection:
 def db_connect(name: str) -> Connection:
     """Open a database connection defined in the settings.
 
+    Args:
+        name: The connection's name under ``connections`` in the settings.
+
+    Returns:
+        A ``Connection`` that wraps the driver's connection object. Use it in a ``with`` block
+        to close it afterward.
+
     Raises:
         FrameworkError: The connection is not configured, its driver package
             is not installed, or connecting failed.
@@ -314,6 +321,18 @@ def db_query(
     Pass values in ``params``, never by formatting them into the SQL. The
     placeholder style is the driver's: ``?`` for sqlite, duckdb, and
     sqlserver, and ``%s`` for postgres and mysql.
+
+    Args:
+        query: The SQL query.
+        connection_name: The connection's name in the settings. The connection is opened for
+            this query and closed afterward.
+        params: Values for the query's placeholders, as a sequence or a mapping.
+
+    Returns:
+        The result rows, with the query's column names.
+
+    Raises:
+        FrameworkError: When the connection cannot be opened or the query fails.
     """
     with db_connect(connection_name) as connection:
         cursor = connection.cursor()
@@ -331,7 +350,21 @@ def db_query(
 def db_execute(
     query: str, connection_name: str, params: Sequence[Any] | Mapping[str, Any] | None = None
 ) -> int:
-    """Run a statement, commit it, and return the number of rows affected."""
+    """Run a statement, commit it, and return the number of rows affected.
+
+    Args:
+        query: The SQL statement.
+        connection_name: The connection's name in the settings. The connection is opened for
+            this statement and closed afterward.
+        params: Values for the statement's placeholders, as a sequence or a mapping. The
+            placeholder style is the driver's, as in ``db_query()``.
+
+    Returns:
+        The driver's count of affected rows, or -1 when the driver does not report one.
+
+    Raises:
+        FrameworkError: When the connection cannot be opened or the statement fails.
+    """
     with db_connect(connection_name) as connection:
         cursor = connection.cursor()
         try:
@@ -355,6 +388,16 @@ def db_transaction(connection: Connection) -> Iterator[Connection]:
 
         with fw.db_connect("warehouse") as conn, fw.db_transaction(conn):
             conn.cursor().execute("UPDATE ...")
+
+    Args:
+        connection: An open connection from ``db_connect()``.
+
+    Returns:
+        A context manager that yields ``connection``.
+
+    Raises:
+        FrameworkError: When the block raises an exception. The transaction is rolled back and
+            the original exception is chained as the cause.
     """
     if connection._in_transaction:
         yield connection

framework/console.py

@@ -10,7 +10,12 @@ from dataclasses import dataclass, field
 
 @dataclass
 class CapturedOutput:
-    """Text printed inside a ``capture_output()`` block, stdout and stderr together."""
+    """Text printed inside a ``capture_output()`` block, stdout and stderr together.
+
+    Attributes:
+        output: Everything printed to stdout and stderr in the block, in order. It is empty
+            until the block ends.
+    """
 
     output: str = ""
     _buffer: io.StringIO = field(default_factory=io.StringIO, repr=False)

framework/create.py

@@ -23,15 +23,15 @@ from typing import Any
 from framework import _yaml, db
 from framework._log import inform, show_messages
 from framework.ai import write_ai_files
-from framework.config_home import global_settings, project_register
+from framework.config_home import config_dir, global_settings, project_register
 from framework.errors import FrameworkError
 from framework.quarto import quarto_generate_all
-from framework.spec.check import spec_check
+from framework.spec.check import ASSISTANTS, spec_check
 from framework.spec.merge import SpecError
 from framework.spec.resolve import TYPES, author_normalize, spec_resolve
 
 CHECKOUT_ALLOWLIST = frozenset({".git", ".github", ".framework", "README.md", "LICENSE"})
-_FILE_LIKE = re.compile(r"\.(qmd|Rmd|R|md|py|ipynb)$")
+FILE_LIKE = re.compile(r"\.(qmd|Rmd|R|md|py|ipynb)$")
 _TEMPLATE_VARS = ("project_name", "author", "author_email", "affiliation", "date")
 _SPLIT_TYPES = ("project", "project_sensitive")
 # Lines that ignore .env (or re-include it with a leading "!"), as in R.
@@ -95,13 +95,13 @@ def _assert_checkout(root: Path) -> None:
 
 def _create_directories(root: Path, spec: Mapping[str, Any]) -> None:
     for path in (spec.get("directories") or {}).values():
-        if path and not path.startswith("outputs/") and not _FILE_LIKE.search(path):
+        if path and not path.startswith("outputs/") and not FILE_LIKE.search(path):
             (root / path).mkdir(parents=True, exist_ok=True)
     for entry in (spec.get("extra_directories") or {}).values():
         if entry.get("path"):
             (root / entry["path"]).mkdir(parents=True, exist_ok=True)
     for key, path in (spec.get("render_dirs") or {}).items():
-        if path and "notebook" in key.lower() and not _FILE_LIKE.search(path):
+        if path and "notebook" in key.lower() and not FILE_LIKE.search(path):
             (root / path).mkdir(parents=True, exist_ok=True)
 
 
@@ -401,6 +401,24 @@ def project_create_from_layers(
 
     With ``interactive=False`` (the default) it runs headless for automation,
     as ``project_create(headless=True)`` describes.
+
+    Args:
+        layers_file: A JSON file with ``spec_version`` (which must be 1), ``name``,
+            ``layers``, and optionally ``author`` and ``source``. The layers are resolved with
+            ``spec_resolve()``.
+        path: The project folder. Default: the current directory. In headless mode it must be
+            a fresh checkout. Otherwise it must not exist.
+        interactive: Create the project in regular mode, with git initialization and a
+            registry entry. Nothing is prompted for either way. Default: False, which creates
+            it headless.
+
+    Returns:
+        The project folder.
+
+    Raises:
+        FrameworkError: When the layers file cannot be read, has an unsupported
+            ``spec_version``, or has no name, or when the project cannot be created there.
+        SpecError: When the resolved spec is invalid.
     """
     source = Path(layers_file)
     try:
@@ -421,14 +439,12 @@ def project_create_from_layers(
     return project_create(spec, path, headless=not interactive)
 
 
-def user_layer(type: str) -> dict[str, Any]:
-    """The user's defaults from the global settings, as a spec layer.
+def _defaults_layer(settings: Mapping[str, Any], type: str) -> dict[str, Any]:
+    """A spec layer from settings in the global-settings format.
 
-    Reads ``author`` and the ``defaults`` section (scaffold, AI assistants,
-    git, and the directories for this project type) of ``settings.yml`` in
-    ``config_dir()``.
+    Reads ``author``, the ``defaults`` section (scaffold, AI assistants, and
+    git), and the directories of this project type.
     """
-    settings = global_settings()
     raw_defaults = settings.get("defaults")
     defaults: dict[str, Any] = raw_defaults if isinstance(raw_defaults, dict) else {}
     layer: dict[str, Any] = {}
@@ -446,8 +462,13 @@ def user_layer(type: str) -> dict[str, Any]:
         layer["ai"] = {
             key: value for key, value in (("enabled", ai.get("enabled")),) if value is not None
         }
-        if isinstance(assistants, list):
-            layer["ai"]["assistants"] = {name: True for name in assistants}
+        if isinstance(assistants, list) and assistants:
+            # The list replaces the selection, as in R: assistants it leaves
+            # out (Claude Code included) are turned off.
+            selected = {name: True for name in assistants}
+            layer["ai"]["assistants"] = {name: False for name in ASSISTANTS} | selected
+        elif isinstance(assistants, dict):
+            layer["ai"]["assistants"] = assistants
     git = defaults.get("git")
     if isinstance(git, dict):
         layer["git"] = {
@@ -462,6 +483,29 @@ def user_layer(type: str) -> dict[str, Any]:
     return layer
 
 
+def user_layer(type: str) -> dict[str, Any]:
+    """The user's defaults from the global settings, as a spec layer.
+
+    Reads ``author`` and the ``defaults`` section (scaffold, AI assistants,
+    git, and the directories for this project type) of ``settings.yml`` in
+    ``config_dir()``. A list in ``defaults.ai.assistants`` replaces the
+    default selection.
+    """
+    return _defaults_layer(global_settings(), type)
+
+
+def catalog_layer(type: str) -> dict[str, Any]:
+    """The user's settings catalog, ``settings-catalog.yml`` in ``config_dir()``, as a spec layer.
+
+    R's ``new()`` starts from the settings catalog with this file merged over
+    it, so Python applies it too, below the global settings. Its keys are read
+    as in the global settings. Without the file the layer is empty.
+    """
+    path = config_dir() / "settings-catalog.yml"
+    catalog = _yaml.read(path) if path.is_file() else None
+    return _defaults_layer(catalog, type) if isinstance(catalog, dict) else {}
+
+
 def new(
     name: str | None = None,
     location: str | Path | None = None,
@@ -510,6 +554,7 @@ def new(
         raise FrameworkError(f"Invalid project type. Must be one of: {', '.join(TYPES)}")
 
     layers: Sequence[Any] = [
+        {"kind": "user", "key": "settings-catalog", "layer": catalog_layer(type)},
         {"kind": "user", "layer": user_layer(type)},
         {"kind": "project", "layer": {"type": type, "language": "python"}},
     ]
@@ -559,7 +604,7 @@ def project_add_directory(project_path: str | Path, key: str, label: str, path:
     """
     from framework.spec.check import is_safe_path
 
-    root = Path(project_path)
+    root = Path(project_path).expanduser()
     if not is_safe_path(path):
         raise FrameworkError(f"Path must be relative and stay inside the project: {path}")
     settings_file = next(

framework/data.py

@@ -22,7 +22,7 @@ import pandas as pd
 from framework import _yaml, db
 from framework._log import inform
 from framework._paths import in_root, relative_to_root
-from framework.catalog import DataInfo, data_info
+from framework.catalog import DataInfo, catalog_locate, data_info
 from framework.errors import (
     DataChangedWarning,
     DataNotFoundError,
@@ -57,7 +57,9 @@ _DIRECT_TYPES = {
     "parquet": "parquet",
 }
 
-# Extension -> type recorded by data_add(), as R's data_add() does.
+# Extension -> type recorded by data_add(). R's data_add() records .por as
+# spss and .xpt as sas, which its own data_read() then opens with the wrong
+# reader. Python records the reader types, which R reads too.
 _ADD_TYPES = {
     "csv": "csv",
     "tsv": "tsv",
@@ -69,12 +71,16 @@ _ADD_TYPES = {
     "dta": "stata",
     "sav": "spss",
     "zsav": "spss",
-    "por": "spss",
+    "por": "spss_por",
     "sas7bdat": "sas",
-    "xpt": "sas",
+    "xpt": "sas_xpt",
     "parquet": "parquet",
 }
 
+# Catalog entries R's data_add() wrote for these extensions carry the generic
+# type, so the extension picks the reader.
+_GENERIC_TYPES = {("spss", ".por"): "spss_por", ("sas", ".xpt"): "sas_xpt"}
+
 _SAVE_TYPES = ("csv", "parquet")
 
 _R_ONLY = (
@@ -236,7 +242,7 @@ def data_read(
             format Python cannot read.
         IntegrityError: A locked file changed since its hash was recorded.
     """
-    direct = Path(path)
+    direct = Path(path).expanduser()
     if direct.is_file():
         extension = direct.suffix.lower().lstrip(".")
         if extension not in _DIRECT_TYPES:
@@ -262,8 +268,9 @@ def data_read(
             FrameworkWarning,
             stacklevel=2,
         )
+    file_type = _GENERIC_TYPES.get((info.type, info.path.suffix.lower()), info.type)
     try:
-        data: pd.DataFrame = _reader(info.type)(info.path, delimiter, keep_attributes, dict(kwargs))
+        data: pd.DataFrame = _reader(file_type)(info.path, delimiter, keep_attributes, dict(kwargs))
     except (FrameworkError, ImportError):
         raise
     except Exception as error:
@@ -306,10 +313,17 @@ def _save_target(path: str, type: str | None) -> tuple[Path, str, bool]:
         )
 
     config = settings_read()
-    directories = config.get("directories") or {}
-    key = f"{parts[0]}_{parts[1]}"
-    folder = directories.get(key) if isinstance(directories, dict) else None
-    if not isinstance(folder, str) or not folder:
+    directories = config.get("directories")
+    if not isinstance(directories, dict):
+        directories = {}
+    # The longest run of leading parts that names a directory key selects the
+    # folder, so inputs.private.raw.x uses inputs_private_raw. R joins only
+    # the first two parts, so it cannot reach three-part keys.
+    for end in range(len(parts) - 1, 1, -1):
+        folder = directories.get("_".join(parts[:end]))
+        if isinstance(folder, str) and folder:
+            break
+    else:
         available = ", ".join(k.replace("_", ".") for k in directories) or "none"
         raise FrameworkError(
             f"Unknown directory '{parts[0]}.{parts[1]}'. Use a configured directory.\n\n"
@@ -318,7 +332,7 @@ def _save_target(path: str, type: str | None) -> tuple[Path, str, bool]:
             "  data_save(df, 'outputs.private.results')"
         )
     type = type or "parquet"
-    target = in_root(folder, project_root()) / f"{'_'.join(parts[2:])}.{type}"
+    target = in_root(folder, project_root()) / f"{'_'.join(parts[end:])}.{type}"
     return target, type, True
 
 
@@ -335,10 +349,11 @@ def data_save(
     Args:
         data: The data to save.
         path: A catalog name such as ``"inputs.intermediate.cleaned"`` or a
-            file path ending in ``.csv`` or ``.parquet``. A name's first two
-            parts select a folder from ``directories`` (``inputs_intermediate``),
-            and the rest becomes the file name. A name is also added to the
-            data catalog, so ``data_read()`` finds it.
+            file path ending in ``.csv`` or ``.parquet``. A name's leading
+            parts select a folder from ``directories``: the longest run that
+            names a key, such as ``inputs_intermediate`` or
+            ``inputs_private_raw``. The rest becomes the file name. A name is
+            also added to the data catalog, so ``data_read()`` finds it.
         type: ``"parquet"`` or ``"csv"``. Default: from the file extension,
             or Parquet for a catalog name.
         delimiter: Delimiter for CSV files: comma, tab, semicolon, or space.
@@ -440,13 +455,18 @@ def _catalog_set(name: str, entry: dict[str, Any]) -> None:
     if not isinstance(current, dict):
         current = {}
 
-    node = current
-    parts = name.split(".")
-    for part in parts[:-1]:
-        if not isinstance(node.get(part), dict):
-            node[part] = {}
-        node = node[part]
-    node[parts[-1]] = entry
+    # An entry already written with dotted keys is updated where it is.
+    existing = catalog_locate(name, current)
+    if existing is not None:
+        existing[0][existing[1]] = entry
+    else:
+        node = current
+        parts = name.split(".")
+        for part in parts[:-1]:
+            if not isinstance(node.get(part), dict):
+                node[part] = {}
+            node = node[part]
+        node[parts[-1]] = entry
 
     if split_path is not None:
         _yaml.write(split_path, {"data": current})
@@ -478,7 +498,7 @@ def data_add(
         The catalog entry. Its path is relative to the project root, so the
         project still works after it is moved or cloned.
     """
-    path = Path(file_path)
+    path = Path(file_path).expanduser()
     if not path.is_file():
         raise DataNotFoundError(f"File not found: {file_path}")
     path = path.resolve()

framework/global_config.py

@@ -143,12 +143,30 @@ _GITIGNORE = {
 
 
 def load_settings_catalog(include_user: bool = True, validate: bool = True) -> dict[str, Any]:
-    """The defaults new Python projects start from.
+    """Framework's package defaults, with the user's settings catalog merged over them.
 
     The catalog has ``defaults`` (Framework's package defaults, layer 0 of
     ``spec_resolve()``) and ``project_types``, each with its ``.gitignore``
     text. With ``include_user``, ``settings-catalog.yml`` in ``config_dir()``
     is merged over it.
+
+    ``new()`` starts from the same package defaults, applies
+    ``settings-catalog.yml`` over them as R's ``new()`` does, and then
+    applies the global ``settings.yml``. It reads ``author``, ``defaults.scaffold``,
+    ``defaults.ai``, ``defaults.git``, and ``project_types.<type>.directories``
+    from the file, with the same keys as the global settings.
+
+    Args:
+        include_user: Merge ``settings-catalog.yml`` from ``config_dir()`` over the shipped
+            catalog when that file exists.
+        validate: Check that the result has ``defaults`` and ``project_types`` mappings.
+
+    Returns:
+        The catalog, with ``meta``, ``defaults``, and ``project_types`` keys.
+
+    Raises:
+        SettingsError: When ``validate`` is true and ``defaults`` or ``project_types`` is
+            missing or not a mapping.
     """
     shipped = files("framework").joinpath("templates")
     defaults = _yaml.load(shipped.joinpath("defaults.yml").read_text(encoding="utf-8")) or {}
@@ -208,12 +226,32 @@ def _template_file(name: str) -> str:
 
 
 def framework_template_path(name: str) -> Path:
-    """The file your version of a template is saved in (it may not exist yet)."""
+    """The file your version of a template is saved in (it may not exist yet).
+
+    Args:
+        name: The template. One of "notebook", "script", or "presentation".
+
+    Returns:
+        The path in ``templates_dir()`` for that template.
+
+    Raises:
+        FrameworkError: When ``name`` is not a known template.
+    """
     return templates_dir() / _template_file(name)
 
 
 def read_framework_template(name: str) -> str:
-    """A template's text: yours when you saved one, otherwise Framework's."""
+    """A template's text: yours when you saved one, otherwise Framework's.
+
+    Args:
+        name: The template. One of "notebook", "script", or "presentation".
+
+    Returns:
+        The template's text.
+
+    Raises:
+        FrameworkError: When ``name`` is not a known template.
+    """
     path = framework_template_path(name)
     if path.is_file():
         return path.read_text(encoding="utf-8")
@@ -222,7 +260,19 @@ def read_framework_template(name: str) -> str:
 
 
 def write_framework_template(name: str, contents: str) -> Path:
-    """Save your version of a template. Returns its path."""
+    """Save your version of a template. Returns its path.
+
+    Args:
+        name: The template. One of "notebook", "script", or "presentation".
+        contents: The template's text. A trailing newline is added when it has none.
+
+    Returns:
+        The path the template was written to.
+
+    Raises:
+        FrameworkError: When ``name`` is not a known template.
+        TypeError: When ``contents`` is not a string.
+    """
     if not isinstance(contents, str):
         raise TypeError("contents must be text")
     path = framework_template_path(name)
@@ -232,7 +282,17 @@ def write_framework_template(name: str, contents: str) -> Path:
 
 
 def reset_framework_template(name: str) -> Path:
-    """Remove your version of a template, so Framework's is used again."""
+    """Remove your version of a template, so Framework's is used again.
+
+    Args:
+        name: The template. One of "notebook", "script", or "presentation".
+
+    Returns:
+        The path of the removed file. Nothing fails when it did not exist.
+
+    Raises:
+        FrameworkError: When ``name`` is not a known template.
+    """
     path = framework_template_path(name)
     path.unlink(missing_ok=True)
     return path

framework/hooks.py

@@ -59,7 +59,7 @@ def hook_script() -> str:
 
 
 def _settings_path(settings_file: str | Path | None) -> Path:
-    path = Path(settings_file) if settings_file is not None else find_settings_file()
+    path = Path(settings_file).expanduser() if settings_file is not None else find_settings_file()
     if path is None or not path.is_file():
         raise FrameworkError("Settings file not found (expected settings.yml or config.yml)")
     return path.resolve()
@@ -150,8 +150,18 @@ def git_hooks_install(
     A pre-commit hook Framework wrote earlier (by R or Python) is replaced. Any
     other pre-commit hook is kept unless ``force`` is True.
 
+    Args:
+        settings_file: The project's settings file. Default: the settings.yml or config.yml
+            found from the current directory.
+        force: Replace a pre-commit hook that Framework did not write.
+        verbose: Print what was done.
+
     Returns:
-        Whether the hook was installed.
+        Whether the hook was installed. False outside a git repository, when no check is
+        enabled, or when another hook is kept.
+
+    Raises:
+        FrameworkError: When the settings file is not found.
     """
     path = _settings_path(settings_file)
     root = path.parent
@@ -181,6 +191,12 @@ def git_hooks_install(
 def git_hooks_uninstall(settings_file: str | Path | None = None, *, verbose: bool = True) -> bool:
     """Remove Framework's pre-commit hook. A hook Framework did not write is kept.
 
+    Args:
+        settings_file: The project's settings file, whose folder holds the repository.
+            Default: the folder of the settings file found from the current directory, or the
+            current directory when there is none.
+        verbose: Print what was done.
+
     Returns:
         Whether a hook was removed.
     """
@@ -207,8 +223,20 @@ def git_hooks_enable(
 ) -> bool:
     """Turn on one hook check in the settings and (re)install the hook.
 
+    Args:
+        hook_name: The check. One of "data_security" or "check_sensitive_dirs". The older
+            name "warn_unignored_sensitive" is accepted for "check_sensitive_dirs".
+        settings_file: The project's settings file. Default: the settings.yml or config.yml
+            found from the current directory.
+        verbose: Print what was done.
+
     Returns:
         Whether the hook is installed.
+
+    Raises:
+        ValueError: When ``hook_name`` is not a known check, including the removed
+            "ai_sync".
+        FrameworkError: When the settings file is not found.
     """
     name = _hook_name(hook_name)
     path = _settings_path(settings_file)
@@ -225,8 +253,21 @@ def git_hooks_disable(
     ``"ai_sync"`` is accepted so older settings can be cleaned up: it is set
     to false.
 
+    Args:
+        hook_name: The check. One of "data_security", "check_sensitive_dirs", or
+            "ai_sync". The older name "warn_unignored_sensitive" is accepted for
+            "check_sensitive_dirs", and turning that check off also turns off the older
+            setting.
+        settings_file: The project's settings file. Default: the settings.yml or config.yml
+            found from the current directory.
+        verbose: Print what was done.
+
     Returns:
         Whether the hook is still installed.
+
+    Raises:
+        ValueError: When ``hook_name`` is not a known check.
+        FrameworkError: When the settings file is not found.
     """
     name = "ai_sync" if hook_name == "ai_sync" else _hook_name(hook_name)
     path = _settings_path(settings_file)
@@ -247,6 +288,10 @@ def git_hooks_disable(
 def git_hooks_list(settings_file: str | Path | None = None) -> pd.DataFrame:
     """The hook checks and whether each is enabled.
 
+    Args:
+        settings_file: The project's settings file. Default: the settings.yml or config.yml
+            found from the current directory.
+
     Returns:
         A DataFrame with columns hook, enabled, and description. Its
         ``attrs["installed"]`` says whether the pre-commit hook is installed.