calkit-python 0.41.11__py3-none-any.whl → 0.41.12__py3-none-any.whl

calkit/cli/list.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import json
 from typing import Annotated, Literal
 
 import typer
@@ -12,6 +13,32 @@ from calkit.cli import AliasGroup, raise_error, warn
 list_app = typer.Typer(cls=AliasGroup, no_args_is_help=True)
 
 
+def _echo_object(obj: dict) -> None:
+    """Print a single object in the human-readable YAML-ish listing format."""
+    # Copy so popping 'path' doesn't mutate the caller's dict.
+    obj = dict(obj)
+    path = obj.pop("path", None)
+    typer.echo(f"- path: {path}")
+    for k, v in obj.items():
+        if isinstance(v, dict):
+            typer.echo(f"    {k}:")
+            for k1, v1 in v.items():
+                typer.echo(f"      {k1}: {v1}")
+        elif isinstance(v, list):
+            typer.echo(f"    {k}:")
+            for item in v:
+                if isinstance(item, dict):
+                    for n, (k1, v1) in enumerate(item.items()):
+                        if n == 0:
+                            typer.echo(f"      - {k1}: {v1}")
+                        else:
+                            typer.echo(f"        {k1}: {v1}")
+                else:
+                    typer.echo(f"        - {item}")
+        else:
+            typer.echo(f"  {k}: {v}")
+
+
 def _list_objects(
     kind: Literal[
         "notebooks",
@@ -21,33 +48,44 @@ def _list_objects(
         "publications",
     ],
 ):
-    """List objects.
+    """List objects."""
+    ck_info = calkit.load_calkit_info()
+    for obj in ck_info.get(kind, []) or []:
+        _echo_object(obj)
+
 
-    TODO: This should probably just use some library to dump YAML to string.
+def _list_artifacts(
+    kind: Literal["figures", "datasets"],
+    json_output: bool,
+    declared_only: bool,
+):
+    """List figures or datasets, optionally including auto-detected ones.
+
+    By default, artifacts declared in ``calkit.yaml`` are merged with any
+    auto-detected from the project's files; ``--declared-only`` returns just the
+    declared ones. Each entry carries a ``detected`` flag so callers can tell
+    declared and auto-detected artifacts apart.
     """
     ck_info = calkit.load_calkit_info()
-    objects = ck_info.get(kind, [])
-    for obj in objects:
-        path = obj.pop("path")
-        typer.echo(f"- path: {path}")
-        for k, v in obj.items():
-            if isinstance(v, dict):
-                typer.echo(f"    {k}:")
-                for k1, v1 in v.items():
-                    typer.echo(f"      {k1}: {v1}")
-            elif isinstance(v, list):
-                typer.echo(f"    {k}:")
-                for item in v:
-                    if isinstance(item, dict):
-                        for n, (k1, v1) in enumerate(item.items()):
-                            if n == 0:
-                                typer.echo(f"      - {k1}: {v1}")
-                            else:
-                                typer.echo(f"        {k1}: {v1}")
-                    else:
-                        typer.echo(f"        - {item}")
-            else:
-                typer.echo(f"  {k}: {v}")
+    declared = ck_info.get(kind, []) or []
+    declared_paths = {o.get("path") for o in declared if isinstance(o, dict)}
+    detected: list[dict] = []
+    if not declared_only:
+        found = calkit.detect.detect_project_artifacts(ck_info=ck_info)
+        for path in found.get(kind, []):
+            if path not in declared_paths:
+                detected.append({"path": path})
+    if json_output:
+        result = [
+            {**o, "detected": False} for o in declared if isinstance(o, dict)
+        ]
+        result += [{**o, "detected": True} for o in detected]
+        typer.echo(json.dumps(result))
+        return
+    for obj in declared:
+        _echo_object(obj)
+    for obj in detected:
+        _echo_object({**obj, "detected": True})
 
 
 @list_app.command(name="notebooks|nb")
@@ -57,15 +95,43 @@ def list_notebooks():
 
 
 @list_app.command(name="figures|figs")
-def list_figures():
+def list_figures(
+    json_output: Annotated[
+        bool, typer.Option("--json", help="Output result as JSON.")
+    ] = False,
+    declared_only: Annotated[
+        bool,
+        typer.Option(
+            "--declared-only",
+            help=(
+                "Only list figures declared in calkit.yaml; "
+                "skip auto-detection."
+            ),
+        ),
+    ] = False,
+):
     """List figures in the project."""
-    _list_objects("figures")
+    _list_artifacts("figures", json_output, declared_only)
 
 
 @list_app.command(name="datasets")
-def list_datasets():
+def list_datasets(
+    json_output: Annotated[
+        bool, typer.Option("--json", help="Output result as JSON.")
+    ] = False,
+    declared_only: Annotated[
+        bool,
+        typer.Option(
+            "--declared-only",
+            help=(
+                "Only list datasets declared in calkit.yaml; "
+                "skip auto-detection."
+            ),
+        ),
+    ] = False,
+):
     """List datasets in the project."""
-    _list_objects("datasets")
+    _list_artifacts("datasets", json_output, declared_only)
 
 
 @list_app.command(name="publications|pubs")

calkit/detect.py

@@ -74,6 +74,7 @@ def detect_r_script_io(
         content,
         script_dir=script_dir,
         wdir=effective_wdir,
+        project_root=effective_wdir,
     )
 
 
@@ -995,7 +996,11 @@ def _resolve_python_import(
 
 
 def _detect_r_code_io(
-    code: str, script_dir: str = ".", wdir: str = "."
+    code: str,
+    script_dir: str = ".",
+    wdir: str = ".",
+    project_root: str = ".",
+    _seen: set[str] | None = None,
 ) -> dict[str, list[str]]:
     """Detect I/O from R code string (used for scripts and notebook cells).
 
@@ -1007,6 +1012,12 @@ def _detect_r_code_io(
         Directory containing the script (for resolving source() includes).
     wdir : str
         Working directory from which the script is executed (for data file paths).
+    project_root : str
+        Directory that ``here::here()`` resolves against, i.e. the project
+        root where the pipeline runs.
+    _seen : set[str] | None
+        Absolute paths of scripts already analyzed, used to guard against
+        cycles when recursing into ``source()``d scripts.
 
     Returns
     -------
@@ -1015,78 +1026,154 @@ def _detect_r_code_io(
     """
     inputs = []
     outputs = []
+    if _seen is None:
+        _seen = set()
     code = re.sub(r"#.*$", "", code, flags=re.MULTILINE)
     r_vars = _extract_r_string_assignments(code)
     fig_dir = r_vars.get("fig_dir")
-    # Detect source() calls for R script includes
-    source_pattern = r'source\s*\(\s*["\']([^"\']+\.R)["\']'
-    source_matches = re.findall(source_pattern, code, flags=re.IGNORECASE)
-    for match in source_matches:
-        if not os.path.isabs(match):
-            full_path = os.path.join(script_dir, match)
-            if os.path.exists(full_path):
-                inputs.append(Path(os.path.relpath(full_path)).as_posix())
-            elif _is_valid_project_path(match):
-                inputs.append(match)
-    # Read patterns that capture variables, file.path() expressions, or literals
-    read_patterns = [
-        r'read\.(?:csv|table|delim|tsv)\s*\(\s*(file\.path\([^\)]*\)|"[^"]+"|\'[^\']+\'|[A-Za-z_][A-Za-z0-9_]*)',
-        r'readRDS\s*\(\s*(file\.path\([^\)]*\)|"[^"]+"|\'[^\']+\'|[A-Za-z_][A-Za-z0-9_]*)',
-        r'load\s*\(\s*(file\.path\([^\)]*\)|"[^"]+"|\'[^\']+\'|[A-Za-z_][A-Za-z0-9_]*)',
-        r'read_csv\s*\(\s*(file\.path\([^\)]*\)|"[^"]+"|\'[^\']+\'|[A-Za-z_][A-Za-z0-9_]*)',
-        r'read_excel\s*\(\s*(file\.path\([^\)]*\)|"[^"]+"|\'[^\']+\'|[A-Za-z_][A-Za-z0-9_]*)',
-        r'fread\s*\(\s*(file\.path\([^\)]*\)|"[^"]+"|\'[^\']+\'|[A-Za-z_][A-Za-z0-9_]*)',
+    # A path argument may be a here::here()/file.path() call, a quoted literal,
+    # or a variable name that resolves to one of those
+    path_expr = (
+        r'(here::here\([^)]*\)|file\.path\([^)]*\)|"[^"]+"|\'[^\']+\'|'
+        r"[A-Za-z_][A-Za-z0-9_]*)"
+    )
+    # Detect source() calls for R script includes, add them as inputs, and
+    # recurse into them so their I/O is attributed to this stage
+    for raw in re.findall(r"source\s*\(\s*" + path_expr, code):
+        resolved = _resolve_r_path_expr(raw, r_vars)
+        if not resolved or not resolved.lower().endswith(".r"):
+            continue
+        if os.path.isabs(resolved):
+            continue
+        # here::here() paths are relative to the project root; everything else
+        # is resolved relative to the sourcing script's directory
+        if raw.strip().startswith("here::here"):
+            fs_path = os.path.join(project_root, resolved)
+        else:
+            fs_path = os.path.join(script_dir, resolved)
+        if os.path.exists(fs_path):
+            inputs.append(Path(os.path.relpath(fs_path)).as_posix())
+        elif _is_valid_project_path(resolved):
+            inputs.append(resolved)
+        # Recurse into the sourced script if we can read it
+        abs_path = os.path.abspath(fs_path)
+        if os.path.exists(fs_path) and abs_path not in _seen:
+            _seen.add(abs_path)
+            try:
+                with open(fs_path, "r", encoding="utf-8") as f:
+                    sub_code = f.read()
+            except (OSError, UnicodeDecodeError):
+                sub_code = None
+            if sub_code is not None:
+                sub_dir = (
+                    os.path.dirname(Path(os.path.relpath(fs_path)).as_posix())
+                    or "."
+                )
+                sub_io = _detect_r_code_io(
+                    sub_code,
+                    script_dir=sub_dir,
+                    wdir=wdir,
+                    project_root=project_root,
+                    _seen=_seen,
+                )
+                inputs.extend(sub_io["inputs"])
+                outputs.extend(sub_io["outputs"])
+    # Read patterns that capture variables, here::here()/file.path()
+    # expressions, or literals
+    read_funcs = [
+        r"read\.(?:csv|table|delim|tsv)",
+        r"readRDS",
+        r"readLines",
+        r"load",
+        r"read_csv",
+        r"read_tsv",
+        r"read_delim",
+        r"read_excel",
+        r"fread",
     ]
-    # Process read patterns with variable resolution
-    for pattern in read_patterns:
-        matches = re.findall(pattern, code)
-        for match in matches:
+    for func in read_funcs:
+        for match in re.findall(func + r"\s*\(\s*" + path_expr, code):
             resolved = _resolve_r_path_expr(match, r_vars)
             if resolved:
                 inputs.append(resolved)
-    # Write patterns for simple cases (literal strings in 2nd argument)
-    simple_write_patterns = [
-        r'write\.(?:csv|table)\s*\(\s*[^,]+,\s*["\']([^"\']+)["\']',
-        r'saveRDS\s*\(\s*[^,]+,\s*["\']([^"\']+)["\']',
-        r'save\s*\([^)]*file\s*=\s*["\']([^"\']+)["\']',
-        r'write_csv\s*\(\s*[^,]+,\s*["\']([^"\']+)["\']',
-        r'write_excel\s*\(\s*[^,]+,\s*["\']([^"\']+)["\']',
-        r'pdf\s*\(\s*["\']([^"\']+)["\']',
-        r'png\s*\(\s*["\']([^"\']+)["\']',
-        r'pdf\s*\([^)]*file\s*=\s*["\']([^"\']+)["\']',
-        r'png\s*\([^)]*filename\s*=\s*["\']([^"\']+)["\']',
-        r'svg\s*\([^)]*file\s*=\s*["\']([^"\']+)["\']',
-        r'svg\s*\([^)]*filename\s*=\s*["\']([^"\']+)["\']',
-        r'jpeg\s*\([^)]*file\s*=\s*["\']([^"\']+)["\']',
-        r'jpeg\s*\([^)]*filename\s*=\s*["\']([^"\']+)["\']',
-        r'tiff\s*\([^)]*file\s*=\s*["\']([^"\']+)["\']',
-        r'tiff\s*\([^)]*filename\s*=\s*["\']([^"\']+)["\']',
-        r'bmp\s*\([^)]*file\s*=\s*["\']([^"\']+)["\']',
-        r'bmp\s*\([^)]*filename\s*=\s*["\']([^"\']+)["\']',
-        r'CairoPNG\s*\([^)]*file\s*=\s*["\']([^"\']+)["\']',
-        r'CairoPDF\s*\([^)]*file\s*=\s*["\']([^"\']+)["\']',
-        r'svglite\s*\([^)]*file\s*=\s*["\']([^"\']+)["\']',
+    # Write functions where the path is the second positional argument
+    second_arg_write_funcs = [
+        r"write\.(?:csv|table)",
+        r"saveRDS",
+        r"write_csv",
+        r"write_tsv",
+        r"write_delim",
+        r"write_excel\w*",
+        r"writeLines",
+        r"fwrite",
     ]
-    for pattern in simple_write_patterns:
-        outputs.extend(re.findall(pattern, code))
-    # ggsave patterns that handle file.path(), variables, or literals
-    ggsave_patterns = [
-        r'ggsave\s*\(\s*(file\.path\([^\)]*\)|"[^"]+"|\'[^\']+\'|[A-Za-z_][A-Za-z0-9_]*)',
-        r'ggsave\s*\([^)]*filename\s*=\s*(file\.path\([^\)]*\)|"[^"]+"|\'[^\']+\'|[A-Za-z_][A-Za-z0-9_]*)',
+    for func in second_arg_write_funcs:
+        pattern = func + r"\s*\(\s*[^,]+,\s*" + path_expr
+        for match in re.findall(pattern, code):
+            resolved = _resolve_r_path_expr(match, r_vars)
+            if resolved:
+                outputs.append(resolved)
+    # Write functions where the path is the (only) first positional argument
+    first_arg_write_funcs = [
+        r"ggsave",
+        r"sink",
+        r"pdf",
+        r"png",
+        r"svg",
+        r"svglite",
+        r"jpeg",
+        r"tiff",
+        r"bmp",
+        r"cairo_pdf",
+        r"CairoPNG",
+        r"CairoPDF",
     ]
-    for pattern in ggsave_patterns:
-        matches = re.findall(pattern, code)
-        for match in matches:
+    for func in first_arg_write_funcs:
+        pattern = func + r"\s*\(\s*" + path_expr
+        for match in re.findall(pattern, code):
+            resolved = _resolve_r_path_expr(match, r_vars)
+            if resolved:
+                outputs.append(resolved)
+    # Write functions whose path is passed as a named argument, regardless of
+    # position (e.g. saveRDS(x, file = ...), pdf(file = ...), save(file = ...))
+    named_arg_write_funcs = [
+        r"write\.(?:csv|table)",
+        r"saveRDS",
+        r"save",
+        r"write_csv",
+        r"write_tsv",
+        r"write_delim",
+        r"write_excel\w*",
+        r"writeLines",
+        r"fwrite",
+        r"ggsave",
+        r"sink",
+        r"pdf",
+        r"png",
+        r"svg",
+        r"svglite",
+        r"jpeg",
+        r"tiff",
+        r"bmp",
+        r"cairo_pdf",
+        r"CairoPNG",
+        r"CairoPDF",
+    ]
+    for func in named_arg_write_funcs:
+        pattern = (
+            func + r"\s*\([^)]*\b(?:file|filename|path|con)\s*=\s*" + path_expr
+        )
+        for match in re.findall(pattern, code):
             resolved = _resolve_r_path_expr(match, r_vars)
             if resolved:
                 outputs.append(resolved)
+    # save_fig() writes into fig_dir when the path isn't already qualified
     save_fig_patterns = [
-        r"save_fig\s*\(\s*[^,]+,\s*(file\.path\([^\)]*\)|\"[^\"]+\"|'[^']+'|[A-Za-z_][A-Za-z0-9_]*)",
-        r"save_fig\s*\([^\)]*filename\s*=\s*(file\.path\([^\)]*\)|\"[^\"]+\"|'[^']+'|[A-Za-z_][A-Za-z0-9_]*)",
+        r"save_fig\s*\(\s*[^,]+,\s*" + path_expr,
+        r"save_fig\s*\([^)]*filename\s*=\s*" + path_expr,
     ]
     for pattern in save_fig_patterns:
-        matches = re.findall(pattern, code)
-        for match in matches:
+        for match in re.findall(pattern, code):
             resolved = _resolve_r_path_expr(match, r_vars)
             if resolved:
                 if fig_dir and not os.path.isabs(resolved):
@@ -1101,13 +1188,27 @@ def _detect_r_code_io(
     # Resolve paths relative to working directory
     inputs = _resolve_paths_to_wdir(inputs, script_dir, wdir)
     outputs = _resolve_paths_to_wdir(outputs, script_dir, wdir)
+    # Normalize to POSIX separators for persistence/display, since paths built
+    # from here::here()/file.path() use the host OS separator
+    inputs = [Path(p).as_posix() for p in inputs]
+    outputs = [Path(p).as_posix() for p in outputs]
     inputs = list(dict.fromkeys(inputs))
     outputs = list(dict.fromkeys(outputs))
+    # A file produced within this stage is not an external input, even if it is
+    # also read back (e.g. an intermediate written by one sourced script and
+    # consumed by another)
+    output_set = set(outputs)
+    inputs = [p for p in inputs if p not in output_set]
     return {"inputs": inputs, "outputs": outputs}
 
 
 def _extract_r_string_assignments(code: str) -> dict[str, str]:
-    """Extract simple string assignments in R code."""
+    """Extract simple string assignments in R code.
+
+    Captures both string-literal assignments (``x <- "path"``) and
+    ``here::here()``/``file.path()`` assignments whose arguments are literals
+    or previously assigned variables (``x <- here::here("a", "b")``).
+    """
     assignments = {}
     pattern = re.compile(
         r"^\s*([A-Za-z_][A-Za-z0-9_]*)\s*(?:<-|=)\s*[\"']([^\"']+)[\"']",
@@ -1115,6 +1216,16 @@ def _extract_r_string_assignments(code: str) -> dict[str, str]:
     )
     for name, value in pattern.findall(code):
         assignments[name] = value
+    # Resolve here::here()/file.path() assignments using the literals above
+    call_pattern = re.compile(
+        r"^\s*([A-Za-z_][A-Za-z0-9_]*)\s*(?:<-|=)\s*"
+        r"(here::here\([^)]*\)|file\.path\([^)]*\))",
+        flags=re.MULTILINE,
+    )
+    for name, expr in call_pattern.findall(code):
+        resolved = _resolve_r_path_expr(expr, assignments)
+        if resolved:
+            assignments[name] = resolved
     return assignments
 
 
@@ -1146,7 +1257,11 @@ def _resolve_julia_path_expr(
 
 
 def _resolve_r_path_expr(expr: str, variables: dict[str, str]) -> str | None:
-    """Resolve simple R path expressions like "x", var, or file.path(...)."""
+    """Resolve simple R path expressions.
+
+    Handles quoted literals, variables, and ``file.path()``/``here::here()``
+    calls whose arguments are literals or known variables.
+    """
     expr = expr.strip()
     if not expr:
         return None
@@ -1154,22 +1269,23 @@ def _resolve_r_path_expr(expr: str, variables: dict[str, str]) -> str | None:
         return expr[1:-1]
     if expr in variables:
         return variables[expr]
-    if expr.startswith("file.path"):
-        match = re.match(r"file\.path\((.*)\)", expr)
-        if not match:
-            return None
-        args = [a.strip() for a in match.group(1).split(",") if a.strip()]
-        if not args:
-            return None
-        parts = []
-        for arg in args:
-            if arg[0] in ['"', "'"] and arg[-1] == arg[0]:
-                parts.append(arg[1:-1])
-            elif arg in variables:
-                parts.append(variables[arg])
-            else:
+    for prefix in ("file.path", "here::here"):
+        if expr.startswith(prefix):
+            match = re.match(re.escape(prefix) + r"\((.*)\)", expr)
+            if not match:
+                return None
+            args = [a.strip() for a in match.group(1).split(",") if a.strip()]
+            if not args:
                 return None
-        return os.path.join(*parts)
+            parts = []
+            for arg in args:
+                if arg[0] in ['"', "'"] and arg[-1] == arg[0]:
+                    parts.append(arg[1:-1])
+                elif arg in variables:
+                    parts.append(variables[arg])
+                else:
+                    return None
+            return os.path.join(*parts)
     return None
 
 
@@ -1687,3 +1803,245 @@ def create_r_description_file(
         os.makedirs(output_dir, exist_ok=True)
     with open(output_path, "w") as f:
         f.write(create_r_description_content(dependencies))
+
+
+# Figure/dataset auto-detection
+# A file is only treated as an auto-detected figure or dataset when it lives in
+# a directory whose name signals its kind. These sets are intentionally narrow
+# (and kept in sync with Calkit Cloud) so we don't flag arbitrary images or
+# data files scattered around a repository.
+FIGURE_EXTENSIONS = {
+    ".png",
+    ".jpg",
+    ".jpeg",
+    ".gif",
+    ".svg",
+    ".pdf",
+    ".eps",
+    ".tiff",
+    ".tif",
+}
+FIGURE_DIRS = {"figures", "figure", "figs", "fig", "plots", "plot", "images"}
+DATASET_EXTENSIONS = {
+    ".csv",
+    ".h5",
+    ".hdf5",
+    ".parquet",
+    ".nc",
+    ".zarr",
+    ".feather",
+    ".arrow",
+    ".avro",
+    ".json",
+    ".jsonl",
+    ".ndjson",
+}
+DATA_DIRS = {"data", "datasets", "dataset"}
+
+
+def _ancestor_dir_names(rel_path: str) -> set[str]:
+    """Lower-cased names of the ancestor directories of a "/"-separated path."""
+    return {p.lower() for p in rel_path.split("/")[:-1]}
+
+
+def _path_ext(rel_path: str) -> str:
+    name = rel_path.rsplit("/", 1)[-1]
+    return ("." + name.rsplit(".", 1)[-1].lower()) if "." in name else ""
+
+
+def is_figure_path(rel_path: str) -> bool:
+    """Whether a repo-relative path looks like an auto-detectable figure."""
+    ancestors = _ancestor_dir_names(rel_path)
+    ext = _path_ext(rel_path)
+    if ext in FIGURE_EXTENSIONS and ancestors & FIGURE_DIRS:
+        return True
+    # A Plotly figure is stored as JSON; treat .json under a figure-only
+    # directory (one that isn't also a data directory) as a figure.
+    if ext == ".json" and ancestors & (FIGURE_DIRS - DATA_DIRS):
+        return True
+    return False
+
+
+def is_dataset_path(rel_path: str) -> bool:
+    """Whether a repo-relative path looks like an auto-detectable dataset."""
+    return (
+        _path_ext(rel_path) in DATASET_EXTENSIONS
+        and bool(_ancestor_dir_names(rel_path) & DATA_DIRS)
+        and not is_figure_path(rel_path)
+    )
+
+
+def _is_hidden_path(rel_path: str) -> bool:
+    return any(part.startswith(".") for part in rel_path.split("/"))
+
+
+def _is_under_any_dir(rel_path: str, dirs: list[str]) -> bool:
+    """Whether ``rel_path`` equals or lives inside any of ``dirs``."""
+    return any(rel_path == d or rel_path.startswith(d + "/") for d in dirs)
+
+
+def _collapse_dataset_folders(paths: list[str]) -> list[str]:
+    """Collapse data files into their containing folder.
+
+    A folder holding two or more data files is reported as a single dataset
+    rather than file-by-file. Folders nested inside another collapsed folder
+    are dropped so only the outermost dataset folder remains.
+    """
+    by_dir: dict[str, list[str]] = {}
+    for p in paths:
+        directory = p.rsplit("/", 1)[0] if "/" in p else ""
+        by_dir.setdefault(directory, []).append(p)
+    collapsed: set[str] = set()
+    for directory, group in by_dir.items():
+        if directory and len(group) >= 2:
+            collapsed.add(directory)
+        else:
+            collapsed.update(group)
+    result = sorted(collapsed)
+    return [
+        p
+        for p in result
+        if not any(o != p and p.startswith(o + "/") for o in result)
+    ]
+
+
+def detect_figures(
+    candidate_paths: list[str],
+    reserved_paths: list[str] | tuple[str, ...] = (),
+) -> list[str]:
+    """Auto-detected figure paths among ``candidate_paths``.
+
+    Paths under a dot-directory or under one of ``reserved_paths`` (declared
+    artifacts and pipeline outputs) are skipped.
+    """
+    reserved = list(reserved_paths)
+    return sorted(
+        {
+            p
+            for p in candidate_paths
+            if not _is_hidden_path(p)
+            and not _is_under_any_dir(p, reserved)
+            and is_figure_path(p)
+        }
+    )
+
+
+def detect_datasets(
+    candidate_paths: list[str],
+    reserved_paths: list[str] | tuple[str, ...] = (),
+    figure_paths: list[str] | tuple[str, ...] = (),
+) -> list[str]:
+    """Auto-detected dataset paths among ``candidate_paths``.
+
+    Anything already detected as a figure (``figure_paths``) is excluded, and a
+    folder full of data files collapses to a single dataset entry.
+    """
+    reserved = list(reserved_paths)
+    figset = set(figure_paths)
+    files = {
+        p
+        for p in candidate_paths
+        if not _is_hidden_path(p)
+        and not _is_under_any_dir(p, reserved)
+        and p not in figset
+        and is_dataset_path(p)
+    }
+    return _collapse_dataset_folders(sorted(files))
+
+
+def list_repo_files(wdir: str | None = None) -> list[str]:
+    """Repo-relative paths of files Git does not ignore.
+
+    Combines tracked and untracked files while honoring ``.gitignore``, so
+    detection skips virtualenvs, ``node_modules``, and other ignored content.
+    Paths are returned relative to ``wdir`` (or the current directory), using
+    POSIX separators; files outside ``wdir`` are omitted.
+    """
+    import calkit.git
+
+    try:
+        repo = calkit.git.get_repo(wdir)
+    except Exception:
+        return []
+    files = calkit.git.ls_files(
+        repo, "--cached", "--others", "--exclude-standard"
+    )
+    repo_root = repo.working_dir
+    base = os.path.abspath(wdir if wdir is not None else ".")
+    rel_files = []
+    for f in files:
+        rel = os.path.relpath(os.path.join(repo_root, f), base)
+        if rel.startswith(".."):
+            continue
+        rel_files.append(Path(rel).as_posix())
+    return rel_files
+
+
+def list_dvc_tracked_files(wdir: str | None = None) -> list[str]:
+    """Repo-relative paths of files tracked by DVC.
+
+    DVC-tracked artifacts (added via ``dvc add`` or produced as pipeline
+    outputs) are Git-ignored, so they don't appear in :func:`list_repo_files`;
+    include them so figures and datasets stored with DVC are still detected.
+    Returns an empty list when DVC isn't available or the project isn't a DVC
+    repo.
+    """
+    from calkit.dvc.core import list_paths
+
+    try:
+        return [
+            p.replace("\\", "/")
+            for p in list_paths(wdir=wdir, recursive=True)
+            if p
+        ]
+    except Exception:
+        return []
+
+
+def _reserved_artifact_paths(
+    wdir: str | None = None, ck_info: dict | None = None
+) -> list[str]:
+    """Paths of artifacts declared in ``calkit.yaml``.
+
+    These are excluded from auto-detection so a declared figure/dataset isn't
+    reported twice and individual files inside a declared artifact folder aren't
+    listed separately. Pipeline/DVC stage outputs are intentionally *not*
+    reserved: a figure or dataset produced by the pipeline should still be
+    detected (and shown with its producing stage as provenance).
+    """
+    import calkit
+
+    if ck_info is None:
+        ck_info = calkit.load_calkit_info(wdir=wdir)
+    paths: list[str] = []
+    for kind in ("figures", "datasets"):
+        for obj in ck_info.get(kind, []) or []:
+            if isinstance(obj, dict) and isinstance(obj.get("path"), str):
+                paths.append(obj["path"])
+    return [p.replace("\\", "/") for p in paths]
+
+
+def detect_project_artifacts(
+    wdir: str | None = None, ck_info: dict | None = None
+) -> dict[str, list[str]]:
+    """Auto-detect figure and dataset paths in the project at ``wdir``.
+
+    Candidates are the files Git does not ignore plus any files tracked by DVC
+    (which are Git-ignored); declared artifacts are reserved so they aren't
+    detected again.
+    """
+    import calkit
+
+    if ck_info is None:
+        ck_info = calkit.load_calkit_info(wdir=wdir)
+    reserved = _reserved_artifact_paths(wdir=wdir, ck_info=ck_info)
+    candidates = list(
+        dict.fromkeys(
+            [*list_repo_files(wdir=wdir), *list_dvc_tracked_files(wdir=wdir)]
+        )
+    )
+    figures = detect_figures(candidates, reserved_paths=reserved)
+    datasets = detect_datasets(
+        candidates, reserved_paths=reserved, figure_paths=figures
+    )
+    return {"figures": figures, "datasets": datasets}

calkit/tests/test_detect.py

@@ -209,6 +209,48 @@ pdf("figure.pdf")
     assert "plot.png" in result["outputs"]
     assert "clean_data.csv" in result["outputs"]
     assert "figure.pdf" in result["outputs"]
+    # An orchestrator script that sources other scripts via here::here()
+    # should recursively attribute their I/O to the stage
+    os.makedirs("code", exist_ok=True)
+    master_content = """
+here::i_am("code/master.R")
+log_file <- here::here("output", "run.log")
+sink(log_file, split = TRUE)
+source(here::here("code", "clean.R"))
+source(here::here("code", "model.R"))
+sink()
+"""
+    clean_content = """
+raw <- read_excel("data/raw.xlsx", sheet = "Master")
+saveRDS(raw, file = "data/clean.rds")
+"""
+    model_content = """
+df <- readRDS("data/clean.rds")
+write.csv(df, file = "output/tables/summary.csv")
+ggsave("output/figures/plot.pdf", p)
+sink("output/tables/model.tex")
+"""
+    with open("code/master.R", "w") as f:
+        f.write(master_content)
+    with open("code/clean.R", "w") as f:
+        f.write(clean_content)
+    with open("code/model.R", "w") as f:
+        f.write(model_content)
+    master = detect_r_script_io("code/master.R")
+    # Sourced scripts are detected as inputs even when wrapped in here::here()
+    assert "code/clean.R" in master["inputs"]
+    assert "code/model.R" in master["inputs"]
+    # I/O from sourced scripts is attributed to the orchestrator
+    assert "data/raw.xlsx" in master["inputs"]
+    # Named file=/filename= write arguments are detected
+    assert "data/clean.rds" in master["outputs"]
+    assert "output/tables/summary.csv" in master["outputs"]
+    # sink(), here::here()-assigned variables, and ggsave() literals
+    assert "output/run.log" in master["outputs"]
+    assert "output/tables/model.tex" in master["outputs"]
+    assert "output/figures/plot.pdf" in master["outputs"]
+    # An intermediate produced and consumed within the stage is not an input
+    assert "data/clean.rds" not in master["inputs"]
 
 
 def test_detect_shell_script_io(tmp_dir):
@@ -1033,3 +1075,77 @@ def test_create_files_in_subdirectory(tmp_dir):
 
     assert os.path.exists(output_path)
     assert os.path.exists(".calkit/envs/test-env")
+
+
+def test_is_figure_path():
+    """Figures must live in a figure-named directory."""
+    from calkit.detect import is_figure_path
+
+    # Detected: image-like extension under a figure directory, at any depth.
+    assert is_figure_path("figures/result.png")
+    assert is_figure_path("paper/figs/plot.pdf")
+    assert is_figure_path("Plots/Fig1.SVG")
+    # Plotly JSON under a figure-only directory counts as a figure.
+    assert is_figure_path("figures/interactive.json")
+    # Not detected: right extension, wrong (or no) directory.
+    assert not is_figure_path("result.png")
+    assert not is_figure_path("results/result.png")
+    assert not is_figure_path("output/plot.pdf")
+    assert not is_figure_path("img/logo.png")
+    # JSON under a data directory is not a figure.
+    assert not is_figure_path("data/records.json")
+    # Non-figure extension under a figure directory.
+    assert not is_figure_path("figures/notes.txt")
+
+
+def test_is_dataset_path():
+    """Datasets must live in a data-named directory."""
+    from calkit.detect import is_dataset_path
+
+    assert is_dataset_path("data/raw.csv")
+    assert is_dataset_path("datasets/measurements.parquet")
+    assert is_dataset_path("project/Data/records.json")
+    # Wrong directory.
+    assert not is_dataset_path("results/raw.csv")
+    assert not is_dataset_path("raw.csv")
+    # A .json under a figure-only dir is a figure, not a dataset.
+    assert not is_dataset_path("figures/plot.json")
+    # Non-dataset extension under a data directory.
+    assert not is_dataset_path("data/readme.md")
+
+
+def test_detect_figures():
+    """detect_figures filters hidden and reserved paths."""
+    from calkit.detect import detect_figures
+
+    candidates = [
+        "figures/a.png",
+        "figs/b.pdf",
+        "result.png",
+        ".cache/figures/c.png",
+        "figures/reserved.png",
+    ]
+    out = detect_figures(candidates, reserved_paths=["figures/reserved.png"])
+    assert out == ["figs/b.pdf", "figures/a.png"]
+    # Files inside a reserved directory are excluded too.
+    out = detect_figures(
+        ["figures/sub/x.png", "figures/y.png"], reserved_paths=["figures/sub"]
+    )
+    assert out == ["figures/y.png"]
+
+
+def test_detect_datasets():
+    """detect_datasets excludes figures and collapses folders."""
+    from calkit.detect import detect_datasets
+
+    candidates = [
+        "data/a.csv",
+        "data/sub/b.csv",
+        "data/sub/c.csv",
+        "figures/plot.json",
+        ".venv/data/d.csv",
+    ]
+    out = detect_datasets(candidates, figure_paths=["figures/plot.json"])
+    # data/sub holds two files, so it collapses to the folder; data/a.csv has
+    # only one file in its folder, so it stays a file.
+    assert out == ["data/a.csv", "data/sub"]

{calkit_python-0.41.11.dist-info → calkit_python-0.41.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: calkit-python
-Version: 0.41.11
+Version: 0.41.12
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://calkit.org
 Project-URL: Issues, https://github.com/calkit/calkit/issues

{calkit_python-0.41.11.dist-info → calkit_python-0.41.12.dist-info}/RECORD

@@ -8,7 +8,7 @@ calkit/config.py,sha256=tkrdAQU1m3__B9efpYgK6IhtLtS3WWFcVpY2zEQpBio,7573
 calkit/core.py,sha256=On8UGUuR0vKBJHdhEvsPJAk5QD9Lzf3LuWl_gmdfUrI,28179
 calkit/datasets.py,sha256=9lOlOreey2aJGMw6MiV9MDzSHp-fYkCf1k1AZbWTdQE,2235
 calkit/dependencies.py,sha256=y5N_4ctwp9rFZVg82Si9_2x8Yh64Ex-WqP0iiN5NNBE,13276
-calkit/detect.py,sha256=w8H8r2p0R38645TUWFz9xtmeinKic_T8E6eTuldN6OM,62418
+calkit/detect.py,sha256=fK-x3uBE2i1VaKu8vO_6Llj0tvLilaM3fGyeI6XbxPg,73436
 calkit/docker.py,sha256=EEcgbXjs-1ttixAZ-G-hmDlvnqSxYbIZmbzeAza37TI,15706
 calkit/environments.py,sha256=y30tKE0KSRM9ANQwrI15ej2BaXe4s0kUeDm1Vr_BhXA,77230
 calkit/fs.py,sha256=tVyiSBfWNZJv5qudR-KzzxajPYR2rf-uTNEJSQjMHfI,43818
@@ -38,7 +38,7 @@ calkit/cli/describe.py,sha256=g6WWqa4nHSDda1Jt4D1g-VUm1k4bMskBfgKwDAhUokU,1817
 calkit/cli/dev.py,sha256=cu4QsfDiA_KurGx4WJcgA2qgfghMoZutE8kl4IZbUcM,1070
 calkit/cli/import_.py,sha256=5j2ANLq6v7HOku9kJUWylwvQnJfkb_OUKCNOdv1BQXU,18264
 calkit/cli/latex.py,sha256=dhcbVNcR3YjVSD5mo-OBtZN1FhH4mv4mlGnuP19EbjU,6220
-calkit/cli/list.py,sha256=jsSq-6_6iikbhHgU2-zgQgeXm_1FbfigQ6Pww__10o4,6812
+calkit/cli/list.py,sha256=gAkcvF4K59cxrtWQZtBK-i3LhcDlO1r6bGqyfXOMOMw,8947
 calkit/cli/new.py,sha256=RmQbHtdlMrO6-4mZANq75RPpjvRqzhZUIi1xo724EUU,126211
 calkit/cli/notebooks.py,sha256=W2EHNLMOx6EuqE1dSFirud6puwayXlg4T8DcC_lSEos,21423
 calkit/cli/office.py,sha256=jVSTvbl91TCzlglST5O2b8hdApZA_QHw_UiEQFJqxdc,1754
@@ -75,7 +75,7 @@ calkit/tests/test_cloud.py,sha256=583oy2zH0fLexBBB_cmrxCgu7qsouBe7tKY-kkbzhXE,10
 calkit/tests/test_conda.py,sha256=t-Oy1mL3Vb5njj_t3LQ8HwMhsmLN64HAzSlebnN1A9s,14204
 calkit/tests/test_core.py,sha256=qZ090GbIu-gqu-0qJqwPj47t7k1Q_N_5mv0sT2VAeZc,5496
 calkit/tests/test_dependencies.py,sha256=HskXLzbvvRJz2TXnKPpPAo9lPhoZGYQFy4BmEF31nvk,11139
-calkit/tests/test_detect.py,sha256=1QLei1SjLdxy3IWwFZi3PIz8WHpKmG50Hjygc_HxYxg,32907
+calkit/tests/test_detect.py,sha256=_dVIMalauvpSFlJQcEuMF33krSsrO9EK2Yelqs0KWpA,37401
 calkit/tests/test_docker.py,sha256=27d0K1GOWiCNv_5TkZSob6EaC0EtQ9DSBIDfkEAEzGk,2077
 calkit/tests/test_environments.py,sha256=ms8avkZy70QPD4GSWAFa04O5rhd2oePaBeo0CBgnXAo,40300
 calkit/tests/test_fs.py,sha256=8XFlC6bsw_aGodhmDq87OnI60L2licz459yUlDcAieY,9448
@@ -118,23 +118,23 @@ calkit/tests/models/test_pipeline.py,sha256=AJsMAX4928U3_Omt0_8K96U5LU_KoNErgpHe
 calkit/agent_skills/add-pipeline-stage/SKILL.md,sha256=kpYYb-rJsS4B9p1Ub3gL21irCG5ZZ4DvWZpdJeEwPZk,3772
 calkit/agent_skills/conventions/SKILL.md,sha256=nAM2FjSMk6ED56Dr5zh2bL_dhfzDra-h2ZgzHU7ruS4,9524
 calkit/agent_skills/create-pipeline/SKILL.md,sha256=2FGw1iDQVRk5FUq79GP3lPdgvgof2Wi1O9O-abGGtgs,5422
-calkit_python-0.41.11.data/data/etc/jupyter/jupyter_server_config.d/calkit.json,sha256=CWrZP--JDGz8fsvbAlKr_duTL1vDriGT48SL1YCtkY0,81
-calkit_python-0.41.11.data/data/share/jupyter/labextensions/calkit/package.json,sha256=INxx8QhvSl7KP6yL7A7-m2-FIsnB__mYEWG_Eq5RP-c,6651
-calkit_python-0.41.11.data/data/share/jupyter/labextensions/calkit/schemas/calkit/package.json.orig,sha256=6w0b5y8LztEv2tErajJ5d39yBryld02crX0CG3zZFNs,6509
-calkit_python-0.41.11.data/data/share/jupyter/labextensions/calkit/schemas/calkit/plugin.json,sha256=YSpIrwpwB-lQBk9Mwv-npedWTOOZreO6nlWZhqlWyGI,840
-calkit_python-0.41.11.data/data/share/jupyter/labextensions/calkit/static/502.9a2c5772a15466e923ef.js,sha256=mixXcqFUZukj7_jQVxHTavsYl7cdZv83sEe4phJgkh0,59893
-calkit_python-0.41.11.data/data/share/jupyter/labextensions/calkit/static/695.2c41003a452d43d2b358.js,sha256=LEEAOkUtQ9KzWEHn-QFv5yGi70B-sBOnrvztzHr0Shw,223
-calkit_python-0.41.11.data/data/share/jupyter/labextensions/calkit/static/867.a42a046aa5108f54f8fb.js,sha256=pCoEaqUQj1T4-zAc9SUd__9ZGm7uL2wHQve2xwL89NI,8156
-calkit_python-0.41.11.data/data/share/jupyter/labextensions/calkit/static/909.e3f9cc3408834a7fdcc3.js,sha256=4_nMNAiDSn_cw7UjIHEwone4fizrvqrqSgbwUWvjmoU,114571
-calkit_python-0.41.11.data/data/share/jupyter/labextensions/calkit/static/946.050af2abf7845cfbdbd2.js,sha256=n_a0yu_gE7l6fauyoXXv9BZ7ZEYt2387mTfs7CbLcs4,51939
-calkit_python-0.41.11.data/data/share/jupyter/labextensions/calkit/static/946.050af2abf7845cfbdbd2.js.LICENSE.txt,sha256=eNJ8gc9n9IF8nW1d9sI9niuHstYzjNz5vqXx9UgWSPc,249
-calkit_python-0.41.11.data/data/share/jupyter/labextensions/calkit/static/b2f1c3efe70cb539d121.png,sha256=svHD7-cMtTnRITFwugwsVaB9nZ-h8A61ose8z32HiRE,24850
-calkit_python-0.41.11.data/data/share/jupyter/labextensions/calkit/static/remoteEntry.65469af996e7a96aa983.js,sha256=ZUaa-ZbnqWqpgxIpcQcRX7auvhOBkPH2YHNID5udGQs,8737
-calkit_python-0.41.11.data/data/share/jupyter/labextensions/calkit/static/style.js,sha256=r89Jlk5v1drcwhCpv9FnC3Jig_JH4k24kZNIvxh6Htk,149
-calkit_python-0.41.11.data/data/share/jupyter/labextensions/calkit/static/third-party-licenses.json,sha256=2BGdItLJwO3fAopLl4eJvp26TEwuscYC0-yFaF-LaLU,13683
-calkit_python-0.41.11.data/data/share/jupyter/labextensions/calkit/install.json,sha256=DK9d8G-q-rMVlcT3rAeGIyo3REWKw6FySBZceLU9yaw,187
-calkit_python-0.41.11.dist-info/METADATA,sha256=GJrq8kJ3EmCVwtkdkfWi3bYksvVr1yAg4U4OJTkGNiE,12071
-calkit_python-0.41.11.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
-calkit_python-0.41.11.dist-info/entry_points.txt,sha256=2iQBBzjTAOdk66CwS-f3ecXXW5DjUqkG1KBRj8mHI1I,133
-calkit_python-0.41.11.dist-info/licenses/LICENSE,sha256=9ZamCaSUTZk9rcrnf-sWFKLOHr3ws-S_dgKMegW4nw8,1056
-calkit_python-0.41.11.dist-info/RECORD,,
+calkit_python-0.41.12.data/data/etc/jupyter/jupyter_server_config.d/calkit.json,sha256=CWrZP--JDGz8fsvbAlKr_duTL1vDriGT48SL1YCtkY0,81
+calkit_python-0.41.12.data/data/share/jupyter/labextensions/calkit/package.json,sha256=INxx8QhvSl7KP6yL7A7-m2-FIsnB__mYEWG_Eq5RP-c,6651
+calkit_python-0.41.12.data/data/share/jupyter/labextensions/calkit/schemas/calkit/package.json.orig,sha256=6w0b5y8LztEv2tErajJ5d39yBryld02crX0CG3zZFNs,6509
+calkit_python-0.41.12.data/data/share/jupyter/labextensions/calkit/schemas/calkit/plugin.json,sha256=YSpIrwpwB-lQBk9Mwv-npedWTOOZreO6nlWZhqlWyGI,840
+calkit_python-0.41.12.data/data/share/jupyter/labextensions/calkit/static/502.9a2c5772a15466e923ef.js,sha256=mixXcqFUZukj7_jQVxHTavsYl7cdZv83sEe4phJgkh0,59893
+calkit_python-0.41.12.data/data/share/jupyter/labextensions/calkit/static/695.2c41003a452d43d2b358.js,sha256=LEEAOkUtQ9KzWEHn-QFv5yGi70B-sBOnrvztzHr0Shw,223
+calkit_python-0.41.12.data/data/share/jupyter/labextensions/calkit/static/867.a42a046aa5108f54f8fb.js,sha256=pCoEaqUQj1T4-zAc9SUd__9ZGm7uL2wHQve2xwL89NI,8156
+calkit_python-0.41.12.data/data/share/jupyter/labextensions/calkit/static/909.e3f9cc3408834a7fdcc3.js,sha256=4_nMNAiDSn_cw7UjIHEwone4fizrvqrqSgbwUWvjmoU,114571
+calkit_python-0.41.12.data/data/share/jupyter/labextensions/calkit/static/946.050af2abf7845cfbdbd2.js,sha256=n_a0yu_gE7l6fauyoXXv9BZ7ZEYt2387mTfs7CbLcs4,51939
+calkit_python-0.41.12.data/data/share/jupyter/labextensions/calkit/static/946.050af2abf7845cfbdbd2.js.LICENSE.txt,sha256=eNJ8gc9n9IF8nW1d9sI9niuHstYzjNz5vqXx9UgWSPc,249
+calkit_python-0.41.12.data/data/share/jupyter/labextensions/calkit/static/b2f1c3efe70cb539d121.png,sha256=svHD7-cMtTnRITFwugwsVaB9nZ-h8A61ose8z32HiRE,24850
+calkit_python-0.41.12.data/data/share/jupyter/labextensions/calkit/static/remoteEntry.65469af996e7a96aa983.js,sha256=ZUaa-ZbnqWqpgxIpcQcRX7auvhOBkPH2YHNID5udGQs,8737
+calkit_python-0.41.12.data/data/share/jupyter/labextensions/calkit/static/style.js,sha256=r89Jlk5v1drcwhCpv9FnC3Jig_JH4k24kZNIvxh6Htk,149
+calkit_python-0.41.12.data/data/share/jupyter/labextensions/calkit/static/third-party-licenses.json,sha256=2BGdItLJwO3fAopLl4eJvp26TEwuscYC0-yFaF-LaLU,13683
+calkit_python-0.41.12.data/data/share/jupyter/labextensions/calkit/install.json,sha256=DK9d8G-q-rMVlcT3rAeGIyo3REWKw6FySBZceLU9yaw,187
+calkit_python-0.41.12.dist-info/METADATA,sha256=SAnLxjNtC4KA3GcwLlonzb98yGDPjHyheSdiK3UuzG0,12071
+calkit_python-0.41.12.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+calkit_python-0.41.12.dist-info/entry_points.txt,sha256=2iQBBzjTAOdk66CwS-f3ecXXW5DjUqkG1KBRj8mHI1I,133
+calkit_python-0.41.12.dist-info/licenses/LICENSE,sha256=9ZamCaSUTZk9rcrnf-sWFKLOHr3ws-S_dgKMegW4nw8,1056
+calkit_python-0.41.12.dist-info/RECORD,,