calkit-python 0.3.3__tar.gz → 0.4.0__tar.gz

{calkit_python-0.3.3 → calkit_python-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: calkit-python
-Version: 0.3.3
+Version: 0.4.0
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://github.com/calkit/calkit
 Project-URL: Issues, https://github.com/calkit/calkit/issues
@@ -31,21 +31,23 @@ Description-Content-Type: text/markdown
 
 # Calkit
 
-[Calkit](https://calkit.io) simplifies reproducibility,
+[Calkit](https://calkit.io) helps simplify reproducibility,
 acting as a layer on top of
 [Git](https://git-scm.com/), [DVC](https://dvc.org/),
-[Docker](https://docker.com), and more,
-such that all all aspects of the research process can be fully described in a
-single repository.
+[Docker](https://docker.com),
+and adds a domain-specific data model
+such that all aspects of the research process can be fully described in a
+single repository and therefore easily consumed by others.
 
 ## Tutorials
 
+- [Microsoft Office (Word and Excel)](https://petebachant.me/office-repro/)
 - [Reproducible OpenFOAM simulations](https://petebachant.me/reproducible-openfoam/)
 
 ## Why does reproducibility matter?
 
 If your work is reproducible, that means that someone else can "run" it and
-get the same results or outputs.
+calculate the same results or outputs.
 This is a major step towards addressing
 [the replication crisis](https://en.wikipedia.org/wiki/Replication_crisis)
 and has some major benefits for both you as an individual and the research
@@ -63,7 +65,8 @@ community:
 ## Why another tool/platform?
 
 Git, GitHub, DVC, Docker et al. are amazing tools/platforms, but their
-use involves multiple fairly difficult learning curves.
+use involves multiple fairly difficult learning curves,
+and tying them together might mean developing something new for each project.
 Our goal is to provide a single tool and platform to unify all of these so
 that there is a single, gentle learning curve.
 However, it is not our goal to hide or replace these underlying components.

{calkit_python-0.3.3 → calkit_python-0.4.0}/README.md

@@ -1,20 +1,22 @@
 # Calkit
 
-[Calkit](https://calkit.io) simplifies reproducibility,
+[Calkit](https://calkit.io) helps simplify reproducibility,
 acting as a layer on top of
 [Git](https://git-scm.com/), [DVC](https://dvc.org/),
-[Docker](https://docker.com), and more,
-such that all all aspects of the research process can be fully described in a
-single repository.
+[Docker](https://docker.com),
+and adds a domain-specific data model
+such that all aspects of the research process can be fully described in a
+single repository and therefore easily consumed by others.
 
 ## Tutorials
 
+- [Microsoft Office (Word and Excel)](https://petebachant.me/office-repro/)
 - [Reproducible OpenFOAM simulations](https://petebachant.me/reproducible-openfoam/)
 
 ## Why does reproducibility matter?
 
 If your work is reproducible, that means that someone else can "run" it and
-get the same results or outputs.
+calculate the same results or outputs.
 This is a major step towards addressing
 [the replication crisis](https://en.wikipedia.org/wiki/Replication_crisis)
 and has some major benefits for both you as an individual and the research
@@ -32,7 +34,8 @@ community:
 ## Why another tool/platform?
 
 Git, GitHub, DVC, Docker et al. are amazing tools/platforms, but their
-use involves multiple fairly difficult learning curves.
+use involves multiple fairly difficult learning curves,
+and tying them together might mean developing something new for each project.
 Our goal is to provide a single tool and platform to unify all of these so
 that there is a single, gentle learning curve.
 However, it is not our goal to hide or replace these underlying components.

{calkit_python-0.3.3 → calkit_python-0.4.0}/calkit/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.3.3"
+__version__ = "0.4.0"
 
 from .core import *
 from . import git
@@ -8,3 +8,4 @@ from . import jupyter
 from . import config
 from . import models
 from . import office
+from . import templates

{calkit_python-0.3.3 → calkit_python-0.4.0}/calkit/cli/list.py

@@ -77,3 +77,10 @@ def list_environments():
         typer.echo(name + ":")
         for k, v in env.items():
             typer.echo(f"    {k}: {v}")
+
+
+@list_app.command(name="templates")
+def list_templates():
+    for kind, tpl_dict in calkit.templates.TEMPLATES.items():
+        for name in tpl_dict:
+            typer.echo(f"{kind}/{name}")

{calkit_python-0.3.3 → calkit_python-0.4.0}/calkit/cli/main.py

@@ -481,11 +481,21 @@ def run_in_env(
             ),
         ),
     ] = None,
+    wdir: Annotated[
+        str,
+        typer.Option(
+            "--wdir",
+            help=(
+                "Working directory. "
+                "By default will run current working directory."
+            ),
+        ),
+    ] = None,
     verbose: Annotated[
         bool, typer.Option("--verbose", "-v", help="Print verbose output.")
     ] = False,
 ):
-    ck_info = calkit.load_calkit_info()
+    ck_info = calkit.load_calkit_info(process_includes=True)
     envs = ck_info.get("environments", {})
     if not envs:
         raise_error("No environments defined in calkit.yaml")
@@ -507,33 +517,42 @@ def run_in_env(
     if env_name is None:
         raise_error("Environment must be specified if there are multiple")
     env = envs[env_name]
-    cwd = os.getcwd()
+    if wdir is not None:
+        cwd = os.path.abspath(wdir)
+    else:
+        cwd = os.getcwd()
     image_name = env.get("image", env_name)
-    wdir = env.get("wdir", "/work")
+    docker_wdir = env.get("wdir", "/work")
+    shell = env.get("shell", "sh")
+    platform = env.get("platform")
     if env["kind"] == "docker":
-        cmd = " ".join(cmd)
-        cmd = [
+        shell_cmd = " ".join(cmd)
+        docker_cmd = [
             "docker",
             "run",
+        ]
+        if platform:
+            docker_cmd += ["--platform", platform]
+        docker_cmd += [
             "-it" if sys.stdin.isatty() else "-i",
             "--rm",
             "-w",
-            wdir,
+            docker_wdir,
             "-v",
-            f"{cwd}:{wdir}",
+            f"{cwd}:{docker_wdir}",
             image_name,
-            "bash",
+            shell,
             "-c",
-            f"{cmd}",
+            f"{shell_cmd}",
         ]
         if verbose:
-            typer.echo(f"Running command: {cmd}")
-        subprocess.call(cmd)
+            typer.echo(f"Running command: {docker_cmd}")
+        subprocess.call(docker_cmd, cwd=wdir)
     elif env["kind"] == "conda":
         cmd = ["conda", "run", "-n", env_name] + cmd
         if verbose:
             typer.echo(f"Running command: {cmd}")
-        subprocess.call(cmd)
+        subprocess.call(cmd, cwd=wdir)
     else:
         raise_error("Environment kind not supported")
 

{calkit_python-0.3.3 → calkit_python-0.4.0}/calkit/cli/new.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import os
+import shutil
 import subprocess
 
 import git
@@ -472,3 +473,189 @@ def new_dataset(
             repo.git.add("dvc.yaml")
         if repo.git.diff("--staged"):
             repo.git.commit(["-m", f"Add dataset {path}"])
+
+
+@new_app.command(name="publication", help="Create a new publication.")
+def new_publication(
+    path: Annotated[
+        str,
+        typer.Argument(
+            help=(
+                "Path for the publication. "
+                "If using a template, this could be a directory."
+            )
+        ),
+    ],
+    title: Annotated[
+        str, typer.Option("--title", help="The title of the publication.")
+    ],
+    description: Annotated[
+        str,
+        typer.Option(
+            "--description", help="A description of the publication."
+        ),
+    ],
+    kind: Annotated[
+        str,
+        typer.Option(
+            "--kind", help="Kind of the publication, e.g., 'journal-article'."
+        ),
+    ],
+    stage_name: Annotated[
+        str,
+        typer.Option(
+            "--stage",
+            help="Name of the pipeline stage to build the output file.",
+        ),
+    ] = None,
+    deps: Annotated[
+        list[str], typer.Option("--dep", help="Path to stage dependency.")
+    ] = [],
+    outs_from_stage: Annotated[
+        str,
+        typer.Option(
+            "--deps-from-stage-outs",
+            help="Stage name from which to add outputs as dependencies.",
+        ),
+    ] = None,
+    template: Annotated[
+        str,
+        typer.Option(
+            "--template",
+            help=(
+                "Template with which to create the source files. "
+                "Should be in the format {type}/{name}."
+            ),
+        ),
+    ] = None,
+    env_name: Annotated[
+        str,
+        typer.Option(
+            "--environment",
+            help="Name of the build environment to create, if desired.",
+        ),
+    ] = None,
+    no_commit: Annotated[
+        bool,
+        typer.Option(
+            "--no-commit", help="Do not commit resulting changes to the repo."
+        ),
+    ] = False,
+    overwrite: Annotated[
+        bool,
+        typer.Option(
+            "--overwrite",
+            "-f",
+            help="Overwrite existing objects if they already exist.",
+        ),
+    ] = False,
+):
+    ck_info = calkit.load_calkit_info(process_includes=False)
+    pubs = ck_info.get("publications", [])
+    envs = ck_info.get("environments", {})
+    pub_paths = [p.get("path") for p in pubs]
+    if template is not None:
+        template_type, _ = template.split("/")
+    else:
+        template_type = None
+    # Check all of our inputs
+    if template_type not in ["latex"]:
+        raise_error(f"Unknown template type '{template_type}'")
+    if env_name is not None and template_type != "latex":
+        raise_error("Environments can only be created for latex templates")
+    if env_name is not None and env_name in envs and not overwrite:
+        raise_error(f"Environment '{env_name}' already exists")
+    if template_type is not None:
+        try:
+            template_obj = calkit.templates.get_template(template)
+        except ValueError:
+            raise_error(f"Template '{template}' does not exist")
+    # Parse outs from stage if specified
+    if outs_from_stage:
+        pipeline = calkit.dvc.read_pipeline()
+        stages = pipeline.get("stages", {})
+        if outs_from_stage not in stages:
+            raise_error(f"Stage {outs_from_stage} does not exist")
+        stage = stages[outs_from_stage]
+        if "foreach" in stage:
+            for val in stage["foreach"]:
+                for out in stage.get("do", {}).get("outs", []):
+                    deps.append(out.replace("${item}", val))
+        else:
+            deps += stage.get("outs", [])
+    # Create publication object
+    if template_type == "latex":
+        pub_fpath = os.path.join(
+            path, template_obj.target.removesuffix(".tex") + ".pdf"
+        )
+    else:
+        pub_fpath = path
+    if not overwrite and pub_fpath in pub_paths:
+        raise_error(f"Publication with path {pub_fpath} already exists")
+    elif overwrite and pub_fpath in pub_paths:
+        pubs = [p for p in pubs if p.get("path") != pub_fpath]
+    pub = dict(
+        path=pub_fpath,
+        kind=kind,
+        title=title,
+        description=description,
+        stage=stage_name,
+    )
+    pubs.append(pub)
+    ck_info["publications"] = pubs
+    repo = git.Repo()
+    # Create environment if applicable
+    if env_name is not None and template_type == "latex":
+        env_path = f".calkit/environments/{env_name}.yaml"
+        os.makedirs(".calkit/environments", exist_ok=True)
+        env = {"_include": env_path}
+        envs[env_name] = env
+        env_remote = dict(
+            kind="docker",
+            image="kjarosh/latex:2024.4",
+            description="TeXlive full from kjarosh.",
+            platform="linux/amd64",
+        )
+        with open(env_path, "w") as f:
+            calkit.ryaml.dump(env_remote, f)
+        ck_info["environments"] = envs
+        repo.git.add(env_path)
+    with open("calkit.yaml", "w") as f:
+        calkit.ryaml.dump(ck_info, f)
+    repo.git.add("calkit.yaml")
+    # Copy in template files if applicable
+    if template_type == "latex":
+        if overwrite and os.path.exists(path):
+            shutil.rmtree(path)
+        calkit.templates.use_template(
+            name=template, dest_dir=path, title=title
+        )
+        repo.git.add(path)
+    # Create stage if applicable
+    if stage_name is not None and template_type == "latex":
+        cmd = f"cd {path} && latexmk -pdf {template_obj.target}"
+        if env_name is not None:
+            cmd = f'calkit runenv -n {env_name} "{cmd}"'
+        target_dep = os.path.join(path, template_obj.target)
+        dvc_cmd = [
+            "dvc",
+            "stage",
+            "add",
+            "-n",
+            stage_name,
+            "-o",
+            pub_fpath,
+            "-d",
+            target_dep,
+        ]
+        if env_name is not None:
+            dvc_cmd += ["-d", env_path]
+        for dep in deps:
+            dvc_cmd += ["-d", dep]
+        if overwrite:
+            dvc_cmd.append("-f")
+        dvc_cmd.append(cmd)
+        subprocess.check_call(dvc_cmd)
+        repo.git.add("dvc.yaml")
+    if not no_commit and repo.git.diff("--staged"):
+        repo.git.commit(["-m", f"Add new publication {pub_fpath}"])

{calkit_python-0.3.3 → calkit_python-0.4.0}/calkit/core.py

@@ -46,11 +46,25 @@ def find_project_dirs(relative=False, max_depth=3) -> list[str]:
     return final_res
 
 
-def load_calkit_info(wdir=None) -> dict:
+def load_calkit_info(wdir=None, process_includes=False) -> dict:
+    """Load Calkit project information."""
+    info = {}
     fpath = "calkit.yaml"
     if wdir is not None:
         fpath = os.path.join(wdir, fpath)
     if os.path.isfile(fpath):
         with open(fpath) as f:
-            return ryaml.load(f)
-    return {}
+            info = ryaml.load(f)
+    # Check for any includes, i.e., entities with an _include key, for which
+    # we should merge in another file
+    # Currently this is only supported with environments because they may need
+    # to be tracked as DVC dependencies
+    if process_includes:
+        if "environments" in info:
+            for env_name, env in info["environments"].items():
+                if "_include" in env:
+                    include_fpath = env.pop("_include")
+                    with open(include_fpath) as f:
+                        include_data = ryaml.load(f)
+                    info["environments"][env_name] |= include_data
+    return info

{calkit_python-0.3.3 → calkit_python-0.4.0}/calkit/models.py

@@ -73,6 +73,8 @@ class DockerEnvironment(Environment):
     kind: str = "docker"
     image: str
     layers: list[str] | None = None
+    shell: Literal["bash", "sh"] = "sh"
+    platform: str | None = None
 
 
 class Software(BaseModel):

calkit_python-0.4.0/calkit/templates/__init__.py

@@ -0,0 +1 @@
+from .core import *

calkit_python-0.4.0/calkit/templates/core.py

@@ -0,0 +1,118 @@
+"""Core functionality for working with templates."""
+
+import os
+import shutil
+from typing import Literal
+
+from pydantic import BaseModel
+
+from calkit.templates.latex import GITIGNORE as LATEX_GITIGNORE
+
+
+class Template(BaseModel):
+    """Model for a template.
+
+    Defines what kind of template it is, its name, and if/how we should allow
+    for replacing strings in the template in the process of copying it to a
+    project, e.g., if it's a LaTeX template and we want to automatically
+    set the title.
+
+    Attributes
+    ----------
+    kind : string
+        What kind of template is this.
+    name : string
+        Kebab-case name of the template.
+    loc : string
+        Location of the template. If not provided, will be inferred from the
+        kind and name.
+    files: list of strings
+        Files to copy. If not specified, will copy all in the directory.
+    gitignore : string
+        Content to put into the ``.gitignore`` file.
+    """
+
+    kind: Literal["latex"]
+    name: str
+    loc: str | None = None  # Can be local (auto-detected by path) or URL
+    files: list[str] | None = None  # Which filenames should be copied
+    gitignore: str | None = None
+
+
+class LatexTemplate(Template):
+    kind: str = "latex"
+    target: str = "paper.tex"
+    gitignore: str = LATEX_GITIGNORE
+
+
+# A registry of available templates, keyed by their kind and subkeyed by their
+# name
+TEMPLATES = {
+    "latex": {
+        "article": LatexTemplate(name="article"),
+        "jfm": LatexTemplate(name="jfm"),
+    },
+    "project": {},
+}
+
+
+def get_template(name: str) -> Template:
+    """Get a template by name, which should include its namespace or type."""
+    template_type, template_name = name.split("/")
+    if template_type not in TEMPLATES:
+        raise ValueError(f"Unknown template type '{template_type}'")
+    templates = TEMPLATES[template_type]
+    if template_name not in templates:
+        raise ValueError(f"Unknown template name '{template_name}'")
+    return templates[template_name]
+
+
+def use_template(name: str, dest_dir: str, **kwargs):
+    """Copy template files into ``dest_dir``.
+
+    The destination directory must be empty if it exists.
+    """
+    template = get_template(name)
+    if template.loc is None:
+        loc = os.path.join(
+            os.path.dirname(os.path.abspath(__file__)),
+            template.kind,
+            template.name,
+        )
+        print(loc)
+    else:
+        loc = template.loc
+    if loc.startswith("http://") or loc.startswith("https://"):
+        raise NotImplementedError("Remote template support not implemented")
+    files = template.files
+    if files is None:
+        files = os.listdir(loc)
+    if isinstance(template, LatexTemplate) and template.target not in files:
+        files.append(template.target)
+    if os.path.exists(dest_dir):
+        if os.path.isfile(dest_dir):
+            raise ValueError("Destination directory already exists as a file")
+        if os.listdir(dest_dir):
+            raise ValueError("Destination directory must be empty")
+    else:
+        os.makedirs(dest_dir)
+    # Copy files into destination
+    for fname in files:
+        fpath = os.path.join(loc, fname)
+        shutil.copy(src=fpath, dst=dest_dir)
+    # Write gitignore if applicable
+    if template.gitignore is not None:
+        with open(os.path.join(dest_dir, ".gitignore"), "w") as f:
+            f.write(template.gitignore)
+    # If there's a title in kwargs and we're using a LaTeX template,
+    # replace that line
+    if isinstance(template, LatexTemplate) and "title" in kwargs:
+        with open(os.path.join(dest_dir, template.target)) as f:
+            lines = f.readlines()
+        txt = ""
+        for line in lines:
+            if line.strip().startswith(r"\title{"):
+                line = r"\title{" + kwargs["title"] + "}\n"
+            txt += line
+        with open(os.path.join(dest_dir, template.target), "w") as f:
+            f.write(txt)

calkit_python-0.4.0/calkit/templates/latex/__init__.py

@@ -0,0 +1 @@
+from .core import *

calkit_python-0.4.0/calkit/templates/latex/article/paper.tex

@@ -0,0 +1,43 @@
+% A generic article template
+\documentclass[11pt]{article}
+
+\usepackage{sectsty}
+\usepackage{graphicx}
+
+% Margins
+\topmargin=-0.45in
+\evensidemargin=0in
+\oddsidemargin=0in
+\textwidth=6.5in
+\textheight=9.0in
+\headsep=0.25in
+
+\title{This is the paper}
+\author{ The Author }
+\date{\today}
+
+\begin{document}
+\maketitle
+\pagebreak
+
+% Optional TOC
+% \tableofcontents
+% \pagebreak
+
+%--Paper--
+
+\section{Section 1}
+
+Lorem Ipsum
+
+% \includegraphics{../figures/my-figure.png}
+
+\pagebreak
+
+\section{Section 2}
+
+Lorem Ipsum \\
+
+%--/Paper--
+
+\end{document}

calkit_python-0.4.0/calkit/templates/latex/core.py

@@ -0,0 +1,11 @@
+"""Core functionality for working with LaTeX templates."""
+
+GITIGNORE = """
+*.aux
+*.fdb_latexmk
+*.fls
+*.log
+*.pdf
+*.out
+*.DS_Store
+"""