calkit-python 0.30.8__tar.gz → 0.30.10__tar.gz

{calkit_python-0.30.8 → calkit_python-0.30.10}/.github/workflows/test.yml

@@ -15,6 +15,8 @@ jobs:
           git config --global user.name github-actions[bot]
           git config --global user.email 41898282+github-actions[bot]@users.noreply.github.com
           git config --global init.defaultBranch main
+      - name: Configure Git push behavior
+        run: git config --global push.autoSetupRemote true
       - name: Setup uv
         uses: astral-sh/setup-uv@v5
       - name: Setup Miniforge

{calkit_python-0.30.8 → calkit_python-0.30.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: calkit-python
-Version: 0.30.8
+Version: 0.30.10
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://calkit.org
 Project-URL: Issues, https://github.com/calkit/calkit/issues

{calkit_python-0.30.8 → calkit_python-0.30.10}/calkit/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.30.8"
+__version__ = "0.30.10"
 
 from .core import *  # noqa: F403, I001
 from . import git  # noqa: F401
@@ -16,4 +16,5 @@ from . import github  # noqa: F401
 from . import invenio  # noqa: F401
 from . import releases  # noqa: F401
 from . import licenses  # noqa: F401
+from . import overleaf  # noqa: F401
 from .notebooks import declare_notebook  # noqa: F401

{calkit_python-0.30.8 → calkit_python-0.30.10}/calkit/cli/main.py

@@ -990,6 +990,9 @@ def run(
     """Check dependencies and run the pipeline."""
     os.environ["CALKIT_PIPELINE_RUNNING"] = "1"
     dotenv.load_dotenv(dotenv_path=".env", verbose=verbose)
+    ck_info = calkit.load_calkit_info()
+    # Set env vars
+    calkit.set_env_vars(ck_info=ck_info)
     if not quiet:
         typer.echo("Getting system information")
     system_info = calkit.get_system_info()
@@ -1013,7 +1016,6 @@ def run(
         os.environ.pop("CALKIT_PIPELINE_RUNNING", None)
         raise_error(str(e))
     # Compile the pipeline
-    ck_info = calkit.load_calkit_info()
     if ck_info.get("pipeline", {}):
         if not quiet:
             typer.echo("Compiling DVC pipeline")
@@ -1251,6 +1253,7 @@ def run_in_env(
 ):
     dotenv.load_dotenv(dotenv_path=".env", verbose=verbose)
     ck_info = calkit.load_calkit_info(process_includes="environments")
+    calkit.set_env_vars(ck_info=ck_info)
     envs = ck_info.get("environments", {})
     if not envs:
         raise_error("No environments defined in calkit.yaml")
@@ -1314,6 +1317,8 @@ def run_in_env(
         if "env-vars" in env:
             warn("The 'env-vars' key is deprecated; use 'env_vars' instead.")
             env_vars.update(env["env-vars"])
+        # Add project-level env vars (non-secret)
+        env_vars.update(ck_info.get("env_vars", {}))
         # Also add any project-level environmental variable dependencies
         project_env_vars = calkit.get_env_var_dep_names()
         if project_env_vars:
@@ -1668,6 +1673,7 @@ def run_procedure(
         return value
 
     ck_info = calkit.load_calkit_info(process_includes="procedures")
+    calkit.set_env_vars(ck_info=ck_info)
     procs = ck_info.get("procedures", {})
     if name not in procs:
         raise_error(f"'{name}' is not defined as a procedure")

{calkit_python-0.30.8 → calkit_python-0.30.10}/calkit/cli/overleaf.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import filecmp
+import json
 import os
 import shutil
 import subprocess
@@ -36,17 +37,6 @@ def import_publication(
             help="Directory at which to save in the project, e.g., 'paper'."
         ),
     ],
-    sync_paths: Annotated[
-        list[str],
-        typer.Option(
-            "--sync-path",
-            "-s",
-            help=(
-                "Paths to sync from the Overleaf project, e.g., 'main.tex'. "
-                "Note that multiple can be specified."
-            ),
-        ),
-    ],
     title: Annotated[
         str,
         typer.Option(
@@ -55,6 +45,14 @@ def import_publication(
             help="Title of the publication.",
         ),
     ],
+    target_path: Annotated[
+        str | None,
+        typer.Option(
+            "--target",
+            "-T",
+            help="Target TeX file path inside Overleaf project.",
+        ),
+    ] = None,
     description: Annotated[
         str | None,
         typer.Option(
@@ -70,6 +68,17 @@ def import_publication(
             help="What of the publication this is, e.g., 'journal-article'.",
         ),
     ] = None,
+    sync_paths: Annotated[
+        list[str],
+        typer.Option(
+            "--sync-path",
+            "-s",
+            help=(
+                "Paths to sync from the Overleaf project, e.g., 'main.tex'. "
+                "Note that multiple can be specified."
+            ),
+        ),
+    ] = [],
     push_paths: Annotated[
         list[str],
         typer.Option(
@@ -82,18 +91,6 @@ def import_publication(
             ),
         ),
     ] = [],
-    pdf_path: Annotated[
-        str | None,
-        typer.Option(
-            "--pdf-path",
-            "-o",
-            help=(
-                "PDF output file in the Overleaf project, e.g., 'main.pdf'. "
-                "If not provided, it will be determined from the first sync "
-                "path."
-            ),
-        ),
-    ] = None,
     no_commit: Annotated[
         bool,
         typer.Option("--no-commit", help="Do not commit changes to repo."),
@@ -127,13 +124,19 @@ def import_publication(
         typer.echo("Storing Overleaf token in Calkit config")
         config.overleaf_token = overleaf_token
         config.write()
-    if not src_url.startswith("https://www.overleaf.com/project/"):
+    if (
+        not src_url.startswith("https://www.overleaf.com/project/")
+        and calkit.config.get_env() != "test"
+    ):
         raise_error(
             "Invalid URL; must start with 'https://www.overleaf.com/project/'"
         )
     overleaf_project_id = src_url.split("/")[-1]
     if not overleaf_project_id:
         raise_error("Invalid Overleaf project ID")
+    # Check target path
+    if target_path is not None and not target_path.endswith(".tex"):
+        raise_error("Target path should have a .tex extension")
     # Make sure destination directory exists, and isn't a file
     if os.path.isfile(dest_dir):
         raise_error("Destination must be a directory, not a file")
@@ -141,21 +144,6 @@ def import_publication(
     ck_info = calkit.load_calkit_info(process_includes="environments")
     pubs = ck_info.get("publications", [])
     # TODO: Don't allow the same Overleaf project ID in multiple publications
-    # Determine the PDF output path
-    if pdf_path is None:
-        # Use the first sync path as the PDF path
-        pdf_path = sync_paths[0].removesuffix(".tex") + ".pdf"
-        typer.echo(f"Using PDF path: {pdf_path}")
-    tex_path = pdf_path.removesuffix(".pdf") + ".tex"
-    pub_path = PurePosixPath(dest_dir, pdf_path).as_posix()
-    pub_paths = [pub.get("path") for pub in pubs]
-    if not overwrite and pub_path in pub_paths:
-        raise_error(
-            f"A publication already exists in this project at {pub_path}"
-        )
-    elif overwrite and pub_path in pub_paths:
-        # Note: This publication will go to the end of the list
-        pubs = [p for p in pubs if p.get("path") != pub_path]
     repo = git.Repo()
     # Clone the Overleaf project into .calkit/overleaf if it doesn't exist
     # otherwise pull
@@ -163,8 +151,8 @@ def import_publication(
     os.makedirs(overleaf_dir, exist_ok=True)
     git_ignore(overleaf_dir, no_commit=no_commit)
     overleaf_project_dir = os.path.join(overleaf_dir, overleaf_project_id)
-    git_clone_url = (
-        f"https://git:{overleaf_token}@git.overleaf.com/{overleaf_project_id}"
+    git_clone_url = calkit.overleaf.get_git_remote_url(
+        project_id=overleaf_project_id, token=overleaf_token
     )
     if os.path.isdir(overleaf_project_dir):
         warn("This Overleaf project has already been cloned; removing")
@@ -176,6 +164,36 @@ def import_publication(
         overleaf_project_dir,
         depth=1,
     )
+    # Detect target path if not specified
+    if target_path is None:
+        ol_contents = os.listdir(overleaf_project_dir)
+        for cand in ["main.tex", "report.tex", "paper.tex"]:
+            if cand in ol_contents:
+                target_path = cand
+                break
+    if target_path is None:
+        # Fall back to lone .tex file if there is one
+        tex_files = [p for p in ol_contents if p.endswith(".tex")]
+        if len(tex_files) == 1:
+            target_path = tex_files[0]
+    if target_path is None:
+        raise_error(
+            "Target TeX file path cannot be detected; "
+            "please specify with --target"
+        )
+    # Determine the PDF output path
+    pdf_path = target_path.removesuffix(".tex") + ".pdf"  # type: ignore
+    typer.echo(f"Using PDF path: {pdf_path}")
+    tex_path = pdf_path.removesuffix(".pdf") + ".tex"
+    pub_path = PurePosixPath(dest_dir, pdf_path).as_posix()
+    pub_paths = [pub.get("path") for pub in pubs]
+    if not overwrite and pub_path in pub_paths:
+        raise_error(
+            f"A publication already exists in this project at {pub_path}"
+        )
+    elif overwrite and pub_path in pub_paths:
+        # Note: This publication will go to the end of the list
+        pubs = [p for p in pubs if p.get("path") != pub_path]
     # Check that we have a LaTeX environment
     typer.echo("Checking that this project has a LaTeX environment")
     envs = ck_info.get("environments", {})
@@ -197,6 +215,8 @@ def import_publication(
             description="TeXlive via Docker.",
         )
         ck_info["environments"] = envs
+        with open("calkit.yaml", "w") as f:
+            calkit.ryaml.dump(ck_info, f)
     # Check that we have a build stage
     # TODO: Use Calkit pipeline for this
     typer.echo("Checking for a build stage in the pipeline")
@@ -224,7 +244,6 @@ def import_publication(
             name=stage_name,
             environment=tex_env_name,
             target_path=PurePosixPath(dest_dir, tex_path).as_posix(),
-            outputs=[pub_path],
             inputs=[
                 os.path.join(dest_dir, p) for p in sync_paths + push_paths
             ],
@@ -234,6 +253,7 @@ def import_publication(
         repo.git.add("calkit.yaml")
     # Add to publications in calkit.yaml
     typer.echo("Adding publication to calkit.yaml")
+    ck_info = calkit.load_calkit_info()
     new_pub = dict(
         path=pub_path,
         title=title,
@@ -305,12 +325,12 @@ def sync(
             help="Enable verbose output.",
         ),
     ] = False,
-    force: Annotated[
+    resolve: Annotated[
         bool,
         typer.Option(
-            "--force",
-            "-f",
-            help="Force push to Overleaf even if there are conflicts.",
+            "--resolve",
+            "-r",
+            help="Mark merge conflicts as resolved before committing.",
         ),
     ] = False,
 ):
@@ -327,11 +347,33 @@ def sync(
     calkit_config = calkit.config.read()
     overleaf_token = calkit_config.overleaf_token
     if not overleaf_token:
+        # See if we can get it from the cloud
+        if calkit_config.token is not None:
+            try:
+                resp = calkit.cloud.get("/user/overleaf-token")
+                overleaf_token = resp["access_token"]
+                calkit_config.overleaf_token = overleaf_token
+                calkit_config.write()
+            except Exception:
+                raise_error(
+                    "Overleaf token not set; "
+                    "Please set it using 'calkit config set overleaf_token'"
+                )
+    repo = git.Repo()
+    conflict_fpath = os.path.join(".calkit", "overleaf", "CONFLICT.json")
+    in_am_session = "in the middle of an am session" in repo.git.status()
+    # Check if we're in the middle of resolving a merge conflict
+    if in_am_session and not resolve:
         raise_error(
-            "Overleaf token not set; "
-            "Please set it using 'calkit config set overleaf_token'"
+            "You are in the middle of resolving a merge conflict; "
+            "use 'calkit overleaf sync --resolve' after editing file(s)"
         )
-    repo = git.Repo()
+    elif resolve:
+        if not os.path.isfile(conflict_fpath):
+            raise_error("No merge conflict to resolve")
+        # Figure out which wdir has the conflict in it
+        with open(conflict_fpath) as f:
+            resolving_info = json.load(f)
     for pub in pubs:
         overleaf_config = pub.get("overleaf", {})
         if not overleaf_config:
@@ -354,6 +396,16 @@ def sync(
                 "No working directory defined for this publication; "
                 "please set it in the publication's Overleaf config"
             )
+        if resolve and wdir == resolving_info["wdir"]:
+            repo.git.add(wdir)
+            if repo.git.diff(["--staged", wdir]):
+                repo.git.commit(
+                    [wdir, "-m", f"Resolve Overleaf merge conflict in {wdir}"]
+                )
+            if in_am_session:
+                repo.git.am("--skip")
+        elif resolve:
+            continue
         # If there are any uncommitted changes in the publication working
         # directory, raise an error
         if repo.git.diff(wdir) or repo.index.diff("HEAD", wdir):
@@ -365,9 +417,8 @@ def sync(
         overleaf_project_dir = os.path.join(
             ".calkit", "overleaf", overleaf_project_id
         )
-        overleaf_remote_url = (
-            f"https://git:{overleaf_token}@git.overleaf.com/"
-            f"{overleaf_project_id}"
+        overleaf_remote_url = calkit.overleaf.get_git_remote_url(
+            project_id=overleaf_project_id, token=str(overleaf_token)
         )
         if not os.path.isdir(overleaf_project_dir):
             overleaf_repo = git.Repo.clone_from(
@@ -389,7 +440,10 @@ def sync(
                 "it matches one in your Overleaf account settings "
                 "(https://overleaf.com/user/settings)"
             )
-        last_sync_commit = pub["overleaf"].get("last_sync_commit")
+        if resolve:
+            last_sync_commit = resolving_info["last_overleaf_commit"]
+        else:
+            last_sync_commit = pub["overleaf"].get("last_sync_commit")
         if last_sync_commit:
             commits_since = list(
                 overleaf_repo.iter_commits(rev=f"{last_sync_commit}..HEAD")
@@ -398,6 +452,8 @@ def sync(
                 f"There have been {len(commits_since)} changes on "
                 "Overleaf since last sync"
             )
+        else:
+            commits_since = []
         # Determine which paths to sync and push
         # TODO: Support glob patterns
         git_sync_paths = pub["overleaf"].get("sync_paths", [])
@@ -405,6 +461,14 @@ def sync(
         dvc_sync_paths = pub["overleaf"].get("dvc_sync_paths", [])
         sync_paths = git_sync_paths + dvc_sync_paths
         push_paths = pub["overleaf"].get("push_paths", [])
+        implicit_sync_paths = os.listdir(overleaf_repo.working_dir)
+        for p in implicit_sync_paths:
+            if p.startswith("."):
+                continue
+            if p not in sync_paths:
+                sync_paths.append(p)
+                if p not in git_sync_paths and p not in dvc_sync_paths:
+                    git_sync_paths.append(p)
         git_sync_paths_in_project = [
             os.path.join(wdir, p) for p in git_sync_paths
         ]
@@ -439,7 +503,6 @@ def sync(
                         "git",
                         "am",
                         "--3way",
-                        "--reject",
                         "--directory",
                         wdir,
                         "-",
@@ -447,25 +510,38 @@ def sync(
                     input=patch,
                     text=True,
                     encoding="utf-8",
+                    capture_output=True,
                 )
-                if process.returncode != 0:
-                    if force:
-                        # Skip any patches we might be in the middle of
-                        try:
-                            repo.git.am("--skip")
-                        except Exception:
-                            pass
-                        typer.echo(
-                            "Failed to apply Overleaf patch to project repo. "
-                            "Proceeding to push project changes to Overleaf "
-                            "anyway."
-                        )
-                    else:
-                        raise_error(
-                            "Failed to apply Overleaf patch to project repo. "
-                            "Check the .rej files and manually apply changes, "
-                            "delete them, then rerun with --force."
+                # Handle merge conflicts
+                if (
+                    process.returncode != 0
+                    and "merge conflict" in process.stdout.lower()
+                ):
+                    msg = ""
+                    for line in process.stdout.split("\n"):
+                        if "merge conflict" in line.lower():
+                            msg += line + "\n"
+                    # Save a file to track this merge conflict
+                    c = overleaf_repo.head.commit.hexsha
+                    with open(conflict_fpath, "w") as f:
+                        json.dump(
+                            {
+                                "wdir": wdir,
+                                "last_overleaf_commit": c,
+                                "pub_path": pub["path"],
+                            },
+                            f,
                         )
+                    raise_error(
+                        f"{msg}Edit the file(s) and then call:\n\n"
+                        "    calkit overleaf sync --resolve"
+                    )
+                elif process.returncode != 0:
+                    raise_error(f"Could not apply:\n{process.stdout}")
+            elif resolve:
+                # We have no patch since the last sync, but we need to update
+                # our latest sync commit
+                typer.echo("Merge conflict resolved")
             else:
                 typer.echo("No changes to apply")
         else:
@@ -535,6 +611,8 @@ def sync(
         with open("calkit.yaml", "w") as f:
             calkit.ryaml.dump(ck_info, f)
         repo.git.add("calkit.yaml")
+        if resolve and os.path.isfile(conflict_fpath):
+            os.remove(conflict_fpath)
         # Stage the changes in the project repo
         # Add any DVC sync paths to DVC
         for dvc_sync_path in dvc_sync_paths_in_project:
@@ -660,9 +738,8 @@ def get_status(
         overleaf_project_dir = os.path.join(
             ".calkit", "overleaf", overleaf_project_id
         )
-        overleaf_remote_url = (
-            f"https://git:{overleaf_token}@git.overleaf.com/"
-            f"{overleaf_project_id}"
+        overleaf_remote_url = calkit.overleaf.get_git_remote_url(
+            project_id=overleaf_project_id, token=str(overleaf_token)
         )
         if not os.path.isdir(overleaf_project_dir):
             overleaf_repo = git.Repo.clone_from(

{calkit_python-0.30.8 → calkit_python-0.30.10}/calkit/cli/update.py

@@ -64,7 +64,7 @@ def update_license(
     ] = False,
 ):
     """Update license with a reasonable default
-    (MIT for code, CC-BY-4.0 for other files)
+    (MIT for code, CC-BY-4.0 for other files).
     """
     with open("LICENSE", "w") as f:
         f.write(
@@ -202,14 +202,38 @@ def update_release(
         # TODO: Enable reuploading artifact releases
         if path != "." or release_type != "project":
             raise_error("Can only handle updating project releases")
-        release_files_dir = f".calkit/releases/{name}/files"
+        release_dir = f".calkit/releases/{name}"
+        release_files_dir = release_dir + "/files"
+        os.makedirs(release_files_dir, exist_ok=True)
+        # Save a metadata file with each DVC file's MD5 checksum
+        dvc_md5s = calkit.releases.make_dvc_md5s(
+            zipfile="archive.zip" if path == "." else None,
+            paths=None if path == "." else [path],
+        )
+        dvc_md5s_path = release_dir + "/dvc-md5s.yaml"
+        typer.echo(f"Saving DVC MD5 info to {dvc_md5s_path}")
+        with open(dvc_md5s_path, "w") as f:
+            calkit.ryaml.dump(dvc_md5s, f)
+        # Create a README for the Invenio release
+        typer.echo("Creating README.md for release")
+        title = ck_info.get("title")
+        if title is None:
+            raise_error("Project has no title")
+        readme_txt = f"# {title}\n"
+        git_rev = repo.git.rev_parse(["--short", "HEAD"])
+        readme_txt += (
+            f"\nThis is a {release_type} release ({name}) generated with "
+            f"Calkit from Git rev {git_rev}.\n"
+        )
+        readme_path = release_files_dir + "/README.md"
+        with open(readme_path, "w") as f:
+            f.write(readme_txt)
         zip_path = release_files_dir + "/archive.zip"
         all_paths = calkit.releases.ls_files()
         typer.echo(f"Adding files to {zip_path}")
         with zipfile.ZipFile(zip_path, "w") as zipf:
             for fpath in all_paths:
                 zipf.write(fpath)
-        files = os.listdir(release_files_dir)
         try:
             files_in_record = [
                 entry["key"]
@@ -224,6 +248,10 @@ def update_release(
                 "Failed to get existing files in record: "
                 f"{e.__class__.__name__}: {e}"
             )
+        # Check size of files dir
+        size = calkit.get_size(release_files_dir)
+        typer.echo(f"Release size: {(size / 1e6):.1f} MB")
+        files = os.listdir(release_files_dir)
         for filename in files:
             if filename in files_in_record:
                 typer.echo(f"Deleting existing file {filename} from draft")

{calkit_python-0.30.8 → calkit_python-0.30.10}/calkit/core.py

@@ -602,3 +602,23 @@ def get_md5(path: str, exclude_files: list[str] | None = None) -> str:
         with open(path) as f:
             content = f.read()
         return hashlib.md5(content.encode()).hexdigest()
+
+
+def set_env_vars(ck_info: dict, cli: bool = True) -> None:
+    """Set environmental variables according to the values read from
+    ``calkit.yaml``.
+    """
+    env_vars = ck_info.get("env_vars", {})
+    if not isinstance(env_vars, dict):
+        msg = (
+            "Environmental variables in Calkit project info must be a "
+            "map/dictionary"
+        )
+        if cli:
+            from calkit.cli import raise_error
+
+            raise_error(msg)
+        else:
+            raise ValueError(msg)
+    for k, v in env_vars.items():
+        os.environ[str(k)] = str(v)

calkit_python-0.30.10/calkit/overleaf.py

@@ -0,0 +1,15 @@
+"""Functionality for working with Overleaf."""
+
+import os
+
+import calkit
+
+
+def get_git_remote_url(project_id: str, token: str) -> str:
+    """Form the Git remote URL for an Overleaf project.
+
+    If running against a test environment, this will use a local directory.
+    """
+    if calkit.config.get_env() == "test":
+        return os.path.join("/tmp", "overleaf", project_id)
+    return f"https://git:{token}@git.overleaf.com/{project_id}"

{calkit_python-0.30.8 → calkit_python-0.30.10}/calkit/tests/cli/test_main.py

@@ -50,6 +50,18 @@ def test_run_in_env(tmp_dir):
         stdin=stdin,
         check=True,
     )
+    # Check that we can pass project env vars into the container
+    ck_info = calkit.load_calkit_info()
+    ck_info["env_vars"] = {"MY_COOL_ENV_VAR": "my cool value"}
+    with open("calkit.yaml", "w") as f:
+        calkit.ryaml.dump(ck_info, f)
+    p = subprocess.run(
+        ["calkit", "xenv", "echo", "$MY_COOL_ENV_VAR"],
+        text=True,
+        capture_output=True,
+        check=True,
+    )
+    assert "my cool value" in p.stdout
     # Now let's create a 2nd Docker env and make sure we need to call it by
     # name when trying to run
     subprocess.check_call(
@@ -452,6 +464,19 @@ def test_run(tmp_dir):
     subprocess.check_call(
         ["calkit", "save", "-am", "Run pipeline", "--no-push"]
     )
+    # Test that we can set env vars at the project level
+    ck_info = calkit.load_calkit_info()
+    ck_info["env_vars"] = {"MY_ENV_VAR": "some-value"}
+    with open("calkit.yaml", "w") as f:
+        calkit.ryaml.dump(ck_info, f)
+    with open("script.py", "w") as f:
+        f.write("import os\nprint(os.environ['MY_ENV_VAR'])")
+    out = subprocess.check_output(["calkit", "run"], text=True)
+    print(out)
+    assert "some-value" in out
+    subprocess.check_call(
+        ["calkit", "save", "-am", "Run pipeline", "--no-push"]
+    )
     # Make sure we can run on a detached head
     repo = git.Repo()
     repo.git.checkout("HEAD^")