penult 0.0.2__py3-none-any.whl

penult/__init__.py

@@ -0,0 +1,56 @@
+"""penult: from an idea to a publication draft, posted where its author can review it.
+
+The seed is a GitHub issue. penult fetches it into a workspace next to it, holds the
+brief, the research notes, the essay, the images and their credits, checks the draft
+with ductus and penlike, renders it as an issue comment, and, once the author says
+so, hands it to ``article`` for an unpublished Substack or Medium draft. The
+thinking in between (research, drafting, the style passes) is an agent's, laid out
+in the ``penult`` skill; the verbs here keep every step from being improvised twice.
+
+>>> import penult
+>>> penult.prefs(files={})["preferences"]["platform"]
+'substack'
+"""
+
+from penult.base import STAGES, PenultError
+from penult.store import DEFAULT_PREFERENCES, data_dir, preferences
+from penult.tools import (
+    TOOLS,
+    approve,
+    check,
+    image_add,
+    image_generate,
+    image_search,
+    mark,
+    post,
+    prefs,
+    prefs_set,
+    publish,
+    render,
+    start,
+    status,
+)
+from penult.workspace import Workspace
+
+__all__ = [
+    "DEFAULT_PREFERENCES",
+    "STAGES",
+    "TOOLS",
+    "PenultError",
+    "Workspace",
+    "approve",
+    "check",
+    "data_dir",
+    "image_add",
+    "image_generate",
+    "image_search",
+    "mark",
+    "post",
+    "preferences",
+    "prefs",
+    "prefs_set",
+    "publish",
+    "render",
+    "start",
+    "status",
+]

penult/__main__.py

@@ -0,0 +1,72 @@
+# PYTHON_ARGCOMPLETE_OK
+"""``penult`` on the command line: every verb of :data:`penult.tools.TOOLS`.
+
+    penult start https://github.com/<owner>/essays/issues/30
+    penult status docs/<slug>
+    penult check docs/<slug>
+    penult image-search "a 1990s modem" --folder docs/<slug>
+    penult render docs/<slug> && penult post docs/<slug>
+    penult approve docs/<slug> --note "looks good, ship it"
+    penult publish docs/<slug>
+
+``--json`` anywhere prints the whole result instead of its text. The exit status is 0
+when the result is ok and 1 when it is not.
+"""
+
+from __future__ import annotations
+
+import functools
+import json
+import sys
+
+import cw
+
+from penult import tools
+from penult.base import PenultError
+
+
+def _command(func):
+    func = tools.without(func)
+
+    @functools.wraps(func)
+    def command(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except PenultError as error:
+            raise cw.CommandError(str(error)) from error
+
+    return command
+
+
+def _egress(as_json):
+    def egress(result, *, out, err):
+        if as_json or not isinstance(result, dict):
+            print(
+                json.dumps(result, indent=2, ensure_ascii=False, default=str), file=out
+            )
+        else:
+            print(result.get("text") or result.get("summary") or "", file=out)
+        return 0 if not isinstance(result, dict) or result.get("ok", True) else 1
+
+    return egress
+
+
+def main(argv=None):
+    """Run the command line. The ``penult`` console script."""
+    for stream in (sys.stdout, sys.stderr):
+        getattr(stream, "reconfigure", lambda **_: None)(errors="backslashreplace")
+    argv = list(sys.argv[1:] if argv is None else argv)
+    as_json = "--json" in argv
+    commands = {f.__name__.replace("_", "-"): _command(f) for f in tools.TOOLS}
+    code = cw.dispatch(
+        commands,
+        [a for a in argv if a != "--json"],
+        prog="penult",
+        convention=cw.MODERN,
+        egress=_egress(as_json),
+    )
+    raise SystemExit(code)
+
+
+if __name__ == "__main__":
+    main()

penult/base.py

@@ -0,0 +1,51 @@
+"""Errors, constants and the stage names shared across penult.
+
+penult takes a seed (a GitHub issue) to a publication draft posted for review. The
+stages below name where a workspace stands; every verb reads and advances them
+through the workspace state file, and nothing here depends on any other module.
+"""
+
+from __future__ import annotations
+
+APP_NAME = "penult"
+DATA_DIR_ENVVAR = "PENULT_DATA_DIR"
+
+#: A seed with fewer words than this (title, body and comments together) is "thin":
+#: the agent asks the user where to take it before researching or drafting.
+THIN_SEED_WORDS = 250
+
+#: The stages a workspace moves through, in order. ``approved`` is set by the user
+#: (or by the agent relaying the user's words) and gates ``published``.
+STAGES = (
+    "seeded",
+    "researched",
+    "drafted",
+    "styled",
+    "illustrated",
+    "posted",
+    "approved",
+    "published",
+)
+
+STATE_FILE = "penult.json"
+SEED_FILE = "seed.md"
+BRIEF_FILE = "brief.md"
+ESSAY_FILE = "essay.md"
+IMAGES_FILE = "images.md"
+COMMENT_FILE = "comment.md"
+CHECK_FILE = "check.md"
+IMG_DIR = "img"
+RESEARCH_DIR = "research"
+
+
+class PenultError(Exception):
+    """An error whose message says what to do next."""
+
+
+def stage_index(stage: str) -> int:
+    """Where a stage sits in :data:`STAGES`; unknown stages sort first.
+
+    >>> stage_index("drafted") > stage_index("seeded")
+    True
+    """
+    return STAGES.index(stage) if stage in STAGES else -1

penult/data/skills/penult/SKILL.md

@@ -0,0 +1,89 @@
+---
+name: penult
+description: Take an idea (a GitHub issue) to a publication draft posted for the author's review, then, once they approve, to an unpublished Substack or Medium draft. Use on "make a draft for <issue url>", "turn this issue into an essay", "write up issue N for Substack", "draft the next essay", "post the draft on the issue", "they approved it, publish the draft", or whenever an essay goes from a seed to a reviewed draft. Drives the whole path with the penult verbs (workspace, state, images within budget, issue comment, publishing hand-off) and the skills that own each step: do-research for checking claims and finding critiques, penlike (and its author-choice guide) and deslop for voice, ductus for machine-sounding passages, illustration for credited images. Posts the draft only where the author asked (the issue thread, public if the repository is); never publishes to a platform, and never sends anything to a platform before the author says the posted draft is good.
+license: MIT
+metadata:
+  audience: users
+---
+
+# penult: from a seed to a draft the author can review
+
+The author gives you a seed, usually an issue in their essays repository, and wants back a link to a **draft posted as a comment on that issue**, with images inline, in their voice, every claim checked and the other side of each argument known. They iterate with you there. When they say it is good, you make an **unpublished** platform draft and give them the link; they press Publish themselves. Nothing you do publishes to a platform. The issue comment is as public as the repository is, which is what the author asked for; if the essay must stay private until it is out, say so before posting and ask where the review should happen.
+
+Everything about *how* the author wants this (voices, platform, image budget, where their credentials are) is in their private preferences: `penult prefs`. Everything about *this* essay (the seed, the brief, the research, the draft, the images and their credits, what was posted) is in the workspace that `penult start` makes next to the issue, in `docs/<slug>/` of its repository, so that it is versioned and searchable later.
+
+## The path
+
+```bash
+penult start <issue url>          # from inside the repository's checkout; prints the workspace and whether the seed is thin
+penult status <workspace>         # at any time: stage, files, what comes next
+```
+
+### 1. Seed and brief
+
+Read `seed.md`. Fill the `[DECIDE: ...]` lines of `brief.md`: thesis, angle and shape, audience, voices, platform, visual style. The essay is **never a report** unless the author asked for one: no executive summary, no numbered sections of findings, no bullet lists of evidence. It is an argument, a story and its moral, a debunking, a field guide.
+
+**If `start` says the seed is thin** (few words, no thesis): do not research or draft. Write the author your questions, each with your recommendation (where to take it, what the point is, who it is for, which style), post them as a comment on the issue, record them in `brief.md`, and stop until they answer. A rich seed (a long body, a worked argument, an existing report) gets no questions: decide, write the assumptions into `brief.md`, and go on. Say in the final hand-over what you assumed.
+
+### 2. Research
+
+Load the `do-research` skill, Deep depth. The themes are fixed by the job: **verify every claim the seed makes** (numbers, quotes, attributions, dates), **find the critiques and counter-evidence** so the essay knows both sides, **the standard names** for the things it talks about, and **candidate images**: figures, tables, charts and public-domain art worth using, each with its source and licence. Run the themes as parallel agents; each writes one Vancouver-referenced note into `research/` of the workspace (lowercase filenames, `[name](url)` references, estimates labelled, "unverified" said plainly). Then `penult mark <workspace> --stage researched`. (Every verb below takes the workspace as its first argument or as `--folder`; the examples leave it out for brevity.)
+
+### 3. Draft
+
+Write `essay.md` with front matter (`title:`, `subtitle:`, optional `tags:`), the title again as the one `#` heading, relative image paths (`img/...`) and `[^n]` footnotes where a source matters. Hyperlink sources inline the way the author's published essays do. What the research corrected, correct in the essay; where a claim stays contested, let the essay say so in one sentence rather than hedge everywhere. Keep the author's own sentences from the seed wherever they are there. `penult mark --stage drafted`.
+
+### 4. Voice
+
+Load the `penlike` skill and run `penlike guide`: it is the author's own rule book for which models fit which subject, audience and part of a text, and what not to overdo. Choose the base voice (the `voice_models` preference, usually the author's own model) and at most two other authors from the guide. Then, passage by passage, with the surrounding paragraphs in view so each rewrite still flows:
+
+1. `ductus gauge essay.md` and your own reading mark the spans that sound machine-made (generalising, balanced-for-balance's-sake, tidy lists of three, summary closers). Those get rewritten first, in the base voice (`penlike brief --like <model> --style essay`).
+2. The passages the guide reserves for another author (an opening jab, a close, a deadpan debunking) are rewritten in that author's register, short, and only where the subject invites it.
+3. The `deslop` skill's passes for rhythm, specificity and ownership; then `penlike check` against the base voice, at most two rounds.
+
+`penult check <workspace>` runs ductus and penlike over the essay and writes a check file under the private data root (it quotes the author's style-model figures, so it never goes into the workspace); read it, but do not chase a zero: human writing is flagged too, and bland prose with no tell is the worse failure. `penult mark --stage styled`.
+
+### 5. Images
+
+For each place an image belongs (the hero, one per major turn of the argument, any figure the research found):
+
+- **Existing images first**: `penult image-search "<what the beat shows>" --folder <workspace>` (the `illustration` package: Openverse, public-domain art, with licence and attribution). Take one with `penult image-add <url> --name <file> --credit "<credit line>" --license <licence>`; the credit goes in the Markdown caption.
+- **Prompts**: write at least `min_image_prompts` (default 5) prompts into `images.md`, each saying the beat, the style, the composition. Generate **at most `max_generated_images` (default 2)** of them with `penult image-generate "<prompt>" --name <file>`: the top one or two, chosen for where an image matters most. The budget is deliberate: generated images are paid and often off; the author tries the other prompts by hand and gives you the files (`penult image-add <file> --name ... --credit "Image generated for this essay" --prompt "<the prompt>"`).
+- Figures and tables from the research are redrawn (matplotlib, a clean table image) and credited to their source.
+
+Place each image in `essay.md` as its own paragraph: `![alt](img/file.png "Caption with the credit")`. `penult mark --stage illustrated`.
+
+### 6. Post the draft
+
+Commit and push the workspace (the comment shows images by their raw GitHub URL, so an unpushed image is a broken picture; `penult post` refuses until they are pushed). Then:
+
+```bash
+penult render <workspace>     # writes comment.md: heading, subtitle, absolute images, a footer naming the research folder
+penult post <workspace>       # posts it; a second run updates the same comment (--new for another)
+```
+
+Give the author the comment link. Iterate on their comments: edit `essay.md`, re-run the voice checks that matter, `render`, `post`. Record anything they say about voice or images that should hold for later essays with `penlike guide-add "<one line>" --source <issue url>` or `penult prefs-set`.
+
+### 7. Approval, then the platform draft
+
+Only when the author says the posted draft is good, in their words:
+
+```bash
+penult approve <workspace> --by <who> --note "<their words>"   # of the text as posted; a later edit needs a new post and approval
+penult publish <workspace>               # substack by default: an UNPUBLISHED draft via `article draft-substack`; prints the editor link
+penult publish <workspace> --dry-run     # what Substack would lose (tables, inline images), no credentials needed
+```
+
+`publish` uses the `article` package with the credentials file named in the preferences (`article_env_file`), never a credential in a repository. Medium is article's Import-a-story step and comes after the Substack post is live. Hand the author the editor link; they review and publish. You do not.
+
+## Hand-over, every time
+
+Say: the comment or draft link; which voices were used for which passages and why (from the guide); what the research corrected in the seed; the images used and their credits; the prompts left for the author to try; what you assumed where the seed was silent; and anything the author must do (a credential, an approval, the Publish button).
+
+## Never
+
+- Publish, schedule, or send anything to readers. The platform step makes a draft and stops.
+- Put a model, a preference, a credential, or anything from `~/.local/share/penlike` or `~/.local/share/penult` into a repository, an issue or a commit.
+- Spend past the image budget without the author asking, or present a generated image as a photograph.
+- Invent a fact, a number, a quote or a source to fill a gap. A gap is a question, written `[ASK: ...]` in the draft and asked.
+- Turn the essay into a report, or sand off the author's opinions into balance.

penult/github.py

@@ -0,0 +1,235 @@
+"""GitHub, through the ``gh`` and ``git`` commands: read an issue, post or edit a comment, find raw URLs.
+
+Every function takes a ``runner`` (the shape of :func:`subprocess.run`) so tests
+run without a network and another transport can be dropped in. The default needs
+an authenticated ``gh``.
+
+>>> parse_issue_url("https://github.com/octo/essays/issues/30")
+('octo', 'essays', 30)
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import subprocess
+from pathlib import Path
+from typing import Any, Callable
+from urllib.parse import quote
+
+from penult.base import PenultError
+
+__all__ = [
+    "edit_comment",
+    "fetch_issue",
+    "parse_issue_url",
+    "post_comment",
+    "raw_url_base",
+    "repo_of_checkout",
+    "unpushed",
+]
+
+Runner = Callable[..., Any]
+
+_ISSUE_URL = re.compile(
+    r"^https?://(?:www\.)?github\.com/([^/]+)/([^/]+)/issues/(\d+)(?:[/#?].*)?$"
+)
+_REMOTE = re.compile(r"[:/]([^/:]+)/([^/]+?)(?:\.git)?/?$")
+_TIMEOUT = 600
+
+
+def parse_issue_url(url: str) -> tuple[str, str, int]:
+    """``(owner, repo, number)`` from an issue URL, or an error that shows the shape."""
+    found = _ISSUE_URL.match(url.strip())
+    if not found:
+        raise PenultError(
+            f"not a GitHub issue URL: {url!r}; expected https://github.com/<owner>/<repo>/issues/<n>"
+        )
+    owner, repo, number = found.groups()
+    return owner, repo, int(number)
+
+
+def _run(runner: Runner, args: list[str], *, what: str, **kwargs: Any) -> str:
+    try:
+        done = runner(args, capture_output=True, text=True, timeout=_TIMEOUT, **kwargs)
+    except FileNotFoundError as error:
+        raise PenultError(
+            f"{args[0]!r} is not installed; it is needed to {what}"
+        ) from error
+    if done.returncode != 0:
+        detail = (done.stderr or done.stdout or "").strip()
+        raise PenultError(f"could not {what}: {detail or 'command failed'}")
+    return done.stdout or ""
+
+
+def fetch_issue(url: str, *, runner: Runner = subprocess.run) -> dict[str, Any]:
+    """The issue's title, body, labels and comments, as one JSON-ready dict."""
+    owner, repo, number = parse_issue_url(url)
+    out = _run(
+        runner,
+        ["gh", "issue", "view", str(number), "-R", f"{owner}/{repo}", "--json",
+         "title,body,labels,comments,url,author,createdAt"],
+        what=f"read issue {owner}/{repo}#{number}",
+    )  # fmt: skip
+    raw = json.loads(out)
+    return {
+        "kind": "github-issue",
+        "owner": owner,
+        "repo": repo,
+        "number": number,
+        "url": raw.get("url") or url,
+        "title": raw.get("title", ""),
+        "body": raw.get("body") or "",
+        "author": (raw.get("author") or {}).get("login", ""),
+        "created": raw.get("createdAt", ""),
+        "labels": [label.get("name", "") for label in raw.get("labels") or []],
+        "comments": [
+            {
+                "author": (c.get("author") or {}).get("login", ""),
+                "created": c.get("createdAt", ""),
+                "body": c.get("body") or "",
+                "url": c.get("url", ""),
+            }
+            for c in raw.get("comments") or []
+        ],
+    }
+
+
+def post_comment(url: str, body_file: Path, *, runner: Runner = subprocess.run) -> str:
+    """Post a file's content as a comment on the issue; returns the comment's URL."""
+    owner, repo, number = parse_issue_url(url)
+    out = _run(
+        runner,
+        ["gh", "issue", "comment", str(number), "-R", f"{owner}/{repo}", "--body-file", str(body_file)],
+        what=f"comment on {owner}/{repo}#{number}",
+    )  # fmt: skip
+    return out.strip().splitlines()[-1] if out.strip() else ""
+
+
+def edit_comment(
+    comment_url: str, body_file: Path, *, runner: Runner = subprocess.run
+) -> str:
+    """Replace an existing comment's body (``comment_url`` ends in ``#issuecomment-<id>``)."""
+    found = re.search(
+        r"github\.com/([^/]+)/([^/]+)/issues/\d+#issuecomment-(\d+)", comment_url
+    )
+    if not found:
+        raise PenultError(f"not an issue-comment URL: {comment_url!r}")
+    owner, repo, comment_id = found.groups()
+    payload = json.dumps({"body": Path(body_file).read_text(encoding="utf-8")})
+    _run(
+        runner,
+        ["gh", "api", "-X", "PATCH", f"repos/{owner}/{repo}/issues/comments/{comment_id}", "--input", "-"],
+        what=f"edit comment {comment_id}",
+        input=payload,
+    )  # fmt: skip
+    return comment_url
+
+
+def repo_of_checkout(
+    path: Path, *, runner: Runner = subprocess.run
+) -> dict[str, Any] | None:
+    """The checkout holding ``path``: its root, GitHub owner/repo, branch and commit, or ``None``."""
+    try:
+        git = ["git", "-C", str(path)]
+        root = _run(
+            runner, [*git, "rev-parse", "--show-toplevel"], what="find the repo"
+        ).strip()
+        git = ["git", "-C", root]
+        remote = _run(
+            runner, [*git, "remote", "get-url", "origin"], what="read the remote"
+        ).strip()
+        branch = _run(
+            runner, [*git, "rev-parse", "--abbrev-ref", "HEAD"], what="read the branch"
+        ).strip()
+        sha = _run(runner, [*git, "rev-parse", "HEAD"], what="read the commit").strip()
+    except PenultError:
+        return None
+    found = _REMOTE.search(remote) if "github" in remote else None
+    if not found:
+        return None
+    owner, repo = found.groups()
+    return {
+        "root": root,
+        "owner": owner,
+        "repo": repo,
+        "branch": branch,
+        "sha": sha or branch,
+    }
+
+
+def raw_url_base(checkout: dict[str, Any], folder: Path) -> str:
+    """The ``raw.githubusercontent.com`` prefix for files in ``folder`` at the checkout's commit.
+
+    The commit, not the branch, so the links keep working after the branch is
+    merged and deleted.
+
+    >>> raw_url_base({"root": "/r", "owner": "o", "repo": "e", "branch": "main", "sha": "abc123"}, Path("/r/docs/an essay"))
+    'https://raw.githubusercontent.com/o/e/abc123/docs/an%20essay/'
+    """
+    rel = (
+        Path(folder).resolve().relative_to(Path(checkout["root"]).resolve()).as_posix()
+    )
+    rel = quote(rel) + "/" if rel not in ("", ".") else ""
+    ref = checkout.get("sha") or checkout["branch"]
+    return f"https://raw.githubusercontent.com/{checkout['owner']}/{checkout['repo']}/{ref}/{rel}"
+
+
+def _status_paths(porcelain_z: str) -> set[str]:
+    """Paths in ``git status --porcelain -z`` output (a rename lists its old path too).
+
+    >>> sorted(_status_paths("?? a.png\\0R  new.png\\0old.png\\0"))
+    ['a.png', 'new.png', 'old.png']
+    """
+    found: set[str] = set()
+    entries = [e for e in porcelain_z.split("\0") if e]
+    i = 0
+    while i < len(entries):
+        entry = entries[i]
+        found.add(entry[3:] if len(entry) > 3 else entry)
+        if entry[:1] in ("R", "C") and i + 1 < len(entries):
+            found.add(entries[i + 1])
+            i += 1
+        i += 1
+    return found
+
+
+def unpushed(
+    checkout: dict[str, Any], paths: list[Path], *, runner: Runner = subprocess.run
+) -> list[str]:
+    """Which of ``paths`` are not yet committed and pushed to the branch's upstream.
+
+    A raw URL only works for a file that is on GitHub, so ``post`` refuses while any
+    image it references is in this list. Paths are compared in git's own form
+    (forward slashes, unescaped thanks to ``-z``).
+    """
+    root = Path(checkout["root"]).resolve()
+    names = [Path(p).resolve().relative_to(root).as_posix() for p in paths]
+    if not names:
+        return []
+    git = ["git", "-C", str(root)]
+    dirty = _run(
+        runner,
+        [*git, "status", "--porcelain", "-z", "--", *names],
+        what="check the tree",
+    )
+    missing = _status_paths(dirty)
+    try:
+        ahead = _run(
+            runner,
+            [
+                *git,
+                "log",
+                "--name-only",
+                "-z",
+                "--pretty=format:",
+                "@{upstream}..HEAD",
+                "--",
+                *names,
+            ],
+            what="compare with the upstream",
+        )
+        missing |= {p.strip() for p in re.split(r"[\0\n]", ahead) if p.strip()}
+    except PenultError:
+        missing |= set(names)  # no upstream: nothing of this branch is on GitHub
+    return sorted(n for n in names if n in missing)