paperpush 0.2.0__tar.gz → 0.2.1__tar.gz

{paperpush-0.2.0 → paperpush-0.2.1}/PKG-INFO

@@ -1,10 +1,10 @@
 Metadata-Version: 2.4
 Name: paperpush
-Version: 0.2.0
+Version: 0.2.1
 Summary: A Python library for automating the submission of manuscripts to academic venues.
 Author-email: Joseph Rich <josephrich98@gmail.com>
 Maintainer-email: Joseph Rich <josephrich98@gmail.com>
-License-Expression: MIT
+License-Expression: BSD-2-Clause
 Project-URL: Homepage, https://github.com/pachterlab/paperpush
 Project-URL: Repository, https://github.com/pachterlab/paperpush
 Project-URL: Documentation, https://github.com/pachterlab/paperpush#readme
@@ -51,7 +51,7 @@ playwright install chromium
 ## Use with AI
 
 ```bash
-git clone https://github.com/paperpush/paperpush.git
+git clone https://github.com/pachterlab/paperpush.git
 cd paperpush
 ```
 
@@ -97,6 +97,7 @@ the tool list and the contract agents should follow.
 
 Helper commands:
 - `paperpush requirements VENUE`: print VENUE's author guidelines, if available.
+- `paperpush update-venues`: Update venues.json and manuscript_requirements.json from the remote venue-data branch.
 - `paperpush options VENUE.FIELD`: print the allowed values for FIELD in VENUE's submission form.
 
 See the YouTube tutorial [PaperPush Tutorial](https://youtu.be/ChCYoud0uBk)
@@ -123,6 +124,8 @@ See `python scripts/paperpush_pipeline.py --help` for the full list of options,
 
 View the list on the command line with `paperpush --venues`.
 
+Venue details (field options, limits, author-guideline rules) are updated independently of releases: paperpush fetches the latest published data at most once a day, and `paperpush update-venues` fetches it right away. New venues arrive with new releases (`pip install -U paperpush`).
+
 For more details, see [`venues.md`](venues.md)
 
 ## Documentation

{paperpush-0.2.0 → paperpush-0.2.1}/README.md

@@ -19,7 +19,7 @@ playwright install chromium
 ## Use with AI
 
 ```bash
-git clone https://github.com/paperpush/paperpush.git
+git clone https://github.com/pachterlab/paperpush.git
 cd paperpush
 ```
 
@@ -65,6 +65,7 @@ the tool list and the contract agents should follow.
 
 Helper commands:
 - `paperpush requirements VENUE`: print VENUE's author guidelines, if available.
+- `paperpush update-venues`: Update venues.json and manuscript_requirements.json from the remote venue-data branch.
 - `paperpush options VENUE.FIELD`: print the allowed values for FIELD in VENUE's submission form.
 
 See the YouTube tutorial [PaperPush Tutorial](https://youtu.be/ChCYoud0uBk)
@@ -91,6 +92,8 @@ See `python scripts/paperpush_pipeline.py --help` for the full list of options,
 
 View the list on the command line with `paperpush --venues`.
 
+Venue details (field options, limits, author-guideline rules) are updated independently of releases: paperpush fetches the latest published data at most once a day, and `paperpush update-venues` fetches it right away. New venues arrive with new releases (`pip install -U paperpush`).
+
 For more details, see [`venues.md`](venues.md)
 
 ## Documentation

{paperpush-0.2.0 → paperpush-0.2.1}/paperpush/__init__.py

@@ -2,7 +2,7 @@
 
 import logging
 
-__version__ = "0.2.0"
+__version__ = "0.2.1"
 __url__ = "https://github.com/pachterlab/paperpush"
 
 # Standard-library convention: a library should not configure logging output.

{paperpush-0.2.0 → paperpush-0.2.1}/paperpush/cli.py

@@ -20,6 +20,7 @@ Implemented so far:
     paperpush login --orcid <j>   store an ORCID iD/password for a journal that
                                       offers "Sign in with ORCID"
     paperpush submit <subfile>    open bioRxiv and run the submission
+    paperpush update-venues       fetch the latest published venue data now
     paperpush --agent-guide       print the guide for AI agents (AGENTS.md)
 
 ``submit`` drives the bioRxiv wizard, typing in the field values read from the
@@ -40,7 +41,7 @@ from pathlib import Path
 
 from pydantic import ConfigDict, validate_call
 
-from . import __url__, __version__, credentials
+from . import __url__, __version__, credentials, venue_data
 from ._logging import configure_logging
 from .database import get_venue, list_venues
 from .subfile import default_filename, write_template
@@ -112,6 +113,37 @@ def _print_venues() -> int:
             print(f"  {venue.name} ({venue.slug})")
 
     print(f"\nFor more details, see {_VENUES_URL}")
+    print(f"Venue data: {venue_data.active_source().describe()} (refresh with 'paperpush update-venues')")
+    return 0
+
+
+@_validate
+def _cmd_update_venues(args: argparse.Namespace) -> int:
+    """Fetch the published venue data now, or clear the cached copy.
+
+    The data is otherwise refreshed on its own at most once a day; this is for
+    picking up a fix right away, and for seeing which copy is in use.
+    """
+    from .database import reload
+
+    if args.clear:
+        venue_data.clear_cache()
+        print(f"Removed the cached venue data; using the data {venue_data.bundled_source().describe()}.")
+        return 0
+
+    if not venue_data.uses_published():
+        print("note: published venue data is not used here (PAPERPUSH_VENUE_DATA, or a source " "checkout); fetching it into the cache anyway.", file=sys.stderr)
+    try:
+        result = venue_data.refresh(force=True)
+    except Exception as exc:
+        print(f"error: could not fetch the venue data from {venue_data.data_url()}: {exc}", file=sys.stderr)
+        return 1
+    reload()
+    print(result.message if result.status != "incompatible" else f"warning: {result.message}")
+    source = venue_data.active_source()
+    print(f"In use: {source.describe()}")
+    if source.kind != "bundled":
+        print(f"Files: {source.root}")
     return 0
 
 
@@ -946,6 +978,7 @@ def _cmd_requirements(args: argparse.Namespace) -> int:
         print(f"  {url}")
     if reqs.inherits:
         print(f"  (starts from {reqs.inherits}'s rules)")
+    print(f"  venue data: {venue_data.active_source().describe()}")
     if article_types and not args.article_type:
         print(f"  article types with their own rules: {', '.join(article_types)} (use --article-type)")
     for section in ("manuscript", "title_page", "abstract", "keywords", "sections", "statements", "figures", "tables", "supplementary", "references", "cover_letter", "upload"):
@@ -1050,7 +1083,7 @@ def build_parser() -> argparse.ArgumentParser:
 
     # metavar lists only the public commands; the internal 'schema' command is
     # registered below but deliberately left out so it does not appear in --help.
-    sub = parser.add_subparsers(dest="command", metavar="{subfile,options,autofill,validate,requirements,login,submit}")
+    sub = parser.add_subparsers(dest="command", metavar="{subfile,options,autofill,validate,requirements,login,submit,update-venues}")
 
     p_subfile = sub.add_parser("subfile", parents=[verbosity], help="create a <venue>.sub submission template")
     p_subfile.add_argument("venue", help="venue slug, e.g. biorxiv")
@@ -1172,6 +1205,10 @@ def build_parser() -> argparse.ArgumentParser:
     )
     p_submit.set_defaults(func=_cmd_submit)
 
+    p_update = sub.add_parser("update-venues", parents=[verbosity], help="fetch the latest published venue data (otherwise refreshed daily)")
+    p_update.add_argument("--clear", action="store_true", help="delete the cached copy and use the data bundled with this version")
+    p_update.set_defaults(func=_cmd_update_venues)
+
     # Internal command: the autofill front-ends (the Claude skill and the API
     # engine) read field roles from here. Hidden from --help (use 'subfile' to
     # inspect a venue's fields by hand).

{paperpush-0.2.0 → paperpush-0.2.1}/paperpush/database.py

@@ -3,6 +3,11 @@
 The database is a single JSON file (``venues.json``) shipped with the
 package. Each top-level key is a venue slug (e.g. ``"biorxiv"``) mapping to a
 venue definition that lists the fields required for submission.
+
+A newer copy may also be published separately from releases and cached locally
+(see :mod:`paperpush.venue_data`). The bundled file is always the baseline: a
+published entry replaces a bundled one only when the installed runner can
+drive it (see :func:`merge_published`).
 """
 
 from __future__ import annotations
@@ -18,8 +23,12 @@ from typing import Annotated, Any, Literal, Optional
 
 from pydantic import Field as PField
 
+from . import venue_data
+
 logger = logging.getLogger(__name__)
 
+# The copies shipped with the package. The files actually read may come from a
+# published copy instead; see paperpush.venue_data.
 DATABASE_PATH = Path(__file__).with_name("venues.json")
 
 # Directory holding shared keyword/vocabulary lists that a field can pull in via
@@ -36,7 +45,10 @@ def _load_options_file(name: str) -> list[str]:
     portal keywords or the PLOS Computational Biology classifications) lives in
     one shared file rather than being inlined in ``venues.json``.
     """
-    path = ASSETS_DIR / name
+    # A published copy may add or update a vocabulary; the bundled one backs it.
+    path = venue_data.active_source().assets / name
+    if not path.is_file():
+        path = ASSETS_DIR / name
     logger.debug("Loading field options from %s", path)
     lines = [line.strip() for line in path.read_text(encoding="utf-8").splitlines()]
     return [line for line in lines if line]
@@ -444,15 +456,118 @@ class Venue:
         )
 
 
+def _read_database(path: Path) -> dict[str, Any]:
+    logger.debug("Loading venue database from %s", path)
+    with path.open(encoding="utf-8") as fh:
+        return json.load(fh)
+
+
 @lru_cache(maxsize=1)
 def _load_raw() -> dict[str, Any]:
-    logger.debug("Loading venue database from %s", DATABASE_PATH)
-    with DATABASE_PATH.open(encoding="utf-8") as fh:
-        data = json.load(fh)
-    logger.info("Loaded %d venue entr(y/ies) from the database", len(data))
+    source = venue_data.active_source()
+    bundled = _read_database(DATABASE_PATH)
+    if source.kind == "bundled":
+        data = bundled
+    else:
+        try:
+            published = _read_database(source.path(venue_data.VENUES_FILE))
+        except (OSError, ValueError) as exc:
+            logger.warning("Could not read the venue data from %s (%s); using the bundled copy", source.root, exc)
+            published = bundled
+        # A local override is the author's own data, used as is; a published
+        # copy is merged venue by venue onto what this version can drive.
+        data = published if source.kind == "override" else merge_published(bundled, published)
+    logger.info("Loaded %d venue entr(y/ies) from the database (%s)", len(data), source.describe())
     return data
 
 
+def _field_shape(slug: str, raw: dict[str, Any]) -> list[tuple[str, str]]:
+    """The (id, type) of each field of ``slug``, after inheritance, in order.
+
+    This is what a venue's runner is written against: it reads fields by id and
+    fills each according to its type. Everything else about a field -- label,
+    help, options, limits, whether it is required -- is data the runner does not
+    hard-code, so it can change without a code release.
+    """
+    return [(f.get("id", ""), f.get("type", "text")) for f in _resolve_entry(slug, raw).get("fields", [])]
+
+
+def merge_published(bundled: dict[str, Any], published: dict[str, Any]) -> dict[str, Any]:
+    """Overlay a published venue database onto the bundled one, venue by venue.
+
+    A published entry replaces the bundled entry only when
+
+    * the bundled database has the venue (a new venue needs its runner, which
+      only a release can bring),
+    * its fields keep the bundled shape (:func:`_field_shape`) -- adding,
+      removing, renaming, reordering or retyping a field needs runner changes,
+    * it resolves (e.g. its ``inherits`` base exists), and
+    * any venue it inherits from is itself taken from the published copy, so an
+      entry is never resolved against a mix of old and new bases.
+
+    Every other venue keeps its bundled entry, and a note is logged so a user on
+    an old version knows an upgrade would bring more.
+    """
+    accepted: set[str] = set()
+    for slug in bundled:
+        if slug not in published:
+            continue
+        try:
+            if _field_shape(slug, published) == _field_shape(slug, bundled):
+                accepted.add(slug)
+        except KeyError as exc:
+            logger.warning("Published venue data: %s does not resolve (%s); keeping the bundled entry", slug, exc)
+
+    # Drop entries whose base was not accepted, until nothing changes (a chain
+    # of inheritance can take several passes).
+    changed = True
+    while changed:
+        changed = False
+        for slug in sorted(accepted):
+            base = str(published[slug].get("inherits", "") or "")
+            if base and base not in accepted and base.lower() not in accepted:
+                accepted.discard(slug)
+                changed = True
+
+    held_back = sorted(set(published) - accepted)
+    if held_back:
+        logger.info("Published venue data for %s needs a newer paperpush; using the bundled entries. " "Run 'pip install -U paperpush' to get them.", ", ".join(held_back))
+    return {slug: published[slug] if slug in accepted else entry for slug, entry in bundled.items()}
+
+
+def check_venue_data(root: Path) -> None:
+    """Raise if the venue data in directory ``root`` does not load.
+
+    Run on a downloaded copy before it replaces the cache: every venue must
+    resolve and build, every ``options_file`` it names must exist, and the
+    manuscript requirements must load too.
+    """
+    from .requirements import check_requirements_file
+
+    raw = _read_database(root / venue_data.VENUES_FILE)
+    if not isinstance(raw, dict) or not raw:
+        raise ValueError("venues.json holds no venues")
+    for slug in raw:
+        entry = _resolve_entry(slug, raw)
+        Venue.from_dict(slug, {**entry, "fields": []})
+        for f in entry.get("fields", []):
+            name = f.get("options_file")
+            if name and not (root / venue_data.ASSETS_SUBDIR / name).is_file() and not (ASSETS_DIR / name).is_file():
+                raise ValueError(f"{slug}.{f.get('id')} names a missing options_file {name!r}")
+            Field.from_dict({k: v for k, v in f.items() if k != "options_file"})
+    check_requirements_file(root / venue_data.REQUIREMENTS_FILE)
+
+
+def reload() -> None:
+    """Re-resolve the venue data source and drop every cached load of it."""
+    from . import requirements
+
+    venue_data.reset()
+    _load_raw.cache_clear()
+    _load_options_file.cache_clear()
+    requirements._load_raw.cache_clear()
+
+
 # Keys an inheriting entry uses to declare how it derives from its base rather
 # than to describe a key of the resulting venue; they are consumed by
 # _resolve_entry and not copied verbatim onto the merged venue.

{paperpush-0.2.0 → paperpush-0.2.1}/paperpush/references.py

@@ -637,7 +637,7 @@ def _resolve_doi(doi: str, *, timeout: float = DOI_CHECK_TIMEOUT) -> tuple[str,
         },
     )
     try:
-        with urllib.request.urlopen(req, timeout=timeout) as resp:  # nosec B310 - scheme is a literal https above
+        with urllib.request.urlopen(req, timeout=timeout) as resp:  # scheme is a literal https above  # nosec B310
             payload = resp.read(MAX_TEXT_BYTES)
     except urllib.error.HTTPError as exc:
         if exc.code in _MISSING_STATUS:

{paperpush-0.2.0 → paperpush-0.2.1}/paperpush/requirements.py

@@ -40,9 +40,14 @@ from pathlib import Path
 from typing import Annotated, Any, Literal, Optional
 
 from pydantic import Field as PField
+from pydantic import TypeAdapter, ValidationError
+
+from . import venue_data
 
 logger = logging.getLogger(__name__)
 
+# The copy shipped with the package; the file actually read may be a newer
+# published copy (see paperpush.venue_data).
 REQUIREMENTS_PATH = Path(__file__).with_name("manuscript_requirements.json")
 
 # Where figures/tables sit for the initial submission.
@@ -383,14 +388,34 @@ def _section_to_dict(section) -> dict[str, Any]:
     return out
 
 
+def _read_requirements(path: Path) -> dict[str, Any]:
+    logger.debug("Loading manuscript requirements from %s", path)
+    with path.open(encoding="utf-8") as fh:
+        data = json.load(fh)
+    if not isinstance(data, dict):
+        raise ValueError(f"{path} is not a JSON object")
+    return data
+
+
 @lru_cache(maxsize=1)
 def _load_raw() -> dict[str, Any]:
-    logger.debug("Loading manuscript requirements from %s", REQUIREMENTS_PATH)
+    source = venue_data.active_source()
     if not REQUIREMENTS_PATH.exists():
         logger.warning("No manuscript requirements database at %s", REQUIREMENTS_PATH)
-        return {}
-    with REQUIREMENTS_PATH.open(encoding="utf-8") as fh:
-        data = json.load(fh)
+        bundled: dict[str, Any] = {}
+    else:
+        bundled = _read_requirements(REQUIREMENTS_PATH)
+    data = bundled
+    if source.kind != "bundled":
+        path = source.path(venue_data.REQUIREMENTS_FILE)
+        try:
+            published = _read_requirements(path)
+        except (OSError, ValueError) as exc:
+            logger.warning("Could not read %s (%s); using the bundled copy", path, exc)
+        else:
+            # A local override is the author's own data, used as is; a published
+            # copy is merged entry by entry onto what this version can read.
+            data = published if source.kind == "override" else merge_published(bundled, published)
     logger.info("Loaded %d manuscript requirement entr(y/ies)", sum(1 for k in data if not k.startswith("$")))
     return data
 
@@ -462,6 +487,144 @@ def _resolve_entry(slug: str, raw: dict[str, Any], seen: tuple[str, ...] = ()) -
     return merged
 
 
+# Keys an entry may carry at its top level (``slug`` is the database key, not a
+# key inside the entry), and keys an ``article_types`` override may carry.
+_ENTRY_KEYS = frozenset(f.name for f in dataclass_fields(ManuscriptRequirements)) - {"slug"}
+_OVERRIDE_KEYS = frozenset(SECTION_TYPES) | {"notes"}
+_ALIASES_ADAPTER = TypeAdapter(dict[str, dict[str, list[str]]])
+
+
+@lru_cache(maxsize=None)
+def _adapter(typ: type) -> TypeAdapter:
+    return TypeAdapter(typ)
+
+
+def _without_nulls(section: dict[str, Any]) -> dict[str, Any]:
+    # A null drops an inherited key (see merge_entries), so it is always allowed.
+    return {k: v for k, v in section.items() if v is not None}
+
+
+def _rules_problem(rules: Any, allowed: frozenset[str], where: str) -> Optional[str]:
+    """Why this version cannot read ``rules`` (an entry or an override), or None."""
+    if not isinstance(rules, dict):
+        return f"{where} is not an object"
+    unknown = sorted(set(rules) - allowed)
+    if unknown:
+        return f"{where} has key(s) this version does not know: {', '.join(unknown)}"
+    for key, typ in SECTION_TYPES.items():
+        section = rules.get(key)
+        if section is None:
+            continue
+        if not isinstance(section, dict):
+            return f"{where}.{key} is not an object"
+        known = {f.name for f in dataclass_fields(typ)}
+        unknown = sorted(set(section) - known)
+        if unknown:
+            return f"{where}.{key} has rule(s) this version does not know: {', '.join(unknown)}"
+        try:
+            _adapter(typ).validate_python(_without_nulls(section))
+        except ValidationError as exc:
+            return f"{where}.{key} does not match this version's rule types ({exc.errors()[0]['msg']})"
+    if not isinstance(rules.get("notes") or [], list):
+        return f"{where}.notes is not a list"
+    return None
+
+
+def _entry_problem(slug: str, entry: Any) -> Optional[str]:
+    """Why this version cannot apply the published ``entry``, or None if it can.
+
+    The rules are only data, but ``validate`` is code: it measures the uploads
+    against the keys and value types the installed section dataclasses define. A
+    newer copy may add a rule this version would silently skip, or change a
+    value's type in a way that would break the check at validation time, so
+    such an entry is caught here, when the copy is loaded, instead.
+    """
+    if slug.startswith("$"):
+        if slug == "$aliases":
+            try:
+                _ALIASES_ADAPTER.validate_python(entry)
+            except ValidationError as exc:
+                return f"$aliases does not match this version's shape ({exc.errors()[0]['msg']})"
+        return None
+    problem = _rules_problem(entry, _ENTRY_KEYS, slug)
+    if problem:
+        return problem
+    top = {k: entry[k] for k in ("source_urls", "retrieved", "inherits", "article_type_field", "notes") if entry.get(k) is not None}
+    try:
+        _adapter(ManuscriptRequirements).validate_python({"slug": slug, **top})
+    except ValidationError as exc:
+        return f"{slug} does not match this version's entry types ({exc.errors()[0]['msg']})"
+    overrides = entry.get("article_types") or {}
+    if not isinstance(overrides, dict):
+        return f"{slug}.article_types is not an object"
+    for name, override in overrides.items():
+        problem = _rules_problem(override, _OVERRIDE_KEYS, f"{slug}.article_types[{name!r}]")
+        if problem:
+            return problem
+    return None
+
+
+def merge_published(bundled: dict[str, Any], published: dict[str, Any]) -> dict[str, Any]:
+    """Overlay a published requirements database onto the bundled one, entry by entry.
+
+    The published copy is authoritative for the data: it may add, change, or
+    drop an entry (no runner depends on these rules, so, unlike a venue in
+    ``venues.json``, a new entry needs no release). Each entry is still checked
+    against this version's rule vocabulary (:func:`_entry_problem`). An entry
+    this version cannot read keeps its bundled version, if there is one. So does
+    an entry that inherits from a held-back or missing base, so an entry is never
+    resolved against a mix of old and new rules.
+    """
+    held_back: dict[str, str] = {}
+    accepted: set[str] = set()
+    for slug, entry in published.items():
+        problem = _entry_problem(slug, entry)
+        if problem:
+            held_back[slug] = problem
+        else:
+            accepted.add(slug)
+
+    # Drop entries whose base was not accepted, until nothing changes (a chain
+    # of inheritance can take several passes).
+    lowered = {s.lower(): s for s in accepted}
+    changed = True
+    while changed:
+        changed = False
+        for slug in sorted(accepted):
+            base = str(published[slug].get("inherits", "") or "") if not slug.startswith("$") else ""
+            if base and base.lower() not in lowered:
+                accepted.discard(slug)
+                lowered.pop(slug.lower(), None)
+                held_back[slug] = f"{slug} inherits from {base!r}, which is held back or missing"
+                changed = True
+
+    for slug, problem in sorted(held_back.items()):
+        logger.info("Published manuscript requirements: %s; using the bundled entry. " "Run 'pip install -U paperpush' to get it.", problem)
+
+    merged: dict[str, Any] = {}
+    for slug, entry in published.items():
+        if slug in accepted:
+            merged[slug] = entry
+        elif slug in bundled:
+            merged[slug] = bundled[slug]
+    return merged
+
+
+def check_requirements_file(path: Path) -> None:
+    """Raise if the requirements database at ``path`` does not load.
+
+    Checks the database this version would actually use -- the file merged onto
+    the bundled copy by :func:`merge_published` -- so every entry must resolve
+    and build.
+    """
+    published = _read_requirements(path)
+    bundled = _read_requirements(REQUIREMENTS_PATH) if REQUIREMENTS_PATH.exists() else {}
+    raw = merge_published(bundled, published)
+    for slug in raw:
+        if not slug.startswith("$"):
+            ManuscriptRequirements.from_dict(slug, _resolve_entry(slug, raw))
+
+
 def list_requirements() -> list[ManuscriptRequirements]:
     """Every venue entry in the database, sorted by slug."""
     raw = _load_raw()

{paperpush-0.2.0 → paperpush-0.2.1}/paperpush/sensitive.py

@@ -694,7 +694,7 @@ def _url_is_reachable(url: str, *, timeout: float = LINK_CHECK_TIMEOUT) -> bool
     for method in ("HEAD", "GET"):
         req = urllib.request.Request(url, method=method, headers={"User-Agent": "paperpush-validate"})
         try:
-            with urllib.request.urlopen(req, timeout=timeout):  # nosec B310 - scheme constrained to http(s) by _URL_RE
+            with urllib.request.urlopen(req, timeout=timeout):  # scheme constrained to http(s) by _URL_RE  # nosec B310
                 return True
         except urllib.error.HTTPError as exc:
             if exc.code in _BROKEN_STATUS:

{paperpush-0.2.0 → paperpush-0.2.1}/paperpush/subfile_templates/iclr_2027.sub

@@ -22,6 +22,7 @@ keywords:
 
 # TL;DR (text, optional)
 #   Optional. A one-sentence summary of the paper ("too long; didn't read").
+#   at most 250 characters
 tldr: 
 
 # Abstract (textarea, REQUIRED)