bibcite-cli 0.6.1__tar.gz → 0.6.3__tar.gz

{bibcite_cli-0.6.1 → bibcite_cli-0.6.3}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: bibcite-cli
-Version: 0.6.1
+Version: 0.6.3
 Summary: Resolve papers (arXiv id / DOI / title) to canonical, normalized BibTeX for agents and humans
 Project-URL: Repository, https://github.com/leo1oel/bibcite
 License-Expression: MIT
@@ -18,7 +18,7 @@ Description-Content-Type: text/markdown
 <h1 align="center">bibcite</h1>
 
 <p align="center">
-  Turn an arXiv ID, DOI, or paper title into clean BibTeX, then keep the whole bibliography normalized and deduplicated.
+  Turn an arXiv ID, DOI, paper title, or web page into clean BibTeX, then keep the whole bibliography normalized and deduplicated.
 </p>
 
 <p align="center">
@@ -105,7 +105,8 @@ uvx --from bibcite-cli bibcite get "Attention is all you need"
 
 ## What it handles
 
-- It accepts arXiv IDs and URLs, arXiv DOIs such as `10.48550/arXiv.1706.03762`, standard DOIs, and paper titles.
+- It accepts arXiv IDs and URLs, arXiv DOIs such as `10.48550/arXiv.1706.03762`, standard DOIs, paper titles, and the URL of a web page.
+- A web page is cited from what the page says about itself, because no index carries blog posts, documentation, or standards. The entry is `@misc` with `howpublished = {\url{...}}`, which every conference `.bst` prints; `@online` is biblatex-only and would be dropped. A page with no byline is attributed to its site, and a page with no date takes the year from its own URL when it has one there.
 - It searches for a published version before falling back to an arXiv preprint, and it reports when source outages make that check incomplete.
 - It canonicalizes journal, conference, and workshop names against the bundled venue table, including year-sensitive names such as NIPS and NeurIPS.
 - It assigns the correct BibTeX entry type and field, such as `@inproceedings` with `booktitle` or `@article` with `journal`.

{bibcite_cli-0.6.1 → bibcite_cli-0.6.3}/Readme.md

@@ -5,7 +5,7 @@
 <h1 align="center">bibcite</h1>
 
 <p align="center">
-  Turn an arXiv ID, DOI, or paper title into clean BibTeX, then keep the whole bibliography normalized and deduplicated.
+  Turn an arXiv ID, DOI, paper title, or web page into clean BibTeX, then keep the whole bibliography normalized and deduplicated.
 </p>
 
 <p align="center">
@@ -92,7 +92,8 @@ uvx --from bibcite-cli bibcite get "Attention is all you need"
 
 ## What it handles
 
-- It accepts arXiv IDs and URLs, arXiv DOIs such as `10.48550/arXiv.1706.03762`, standard DOIs, and paper titles.
+- It accepts arXiv IDs and URLs, arXiv DOIs such as `10.48550/arXiv.1706.03762`, standard DOIs, paper titles, and the URL of a web page.
+- A web page is cited from what the page says about itself, because no index carries blog posts, documentation, or standards. The entry is `@misc` with `howpublished = {\url{...}}`, which every conference `.bst` prints; `@online` is biblatex-only and would be dropped. A page with no byline is attributed to its site, and a page with no date takes the year from its own URL when it has one there.
 - It searches for a published version before falling back to an arXiv preprint, and it reports when source outages make that check incomplete.
 - It canonicalizes journal, conference, and workshop names against the bundled venue table, including year-sensitive names such as NIPS and NeurIPS.
 - It assigns the correct BibTeX entry type and field, such as `@inproceedings` with `booktitle` or `@article` with `journal`.

{bibcite_cli-0.6.1 → bibcite_cli-0.6.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "bibcite-cli"
-version = "0.6.1"
+version = "0.6.3"
 description = "Resolve papers (arXiv id / DOI / title) to canonical, normalized BibTeX for agents and humans"
 readme = "Readme.md"
 license = "MIT"

{bibcite_cli-0.6.1 → bibcite_cli-0.6.3}/src/bibcite/resolve.py

@@ -22,6 +22,9 @@ from .sources import (
     Match,
     arxiv_metadata,
     crossref_by_doi,
+    PageUnreadable,
+    WebPage,
+    fetch_web_page,
     find_published,
 )
 from .venues import canonicalize
@@ -43,6 +46,7 @@ ARXIV_URL = re.compile(
     re.I,
 )
 DOI_URL = re.compile(r"doi\.org/(10\.\S+)", re.I)
+WEB_URL = re.compile(r"^https?://\S+$", re.I)
 DOI_RE = re.compile(r"^10\.\d{4,9}/\S+$")
 ARXIV_DOI = re.compile(
     r"^10\.48550/arxiv\."
@@ -74,6 +78,12 @@ def classify(query: str) -> tuple[str, str]:
         return "arxiv", m.group(1)
     if DOI_RE.match(q):
         return "doi", q
+    # A URL that is not arXiv and not a DOI is a page: a blog post, a standard,
+    # a documentation page. Nothing indexes those, so searching for it as a
+    # title only ever returns "no match anywhere" — the page itself is the
+    # source.
+    if WEB_URL.match(q):
+        return "webpage", q
     return "title", query.strip()
 
 
@@ -198,6 +208,31 @@ def _arxiv_only_entry(meta: ArxivMeta) -> dict:
     }
 
 
+def _web_entry(page: WebPage) -> dict:
+    """A page cited as @misc, the one type every conference .bst understands.
+
+    @online is biblatex-only; a NeurIPS or IEEE style would drop the entry
+    entirely. howpublished carries the link for the same reason the arXiv
+    preprint entry uses it: classic styles print it and ignore `url`. No access
+    date, because the tidy step omits `note` from every entry in the file.
+    """
+    entry = {
+        "ENTRYTYPE": "misc",
+        "title": page.title,
+        "howpublished": f"\\url{{{page.url}}}",
+        "url": page.url,
+    }
+    if page.authors:
+        entry["author"] = " and ".join(page.authors)
+    elif page.site:
+        # No byline: the site is the closest thing to a corporate author, and a
+        # key of `anonymousXXXX…` is worse than one naming where it came from.
+        entry["author"] = f"{{{page.site}}}"
+    if page.year:
+        entry["year"] = page.year
+    return entry
+
+
 def resolve(query: str, require_published: bool = False) -> Resolved:
     kind, value = classify(query)
     _log(f"[bibcite] query understood as {kind}: {value}")
@@ -248,6 +283,16 @@ def resolve(query: str, require_published: bool = False) -> Resolved:
         entry = _arxiv_only_entry(meta)
         return Resolved(_finalize(entry, meta), "arxiv", "", False, check)
 
+    if kind == "webpage":
+        try:
+            page = fetch_web_page(value)
+        except PageUnreadable as e:
+            raise NotFound(f"could not read {value}: {e}") from e
+        if not page.title:
+            raise NotFound(f"No title found on the page: {value}")
+        _log(f"[web] {page.title}" + (f" ({page.year})" if page.year else ""))
+        return Resolved(_finalize(_web_entry(page), None), "webpage", page.site, False)
+
     if kind == "doi":
         match = crossref_by_doi(value)
         if not match or not match.title:

{bibcite_cli-0.6.1 → bibcite_cli-0.6.3}/src/bibcite/sources.py

@@ -10,6 +10,7 @@ import html
 import os
 import re
 import sys
+import threading
 import time
 import xml.etree.ElementTree as ET
 from concurrent.futures import ThreadPoolExecutor, as_completed
@@ -30,6 +31,10 @@ BROWSER_UA = (
 # never a false "not published". The arXiv metadata fetch sets its own longer
 # timeout on the request itself, so this does not affect it.
 TIMEOUT = 8.0
+# Publication matching is enrichment on top of a valid arXiv citation. Keep
+# the entire concurrent cascade, including its DBLP title-drift fallback,
+# within an interactive budget instead of letting sequential retries add up.
+PUBLICATION_TIMEOUT = 10.0
 
 PREPRINT_VENUES = re.compile(r"arxiv|corr|biorxiv|medrxiv|chemrxiv|ssrn|preprint", re.I)
 ARXIV_DOI = re.compile(r"^10\.48550/", re.I)
@@ -43,11 +48,67 @@ class SourceUnavailable(Exception):
     """Raised when a source rate-limits/blocks us; the cascade skips it."""
 
 
+class PageUnreadable(Exception):
+    """A page refused to be read and will refuse again — a different
+    identifier, or a hand-written entry, is the next step rather than a
+    retry."""
+
+
 class TransientSourceError(SourceUnavailable):
     """Raised after request retries are exhausted without tripping the
     process-wide circuit breaker for later batch entries."""
 
 
+class PublicationTimeout(TransientSourceError):
+    """The total publication-matching budget was exhausted."""
+
+
+_REQUEST_DEADLINE = threading.local()
+
+
+def _request_timeout(cap: float = TIMEOUT) -> float:
+    deadline = getattr(_REQUEST_DEADLINE, "value", None)
+    if deadline is None:
+        return cap
+    remaining = deadline - time.monotonic()
+    if remaining <= 0:
+        raise PublicationTimeout("publication lookup timed out")
+    return max(0.001, min(cap, remaining))
+
+
+def _get(
+    c: httpx.Client, url: str, *, timeout: float = TIMEOUT, **kwargs
+) -> httpx.Response:
+    """GET with the per-request cap narrowed by the active total deadline."""
+    return c.get(url, timeout=_request_timeout(timeout), **kwargs)
+
+
+def _sleep(delay: float):
+    """Sleep for pacing/backoff without crossing the publication deadline."""
+    deadline = getattr(_REQUEST_DEADLINE, "value", None)
+    if deadline is None:
+        time.sleep(delay)
+        return
+    remaining = deadline - time.monotonic()
+    if remaining <= 0:
+        raise PublicationTimeout("publication lookup timed out")
+    time.sleep(min(delay, remaining))
+    if delay >= remaining:
+        raise PublicationTimeout("publication lookup timed out")
+
+
+def _with_deadline(deadline: float, fn, *args):
+    previous = getattr(_REQUEST_DEADLINE, "value", None)
+    _REQUEST_DEADLINE.value = deadline
+    try:
+        return fn(*args)
+    finally:
+        if previous is None:
+            del _REQUEST_DEADLINE.value
+        else:
+            _REQUEST_DEADLINE.value = previous
+
+
 def _client(browser: bool = False) -> httpx.Client:
     return httpx.Client(
         headers={"User-Agent": BROWSER_UA if browser else UA},
@@ -123,7 +184,8 @@ def arxiv_api_get(params: dict) -> httpx.Response:
             time.sleep(3 * attempt)
         try:
             with _client() as c:
-                r = c.get(
+                r = _get(
+                    c,
                     "https://export.arxiv.org/api/query",
                     params=params,
                     timeout=30.0,
@@ -192,13 +254,13 @@ def _paced_get(
     for attempt in range(2):
         wait = min_interval - (time.monotonic() - _LAST_REQUEST.get(source, 0.0))
         if wait > 0:
-            time.sleep(wait)
+            _sleep(wait)
         _LAST_REQUEST[source] = time.monotonic()
         try:
-            r = c.get(url, params=params, headers=headers)
+            r = _get(c, url, params=params, headers=headers)
         except httpx.HTTPError as e:  # Retry transport errors once before failing.
             if attempt < 1:
-                time.sleep(1)
+                _sleep(1)
                 continue
             raise TransientSourceError(
                 f"{source} unreachable ({type(e).__name__})"
@@ -212,7 +274,7 @@ def _paced_get(
             # skip this source for the rest of the run.
             retry_after = int(r.headers.get("Retry-After") or 0)
             if attempt < 1 and retry_after <= 2:
-                time.sleep(max(retry_after, 1))
+                _sleep(max(retry_after, 1))
                 continue
             raise SourceUnavailable(f"{source} rate-limited (429)")
         return r
@@ -392,7 +454,7 @@ def arxiv_abs_metadata(arxiv_id: str) -> ArxivMeta | None:
     """Scrape the arxiv.org abs page's Highwire meta tags — the abs pages stay
     up when the export API throttles."""
     with _client(browser=True) as c:
-        r = c.get(f"https://arxiv.org/abs/{arxiv_id}")
+        r = _get(c, f"https://arxiv.org/abs/{arxiv_id}")
         if r.status_code != 200:
             return None
         page = r.text
@@ -483,7 +545,8 @@ def try_semantic_scholar(
 
 def try_google_scholar(title: str) -> Match | None:
     with _client(browser=True) as c:
-        r = c.get(
+        r = _get(
+            c,
             "https://scholar.google.com/scholar",
             params={"q": title, "hl": "en"},
         )
@@ -510,12 +573,12 @@ def try_google_scholar(title: str) -> Match | None:
             "https://scholar.google.com/scholar?q=info:"
             f"{data_id}:scholar.google.com/&output=cite&scirp=0&hl=en"
         )
-        cite_html = c.get(cite_url).text
+        cite_html = _get(c, cite_url).text
         bm = re.search(r'<a[^>]*href="([^">]+)"[^>]*>BibTex</a>', cite_html, re.I)
         if not bm:
             return None
         bib_url = re.sub(r"\s+", "", bm.group(1).replace("&amp;", "&"))
-        bibtex = c.get(bib_url).text
+        bibtex = _get(c, bib_url).text
     from .bibfile import parse_bibtex_entry  # local import to avoid cycle
 
     entry = parse_bibtex_entry(bibtex)
@@ -538,7 +601,8 @@ def try_google_scholar(title: str) -> Match | None:
 
 def try_crossref(title: str) -> Match | None:
     with _client() as c:
-        r = c.get(
+        r = _get(
+            c,
             "https://api.crossref.org/works",
             params={
                 "rows": 3,
@@ -577,8 +641,9 @@ def try_crossref(title: str) -> Match | None:
                 year = str(parts[0][0])
             bibtex = ""
             if doi:
-                br = c.get(
-                    f"https://api.crossref.org/works/{doi}/transform/application/x-bibtex"
+                br = _get(
+                    c,
+                    f"https://api.crossref.org/works/{doi}/transform/application/x-bibtex",
                 )
                 if br.status_code == 200:
                     bibtex = br.text
@@ -600,7 +665,8 @@ def try_crossref(title: str) -> Match | None:
 
 def try_unpaywall(title: str) -> Match | None:
     with _client() as c:
-        r = c.get(
+        r = _get(
+            c,
             "https://api.unpaywall.org/v2/search",
             params={"query": title, "is_oa": "true", "email": _mailto()},
         )
@@ -647,7 +713,8 @@ def try_unpaywall(title: str) -> Match | None:
 def openalex_search(title: str) -> dict | None:
     """OpenAlex work with an exactly-matching normalized title, or None."""
     with _client() as c:
-        r = c.get(
+        r = _get(
+            c,
             "https://api.openalex.org/works",
             params=_openalex_params({"search": title, "per-page": 5}),
         )
@@ -719,7 +786,9 @@ def try_openalex(title: str) -> Match | None:
 
 def crossref_by_doi(doi: str) -> Match | None:
     with _client() as c:
-        r = c.get(f"https://api.crossref.org/works/{doi}", params={"mailto": _mailto()})
+        r = _get(
+            c, f"https://api.crossref.org/works/{doi}", params={"mailto": _mailto()}
+        )
         if r.status_code != 200:
             return None
         data = r.json().get("message", {})
@@ -731,7 +800,9 @@ def crossref_by_doi(doi: str) -> Match | None:
         if parts and parts[0]:
             year = str(parts[0][0])
         bibtex = ""
-        br = c.get(f"https://api.crossref.org/works/{doi}/transform/application/x-bibtex")
+        br = _get(
+            c, f"https://api.crossref.org/works/{doi}/transform/application/x-bibtex"
+        )
         if br.status_code == 200:
             bibtex = br.text
         authors = [
@@ -809,12 +880,21 @@ def find_published(
     # version (the common case) misses everywhere, and used to pay the *sum* of
     # each source's latency; now the wall-clock is the slowest single source.
     # The first verified hit by CASCADE priority still wins.
+    deadline = time.monotonic() + PUBLICATION_TIMEOUT
     active = [(name, fn) for name, fn in CASCADE if name not in _DISABLED]
     outcomes: dict[str, tuple] = {}
     if active:
         with ThreadPoolExecutor(max_workers=len(active)) as pool:
             futures = {
-                pool.submit(fn, title, year, arxiv_id, author_hint): name
+                pool.submit(
+                    _with_deadline,
+                    deadline,
+                    fn,
+                    title,
+                    year,
+                    arxiv_id,
+                    author_hint,
+                ): name
                 for name, fn in active
             }
             for future in as_completed(futures):
@@ -852,9 +932,18 @@ def find_published(
     # Exact-title search missed everywhere. Before concluding "no published
     # version", try the title-drift fallback — camera-ready titles frequently
     # differ from the arXiv ones, which is precisely the upgrade scenario.
-    if author_hint and "dblp" not in _DISABLED:
+    # Only a clean exact DBLP miss justifies another query. A timeout or other
+    # failure has already spent its chance for this entry, and retrying the
+    # fuzzy form was doubling the worst-case interactive latency.
+    dblp_outcome = outcomes.get("dblp")
+    if (
+        author_hint
+        and dblp_outcome is not None
+        and dblp_outcome[0] == "miss"
+        and time.monotonic() < deadline
+    ):
         try:
-            m = try_dblp_fuzzy(title, author_hint, year)
+            m = _with_deadline(deadline, try_dblp_fuzzy, title, author_hint, year)
             if m:
                 cache.put(cache_key, m.__dict__)
                 return m, "found"
@@ -871,3 +960,136 @@ def find_published(
     if not clean_misses:
         return None, "unavailable"
     return None, ("incomplete" if incomplete else "not_found")
+
+
+@dataclass
+class WebPage:
+    """What a web page can say about itself, for citing it as an @misc."""
+
+    url: str
+    title: str
+    authors: list[str] = field(default_factory=list)
+    year: str = ""
+    site: str = ""
+
+
+def _meta_content(html_text: str, *keys: str) -> str:
+    """The content of the first <meta> whose name/property matches a key.
+
+    Attribute order varies between generators, so both orders are tried rather
+    than assuming content comes last.
+    """
+    for key in keys:
+        for pattern in (
+            rf'<meta[^>]+(?:name|property)=["\']{re.escape(key)}["\'][^>]*'
+            rf'content=["\'](.*?)["\']',
+            rf'<meta[^>]+content=["\'](.*?)["\'][^>]*'
+            rf'(?:name|property)=["\']{re.escape(key)}["\']',
+        ):
+            m = re.search(pattern, html_text, re.I | re.S)
+            if m and m.group(1).strip():
+                return html.unescape(m.group(1).strip())
+    return ""
+
+
+def fetch_web_page(url: str) -> WebPage:
+    """Read a page's own description of itself.
+
+    Blogs, documentation and standards pages are cited constantly and are in
+    none of the academic indexes, so there is nothing to look them up in — the
+    page itself is the only source. Highwire and Dublin Core tags come first
+    because sites that carry them mean them; Open Graph and <title> are the
+    fallback every site has.
+    """
+    try:
+        with _client(browser=True) as client:
+            response = _get(client, url)
+            response.raise_for_status()
+            body = response.text[:400_000]
+    except httpx.HTTPStatusError as e:
+        status = e.response.status_code
+        # 403 and 404 are settled answers: the page will not become readable on
+        # a retry, so say so rather than sending the caller back to wait.
+        if status in (401, 403, 404, 410) or 400 <= status < 500:
+            raise PageUnreadable(
+                f"the page answered {status} — cite it with --bibtex, or use its DOI if it has one"
+            ) from e
+        raise SourceUnavailable(f"could not fetch {url}: {e}") from e
+    except httpx.HTTPError as e:
+        raise SourceUnavailable(f"could not fetch {url}: {e}") from e
+
+    title = (
+        _meta_content(body, "citation_title", "DC.title", "og:title", "twitter:title")
+        or _title_tag(body)
+    )
+    authors = [
+        author
+        for author in (
+            _meta_content(body, "citation_author", "DC.creator", "author", "article:author"),
+        )
+        if author
+    ]
+    date = _meta_content(
+        body,
+        "citation_publication_date",
+        "citation_date",
+        "DC.date",
+        "article:published_time",
+        "og:updated_time",
+        "date",
+    )
+    year = _year_in(date) or _year_in_path(url)
+    site = _meta_content(body, "og:site_name") or _site_author(_host(url))
+    return WebPage(url=str(response.url), title=title, authors=authors, year=year, site=site)
+
+
+def _year_in(text: str) -> str:
+    m = re.search(r"(?:19|20)\d{2}", text)
+    return m.group(0) if m else ""
+
+
+def _year_in_path(url: str) -> str:
+    """The year a dateless page puts in its own URL.
+
+    Blog engines write `/2015/05/21/title`, and a citation key of
+    `karpathyXXXXunreasonable` is worse than one carrying the year the post
+    announces about itself. Only the path is read: a query string can hold any
+    number at all.
+    """
+    path = re.sub(r"^https?://[^/]+", "", url).split("?")[0].split("#")[0]
+    for segment in path.split("/"):
+        if re.fullmatch(r"(?:19|20)\d{2}", segment):
+            return segment
+    return ""
+
+
+def _title_tag(html_text: str) -> str:
+    m = re.search(r"<title[^>]*>(.*?)</title>", html_text, re.I | re.S)
+    if not m:
+        return ""
+    # Strip the trailing " — Site Name" many templates append; the site name is
+    # recorded separately, and repeating it in the title reads badly in a
+    # bibliography.
+    title = html.unescape(re.sub(r"\s+", " ", m.group(1))).strip()
+    return re.sub(r"\s*[|·—–-]\s*[^|·—–-]{1,40}$", "", title).strip() or title
+
+
+def _host(url: str) -> str:
+    m = re.match(r"https?://(?:www\.)?([^/:]+)", url, re.I)
+    return m.group(1) if m else ""
+
+
+def _site_author(host: str) -> str:
+    """A readable stand-in author for a page with no byline.
+
+    The bare host makes an unreadable key — `karpathygithubioXXXXunreasonable`
+    — so the hosting suffix goes and the name that identifies the site stays:
+    `karpathy.github.io` reads as Karpathy, `docs.python.org` as Python.
+    """
+    labels = [label for label in host.lower().split(".") if label]
+    if not labels:
+        return host
+    generic = {"github", "io", "com", "org", "net", "edu", "gov", "ai", "dev",
+               "co", "uk", "cn", "blog", "www", "docs", "pages", "medium"}
+    named = [label for label in labels if label not in generic]
+    return (named[0] if named else labels[0]).capitalize()

{bibcite_cli-0.6.1 → bibcite_cli-0.6.3}/tests/test_source_retries.py

@@ -5,7 +5,12 @@ import bibcite.sources as sources
 from bibcite import cache
 from bibcite.bibfile import load_bib_file
 from bibcite.cli import _upgrade_entries
-from bibcite.sources import Match, SourceUnavailable, find_published
+from bibcite.sources import (
+    Match,
+    SourceUnavailable,
+    TransientSourceError,
+    find_published,
+)
 
 
 @pytest.fixture(autouse=True)
@@ -22,7 +27,7 @@ class _ReadErrorClient:
         self.failures = failures
         self.calls = 0
 
-    def get(self, url, params=None, headers=None):
+    def get(self, url, params=None, headers=None, timeout=None):
         self.calls += 1
         request = httpx.Request("GET", url, params=params, headers=headers)
         if self.calls <= self.failures:
@@ -70,6 +75,41 @@ def test_dblp_read_failures_do_not_disable_later_batch_entries(monkeypatch):
     assert second_match.venue == "TMLR"
 
 
+def test_dblp_transport_failure_skips_the_fuzzy_retry(monkeypatch):
+    fuzzy_calls = 0
+
+    def dblp(*args):
+        raise TransientSourceError("simulated timeout")
+
+    def fuzzy(*args):
+        nonlocal fuzzy_calls
+        fuzzy_calls += 1
+
+    monkeypatch.setattr(
+        sources,
+        "CASCADE",
+        (
+            ("dblp", dblp),
+            ("crossref", lambda *args: None),
+        ),
+    )
+    monkeypatch.setattr(sources, "try_dblp_fuzzy", fuzzy)
+
+    match, status = find_published("First paper", author_hint="doe")
+
+    assert (match, status) == (None, "incomplete")
+    assert fuzzy_calls == 0
+
+
+def test_requests_use_only_the_remaining_publication_budget(monkeypatch):
+    monkeypatch.setattr(sources.time, "monotonic", lambda: 7.0)
+    sources._REQUEST_DEADLINE.value = 10.0
+    try:
+        assert sources._request_timeout(8.0) == 3.0
+    finally:
+        del sources._REQUEST_DEADLINE.value
+
+
 def test_upgrade_retries_dblp_after_previous_entry_read_failures(
     tmp_path, monkeypatch
 ):

bibcite_cli-0.6.3/tests/test_webpages.py

@@ -0,0 +1,69 @@
+"""Citing a page that no index knows about.
+
+Blogs, documentation and standards pages are cited constantly and appear in
+none of the academic sources, so the page itself has to be the source.
+"""
+
+from bibcite.resolve import _web_entry, classify
+from bibcite.sources import WebPage, _meta_content, _site_author, _title_tag, _year_in_path
+
+
+def test_a_plain_url_is_a_webpage_and_the_others_still_are_not():
+    assert classify("https://karpathy.github.io/2015/05/21/rnn-effectiveness/")[0] == "webpage"
+    assert classify("http://example.edu/notes")[0] == "webpage"
+    # The identifiers that resolve properly must keep their own paths.
+    assert classify("https://arxiv.org/abs/1706.03762") == ("arxiv", "1706.03762")
+    assert classify("https://doi.org/10.1038/s41586-021-03819-2")[0] == "doi"
+    assert classify("10.1109/CVPR.2016.90")[0] == "doi"
+    assert classify("Attention Is All You Need")[0] == "title"
+
+
+def test_entry_is_misc_so_conference_styles_can_print_it():
+    # @online is biblatex-only: a NeurIPS or IEEE .bst drops the entry.
+    entry = _web_entry(WebPage(url="https://example.org/post", title="A Post", year="2024"))
+    assert entry["ENTRYTYPE"] == "misc"
+    assert entry["howpublished"] == r"\url{https://example.org/post}"
+    assert entry["url"] == "https://example.org/post"
+    assert entry["year"] == "2024"
+
+
+def test_a_page_with_no_byline_is_attributed_to_its_site():
+    entry = _web_entry(WebPage(url="https://example.org/post", title="A Post", site="Example"))
+    # Braced, so the .bst treats it as a corporate name and does not invert it.
+    assert entry["author"] == "{Example}"
+
+
+def test_a_byline_wins_over_the_site():
+    entry = _web_entry(
+        WebPage(url="https://example.org/post", title="A Post", authors=["Ada Lovelace"], site="Example")
+    )
+    assert entry["author"] == "Ada Lovelace"
+
+
+def test_the_site_author_drops_hosting_suffixes():
+    # `karpathygithubio…` makes an unreadable citation key.
+    assert _site_author("karpathy.github.io") == "Karpathy"
+    assert _site_author("www.distill.pub") == "Distill"
+    assert _site_author("docs.python.org") == "Python"
+
+
+def test_a_dateless_page_takes_the_year_from_its_own_url():
+    assert _year_in_path("https://karpathy.github.io/2015/05/21/rnn-effectiveness/") == "2015"
+    assert _year_in_path("https://example.org/blog/2016/misread-tsne/") == "2016"
+    # Not from a query string, where any number at all can appear.
+    assert _year_in_path("https://example.org/page?id=2019") == ""
+    assert _year_in_path("https://example.org/about") == ""
+
+
+def test_the_title_loses_the_site_name_templates_append():
+    assert _title_tag("<title>How to Use t-SNE | Distill</title>") == "How to Use t-SNE"
+    assert _title_tag("<title>Plain Title</title>") == "Plain Title"
+    # A title that is only a site name must survive rather than become empty.
+    assert _title_tag("<title>Distill</title>") == "Distill"
+
+
+def test_meta_is_read_in_either_attribute_order():
+    assert _meta_content('<meta property="og:title" content="Hello">', "og:title") == "Hello"
+    assert _meta_content('<meta content="Hello" name="og:title">', "og:title") == "Hello"
+    assert _meta_content('<meta property="og:title" content="A &amp; B">', "og:title") == "A & B"
+    assert _meta_content("<html></html>", "og:title") == ""

{bibcite_cli-0.6.1 → bibcite_cli-0.6.3}/uv.lock

@@ -18,7 +18,7 @@ wheels = [
 
 [[package]]
 name = "bibcite-cli"
-version = "0.6.1"
+version = "0.6.3"
 source = { editable = "." }
 dependencies = [
     { name = "bibtexparser" },