bibcite-cli 0.6.1__tar.gz → 0.6.2__tar.gz

{bibcite_cli-0.6.1 → bibcite_cli-0.6.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bibcite-cli
-Version: 0.6.1
+Version: 0.6.2
 Summary: Resolve papers (arXiv id / DOI / title) to canonical, normalized BibTeX for agents and humans
 Project-URL: Repository, https://github.com/leo1oel/bibcite
 License-Expression: MIT
@@ -18,7 +18,7 @@ Description-Content-Type: text/markdown
 <h1 align="center">bibcite</h1>
 
 <p align="center">
-  Turn an arXiv ID, DOI, or paper title into clean BibTeX, then keep the whole bibliography normalized and deduplicated.
+  Turn an arXiv ID, DOI, paper title, or web page into clean BibTeX, then keep the whole bibliography normalized and deduplicated.
 </p>
 
 <p align="center">
@@ -105,7 +105,8 @@ uvx --from bibcite-cli bibcite get "Attention is all you need"
 
 ## What it handles
 
-- It accepts arXiv IDs and URLs, arXiv DOIs such as `10.48550/arXiv.1706.03762`, standard DOIs, and paper titles.
+- It accepts arXiv IDs and URLs, arXiv DOIs such as `10.48550/arXiv.1706.03762`, standard DOIs, paper titles, and the URL of a web page.
+- A web page is cited from what the page says about itself, because no index carries blog posts, documentation, or standards. The entry is `@misc` with `howpublished = {\url{...}}`, which every conference `.bst` prints; `@online` is biblatex-only and would be dropped. A page with no byline is attributed to its site, and a page with no date takes the year from its own URL when it has one there.
 - It searches for a published version before falling back to an arXiv preprint, and it reports when source outages make that check incomplete.
 - It canonicalizes journal, conference, and workshop names against the bundled venue table, including year-sensitive names such as NIPS and NeurIPS.
 - It assigns the correct BibTeX entry type and field, such as `@inproceedings` with `booktitle` or `@article` with `journal`.

{bibcite_cli-0.6.1 → bibcite_cli-0.6.2}/Readme.md

@@ -5,7 +5,7 @@
 <h1 align="center">bibcite</h1>
 
 <p align="center">
-  Turn an arXiv ID, DOI, or paper title into clean BibTeX, then keep the whole bibliography normalized and deduplicated.
+  Turn an arXiv ID, DOI, paper title, or web page into clean BibTeX, then keep the whole bibliography normalized and deduplicated.
 </p>
 
 <p align="center">
@@ -92,7 +92,8 @@ uvx --from bibcite-cli bibcite get "Attention is all you need"
 
 ## What it handles
 
-- It accepts arXiv IDs and URLs, arXiv DOIs such as `10.48550/arXiv.1706.03762`, standard DOIs, and paper titles.
+- It accepts arXiv IDs and URLs, arXiv DOIs such as `10.48550/arXiv.1706.03762`, standard DOIs, paper titles, and the URL of a web page.
+- A web page is cited from what the page says about itself, because no index carries blog posts, documentation, or standards. The entry is `@misc` with `howpublished = {\url{...}}`, which every conference `.bst` prints; `@online` is biblatex-only and would be dropped. A page with no byline is attributed to its site, and a page with no date takes the year from its own URL when it has one there.
 - It searches for a published version before falling back to an arXiv preprint, and it reports when source outages make that check incomplete.
 - It canonicalizes journal, conference, and workshop names against the bundled venue table, including year-sensitive names such as NIPS and NeurIPS.
 - It assigns the correct BibTeX entry type and field, such as `@inproceedings` with `booktitle` or `@article` with `journal`.

{bibcite_cli-0.6.1 → bibcite_cli-0.6.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "bibcite-cli"
-version = "0.6.1"
+version = "0.6.2"
 description = "Resolve papers (arXiv id / DOI / title) to canonical, normalized BibTeX for agents and humans"
 readme = "Readme.md"
 license = "MIT"

{bibcite_cli-0.6.1 → bibcite_cli-0.6.2}/src/bibcite/resolve.py

@@ -22,6 +22,9 @@ from .sources import (
     Match,
     arxiv_metadata,
     crossref_by_doi,
+    PageUnreadable,
+    WebPage,
+    fetch_web_page,
     find_published,
 )
 from .venues import canonicalize
@@ -43,6 +46,7 @@ ARXIV_URL = re.compile(
     re.I,
 )
 DOI_URL = re.compile(r"doi\.org/(10\.\S+)", re.I)
+WEB_URL = re.compile(r"^https?://\S+$", re.I)
 DOI_RE = re.compile(r"^10\.\d{4,9}/\S+$")
 ARXIV_DOI = re.compile(
     r"^10\.48550/arxiv\."
@@ -74,6 +78,12 @@ def classify(query: str) -> tuple[str, str]:
         return "arxiv", m.group(1)
     if DOI_RE.match(q):
         return "doi", q
+    # A URL that is not arXiv and not a DOI is a page: a blog post, a standard,
+    # a documentation page. Nothing indexes those, so searching for it as a
+    # title only ever returns "no match anywhere" — the page itself is the
+    # source.
+    if WEB_URL.match(q):
+        return "webpage", q
     return "title", query.strip()
 
 
@@ -198,6 +208,31 @@ def _arxiv_only_entry(meta: ArxivMeta) -> dict:
     }
 
 
+def _web_entry(page: WebPage) -> dict:
+    """A page cited as @misc, the one type every conference .bst understands.
+
+    @online is biblatex-only; a NeurIPS or IEEE style would drop the entry
+    entirely. howpublished carries the link for the same reason the arXiv
+    preprint entry uses it: classic styles print it and ignore `url`. No access
+    date, because the tidy step omits `note` from every entry in the file.
+    """
+    entry = {
+        "ENTRYTYPE": "misc",
+        "title": page.title,
+        "howpublished": f"\\url{{{page.url}}}",
+        "url": page.url,
+    }
+    if page.authors:
+        entry["author"] = " and ".join(page.authors)
+    elif page.site:
+        # No byline: the site is the closest thing to a corporate author, and a
+        # key of `anonymousXXXX…` is worse than one naming where it came from.
+        entry["author"] = f"{{{page.site}}}"
+    if page.year:
+        entry["year"] = page.year
+    return entry
+
+
 def resolve(query: str, require_published: bool = False) -> Resolved:
     kind, value = classify(query)
     _log(f"[bibcite] query understood as {kind}: {value}")
@@ -248,6 +283,16 @@ def resolve(query: str, require_published: bool = False) -> Resolved:
         entry = _arxiv_only_entry(meta)
         return Resolved(_finalize(entry, meta), "arxiv", "", False, check)
 
+    if kind == "webpage":
+        try:
+            page = fetch_web_page(value)
+        except PageUnreadable as e:
+            raise NotFound(f"could not read {value}: {e}") from e
+        if not page.title:
+            raise NotFound(f"No title found on the page: {value}")
+        _log(f"[web] {page.title}" + (f" ({page.year})" if page.year else ""))
+        return Resolved(_finalize(_web_entry(page), None), "webpage", page.site, False)
+
     if kind == "doi":
         match = crossref_by_doi(value)
         if not match or not match.title:

{bibcite_cli-0.6.1 → bibcite_cli-0.6.2}/src/bibcite/sources.py

@@ -43,6 +43,12 @@ class SourceUnavailable(Exception):
     """Raised when a source rate-limits/blocks us; the cascade skips it."""
 
 
+class PageUnreadable(Exception):
+    """A page refused to be read and will refuse again — a different
+    identifier, or a hand-written entry, is the next step rather than a
+    retry."""
+
+
 class TransientSourceError(SourceUnavailable):
     """Raised after request retries are exhausted without tripping the
     process-wide circuit breaker for later batch entries."""
@@ -871,3 +877,136 @@ def find_published(
     if not clean_misses:
         return None, "unavailable"
     return None, ("incomplete" if incomplete else "not_found")
+
+
+@dataclass
+class WebPage:
+    """What a web page can say about itself, for citing it as an @misc."""
+
+    url: str
+    title: str
+    authors: list[str] = field(default_factory=list)
+    year: str = ""
+    site: str = ""
+
+
+def _meta_content(html_text: str, *keys: str) -> str:
+    """The content of the first <meta> whose name/property matches a key.
+
+    Attribute order varies between generators, so both orders are tried rather
+    than assuming content comes last.
+    """
+    for key in keys:
+        for pattern in (
+            rf'<meta[^>]+(?:name|property)=["\']{re.escape(key)}["\'][^>]*'
+            rf'content=["\'](.*?)["\']',
+            rf'<meta[^>]+content=["\'](.*?)["\'][^>]*'
+            rf'(?:name|property)=["\']{re.escape(key)}["\']',
+        ):
+            m = re.search(pattern, html_text, re.I | re.S)
+            if m and m.group(1).strip():
+                return html.unescape(m.group(1).strip())
+    return ""
+
+
+def fetch_web_page(url: str) -> WebPage:
+    """Read a page's own description of itself.
+
+    Blogs, documentation and standards pages are cited constantly and are in
+    none of the academic indexes, so there is nothing to look them up in — the
+    page itself is the only source. Highwire and Dublin Core tags come first
+    because sites that carry them mean them; Open Graph and <title> are the
+    fallback every site has.
+    """
+    try:
+        with _client(browser=True) as client:
+            response = client.get(url)
+            response.raise_for_status()
+            body = response.text[:400_000]
+    except httpx.HTTPStatusError as e:
+        status = e.response.status_code
+        # 403 and 404 are settled answers: the page will not become readable on
+        # a retry, so say so rather than sending the caller back to wait.
+        if status in (401, 403, 404, 410) or 400 <= status < 500:
+            raise PageUnreadable(
+                f"the page answered {status} — cite it with --bibtex, or use its DOI if it has one"
+            ) from e
+        raise SourceUnavailable(f"could not fetch {url}: {e}") from e
+    except httpx.HTTPError as e:
+        raise SourceUnavailable(f"could not fetch {url}: {e}") from e
+
+    title = (
+        _meta_content(body, "citation_title", "DC.title", "og:title", "twitter:title")
+        or _title_tag(body)
+    )
+    authors = [
+        author
+        for author in (
+            _meta_content(body, "citation_author", "DC.creator", "author", "article:author"),
+        )
+        if author
+    ]
+    date = _meta_content(
+        body,
+        "citation_publication_date",
+        "citation_date",
+        "DC.date",
+        "article:published_time",
+        "og:updated_time",
+        "date",
+    )
+    year = _year_in(date) or _year_in_path(url)
+    site = _meta_content(body, "og:site_name") or _site_author(_host(url))
+    return WebPage(url=str(response.url), title=title, authors=authors, year=year, site=site)
+
+
+def _year_in(text: str) -> str:
+    m = re.search(r"(?:19|20)\d{2}", text)
+    return m.group(0) if m else ""
+
+
+def _year_in_path(url: str) -> str:
+    """The year a dateless page puts in its own URL.
+
+    Blog engines write `/2015/05/21/title`, and a citation key of
+    `karpathyXXXXunreasonable` is worse than one carrying the year the post
+    announces about itself. Only the path is read: a query string can hold any
+    number at all.
+    """
+    path = re.sub(r"^https?://[^/]+", "", url).split("?")[0].split("#")[0]
+    for segment in path.split("/"):
+        if re.fullmatch(r"(?:19|20)\d{2}", segment):
+            return segment
+    return ""
+
+
+def _title_tag(html_text: str) -> str:
+    m = re.search(r"<title[^>]*>(.*?)</title>", html_text, re.I | re.S)
+    if not m:
+        return ""
+    # Strip the trailing " — Site Name" many templates append; the site name is
+    # recorded separately, and repeating it in the title reads badly in a
+    # bibliography.
+    title = html.unescape(re.sub(r"\s+", " ", m.group(1))).strip()
+    return re.sub(r"\s*[|·—–-]\s*[^|·—–-]{1,40}$", "", title).strip() or title
+
+
+def _host(url: str) -> str:
+    m = re.match(r"https?://(?:www\.)?([^/:]+)", url, re.I)
+    return m.group(1) if m else ""
+
+
+def _site_author(host: str) -> str:
+    """A readable stand-in author for a page with no byline.
+
+    The bare host makes an unreadable key — `karpathygithubioXXXXunreasonable`
+    — so the hosting suffix goes and the name that identifies the site stays:
+    `karpathy.github.io` reads as Karpathy, `docs.python.org` as Python.
+    """
+    labels = [label for label in host.lower().split(".") if label]
+    if not labels:
+        return host
+    generic = {"github", "io", "com", "org", "net", "edu", "gov", "ai", "dev",
+               "co", "uk", "cn", "blog", "www", "docs", "pages", "medium"}
+    named = [label for label in labels if label not in generic]
+    return (named[0] if named else labels[0]).capitalize()

bibcite_cli-0.6.2/tests/test_webpages.py

@@ -0,0 +1,69 @@
+"""Citing a page that no index knows about.
+
+Blogs, documentation and standards pages are cited constantly and appear in
+none of the academic sources, so the page itself has to be the source.
+"""
+
+from bibcite.resolve import _web_entry, classify
+from bibcite.sources import WebPage, _meta_content, _site_author, _title_tag, _year_in_path
+
+
+def test_a_plain_url_is_a_webpage_and_the_others_still_are_not():
+    assert classify("https://karpathy.github.io/2015/05/21/rnn-effectiveness/")[0] == "webpage"
+    assert classify("http://example.edu/notes")[0] == "webpage"
+    # The identifiers that resolve properly must keep their own paths.
+    assert classify("https://arxiv.org/abs/1706.03762") == ("arxiv", "1706.03762")
+    assert classify("https://doi.org/10.1038/s41586-021-03819-2")[0] == "doi"
+    assert classify("10.1109/CVPR.2016.90")[0] == "doi"
+    assert classify("Attention Is All You Need")[0] == "title"
+
+
+def test_entry_is_misc_so_conference_styles_can_print_it():
+    # @online is biblatex-only: a NeurIPS or IEEE .bst drops the entry.
+    entry = _web_entry(WebPage(url="https://example.org/post", title="A Post", year="2024"))
+    assert entry["ENTRYTYPE"] == "misc"
+    assert entry["howpublished"] == r"\url{https://example.org/post}"
+    assert entry["url"] == "https://example.org/post"
+    assert entry["year"] == "2024"
+
+
+def test_a_page_with_no_byline_is_attributed_to_its_site():
+    entry = _web_entry(WebPage(url="https://example.org/post", title="A Post", site="Example"))
+    # Braced, so the .bst treats it as a corporate name and does not invert it.
+    assert entry["author"] == "{Example}"
+
+
+def test_a_byline_wins_over_the_site():
+    entry = _web_entry(
+        WebPage(url="https://example.org/post", title="A Post", authors=["Ada Lovelace"], site="Example")
+    )
+    assert entry["author"] == "Ada Lovelace"
+
+
+def test_the_site_author_drops_hosting_suffixes():
+    # `karpathygithubio…` makes an unreadable citation key.
+    assert _site_author("karpathy.github.io") == "Karpathy"
+    assert _site_author("www.distill.pub") == "Distill"
+    assert _site_author("docs.python.org") == "Python"
+
+
+def test_a_dateless_page_takes_the_year_from_its_own_url():
+    assert _year_in_path("https://karpathy.github.io/2015/05/21/rnn-effectiveness/") == "2015"
+    assert _year_in_path("https://example.org/blog/2016/misread-tsne/") == "2016"
+    # Not from a query string, where any number at all can appear.
+    assert _year_in_path("https://example.org/page?id=2019") == ""
+    assert _year_in_path("https://example.org/about") == ""
+
+
+def test_the_title_loses_the_site_name_templates_append():
+    assert _title_tag("<title>How to Use t-SNE | Distill</title>") == "How to Use t-SNE"
+    assert _title_tag("<title>Plain Title</title>") == "Plain Title"
+    # A title that is only a site name must survive rather than become empty.
+    assert _title_tag("<title>Distill</title>") == "Distill"
+
+
+def test_meta_is_read_in_either_attribute_order():
+    assert _meta_content('<meta property="og:title" content="Hello">', "og:title") == "Hello"
+    assert _meta_content('<meta content="Hello" name="og:title">', "og:title") == "Hello"
+    assert _meta_content('<meta property="og:title" content="A &amp; B">', "og:title") == "A & B"
+    assert _meta_content("<html></html>", "og:title") == ""

{bibcite_cli-0.6.1 → bibcite_cli-0.6.2}/uv.lock

@@ -18,7 +18,7 @@ wheels = [
 
 [[package]]
 name = "bibcite-cli"
-version = "0.6.1"
+version = "0.6.2"
 source = { editable = "." }
 dependencies = [
     { name = "bibtexparser" },