sluicer 0.2.0__py3-none-any.whl

sluicer/__init__.py

@@ -0,0 +1,22 @@
+"""Deterministic extraction of the data a web page already declares."""
+
+from sluicer.api import Extraction, extract
+from sluicer.declared.merge import Field, Record
+from sluicer.declared.microformats import MicroformatsExtraMissing
+from sluicer.markdown import MarkdownExtraMissing, to_markdown
+from sluicer.structure import induce
+from sluicer.summary import SummaryField
+
+__all__ = [
+    "Extraction",
+    "Field",
+    "MarkdownExtraMissing",
+    "MicroformatsExtraMissing",
+    "Record",
+    "SummaryField",
+    "__version__",
+    "extract",
+    "induce",
+    "to_markdown",
+]
+__version__ = "0.2.0"

sluicer/api.py

@@ -0,0 +1,128 @@
+"""The one call most people will make."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from sluicer.declared.dublincore import read_dublincore
+from sluicer.declared.htmlmeta import read_htmlmeta
+from sluicer.declared.jsonld import read_jsonld
+from sluicer.declared.merge import ABOUT_A_THING, Record, merge
+from sluicer.declared.microdata import read_microdata
+from sluicer.declared.microformats import read_microformats
+from sluicer.declared.opengraph import read_opengraph
+from sluicer.declared.rdfa import read_rdfa
+from sluicer.declared.twitter import read_twitter
+from sluicer.document import load
+from sluicer.structure import induce as induce_records
+from sluicer.summary import SummaryField, summarise
+
+
+@dataclass
+class Extraction:
+    """What Sluicer found in one page, and where it came from.
+
+    ``url`` is the address given to ``extract``. ``summary`` answers the
+    questions most callers ask -- title, author, date, price -- one value each,
+    chosen from the records by fixed rules, each naming its reader and key (see
+    ``sluicer.summary.FIELDS``). ``records`` is everything the page declared.
+    ``sources`` names every reader that found something.
+    """
+
+    url: str | None = None
+    summary: dict[str, SummaryField] = field(default_factory=dict)
+    records: list[Record] = field(default_factory=list)
+    sources: list[str] = field(default_factory=list)
+
+
+def extract(
+    html: str | bytes,
+    url: str | None = None,
+    induce: bool = False,
+    microformats: bool = False,
+) -> Extraction:
+    """Read the structured data ``html`` declares, merged, with its provenance.
+
+    Args:
+        html: the page. Bytes are best: the page's own charset is then honoured
+            (see ``sluicer.document.load``).
+        url: the address the page came from, used to resolve its links.
+        induce: when the page declares nothing about the things on it, also
+            read the rows its markup repeats; those fields say
+            ``source="induced"``. Never fills a gap in a declared record.
+        microformats: also read microformats2. Off by default; needs
+            ``sluicer[microformats]``.
+
+    Returns:
+        An ``Extraction``: the ``summary``, the ``records`` (a record with no
+        field is never reported), and the ``sources`` that found something,
+        in the order of precedence -- JSON-LD, microdata, microformats, RDFa,
+        Dublin Core, OpenGraph, the Twitter card, HTML's own meta names.
+
+    Raises:
+        MicroformatsExtraMissing: ``microformats=True`` without the extra.
+        Nothing else: any input, however broken, is read or reported empty.
+
+    Why the order is what it is, and when induction runs, is in
+    ``docs/design-notes.md``.
+    """
+    doc = load(html, url=url)
+    jsonld = read_jsonld(doc)
+    microdata = read_microdata(doc)
+    # Not calling the reader is what keeps mf2py unimported on a base install.
+    found_microformats = read_microformats(doc) if microformats else []
+    rdfa = read_rdfa(doc)
+    dublincore = read_dublincore(doc)
+    opengraph = read_opengraph(doc)
+    twitter = read_twitter(doc)
+    htmlmeta = read_htmlmeta(doc)
+
+    sources = [
+        name
+        for name, found in (
+            ("jsonld", jsonld),
+            ("microdata", microdata),
+            ("microformats", found_microformats),
+            ("rdfa", rdfa),
+            ("dublincore", dublincore),
+            ("opengraph", opengraph),
+            ("twitter", twitter),
+            ("html", htmlmeta),
+        )
+        if found
+    ]
+    records = merge(
+        jsonld=jsonld,
+        microdata=microdata,
+        microformats=found_microformats,
+        rdfa=rdfa,
+        dublincore=dublincore,
+        opengraph=opengraph,
+        twitter=twitter,
+        htmlmeta=htmlmeta,
+    )
+    # A record carrying no field is a type and nothing else -- a ``WebPage``
+    # with only an ``@id``, a microformats root that was a CSS class -- and an
+    # empty value is not a value, whole records included.
+    records = [record for record in records if record.fields]
+    summary = summarise(doc, records, dublincore, opengraph, twitter, htmlmeta)
+    if induce and not _declared_about_its_things(records):
+        induced = induce_records(doc)
+        if induced:
+            records = [*records, *induced]
+            sources = [*sources, "induced"]
+    return Extraction(url=url, summary=summary, records=records, sources=sources)
+
+
+def _declared_about_its_things(records: list[Record]) -> bool:
+    """True when a reader produced a field about a thing on the page.
+
+    A field, not a parse: an empty declaration produces a record with nothing in
+    it. And a field about a thing: OpenGraph describes the page it sits on, so
+    its fields answer "what is this page", never "what is in this list".
+    """
+    return any(
+        field.source in ABOUT_A_THING
+        for record in records
+        for field in record.fields.values()
+    )

sluicer/cli.py

@@ -0,0 +1,389 @@
+"""Command line front door.
+
+Exit codes follow grep: 0 when something was found -- a record, or at least one
+summary answer -- 1 when the page was read and gives nothing at all, 2 when it
+could not be read. A script can tell "this page gives nothing" from "the fetch
+failed" without parsing English. ``run`` and ``heal`` add 3: a page broke the
+extractor's contract, or healing lost a field, and that is never a success.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import sys
+from dataclasses import asdict
+from pathlib import Path
+from typing import NoReturn
+
+import click
+
+from sluicer.api import extract as extract_html
+from sluicer.declared.microformats import MicroformatsExtraMissing
+from sluicer.extractor import (
+    LOSSES,
+    Extractor,
+    NothingToLearn,
+    compile_extractor,
+    heal as heal_extractor,
+    run_extractor,
+)
+from sluicer.fetch import FetchFailed, RobotsRefused, fetch as fetch_url
+from sluicer.fetch.result import Fetched
+from sluicer.fetch.scrapling_rungs import FetchExtraMissing
+from sluicer.markdown import MarkdownExtraMissing, to_markdown
+
+NOTHING_FOUND = 1
+COULD_NOT_READ = 2
+CONTRACT_BROKEN = 3
+
+
+@click.group()
+@click.version_option(package_name="sluicer")
+def main() -> None:
+    """Turn a web page into structured data with no model in the loop.
+
+    SOURCE is a URL, a saved HTML file, or - for standard input.
+    """
+    _quiet_scrapling()
+
+
+def _quiet_scrapling() -> None:
+    """Keep scrapling's own log lines out of this command's stderr.
+
+    Every one of them is said again here, better: a request is part of the
+    ladder this command reports, and a failure is the message it exits with.
+    Printed as well, a failed fetch read twice, once in scrapling's words.
+
+    A filter and not a level: scrapling sets its logger to INFO when it is
+    imported, which happens later, at the first fetch, and would undo a level
+    set here. A filter on the logger survives that.
+    """
+    logging.getLogger("scrapling").addFilter(
+        lambda record: record.levelno >= logging.CRITICAL
+    )
+
+
+def _fail(message: str, cause: BaseException | None = None) -> NoReturn:
+    click.echo(message, err=True)
+    raise SystemExit(COULD_NOT_READ) from cause
+
+
+_fetch_options = [
+    click.option(
+        "--stealth",
+        is_flag=True,
+        help="Allow the stealth rung, which does not announce itself.",
+    ),
+    click.option(
+        "--no-robots",
+        is_flag=True,
+        help="Fetch even where the site's robots.txt says no.",
+    ),
+    click.option(
+        "--url",
+        "base_url",
+        metavar="URL",
+        help="The address a file or stdin came from, to resolve its links.",
+    ),
+]
+
+
+def _with_fetch_options(command: click.decorators.FC) -> click.decorators.FC:
+    for option in reversed(_fetch_options):
+        command = option(command)
+    return command
+
+
+def _read_source(
+    source: str,
+    stealth: bool = False,
+    no_robots: bool = False,
+    base_url: str | None = None,
+) -> tuple[str | bytes, str | None, Fetched | None]:
+    """Return the HTML of ``source``, the URL to attribute it to, and the fetch record.
+
+    ``source`` is a URL, a path, or ``-`` for standard input. Every way of
+    failing to read it -- a missing extra, a refusal, a failed fetch, a missing,
+    empty or non-file path -- exits here with a message and ``COULD_NOT_READ``,
+    one place for both commands. The fetch record is None for a file or stdin.
+    """
+    if source.lower().startswith(("http://", "https://")):
+        # Only FetchExtraMissing, not ImportError: an import failure inside a
+        # working scrapling install is a bug and keeps its traceback.
+        try:
+            fetched = fetch_url(source, stealth=stealth, obey_robots=not no_robots)
+        except FetchExtraMissing as missing:
+            _fail(str(missing), missing)
+        except RobotsRefused as refused:
+            # The site told us no: an answer, not a malfunction.
+            _fail(str(refused), refused)
+        except FetchFailed as failed:
+            # Every rung failed, whatever library it was built on: a browser's
+            # timeout, for one, is not an OSError.
+            _fail(str(failed), failed)
+        except (OSError, ValueError) as failure:
+            # An operational failure is a message and a bug is a traceback.
+            # OSError covers down, unresolvable and timed out; ValueError is a
+            # rung that came back with no HTML. Anything else keeps its
+            # traceback, deliberately.
+            _fail(
+                f"Could not fetch {source}: {type(failure).__name__}: {failure}",
+                failure,
+            )
+        return fetched.html, fetched.url, fetched
+
+    if source == "-":
+        data = sys.stdin.buffer.read()
+        if not data.strip():
+            _fail("Standard input contains no HTML.")
+        return data, base_url, None
+
+    path = Path(source)
+
+    # By hand, not click.Path(exists=True): the same argument also takes a URL.
+    if not path.is_file():
+        _fail(
+            f"{source} is not a file." if path.exists() else f"{source} does not exist."
+        )
+
+    # Bytes, not text: decoding here would pick the process default before
+    # the page's own charset declaration is read, and destroy every byte that
+    # was not UTF-8. extract() and to_markdown() both decode bytes properly.
+    data = path.read_bytes()
+
+    # An empty file is a user mistake and gets a message about the file;
+    # load() would quietly read it as a page that declares nothing.
+    if not data.strip():
+        _fail("This file contains no HTML.")
+
+    # A path is not an address: handed to the readers as the page's URL it
+    # would resolve every relative link against the file name.
+    return data, base_url, None
+
+
+@main.command()
+@click.argument("source")
+@click.option(
+    "--induce",
+    is_flag=True,
+    help="Also read the rows a page repeats when it declares nothing about them.",
+)
+@click.option(
+    "--microformats",
+    is_flag=True,
+    help="Also read microformats2 (needs sluicer[microformats]).",
+)
+@_with_fetch_options
+def extract(
+    source: str,
+    induce: bool,
+    microformats: bool,
+    stealth: bool,
+    no_robots: bool,
+    base_url: str | None,
+) -> None:
+    """Read the structured data a URL, a file or stdin declares."""
+    html, url, fetched = _read_source(source, stealth, no_robots, base_url)
+    try:
+        result = extract_html(html, url=url, induce=induce, microformats=microformats)
+    except MicroformatsExtraMissing as missing:
+        _fail(str(missing), missing)
+    if not result.records and not result.summary:
+        click.echo("This page gives nothing: no record and no summary.", err=True)
+        if fetched is not None:
+            # What the page cost is reported even when it declared nothing.
+            click.echo(f"Fetch reached the '{fetched.rung}' rung.", err=True)
+            for climb in fetched.climbs:
+                click.echo(
+                    f"  {climb.from_rung} -> {climb.to_rung}: {climb.reason}",
+                    err=True,
+                )
+        raise SystemExit(NOTHING_FOUND)
+    payload = asdict(result)
+    if fetched is not None:
+        payload["fetch"] = {
+            "rung": fetched.rung,
+            "status": fetched.status,
+            "climbs": [asdict(climb) for climb in fetched.climbs],
+        }
+    click.echo(json.dumps(payload, indent=2, ensure_ascii=False))
+
+
+@main.command()
+@click.argument("source")
+@_with_fetch_options
+def markdown(source: str, stealth: bool, no_robots: bool, base_url: str | None) -> None:
+    """Print the main content of a URL, a file or stdin as markdown."""
+    html, url, _fetched = _read_source(source, stealth, no_robots, base_url)
+    # MarkdownExtraMissing: trafilatura is not installed; the message says how.
+    try:
+        content = to_markdown(html, url=url)
+    except MarkdownExtraMissing as missing:
+        _fail(str(missing), missing)
+    if not content:
+        click.echo("This page has no main content.", err=True)
+        raise SystemExit(NOTHING_FOUND)
+    click.echo(content)
+
+
+def _read_pages(
+    sources: tuple[str, ...], stealth: bool, no_robots: bool
+) -> list[tuple[str | bytes, str | None]]:
+    return [_read_source(source, stealth, no_robots, None)[:2] for source in sources]
+
+
+def _load_extractor(path: str) -> Extractor:
+    try:
+        return Extractor.from_json(Path(path).read_text(encoding="utf-8"))
+    except (OSError, ValueError) as failure:
+        _fail(f"{path} is not an extractor: {failure}", failure)
+
+
+@main.command("compile")
+@click.argument("sources", nargs=-1, required=True)
+@click.option("-o", "--output", required=True, help="Where to write the extractor.")
+@click.option(
+    "--listing/--no-listing",
+    default=None,
+    help="Learn the rows the pages repeat (default: only if they declare no thing).",
+)
+@click.option("--stealth", is_flag=True, help="Allow the stealth rung.")
+@click.option("--no-robots", is_flag=True, help="Fetch even where robots.txt says no.")
+def compile_command(
+    sources: tuple[str, ...],
+    output: str,
+    listing: bool | None,
+    stealth: bool,
+    no_robots: bool,
+) -> None:
+    """Learn an extractor from pages of one template, and write it to a file."""
+    pages = _read_pages(sources, stealth, no_robots)
+    try:
+        extractor = compile_extractor(pages, listing=listing, names=list(sources))
+    except NothingToLearn as nothing:
+        click.echo(f"Learnt nothing: {nothing}.", err=True)
+        raise SystemExit(NOTHING_FOUND) from nothing
+    _write(output, extractor.to_json())
+    learnt = []
+    if extractor.listing is not None:
+        rows = extractor.listing.rows
+        learnt.append(
+            f"a listing at {extractor.listing.container}, {rows[0]}-{rows[1]} rows "
+            f"of {len(extractor.listing.fields)} fields"
+        )
+    if extractor.summary:
+        learnt.append(f"{len(extractor.summary)} summary answers")
+    if extractor.types:
+        learnt.append("declared " + ", ".join(extractor.types))
+    click.echo(f"Learnt {'; '.join(learnt)}. Wrote {output}.", err=True)
+    for note in extractor.notes:
+        click.echo(f"Note: {note}.", err=True)
+
+
+def _write(path: str, text: str) -> None:
+    try:
+        Path(path).write_text(text, encoding="utf-8")
+    except OSError as failure:
+        _fail(f"Could not write {path}: {failure.strerror or failure}", failure)
+
+
+@main.command("run")
+@click.argument("extractor_file")
+@click.argument("sources", nargs=-1, required=True)
+@click.option("--stealth", is_flag=True, help="Allow the stealth rung.")
+@click.option("--no-robots", is_flag=True, help="Fetch even where robots.txt says no.")
+def run_command(
+    extractor_file: str, sources: tuple[str, ...], stealth: bool, no_robots: bool
+) -> None:
+    """Replay an extractor on pages, and exit 3 if any page broke its contract."""
+    extractor = _load_extractor(extractor_file)
+    pages = []
+    broken = False
+    for source, (html, url) in zip(
+        sources, _read_pages(sources, stealth, no_robots), strict=True
+    ):
+        run = run_extractor(extractor, html, url=url)
+        failed = [asdict(check) for check in run.checks if not check.ok]
+        pages.append(
+            {
+                "source": source,
+                "url": run.url,
+                "ok": run.ok,
+                "rows": run.rows,
+                "summary": run.summary,
+                "failed": failed,
+            }
+        )
+        for check in run.checks:
+            if not check.ok:
+                broken = True
+                click.echo(
+                    f"FAILED {source}: expected {check.expected}, got {check.got}",
+                    err=True,
+                )
+    click.echo(
+        json.dumps(
+            {"extractor": extractor_file, "pages": pages}, indent=2, ensure_ascii=False
+        )
+    )
+    if broken:
+        raise SystemExit(CONTRACT_BROKEN)
+
+
+@main.command("heal")
+@click.argument("extractor_file")
+@click.argument("sources", nargs=-1, required=True)
+@click.option("-o", "--output", help="Where to write the healed extractor.")
+@click.option(
+    "--force",
+    is_flag=True,
+    help="Write the healed extractor even when healing lost something.",
+)
+@click.option("--stealth", is_flag=True, help="Allow the stealth rung.")
+@click.option("--no-robots", is_flag=True, help="Fetch even where robots.txt says no.")
+def heal_command(
+    extractor_file: str,
+    sources: tuple[str, ...],
+    output: str | None,
+    force: bool,
+    stealth: bool,
+    no_robots: bool,
+) -> None:
+    """Learn pages again and say what moved; write the result only with -o.
+
+    Exits 3 when a field, a summary answer, a type or the listing was lost for
+    good: healing moved what it could, and what it could not needs a person.
+    Nothing is written then without --force, so a lossy extractor never
+    quietly replaces the one that would have kept failing.
+    """
+    extractor = _load_extractor(extractor_file)
+    pages = _read_pages(sources, stealth, no_robots)
+    try:
+        healed, changes = heal_extractor(extractor, pages, names=list(sources))
+    except NothingToLearn as nothing:
+        click.echo(f"The pages hold nothing to heal from: {nothing}.", err=True)
+        raise SystemExit(CONTRACT_BROKEN) from nothing
+    for change in changes:
+        if change.kind == "kept":
+            continue
+        if change.before and change.after:
+            click.echo(f"{change.kind}: {change.before} -> {change.after}", err=True)
+        else:
+            click.echo(f"{change.kind}: {change.before or change.after}", err=True)
+    lost = any(c.kind in LOSSES for c in changes)
+    if output and (force or not lost):
+        _write(output, healed.to_json())
+        click.echo(f"Wrote {output}.", err=True)
+    elif output:
+        click.echo(
+            f"Did not write {output}: healing lost data. Pass --force to write it.",
+            err=True,
+        )
+    click.echo(
+        json.dumps(
+            {"changes": [asdict(c) for c in changes]}, indent=2, ensure_ascii=False
+        )
+    )
+    if lost:
+        raise SystemExit(CONTRACT_BROKEN)

sluicer/declared/__init__.py

@@ -0,0 +1 @@
+"""Readers for the structured data a page already declares."""

sluicer/declared/dublincore.py

@@ -0,0 +1,37 @@
+"""Read Dublin Core meta tags.
+
+Older than schema.org, and still the only structured data on many library,
+university, repository and government pages: flat ``<meta name="DC.title">``
+tags, sometimes with a ``scheme`` attribute naming the value's encoding rather
+than adding a field. ``DC.`` (the fifteen original elements) and ``DCTERMS.``
+(the refined set) turn up on the same page and answer the same questions, so
+both are read into one flat mapping.
+
+The whole name is matched case-insensitively and the key lowercased: the 1997
+examples wrote ``DC.Title``, the HTML that copied them ``dc.title``, and
+first-wins only means something across one spelling.
+
+Dublin Core describes the document, so it is not in
+``sluicer.declared.merge.ABOUT_A_THING``.
+"""
+
+from __future__ import annotations
+
+from sluicer.document import Document
+
+_PREFIXES = ("dc.", "dcterms.")
+
+
+def read_dublincore(doc: Document) -> dict[str, str]:
+    """Return the Dublin Core meta tags, prefixes stripped, keys lowercased."""
+    found: dict[str, str] = {}
+    for meta in doc.tree.xpath("//meta[@name]"):
+        name = (meta.get("name") or "").strip().lower()
+        content = (meta.get("content") or "").strip()
+        if not content:
+            continue
+        for prefix in _PREFIXES:
+            if name.startswith(prefix):
+                found.setdefault(name[len(prefix) :], content)
+                break
+    return found

sluicer/declared/htmlmeta.py

@@ -0,0 +1,44 @@
+"""Read the metadata names HTML defines for itself.
+
+The oldest declaration on the web and still the most common: pages with no
+JSON-LD, microdata or OpenGraph very often carry ``<meta name="description">``
+and ``<meta name="author">``. Measured on 2026-09-22 across the 359 WCXB pages
+Sluicer targets, ``description`` is on 87% of them and ``author`` on 29%.
+
+It is a closed list: ``author``, ``description``, ``keywords``, ``generator``,
+``application-name`` and ``theme-color``. Every ``name=`` attribute would
+report a page's ``csrf-token`` and build id as statements about it; browser
+directives (``robots``, ``viewport``) are instructions to a client, not
+statements about the page's subject.
+
+These values arrive as ``source="html"``: no vocabulary, no schema, no type.
+Extruct files ``<meta name="description">`` under Dublin Core, which never
+claimed it. The reader runs last, so any real vocabulary wins and a bare tag
+only fills a gap. It describes the document, so it is not in
+``sluicer.declared.merge.ABOUT_A_THING``.
+"""
+
+from __future__ import annotations
+
+from sluicer.declared.meta import read_named_meta
+from sluicer.document import Document
+
+# The metadata names the HTML standard defines, and nothing else. Written as
+# a frozenset because it is a membership test and a closed list, and the
+# closedness is the design: every name added here is a claim that the name
+# says something about the page's subject rather than about its plumbing.
+NAMES = frozenset(
+    {
+        "author",
+        "description",
+        "keywords",
+        "generator",
+        "application-name",
+        "theme-color",
+    }
+)
+
+
+def read_htmlmeta(doc: Document) -> dict[str, str]:
+    """Return the standard ``<meta name=...>`` metadata, keys lowercased."""
+    return read_named_meta(doc, NAMES)