refsource 0.1.0__py3-none-any.whl

refsource/__init__.py

@@ -0,0 +1,636 @@
+"""refsource — look up reference data that carries its own source.
+
+    >>> import refsource
+    >>> rows = refsource.lookup("conforming-loan-limits", state="AL",
+    ...                         county_name="AUTAUGA COUNTY")
+    >>> rows[0]["limit_1_unit"]
+    '$832,750'
+    >>> rows[0]["limit_1_unit"].source
+    'https://www.fhfa.gov/document/d/cll/fullcountyloanlimitlist2026_hera-based_final_flat.csv'
+    >>> print(rows[0].cite())
+
+Every value comes back with the URL it was read from, a verbatim quote from
+that page, and the date it was last checked. That is the whole point: a value
+you can check is worth more than a value you have to trust, and a language
+model asked the same question invents a plausible answer about a fifth of the
+time.
+
+The package ships no records. It ships a manifest of what exists — 191
+datasets, their fields, record counts, verification dates and the SHA-256 of
+each bundle at release time — and fetches a dataset's records from
+referencesource.org the first time you ask for one, then caches them.
+
+Full catalogue and method: https://referencesource.org/
+"""
+from __future__ import annotations
+
+import datetime
+import hashlib
+import json
+import os
+import pathlib
+import re
+import sys
+import tempfile
+import urllib.error
+import urllib.request
+import warnings
+
+__version__ = "0.1.0"
+__all__ = [
+    "lookup", "search", "get", "dataset", "datasets", "fields", "configure",
+    "Record", "Value", "Dataset",
+    "RefsourceError", "NoSuchDataset", "NoSuchField", "OfflineError",
+    "IntegrityError", "StaleDataWarning", "ChangedUpstreamWarning",
+]
+
+_HERE = pathlib.Path(__file__).resolve().parent
+_MANIFEST = json.loads((_HERE / "manifest.json").read_text(encoding="utf-8"))
+_BY_SLUG = {d["slug"]: d for d in _MANIFEST["datasets"]}
+
+USER_AGENT = "refsource-python/{0} (+https://referencesource.org)".format(__version__)
+
+
+# --------------------------------------------------------------------- errors
+
+class RefsourceError(Exception):
+    """Base class for everything this package raises."""
+
+
+class NoSuchDataset(RefsourceError):
+    pass
+
+
+class NoSuchField(RefsourceError):
+    """A filter named a field the dataset does not have.
+
+    Raised rather than returning nothing, because an empty result from a typo
+    reads exactly like an empty result from a real absence, and one of those
+    two is a wrong answer.
+    """
+
+
+class OfflineError(RefsourceError):
+    pass
+
+
+class IntegrityError(RefsourceError):
+    pass
+
+
+class StaleDataWarning(UserWarning):
+    """The dataset is past the date by which it said it should be re-checked."""
+
+
+class ChangedUpstreamWarning(UserWarning):
+    """The bundle fetched is not byte-identical to the one this release pinned.
+
+    Normal and expected — it means the dataset was re-verified upstream since
+    this version of the package was cut. The live copy is the current one; the
+    hash is here so a change is visible rather than silent.
+    """
+
+
+# --------------------------------------------------------------------- config
+
+class _Config(object):
+    def __init__(self):
+        self.base_url = os.environ.get("REFSOURCE_BASE_URL", "").rstrip("/")
+        self.cache_dir = os.environ.get("REFSOURCE_CACHE") or _default_cache()
+        self.cache_ttl = _int_env("REFSOURCE_CACHE_TTL", 86400)
+        self.timeout = _int_env("REFSOURCE_TIMEOUT", 30)
+        self.offline = _bool_env("REFSOURCE_OFFLINE", False)
+        self.strict = _bool_env("REFSOURCE_STRICT", False)
+
+
+def _int_env(name, default):
+    try:
+        return int(os.environ[name])
+    except (KeyError, ValueError):
+        return default
+
+
+def _bool_env(name, default):
+    v = os.environ.get(name)
+    if v is None:
+        return default
+    return v.strip().lower() in ("1", "true", "yes", "on")
+
+
+def _default_cache():
+    if sys.platform == "win32":
+        base = os.environ.get("LOCALAPPDATA") or tempfile.gettempdir()
+    elif sys.platform == "darwin":
+        base = os.path.expanduser("~/Library/Caches")
+    else:
+        base = os.environ.get("XDG_CACHE_HOME") or os.path.expanduser("~/.cache")
+    return os.path.join(base, "refsource")
+
+
+_config = _Config()
+
+
+def configure(base_url=None, cache_dir=None, cache_ttl=None, timeout=None,
+              offline=None, strict=None):
+    """Change how records are fetched and cached.
+
+    These live here rather than as keyword arguments on `lookup()` so that
+    `lookup(slug, **filters)` can stay entirely field names — a dataset with a
+    column called `timeout` should not fight the library for it.
+
+    - `base_url`: serve records from somewhere else (a local render, a mirror).
+      A `file://` URL works, which is how the tests run without a network.
+    - `cache_ttl`: seconds before a cached bundle is re-fetched. 0 = always
+      re-fetch, negative = never expire.
+    - `offline`: use the cache and never open a connection.
+    - `strict`: raise `IntegrityError` when a fetched bundle does not match the
+      SHA-256 this release pinned, instead of warning.
+
+    Every one of these also has an environment variable — REFSOURCE_BASE_URL,
+    REFSOURCE_CACHE, REFSOURCE_CACHE_TTL, REFSOURCE_TIMEOUT, REFSOURCE_OFFLINE,
+    REFSOURCE_STRICT.
+    """
+    if base_url is not None:
+        _config.base_url = base_url.rstrip("/")
+    if cache_dir is not None:
+        _config.cache_dir = cache_dir
+    if cache_ttl is not None:
+        _config.cache_ttl = int(cache_ttl)
+    if timeout is not None:
+        _config.timeout = int(timeout)
+    if offline is not None:
+        _config.offline = bool(offline)
+    if strict is not None:
+        _config.strict = bool(strict)
+
+
+# ---------------------------------------------------------------------- value
+
+class Value(str):
+    """A value that knows where it came from.
+
+    It behaves as an ordinary string everywhere a string is expected, and
+    carries the citation for *this field* — which is not always the record's
+    own source. A record whose key was reported by a second publisher keeps
+    that publisher's values with that publisher's URL and quote, and this is
+    how you get at them without attributing one source's number to another.
+    """
+
+    # No __slots__: a str subclass cannot have them (variable-length type).
+
+    def __new__(cls, text, field="", source="", quote="", as_of="",
+                confirmed=False, derived=False, record=None):
+        self = str.__new__(cls, "" if text is None else str(text))
+        self.field = field
+        self.source = source
+        self.quote = quote
+        self.as_of = as_of
+        self.confirmed = confirmed
+        self.derived = derived
+        self.record = record
+        return self
+
+    @property
+    def disagreement(self):
+        """Other readings of this same field — never this one.
+
+        A list of dicts with `value`, `source`, `source_quote`. Empty for the
+        overwhelming majority of values. When it is not empty, the honest thing
+        to report is that the sources differ, not one of them.
+
+        The bundle lists every version including the one in hand, so that is
+        dropped here: an entry repeating the value you already have reads as a
+        second opinion when it is the same opinion.
+        """
+        if self.record is None:
+            return []
+        return [v for v in self.record.disagreements.get(self.field, [])
+                if str(v.get("value", "")) != str(self)]
+
+    def cite(self):
+        """One line you can paste into whatever you are writing."""
+        page = self.record.url if self.record is not None else ""
+        parts = ["{0}: {1}".format(self.field, str(self))]
+        if self.source:
+            parts.append("source: {0}".format(self.source))
+        if self.quote:
+            parts.append('quoted: "{0}"'.format(_clip(self.quote, 200)))
+        if page:
+            parts.append(page)
+        if self.as_of:
+            parts.append("verified {0}".format(self.as_of))
+        return " — ".join(parts)
+
+
+def _clip(text, n):
+    t = " ".join(str(text).split())
+    return t if len(t) <= n else t[: n - 1] + "\u2026"
+
+
+# --------------------------------------------------------------------- record
+
+class Record(object):
+    """One published record: its values, each with its own citation."""
+
+    def __init__(self, raw, ds):
+        self._raw = raw
+        self.dataset = ds
+        self.id = raw.get("id", "")
+        self.url = raw.get("url", "")
+        self.source_url = raw.get("source", "")
+        self.source_quote = raw.get("source_quote", "")
+        self.verified = ds.last_verified
+        self.stale_after = ds.stale_after
+
+        self.disagreements = {}
+        for d in raw.get("disagreements") or []:
+            if d.get("field"):
+                self.disagreements[d["field"]] = d.get("versions") or []
+
+        confirmed = set(raw.get("verified_fields") or [])
+        derived = set(raw.get("derived_fields") or [])
+        self.values = {}
+        for k, v in raw.items():
+            if k in _META_KEYS or isinstance(v, (list, dict)):
+                continue
+            self.values[k] = Value(
+                v, field=k, source=self.source_url, quote=self.source_quote,
+                as_of=ds.last_verified, confirmed=k in confirmed,
+                derived=k in derived, record=self)
+        # A second publisher's values are kept apart in the bundle so nobody
+        # reads them as the first publisher's. Same treatment here: they are
+        # available by name, and they carry their own source and quote.
+        for extra in raw.get("also_reported") or []:
+            f = extra.get("field")
+            if not f:
+                continue
+            self.values[f] = Value(
+                extra.get("value"), field=f, source=extra.get("source", ""),
+                quote=extra.get("source_quote", ""),
+                as_of=extra.get("as_of") or ds.last_verified,
+                confirmed=True, record=self)
+
+    # dict-ish access, because that is what people try first
+    def __getitem__(self, field):
+        try:
+            return self.values[field]
+        except KeyError:
+            raise NoSuchField("{0!r} is not a field of {1}. Fields: {2}".format(
+                field, self.dataset.slug, ", ".join(sorted(self.values))))
+
+    def get(self, field, default=None):
+        return self.values.get(field, default)
+
+    def __contains__(self, field):
+        return field in self.values
+
+    def __iter__(self):
+        return iter(self.values)
+
+    def keys(self):
+        return self.values.keys()
+
+    def items(self):
+        return self.values.items()
+
+    def __repr__(self):
+        head = ", ".join("{0}={1!r}".format(k, str(v))
+                         for k, v in list(self.values.items())[:3])
+        return "<Record {0} {1}>".format(self.dataset.slug, head)
+
+    def cite(self):
+        """A citation for the record as a whole."""
+        line = "{0} — {1}".format(self.dataset.title, self.url or self.dataset.url)
+        if self.source_url:
+            line += "\n  source: {0}".format(self.source_url)
+        if self.source_quote:
+            line += '\n  quoted: "{0}"'.format(_clip(self.source_quote, 300))
+        line += "\n  verified {0}".format(self.verified)
+        if self.dataset.stale:
+            line += " (past its stale_after of {0} — re-check the source)".format(
+                self.stale_after)
+        return line
+
+    def to_dict(self):
+        """Plain data, citations included, for JSON output."""
+        return {
+            "id": self.id,
+            "url": self.url,
+            "dataset": self.dataset.slug,
+            "verified": self.verified,
+            "stale_after": self.stale_after,
+            "values": {
+                k: {
+                    "value": str(v), "source": v.source, "source_quote": v.quote,
+                    "confirmed": v.confirmed, "derived": v.derived,
+                    "disagreement": v.disagreement,
+                }
+                for k, v in self.values.items()
+            },
+        }
+
+
+_META_KEYS = frozenset((
+    "id", "url", "source", "source_quote",
+    "verified_fields", "derived_fields", "also_reported", "disagreements",
+))
+
+
+# -------------------------------------------------------------------- dataset
+
+class Dataset(object):
+    """One published dataset. Metadata is offline; records are fetched."""
+
+    def __init__(self, entry):
+        self._entry = entry
+        self.slug = entry["slug"]
+        self.title = entry.get("title", "")
+        self.summary = entry.get("summary", "")
+        self.url = entry.get("url", "")
+        self.data_url = entry.get("data", "")
+        self.record_count = entry.get("records", 0)
+        self.last_verified = entry.get("last_verified", "")
+        self.stale_after = entry.get("stale_after", "")
+        self.fields = list(entry.get("fields") or [])
+        self.bytes = entry.get("bytes", 0)
+        self.licence = ""          # filled in from the bundle once fetched
+        self.checksum = "unknown"  # "pinned" | "changed" | "unknown"
+        self._records = None
+
+    @property
+    def stale(self):
+        """Past the date the publisher's own cadence says it should be re-checked."""
+        if not self.stale_after:
+            return False
+        try:
+            due = datetime.date(*[int(x) for x in self.stale_after.split("-")])
+        except (ValueError, TypeError):
+            return False
+        return datetime.date.today() > due
+
+    def records(self):
+        """Every record, fetched on first call and cached thereafter."""
+        if self._records is None:
+            self._records = [Record(r, self) for r in _fetch_records(self)]
+            if self.stale:
+                warnings.warn(
+                    "{0} is past its stale_after date ({1}); the values may have "
+                    "moved on. Check {2} for the current copy.".format(
+                        self.slug, self.stale_after, self.url),
+                    StaleDataWarning, stacklevel=3)
+        return self._records
+
+    def lookup(self, **filters):
+        return _filter(self.records(), self, filters)
+
+    def search(self, text, limit=None):
+        needle = _norm(text)
+        if not needle:
+            return []
+        out = []
+        for rec in self.records():
+            hay = _norm(" ".join(str(v) for v in rec.values.values()))
+            if needle in hay:
+                out.append(rec)
+                if limit and len(out) >= limit:
+                    break
+        return out
+
+    def get(self, record_id):
+        for rec in self.records():
+            if rec.id == record_id:
+                return rec
+        return None
+
+    def values_of(self, field, limit=None):
+        """The distinct values a field takes, in first-seen order.
+
+        The reason this exists: one dataset spells a state "TX" and another
+        spells it "Texas", so a filter that looks obviously right can match
+        nothing. This is how you find out which it is, rather than concluding
+        the data is missing.
+        """
+        seen = []
+        for rec in self.records():
+            v = rec.values.get(field)
+            if v is None:
+                continue
+            s = str(v)
+            if s not in seen:
+                seen.append(s)
+                if limit and len(seen) >= limit:
+                    break
+        return seen
+
+    def __repr__(self):
+        return "<Dataset {0} ({1} records, verified {2})>".format(
+            self.slug, self.record_count, self.last_verified)
+
+
+_datasets_cache = {}
+
+
+def dataset(slug):
+    """The Dataset object for a slug. Nothing is fetched until you ask for records."""
+    if slug not in _BY_SLUG:
+        near = [s for s in _BY_SLUG if slug in s or s in slug][:5]
+        hint = (" Did you mean: {0}?".format(", ".join(near)) if near
+                else " Call refsource.datasets() for the list.")
+        raise NoSuchDataset("No dataset {0!r}.{1}".format(slug, hint))
+    if slug not in _datasets_cache:
+        _datasets_cache[slug] = Dataset(_BY_SLUG[slug])
+    return _datasets_cache[slug]
+
+
+def datasets(query=None):
+    """Every published dataset, or the ones matching `query`. No network.
+
+    Matches the slug, title, summary and field names, so
+    `datasets("loan limit")` and `datasets("fips")` both find things.
+    """
+    out = [dataset(s) for s in _BY_SLUG]
+    if not query:
+        return out
+    terms = [t for t in _norm(query).split() if t]
+    hits = []
+    for ds in out:
+        hay = _norm(" ".join([ds.slug.replace("-", " "), ds.title, ds.summary,
+                              " ".join(ds.fields)]))
+        if all(t in hay for t in terms):
+            hits.append(ds)
+    return hits
+
+
+def fields(slug):
+    """The field names of a dataset, for building a lookup. No network."""
+    return dataset(slug).fields
+
+
+# --------------------------------------------------------------------- lookup
+
+def lookup(slug, **filters):
+    """Records from `slug` whose fields match every filter given.
+
+        refsource.lookup("conforming-loan-limits", state="AL")
+        refsource.lookup("auto-insurance-minimums", state=["TX", "NM"])
+
+    Matching is case-insensitive and ignores surrounding punctuation and
+    repeated spaces, so "Autauga County" finds "AUTAUGA COUNTY". A list matches
+    any of its members. With no filters you get every record.
+
+    A filter naming a field the dataset does not have raises `NoSuchField`
+    rather than returning an empty list — silence is indistinguishable from a
+    real absence, and one of those is a wrong answer.
+    """
+    ds = dataset(slug)
+    return ds.lookup(**filters)
+
+
+def search(slug, text, limit=None):
+    """Records from `slug` where `text` appears anywhere in the values."""
+    return dataset(slug).search(text, limit=limit)
+
+
+def get(slug, record_id):
+    """One record by its id, or None."""
+    return dataset(slug).get(record_id)
+
+
+_PUNCT = re.compile(r"[^\w\s%.$/+-]+", re.UNICODE)
+
+
+def _norm(text):
+    """Same shape of comparison the site's own verifier uses: loose but not lossy."""
+    return " ".join(_PUNCT.sub(" ", str(text).lower()).split())
+
+
+def _filter(records, ds, filters):
+    if not filters:
+        return list(records)
+    known = set(ds.fields)
+    for f in filters:
+        if known and f not in known:
+            raise NoSuchField(
+                "{0!r} is not a field of {1}. Fields: {2}".format(
+                    f, ds.slug, ", ".join(ds.fields)))
+    wanted = {}
+    for f, v in filters.items():
+        vals = v if isinstance(v, (list, tuple, set)) else [v]
+        wanted[f] = set(_norm(x) for x in vals)
+    out = []
+    for rec in records:
+        for f, vals in wanted.items():
+            got = rec.values.get(f)
+            if got is None or _norm(got) not in vals:
+                break
+        else:
+            out.append(rec)
+    return out
+
+
+# ---------------------------------------------------------------------- fetch
+
+_warned = set()
+
+
+def _url_for(url):
+    """Rewrite a canonical URL onto a configured base (a local render, a mirror)."""
+    if not _config.base_url:
+        return url
+    tail = url.split("://", 1)[-1].split("/", 1)[-1]
+    return "{0}/{1}".format(_config.base_url, tail)
+
+
+def _cache_path(url):
+    name = hashlib.sha256(url.encode("utf-8")).hexdigest()[:16]
+    return pathlib.Path(_config.cache_dir) / (name + ".json")
+
+
+def _read_cache(path):
+    if not path.exists():
+        return None
+    if _config.cache_ttl == 0:
+        return None
+    if _config.cache_ttl > 0:
+        import time
+        if time.time() - path.stat().st_mtime > _config.cache_ttl:
+            return None
+    return path.read_bytes()
+
+
+def _write_cache(path, blob):
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        tmp = path.with_suffix(".tmp")
+        tmp.write_bytes(blob)
+        tmp.replace(path)
+    except OSError:
+        pass  # an unwritable cache is a slow package, not a broken one
+
+
+def _download(url):
+    if url.startswith("file://") or url.startswith("/"):
+        path = url[7:] if url.startswith("file://") else url
+        return pathlib.Path(path).read_bytes()
+    req = urllib.request.Request(url, headers={
+        "User-Agent": USER_AGENT,
+        "Accept": "application/json",
+    })
+    try:
+        with urllib.request.urlopen(req, timeout=_config.timeout) as resp:
+            return resp.read()
+    except urllib.error.HTTPError as exc:
+        raise RefsourceError("{0} returned HTTP {1}".format(url, exc.code))
+    except urllib.error.URLError as exc:
+        raise RefsourceError("could not reach {0}: {1}".format(url, exc.reason))
+
+
+def _get_bytes(url, expect_sha=None, label=""):
+    target = _url_for(url)
+    path = _cache_path(target)
+    blob = _read_cache(path)
+    if blob is None:
+        if _config.offline:
+            cached = path.exists()
+            raise OfflineError(
+                "offline and {0} is {1} in the cache ({2})".format(
+                    label or url, "expired" if cached else "absent", path))
+        blob = _download(target)
+        _write_cache(path, blob)
+    if expect_sha:
+        got = hashlib.sha256(blob).hexdigest()
+        if got != expect_sha:
+            msg = ("{0} has changed since refsource {1} was released — it was "
+                   "re-verified upstream. The copy you have is the live one; "
+                   "the pinned hash is only how you find out."
+                   .format(label or url, __version__))
+            if _config.strict:
+                raise IntegrityError(msg)
+            if url not in _warned:
+                _warned.add(url)
+                warnings.warn(msg, ChangedUpstreamWarning, stacklevel=4)
+            return blob, "changed"
+        return blob, "pinned"
+    return blob, "unknown"
+
+
+def _fetch_records(ds):
+    entry = ds._entry
+    blob, state = _get_bytes(ds.data_url, entry.get("sha256"), ds.slug)
+    ds.checksum = state
+    body = json.loads(blob.decode("utf-8"))
+    ds.licence = body.get("licence", "")
+    if isinstance(body.get("records"), list):
+        return body["records"]
+    # The biggest datasets are published as numbered parts, because no single
+    # file may exceed the host's size cap. Take all of them: a library on a
+    # laptop can afford what a request handler cannot, and half a register
+    # silently returned is a wrong answer.
+    pinned = {p["url"]: p.get("sha256") for p in (entry.get("parts") or [])}
+    records = []
+    for url in body.get("parts") or []:
+        part, _ = _get_bytes(url, pinned.get(url), "{0} part".format(ds.slug))
+        records.extend(json.loads(part.decode("utf-8")).get("records") or [])
+    if not records:
+        raise RefsourceError("{0} returned no readable records".format(ds.slug))
+    return records