betteroff 0.1.0__py3-none-any.whl

betteroff/__init__.py

@@ -0,0 +1,54 @@
+"""Betteroff: clean data and charts from the Michigan consumer surveys.
+
+Independent project; not affiliated with or endorsed by the University of Michigan.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+try:
+    from ._version import __version__
+except ImportError:  # running from a source checkout without a build
+    from importlib.metadata import PackageNotFoundError, version
+
+    try:
+        __version__ = version('betteroff')
+    except PackageNotFoundError:
+        __version__ = '0.0.0+unknown'
+
+if TYPE_CHECKING:
+    from .api import BetteroffWarning, describe, get, list_questions, search
+    from .catalog import Question
+    from .models import Dataset
+
+__all__ = [
+    'BetteroffWarning',
+    'Dataset',
+    'Question',
+    '__version__',
+    'describe',
+    'get',
+    'list_questions',
+    'search',
+]
+
+_LAZY = {
+    'BetteroffWarning': 'api',
+    'describe': 'api',
+    'get': 'api',
+    'list_questions': 'api',
+    'search': 'api',
+    'Question': 'catalog',
+    'Dataset': 'models',
+}
+
+
+def __getattr__(name: str) -> Any:
+    # Keeps `import betteroff` and the fast CLI paths free of pandas and HTTPX.
+    if name in _LAZY:
+        import importlib
+
+        module = importlib.import_module(f'.{_LAZY[name]}', __name__)
+        return getattr(module, name)
+    raise AttributeError(f'module {__name__!r} has no attribute {name!r}')

betteroff/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+main()

betteroff/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.1.0'
+__version_tuple__ = version_tuple = (0, 1, 0)
+
+__commit_id__ = commit_id = None

betteroff/api.py

@@ -0,0 +1,280 @@
+"""Orchestration shared by the CLI, Python API and recipes."""
+
+from __future__ import annotations
+
+import logging
+import warnings
+from dataclasses import replace
+from datetime import date
+from typing import Any
+
+import pandas as pd
+
+from . import __version__
+from .cache import Cache, Snapshot, now_utc
+from .catalog import Question, load_catalog
+from .errors import FetchError, InputError
+from .models import Dataset, check_measure, parse_date
+from .parsers import historical
+
+log = logging.getLogger('betteroff')
+METADATA_SCHEMA_VERSION = 1
+
+
+class BetteroffWarning(UserWarning):
+    """A condition that changes what the returned data means."""
+
+
+def search(text: str) -> list[Question]:
+    return load_catalog().search(text)
+
+
+def list_questions() -> list[Question]:
+    return list(load_catalog().questions)
+
+
+def describe(identifier: str) -> Question:
+    return load_catalog().resolve(identifier)
+
+
+def _period(ts: pd.Timestamp) -> str:
+    return f'{ts:%Y-%m}'
+
+
+def compare_snapshots(old: pd.DataFrame, new: pd.DataFrame, columns: list[str]) -> dict[str, Any]:
+    """Describe appended, removed and revised months between two parsed frames."""
+    o = old.set_index('date')[columns]
+    n = new.set_index('date')[columns]
+    appended = n.index.difference(o.index)
+    removed = o.index.difference(n.index)
+    common = n.index.intersection(o.index)
+    a, b = o.loc[common], n.loc[common]
+    differs = ~((a == b) | (a.isna() & b.isna()))
+    changed = differs.any(axis=1)
+    return {
+        'appended_months': [_period(d) for d in appended],
+        'removed_months': [_period(d) for d in removed],
+        'changed_months': [_period(d) for d in changed[changed].index],
+        'changed_values': int(differs.to_numpy().sum()),
+    }
+
+
+def _parse_and_validate(content: bytes, question: Question) -> tuple[Any, list[dict]]:
+    from .validate import validate
+
+    parsed = historical.parse(content, question)
+    flags = validate(parsed, question)
+    return parsed, flags
+
+
+def _fetch(question: Question, client: Any = None) -> Any:
+    from .fetch import download_table
+
+    return download_table(question.table, load_catalog().landing_url, client=client)
+
+
+def load(
+    question: Question,
+    *,
+    refresh: bool = False,
+    offline: bool = False,
+    cache: Cache | None = None,
+    client: Any = None,
+) -> tuple[Any, list[dict], Snapshot, str, dict[str, Any] | None]:
+    """Return parsed data, flags, snapshot, retrieval mode and revisions."""
+    if refresh and offline:
+        raise InputError('--refresh and --offline cannot be used together.', 'Choose one.')
+    cache = cache or Cache()
+    current = cache.latest(question.id)
+
+    if offline:
+        if current is None:
+            raise FetchError(
+                f'No saved snapshot for {question.id}.',
+                f'Run `betteroff get {question.id}` once while online.',
+            )
+        parsed, flags = _parse_and_validate(cache.read_bytes(current), question)
+        warnings.warn(
+            f'Offline: using {question.id} snapshot {current.id} retrieved '
+            f'{current.retrieved_at}; latest month {current.latest_period}.',
+            BetteroffWarning,
+            stacklevel=3,
+        )
+        return parsed, flags, current, 'offline', None
+
+    if current is not None and not refresh and current.is_fresh():
+        parsed, flags = _parse_and_validate(cache.read_bytes(current), question)
+        return parsed, flags, current, 'cached', None
+
+    log.info('Checking source for table %s (%s)...', question.table, question.id)
+    try:
+        download = _fetch(question, client)
+    except FetchError:
+        if refresh or current is None:
+            raise
+        parsed, flags = _parse_and_validate(cache.read_bytes(current), question)
+        warnings.warn(
+            f'Could not reach the source; using {question.id} snapshot {current.id} '
+            f'last checked {current.checked_at}. Use --refresh to require fresh data.',
+            BetteroffWarning,
+            stacklevel=3,
+        )
+        return parsed, flags, current, 'stale', None
+
+    parsed, flags = _parse_and_validate(download.content, question)
+    sha = Cache.digest(download.content)
+    stamp = now_utc().isoformat()
+    frame = parsed.frame
+    snapshot = Snapshot(
+        sha256=sha,
+        question_id=question.id,
+        table=question.table,
+        url=download.url,
+        source_period=download.source_period,
+        retrieved_at=stamp,
+        checked_at=stamp,
+        first_period=_period(frame['date'].min()),
+        latest_period=_period(frame['date'].max()),
+        observations=len(frame),
+        parser_version=historical.PARSER_VERSION,
+    )
+    revisions = None
+    if current is not None and current.sha256 == sha:
+        snapshot = replace(snapshot, retrieved_at=current.retrieved_at)
+        log.info(
+            'No change since %s; latest month %s.', current.retrieved_at, snapshot.latest_period
+        )
+    elif current is not None:
+        try:
+            old, _ = _parse_and_validate(cache.read_bytes(current), question)
+        except Exception:  # an unreadable old snapshot shouldn't block a good new one
+            old = None
+        if old is not None:
+            columns = [*(r.id for r in question.responses), 'index']
+            revisions = compare_snapshots(old.frame, frame, columns)
+            revisions['previous_snapshot'] = current.id
+            _report_revisions(question, revisions)
+    cache.store(snapshot, download.content)
+    return parsed, flags, snapshot, 'fresh', revisions
+
+
+def _report_revisions(question: Question, rev: dict[str, Any]) -> None:
+    if rev['appended_months']:
+        log.info(
+            '%s: %d new month(s) through %s.',
+            question.id,
+            len(rev['appended_months']),
+            rev['appended_months'][-1],
+        )
+    else:
+        log.info('%s: no new months.', question.id)
+    if rev['changed_months'] or rev['removed_months']:
+        months = rev['changed_months'] + rev['removed_months']
+        shown = ', '.join(months[:6]) + (' ...' if len(months) > 6 else '')
+        warnings.warn(
+            f'{question.id}: source revised {len(rev["changed_months"])} month(s) and removed '
+            f'{len(rev["removed_months"])} since snapshot {rev["previous_snapshot"]} ({shown}).',
+            BetteroffWarning,
+            stacklevel=4,
+        )
+
+
+def build_metadata(
+    question: Question,
+    parsed: Any,
+    flags: list[dict],
+    snapshot: Snapshot,
+    mode: str,
+    revisions: dict[str, Any] | None,
+) -> dict[str, Any]:
+    catalog = load_catalog()
+    quirks = [f for f in flags if f['check'] != 'wording_changed']
+    if quirks and mode == 'fresh':
+        log.info(
+            '%s: %d source quirk(s) flagged in metadata (quality_flags).', question.id, len(quirks)
+        )
+    for f in flags:
+        if f['check'] == 'wording_changed':
+            warnings.warn(
+                f'{question.id}: the question wording in the source file no longer matches the '
+                f'catalog. {f["detail"]}',
+                BetteroffWarning,
+                stacklevel=4,
+            )
+    return {
+        'schema_version': METADATA_SCHEMA_VERSION,
+        'betteroff_version': __version__,
+        'parser_version': historical.PARSER_VERSION,
+        'question': {
+            **question.to_dict(),
+            'index': {'label': catalog.index_label, 'definition': catalog.index_definition},
+        },
+        'population': catalog.population,
+        'units': {'share': 'percent', 'index': 'index'},
+        'source': {
+            'publisher': catalog.attribution,
+            'landing_url': catalog.landing_url,
+            'download_url': snapshot.url,
+            'table': question.table,
+            'source_period': snapshot.source_period,
+            'release_status': 'unknown',
+            'usage_agreement': catalog.agreement_url,
+        },
+        'snapshot': {
+            'id': snapshot.id,
+            'sha256': snapshot.sha256,
+            'retrieved_at': snapshot.retrieved_at,
+            'checked_at': snapshot.checked_at,
+            'mode': mode,
+        },
+        'coverage': {
+            'first_period': snapshot.first_period,
+            'latest_period': snapshot.latest_period,
+            'observations': snapshot.observations,
+            'monthly_since': (question.monthly_since or catalog.monthly_since).isoformat()[:7],
+        },
+        'notes': [*catalog.notes, *question.notes],
+        'source_notes': parsed.source_notes,
+        'quality_flags': flags,
+        'revisions': revisions,
+    }
+
+
+def get(
+    identifier: str,
+    *,
+    refresh: bool = False,
+    offline: bool = False,
+    start: str | date | None = None,
+    end: str | date | None = None,
+    measure: str = 'share',
+    cache: Cache | None = None,
+    client: Any = None,
+) -> Dataset:
+    """Fetch (or reuse a cached copy of) a table, validate it and return a Dataset."""
+    question = load_catalog().resolve(identifier)
+    check_measure(measure)
+    start_d, end_d = parse_date(start, 'start'), parse_date(end, 'end')
+    parsed, flags, snapshot, mode, revisions = load(
+        question, refresh=refresh, offline=offline, cache=cache, client=client
+    )
+    metadata = build_metadata(question, parsed, flags, snapshot, mode, revisions)
+    return Dataset(
+        question=question,
+        frame=parsed.frame,
+        metadata=metadata,
+        measure=measure,
+        start=start_d,
+        end=end_d,
+    )
+
+
+def cached_dataset(question: Question, cache: Cache | None = None) -> Dataset | None:
+    """Dataset from the latest snapshot without touching the network, or None."""
+    cache = cache or Cache()
+    snapshot = cache.latest(question.id)
+    if snapshot is None:
+        return None
+    parsed, flags = _parse_and_validate(cache.read_bytes(snapshot), question)
+    metadata = build_metadata(question, parsed, flags, snapshot, 'cached', None)
+    return Dataset(question=question, frame=parsed.frame, metadata=metadata)

betteroff/cache.py

@@ -0,0 +1,142 @@
+"""Content-addressed snapshots of source downloads plus a JSON manifest."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import tempfile
+from dataclasses import asdict, dataclass
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+
+from .errors import FetchError
+
+MANIFEST_VERSION = 1
+FRESH_FOR = timedelta(hours=24)
+
+
+def cache_dir() -> Path:
+    env = os.environ.get('BETTEROFF_CACHE_DIR')
+    if env:
+        return Path(env).expanduser()
+    from platformdirs import user_cache_path
+
+    return user_cache_path('betteroff', appauthor=False)
+
+
+def now_utc() -> datetime:
+    return datetime.now(UTC).replace(microsecond=0)
+
+
+def _atomic_write(path: Path, data: bytes) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    fd, tmp = tempfile.mkstemp(dir=path.parent, prefix='.tmp-')
+    try:
+        with os.fdopen(fd, 'wb') as fh:
+            fh.write(data)
+        os.replace(tmp, path)
+    except BaseException:
+        Path(tmp).unlink(missing_ok=True)
+        raise
+
+
+@dataclass
+class Snapshot:
+    sha256: str
+    question_id: str
+    table: str
+    url: str
+    source_period: str | None
+    retrieved_at: str
+    checked_at: str
+    first_period: str
+    latest_period: str
+    observations: int
+    parser_version: str
+
+    @property
+    def id(self) -> str:
+        return self.sha256[:12]
+
+    def is_fresh(self, now: datetime | None = None) -> bool:
+        checked = datetime.fromisoformat(self.checked_at)
+        return (now or now_utc()) - checked < FRESH_FOR
+
+
+class Cache:
+    def __init__(self, root: Path | None = None) -> None:
+        self.root = root or cache_dir()
+
+    @property
+    def manifest_path(self) -> Path:
+        return self.root / 'manifest.json'
+
+    def snapshot_path(self, sha256: str) -> Path:
+        return self.root / 'snapshots' / f'{sha256}.xls'
+
+    def _read(self) -> dict:
+        try:
+            data = json.loads(self.manifest_path.read_text('utf-8'))
+        except FileNotFoundError:
+            return {'schema_version': MANIFEST_VERSION, 'questions': {}}
+        except (OSError, json.JSONDecodeError) as exc:
+            raise FetchError(
+                f'The cache manifest at {self.manifest_path} is unreadable ({exc}).',
+                'Delete the cache directory (see `betteroff cache path`) and run again.',
+            ) from exc
+        return data
+
+    def _write(self, data: dict) -> None:
+        _atomic_write(self.manifest_path, json.dumps(data, indent=2).encode('utf-8'))
+
+    def snapshots(self, question_id: str) -> list[Snapshot]:
+        entry = self._read()['questions'].get(question_id, {})
+        return [Snapshot(**s) for s in entry.get('snapshots', [])]
+
+    def latest(self, question_id: str) -> Snapshot | None:
+        entry = self._read()['questions'].get(question_id)
+        if not entry:
+            return None
+        for s in entry['snapshots']:
+            if s['sha256'] == entry['latest']:
+                return Snapshot(**s)
+        return None
+
+    def all_latest(self) -> list[Snapshot]:
+        out = []
+        for qid in self._read()['questions']:
+            snap = self.latest(qid)
+            if snap:
+                out.append(snap)
+        return out
+
+    def read_bytes(self, snapshot: Snapshot) -> bytes:
+        path = self.snapshot_path(snapshot.sha256)
+        try:
+            content = path.read_bytes()
+        except FileNotFoundError:
+            content = b''
+        if hashlib.sha256(content).hexdigest() != snapshot.sha256:
+            raise FetchError(
+                f'The cached snapshot for {snapshot.question_id} is missing or damaged.',
+                'Run the command again with --refresh.',
+            )
+        return content
+
+    def store(self, snapshot: Snapshot, content: bytes) -> Snapshot:
+        """Save validated bytes and make them the latest snapshot for the question."""
+        path = self.snapshot_path(snapshot.sha256)
+        if not path.exists():
+            _atomic_write(path, content)
+        data = self._read()
+        entry = data['questions'].setdefault(snapshot.question_id, {'snapshots': []})
+        entry['snapshots'] = [s for s in entry['snapshots'] if s['sha256'] != snapshot.sha256]
+        entry['snapshots'].append(asdict(snapshot))
+        entry['latest'] = snapshot.sha256
+        self._write(data)
+        return snapshot
+
+    @staticmethod
+    def digest(content: bytes) -> str:
+        return hashlib.sha256(content).hexdigest()

betteroff/catalog.py

@@ -0,0 +1,159 @@
+"""Packaged catalog of supported tables. Reads only TOML, so it stays fast."""
+
+from __future__ import annotations
+
+import difflib
+import re
+import tomllib
+from dataclasses import dataclass, field
+from datetime import date
+from functools import cache
+from importlib import resources
+
+from .errors import InputError
+
+POLARITIES = ('positive', 'neutral', 'negative', 'nonresponse')
+
+
+@dataclass(frozen=True)
+class Response:
+    id: str
+    label: str
+    source_label: str
+    polarity: str
+
+
+@dataclass(frozen=True)
+class Break:
+    date: date
+    label: str
+    comparable: bool
+
+
+@dataclass(frozen=True)
+class Question:
+    id: str
+    table: str
+    title: str
+    wording: str
+    responses: tuple[Response, ...]
+    keywords: tuple[str, ...] = ()
+    notes: tuple[str, ...] = ()
+    breaks: tuple[Break, ...] = ()
+    monthly_since: date | None = None  # Only when later than the survey-wide date
+
+    def response(self, response_id: str) -> Response:
+        for r in self.responses:
+            if r.id == response_id:
+                return r
+        raise KeyError(response_id)
+
+    def to_dict(self) -> dict[str, object]:
+        return {
+            'id': self.id,
+            'table': self.table,
+            'title': self.title,
+            'wording': self.wording,
+            'responses': [r.__dict__ for r in self.responses],
+            'notes': list(self.notes),
+            'breaks': [
+                {'date': b.date.isoformat(), 'label': b.label, 'comparable': b.comparable}
+                for b in self.breaks
+            ],
+            'monthly_since': self.monthly_since.isoformat() if self.monthly_since else None,
+        }
+
+
+@dataclass(frozen=True)
+class Catalog:
+    landing_url: str
+    attribution: str
+    agreement_url: str
+    population: str
+    monthly_since: date
+    index_label: str
+    index_definition: str
+    notes: tuple[str, ...]
+    questions: tuple[Question, ...] = field(default_factory=tuple)
+
+    def resolve(self, identifier: str) -> Question:
+        key = identifier.strip().lower()
+        key = re.sub(r'^table\s*', '', key)
+        for q in self.questions:
+            if key in (q.id, q.table.lower()):
+                return q
+        ids = [q.id for q in self.questions]
+        close = difflib.get_close_matches(key, ids, n=3)
+        hint = f'Did you mean {", ".join(close)}?' if close else ''
+        raise InputError(
+            f'Unknown question or table "{identifier}". {hint}'.strip(),
+            'Run `betteroff list` or `betteroff search TEXT` to find an identifier.',
+        )
+
+    def search(self, text: str) -> list[Question]:
+        terms = [t for t in re.split(r'\W+', text.lower()) if t]
+        if not terms:
+            return list(self.questions)
+        scored = []
+        for q in self.questions:
+            fields = {
+                'id': q.id.replace('_', ' '),
+                'title': q.title.lower(),
+                'keywords': ' '.join(q.keywords).lower(),
+                'wording': q.wording.lower(),
+                'responses': ' '.join(r.label.lower() for r in q.responses),
+            }
+            weights = {'id': 3, 'title': 3, 'keywords': 2, 'wording': 1, 'responses': 1}
+            score = 0
+            for term in terms:
+                hit = 0
+                for name, value in fields.items():
+                    if re.search(rf'\b{re.escape(term)}', value):
+                        hit = max(hit, weights[name])
+                if term == q.table.lower():
+                    hit = max(hit, 3)
+                if not hit:
+                    break
+                score += hit
+            else:
+                scored.append((score, q))
+        scored.sort(key=lambda pair: -pair[0])
+        return [q for _, q in scored]
+
+
+def _question(raw: dict) -> Question:
+    responses = tuple(Response(**r) for r in raw['responses'])
+    for r in responses:
+        if r.polarity not in POLARITIES:
+            raise ValueError(f'{raw["id"]}: bad polarity {r.polarity!r}')
+    return Question(
+        id=raw['id'],
+        table=str(raw['table']),
+        title=raw['title'],
+        wording=raw['wording'],
+        responses=responses,
+        keywords=tuple(raw.get('keywords', ())),
+        notes=tuple(raw.get('notes', ())),
+        breaks=tuple(
+            Break(date.fromisoformat(b['date']), b['label'], bool(b['comparable']))
+            for b in raw.get('breaks', ())
+        ),
+        monthly_since=date.fromisoformat(raw['monthly_since']) if 'monthly_since' in raw else None,
+    )
+
+
+@cache
+def load_catalog() -> Catalog:
+    text = resources.files('betteroff').joinpath('data/catalog.toml').read_text('utf-8')
+    raw = tomllib.loads(text)
+    return Catalog(
+        landing_url=raw['landing_url'],
+        attribution=raw['attribution'],
+        agreement_url=raw['agreement_url'],
+        population=raw['population'],
+        monthly_since=date.fromisoformat(raw['monthly_since']),
+        index_label=raw['index_label'],
+        index_definition=raw['index_definition'],
+        notes=tuple(raw['notes']),
+        questions=tuple(_question(q) for q in raw['questions']),
+    )