lgd-query 0.1.0__py3-none-any.whl

lgd_query/__init__.py

@@ -0,0 +1,12 @@
+"""Public APIs for LGD release metadata, downloads and optional DuckDB access."""
+
+from .metadata import Asset, Dataset, Release, DEFAULT_METADATA_URL, load_manifest, load_release
+from .files import download
+from .query import connect
+from .errors import DatasetUnavailableError, IntegrityError, MetadataError, ObservationDateError
+
+__all__ = [
+    'Asset', 'Dataset', 'Release', 'DEFAULT_METADATA_URL', 'load_release', 'load_manifest',
+    'download', 'connect', 'DatasetUnavailableError', 'IntegrityError', 'MetadataError',
+    'ObservationDateError',
+]

lgd_query/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

lgd_query/_http.py

@@ -0,0 +1,66 @@
+"""Anonymous HTTP transport shared by metadata and file downloads."""
+
+import json
+import time
+from urllib.error import HTTPError, URLError
+from urllib.parse import urlsplit
+from urllib.request import Request, urlopen
+
+from .errors import MetadataError
+
+
+MAX_METADATA_BYTES = 8 * 1024 * 1024
+
+
+def http_url(value):
+    if not isinstance(value, str):
+        raise MetadataError('Expected an HTTP(S) URL')
+    try:
+        parts = urlsplit(value)
+        # Accessing port performs validation; urlsplit alone accepts bad ports.
+        parts.port
+        valid = (parts.scheme in ('https', 'http') and parts.hostname and
+                 parts.username is None and parts.password is None and not parts.fragment and
+                 not any(char.isspace() or ord(char) < 32 or ord(char) == 127 for char in value))
+    except ValueError as error:
+        raise MetadataError(f'Invalid public HTTP(S) URL: {value!r}') from error
+    if not valid:
+        raise MetadataError(f'Invalid public HTTP(S) URL: {value!r}')
+    return value
+
+
+def open_url(url, *, timeout=30, retries=2):
+    http_url(url)
+    if timeout <= 0 or type(retries) is not int or retries < 0:
+        raise ValueError('timeout must be positive and retries a nonnegative integer')
+    request = Request(url, headers={'User-Agent': 'lgd-query', 'Accept-Encoding': 'identity'})
+    for attempt in range(retries + 1):
+        try:
+            return urlopen(request, timeout=timeout)
+        except HTTPError as error:
+            # metadata.json can briefly disappear during release replacement.
+            error.close()
+            if error.code not in (404, 408, 429, 500, 502, 503, 504) or attempt == retries:
+                raise
+        except (URLError, TimeoutError):
+            if attempt == retries:
+                raise
+        time.sleep(0.5 * 2**attempt)
+
+
+def read_json(url, *, asset=None, timeout=30, retries=2):
+    if asset is not None and asset.size > MAX_METADATA_BYTES:
+        raise MetadataError('Manifest exceeds the metadata size limit')
+    with open_url(url, timeout=timeout, retries=retries) as response:
+        content = response.read(MAX_METADATA_BYTES + 1)
+    if len(content) > MAX_METADATA_BYTES:
+        raise MetadataError('Response exceeds the metadata size limit')
+    if asset is not None:
+        asset.verify(content)
+    try:
+        result = json.loads(content)
+    except (ValueError, UnicodeError) as error:
+        raise MetadataError(f'Invalid JSON metadata: {url}') from error
+    if not isinstance(result, dict):
+        raise MetadataError('Metadata must be a JSON object')
+    return result

lgd_query/cli.py

@@ -0,0 +1,151 @@
+"""Command-line wrappers around the three public reader APIs."""
+
+import argparse
+import json
+import math
+import sys
+
+from . import DEFAULT_METADATA_URL, connect, download, load_manifest, load_release
+
+
+def _json_value(value):
+    if isinstance(value, float) and not math.isfinite(value):
+        return 'NaN' if math.isnan(value) else ('Infinity' if value > 0 else '-Infinity')
+    if isinstance(value, (list, tuple)):
+        return [_json_value(item) for item in value]
+    if isinstance(value, dict):
+        if any(not isinstance(key, str) for key in value):
+            # Match DuckDB's representation for MAP keys that cannot be dict
+            # keys. JSON objects would stringify keys and can lose identity.
+            return {'key': [_json_value(key) for key in value],
+                    'value': [_json_value(item) for item in value.values()]}
+        return {key: _json_value(item) for key, item in value.items()}
+    return value
+
+
+def _print(value):
+    print(json.dumps(_json_value(value), ensure_ascii=False, default=str, allow_nan=False, indent=2))
+
+
+def _rich():
+    try:
+        from rich.console import Console
+        from rich.table import Column, Table
+        from rich.text import Text
+    except ImportError as error:
+        raise ImportError('Table output requires: pip install "lgd-query[cli]"; '
+                          'or use --format json') from error
+    return Console, Column, Table, Text
+
+
+def _table(columns, rows, *, title=None):
+    Console, Column, Table, Text = _rich()
+    table = Table(*(Column(Text(str(c))) for c in columns), title=Text(title) if title else None)
+    for row in rows:
+        table.add_row(*(Text('—' if value is None else str(value)) for value in row))
+    Console().print(table)
+
+
+def _query_output(cursor, format):
+    columns = [column[0] for column in cursor.description]
+    if format == 'table':
+        # Bound Rich's table buffer as well as DuckDB's fetched row batches.
+        batch = cursor.fetchmany(1000)
+        _table(columns, batch)
+        while batch := cursor.fetchmany(1000):
+            _table(columns, batch)
+    else:
+        # Column/value arrays retain repeated column names in joins.
+        sys.stdout.write('{"columns": ' + json.dumps(columns, ensure_ascii=False) + ', "rows": [')
+        separator = ''
+        while batch := cursor.fetchmany(1000):
+            for row in batch:
+                sys.stdout.write(separator + json.dumps(_json_value(row), ensure_ascii=False,
+                                                       default=str, allow_nan=False))
+                separator = ',\n'
+        sys.stdout.write(']}\n')
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description='Read public LGD Parquet release data')
+    parser.add_argument('--metadata-url', default=DEFAULT_METADATA_URL,
+                        help='Current metadata or an immutable saved-generation URL')
+    commands = parser.add_subparsers(dest='command', required=True)
+    listing = commands.add_parser('list', help='List datasets, dates and availability')
+    listing.add_argument('--available-only', action='store_true')
+    resolve = commands.add_parser('resolve', help='Resolve a dataset to its immutable file references')
+    resolve.add_argument('dataset')
+    resolve.add_argument('--manifest', action='store_true', help='Also fetch and verify schema/source metadata')
+    fetching = commands.add_parser('download', help='Download and verify selected Parquet files')
+    fetching.add_argument('datasets', nargs='+')
+    fetching.add_argument('--directory', required=True)
+    querying = commands.add_parser('query', help='Run SQL over selected datasets')
+    querying.add_argument('sql')
+    querying.add_argument('--datasets', nargs='+', required=True)
+    querying.add_argument('--cache-dir', help='Use verified local files instead of remote range reads')
+    querying.add_argument('--params', help='SQL parameters as a JSON array or object')
+    for command in (listing, resolve, fetching, querying):
+        command.add_argument('--format', choices=('table', 'json'), default='table',
+                             help='Output format (default: table, requires the cli extra)')
+    for command in (resolve, fetching, querying):
+        command.add_argument('--observed-on', help='Require this exact YYYY-MM-DD report date')
+    args = parser.parse_args(argv)
+    try:
+        if args.format == 'table':
+            _rich()  # Fail before metadata requests, downloads or SQL work.
+        release = load_release(args.metadata_url)
+        if args.command == 'list':
+            result = dict(generation=release.generation, published_at=release.published_at,
+                        immutable_url=release.immutable_url,
+                        datasets=[dict(name=d.name, available=d.available, observed_on=d.observed_on,
+                                       uploaded_at=d.uploaded_at, rows=d.rows,
+                                       representation=d.representation, status=d.status, reason=d.reason)
+                                  for d in release.datasets.values()
+                                  if not args.available_only or d.available])
+            if args.format == 'json':
+                _print(result)
+            else:
+                _table(['Dataset', 'Observed on', 'Rows', 'Status'],
+                       [(d['name'], d['observed_on'], d['rows'], d['status']) for d in result['datasets']],
+                       title=f'LGD · {release.generation}')
+        elif args.command == 'resolve':
+            dataset = release.dataset(args.dataset, observed_on=args.observed_on)
+            result = dict(generation=release.generation, immutable_url=release.immutable_url,
+                          dataset=dataset.name, entry=dataset.metadata)
+            if args.manifest:
+                result['manifest'] = load_manifest(dataset)
+            if args.format == 'json':
+                _print(result)
+            else:
+                _table(['Field', 'Value'], [
+                    ('Dataset', dataset.name), ('Representation', dataset.representation),
+                    ('Observed on', dataset.observed_on), ('Uploaded at', dataset.uploaded_at),
+                    ('Rows', dataset.rows), ('Outcome', dataset.status), ('Reason', dataset.reason),
+                    ('Parquet URL', dataset.parquet.url), ('Bytes', dataset.parquet.size),
+                    ('SHA-256', dataset.parquet.sha256), ('Manifest URL', dataset.manifest.url),
+                    ('Generation', release.generation), ('Index URL', release.immutable_url),
+                ])
+                if args.manifest:
+                    _table(['ID', 'Column', 'Type'],
+                           [(f['id'], f['name'], f['type']) for f in result['manifest']['schema']],
+                           title='Canonical schema')
+        else:
+            datasets = [release.dataset(name, observed_on=args.observed_on) for name in args.datasets]
+            if args.command == 'download':
+                result = [dict(dataset=d.name, observed_on=d.observed_on,
+                               path=str(download(d, args.directory))) for d in datasets]
+                if args.format == 'json':
+                    _print(result)
+                else:
+                    _table(['Dataset', 'Observed on', 'Verified file'],
+                           [(r['dataset'], r['observed_on'], r['path']) for r in result])
+            else:
+                params = json.loads(args.params) if args.params is not None else None
+                if params is not None and not isinstance(params, (list, dict)):
+                    raise ValueError('--params must be a JSON array or object')
+                with connect(datasets, cache_dir=args.cache_dir) as connection:
+                    cursor = connection.execute(args.sql, params)
+                    _query_output(cursor, args.format)
+    except Exception as error:
+        parser.exit(1, f'lgd-query: {error}\n')
+    return 0

lgd_query/errors.py

@@ -0,0 +1,17 @@
+"""Errors exposed by the reader APIs."""
+
+
+class MetadataError(ValueError):
+    """The discovery index or dataset manifest does not match its contract."""
+
+
+class IntegrityError(ValueError):
+    """Downloaded bytes differ from their advertised size or SHA-256."""
+
+
+class DatasetUnavailableError(LookupError):
+    """A known dataset has no published Parquet file."""
+
+
+class ObservationDateError(ValueError):
+    """The published report does not have the requested observation date."""

lgd_query/files.py

@@ -0,0 +1,60 @@
+"""Verified downloads, independent of DuckDB and metadata fetching."""
+
+from __future__ import annotations
+
+import hashlib
+import os
+from pathlib import Path
+import tempfile
+
+from ._http import open_url
+from .errors import IntegrityError
+from .metadata import Dataset
+
+
+def _digest(path):
+    digest = hashlib.sha256()
+    with path.open('rb') as source:
+        while chunk := source.read(1024 * 1024):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def download(dataset: Dataset, directory: str | os.PathLike, *,
+             timeout: float = 30, retries: int = 2) -> Path:
+    """Return a size/SHA-256 verified Parquet file in the requested directory.
+
+    Names contain the full content hash. Valid files are reused; corrupt cached
+    files are replaced only after a complete replacement passes verification.
+    Interrupted transfers never become final cache entries.
+    """
+    dataset.require_available()
+    asset = dataset.parquet
+    directory = Path(directory).expanduser().resolve()
+    directory.mkdir(parents=True, exist_ok=True)
+    destination = directory / f'{dataset.name}.{asset.sha256}.parquet'
+    if (destination.exists() and destination.stat().st_size == asset.size and
+            _digest(destination) == asset.sha256):
+        return destination
+    temporary = None
+    try:
+        with tempfile.NamedTemporaryFile(dir=directory, prefix=f'.{dataset.name}.',
+                                         suffix='.tmp', delete=False) as target:
+            temporary = Path(target.name)
+            digest, size = hashlib.sha256(), 0
+            with open_url(asset.url, timeout=timeout, retries=retries) as response:
+                while chunk := response.read(1024 * 1024):
+                    size += len(chunk)
+                    if size > asset.size:
+                        raise IntegrityError(f'{dataset.name}: download exceeds advertised size')
+                    digest.update(chunk)
+                    target.write(chunk)
+            if size != asset.size or digest.hexdigest() != asset.sha256:
+                raise IntegrityError(f'{dataset.name}: download differs from advertised size/SHA-256')
+            target.flush()
+            os.fsync(target.fileno())
+        temporary.replace(destination)
+        return destination
+    finally:
+        if temporary is not None:
+            temporary.unlink(missing_ok=True)

lgd_query/metadata.py

@@ -0,0 +1,215 @@
+"""Resolve the daily release index without importing DuckDB or downloader code."""
+
+from __future__ import annotations
+
+from copy import deepcopy
+from dataclasses import dataclass, field
+from datetime import date, datetime
+import hashlib
+import re
+from types import MappingProxyType
+from typing import Any, Mapping
+
+from ._http import http_url, read_json
+from .errors import DatasetUnavailableError, IntegrityError, MetadataError, ObservationDateError
+
+
+DEFAULT_METADATA_URL = ('https://github.com/ramSeraph/opendata/releases/download/'
+                        'lgd-parquet-latest/metadata.json')
+
+
+def _require(condition, message):
+    if not condition:
+        raise MetadataError(message)
+
+
+def _object(value, label):
+    _require(isinstance(value, dict), f'{label} must be an object')
+    return value
+
+
+def _text(value, label):
+    _require(isinstance(value, str) and bool(value), f'Missing/invalid {label}')
+    return value
+
+
+def _date(value):
+    try:
+        result = date.fromisoformat(value)
+        if result.isoformat() == value:
+            return result
+    except (TypeError, ValueError):
+        pass
+    raise MetadataError(f'Expected a YYYY-MM-DD observation date: {value!r}')
+
+
+def _timestamp(value):
+    try:
+        result = datetime.fromisoformat(value.replace('Z', '+00:00'))
+        if result.tzinfo is not None:
+            return result
+    except (AttributeError, TypeError, ValueError):
+        pass
+    raise MetadataError(f'Expected a timestamp with a timezone: {value!r}')
+
+
+def _contract(document):
+    _object(document, 'Metadata')
+    _require(document.get('kind') == 'lgd-daily-parquet' and
+             type(document.get('format_version')) is int and document['format_version'] == 1,
+             'Unsupported daily Parquet metadata kind/version')
+
+
+@dataclass(frozen=True)
+class Asset:
+    """An immutable file reference advertised by the release metadata."""
+
+    url: str
+    size: int
+    sha256: str
+
+    def __post_init__(self):
+        http_url(self.url)
+        _require(type(self.size) is int and self.size > 0, 'Invalid asset size')
+        _require(isinstance(self.sha256, str) and re.fullmatch('[a-f0-9]{64}', self.sha256),
+                 'Invalid asset SHA-256')
+
+    @classmethod
+    def from_dict(cls, value: dict) -> Asset:
+        value = _object(value, 'Asset')
+        return cls(value.get('url'), value.get('size'), value.get('sha256'))
+
+    def verify(self, content: bytes) -> None:
+        if len(content) != self.size or hashlib.sha256(content).hexdigest() != self.sha256:
+            raise IntegrityError(f'Content differs from advertised size/SHA-256: {self.url}')
+
+
+@dataclass(frozen=True)
+class Dataset:
+    """One dataset in a pinned index, including its actual date and outcome."""
+
+    name: str
+    status: str
+    reason: str | None
+    representation: str | None
+    observed_on: date | None
+    uploaded_at: datetime | None
+    rows: int | None
+    parquet: Asset | None
+    manifest: Asset | None
+    identity: str | None
+    _entry: dict = field(repr=False, compare=False)
+
+    @property
+    def available(self) -> bool:
+        """True even if a failed update retained an older, usable file."""
+        return self.parquet is not None
+
+    @property
+    def metadata(self) -> dict:
+        """A detached copy of the dataset's original index entry."""
+        return deepcopy(self._entry)
+
+    @property
+    def coverage(self) -> dict | None:
+        """Event date/collection coverage; snapshot reports normally have none."""
+        return deepcopy(self._entry.get('coverage'))
+
+    def require_available(self) -> Dataset:
+        if not self.available:
+            raise DatasetUnavailableError(f'{self.name}: {self.reason or self.status}')
+        return self
+
+    @classmethod
+    def _parse(cls, name, entry):
+        _require(isinstance(name, str) and re.fullmatch('[a-z][a-z0-9_]*', name),
+                 f'Invalid dataset name: {name!r}')
+        entry = deepcopy(_object(entry, f'{name} entry'))
+        outcome = _object(entry.get('outcome'), f'{name} outcome')
+        status = outcome.get('status')
+        _require(status in ('updated', 'unchanged', 'unavailable', 'blocked', 'failed'),
+                 f'{name}: unknown publication outcome')
+        reason = outcome.get('reason')
+        _require(reason is None or isinstance(reason, str), f'{name}: invalid failure reason')
+        if entry.get('daily') is None:
+            _require(entry.get('manifest') is None and status in ('unavailable', 'blocked', 'failed'),
+                     f'{name}: published dataset has no daily file')
+            return cls(name, status, reason, None, None, None, None, None, None, None, entry)
+        daily = _object(entry['daily'], f'{name} daily')
+        representation = entry.get('representation')
+        _require(representation in ('snapshot', 'events'), f'{name}: unknown representation')
+        observed = daily.get('observed_on')
+        if representation == 'snapshot':
+            observed = _date(observed)
+        else:
+            _require(observed is None, f'{name}: event rows must not invent an observation date')
+        rows = daily.get('rows')
+        _require(type(rows) is int and rows >= 0, f'{name}: invalid row count')
+        coverage = entry.get('coverage')
+        _require(coverage is None or isinstance(coverage, dict), f'{name}: invalid coverage')
+        return cls(name, status, reason, representation, observed, _timestamp(daily.get('uploaded_at')),
+                   rows, Asset.from_dict(daily.get('file')), Asset.from_dict(entry.get('manifest')),
+                   _text(entry.get('identity'), f'{name} identity'), entry)
+
+
+@dataclass(frozen=True)
+class Release:
+    """A single metadata generation; accessing it never refetches latest."""
+
+    generation: str
+    published_at: datetime
+    immutable_url: str
+    repository: str
+    datasets: Mapping[str, Dataset]
+    source_url: str | None
+    _document: dict = field(repr=False, compare=False)
+
+    @classmethod
+    def from_dict(cls, document: dict, *, source_url: str | None = None) -> Release:
+        """Resolve a previously saved JSON index without network requests."""
+        _contract(document)
+        document = deepcopy(document)
+        entries = _object(document.get('datasets'), 'datasets')
+        datasets = MappingProxyType({name: Dataset._parse(name, entry) for name, entry in entries.items()})
+        return cls(_text(document.get('generation'), 'generation'),
+                   _timestamp(document.get('published_at')), http_url(document.get('immutable_url')),
+                   _text(document.get('repository'), 'repository'), datasets, source_url, document)
+
+    @property
+    def metadata(self) -> dict:
+        """A detached JSON-serializable index, suitable for saving and reopening."""
+        return deepcopy(self._document)
+
+    def dataset(self, name: str, *, observed_on: date | str | None = None) -> Dataset:
+        """Resolve a usable dataset, optionally requiring an exact source date.
+
+        The date requirement never falls back to an older report and does not
+        perform a historical lookup. Cumulative events have no observation date.
+        """
+        try:
+            result = self.datasets[name].require_available()
+        except KeyError:
+            raise KeyError(f'Unknown dataset: {name}') from None
+        if observed_on is not None:
+            expected = observed_on if type(observed_on) is date else _date(observed_on)
+            if result.observed_on != expected:
+                raise ObservationDateError(f'{name}: requested {expected}, published observation '
+                                           f'is {result.observed_on} (outcome: {result.status})')
+        return result
+
+
+def load_release(url: str = DEFAULT_METADATA_URL, *, timeout: float = 30, retries: int = 2) -> Release:
+    """Fetch the current index once, or pin an explicitly supplied immutable URL."""
+    return Release.from_dict(read_json(url, timeout=timeout, retries=retries), source_url=url)
+
+
+def load_manifest(dataset: Dataset, *, timeout: float = 30, retries: int = 2) -> dict[str, Any]:
+    """Fetch schema/source details, verify bytes, and match them to the pinned index."""
+    dataset.require_available()
+    ref = dataset.manifest
+    manifest = read_json(ref.url, asset=ref, timeout=timeout, retries=retries)
+    _contract(manifest)
+    _require(manifest.get('dataset') == dataset.name, 'Manifest belongs to another dataset')
+    for key in ('identity', 'representation', 'daily', 'coverage'):
+        _require(manifest.get(key) == dataset._entry.get(key), f'Manifest differs from index: {key}')
+    return manifest

lgd_query/query.py

@@ -0,0 +1,45 @@
+"""DuckDB access layered on resolved datasets and optional verified downloads."""
+
+from __future__ import annotations
+
+import os
+from typing import TYPE_CHECKING, Iterable, Mapping
+
+from .files import download
+from .metadata import Dataset
+
+if TYPE_CHECKING:
+    from duckdb import DuckDBPyConnection
+
+
+def connect(datasets: Iterable[Dataset], *, cache_dir: str | os.PathLike | None = None,
+            config: Mapping | None = None, timeout: float = 30,
+            retries: int = 2) -> DuckDBPyConnection:
+    """Open an in-memory DuckDB connection with a view named for each dataset.
+
+    Without cache_dir, DuckDB reads immutable remote URLs using HTTP range
+    requests. With cache_dir, download() verifies complete local files first.
+    Close the connection or use it as a context manager. No index is refetched.
+    timeout/retries apply to downloads; DuckDB controls remote query transport.
+    """
+    datasets = tuple(datasets)
+    if not datasets or any(not isinstance(d, Dataset) for d in datasets):
+        raise ValueError('Pass one or more resolved Dataset objects')
+    if len({d.name for d in datasets}) != len(datasets):
+        raise ValueError('Dataset view names must be unique')
+    for dataset in datasets:
+        dataset.require_available()
+    try:
+        import duckdb
+    except ImportError as error:
+        raise ImportError('SQL access requires: pip install "lgd-query[duckdb]"') from error
+    connection = duckdb.connect(config=dict(config or {}))
+    try:
+        for dataset in datasets:
+            path = (str(download(dataset, cache_dir, timeout=timeout, retries=retries))
+                    if cache_dir is not None else dataset.parquet.url)
+            connection.read_parquet(path).create_view(dataset.name, replace=False)
+        return connection
+    except BaseException:
+        connection.close()
+        raise

lgd_query-0.1.0.dist-info/METADATA

@@ -0,0 +1,227 @@
+Metadata-Version: 2.4
+Name: lgd-query
+Version: 0.1.0
+Summary: Resolve, download and query public Local Government Directory Parquet releases
+Project-URL: Repository, https://github.com/ramSeraph/opendata
+Project-URL: Documentation, https://github.com/ramSeraph/opendata/tree/master/lgd-query
+Project-URL: Issues, https://github.com/ramSeraph/opendata/issues
+Keywords: india,lgd,local-government-directory,parquet,duckdb
+Classifier: Development Status :: 3 - Alpha
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Database
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Provides-Extra: duckdb
+Requires-Dist: duckdb>=2.0.0; extra == "duckdb"
+Provides-Extra: cli
+Requires-Dist: rich>=14; extra == "cli"
+
+# lgd-query
+
+A small Python client for the public [LGD Parquet releases](https://github.com/ramSeraph/opendata/releases/tag/lgd-parquet-latest).
+It has three public APIs: metadata resolution, verified downloading and DuckDB
+access. The CLI uses those same APIs. No GitHub account or API server is needed.
+
+This directory is a standalone distribution. Its wheel contains only
+`lgd_query`; it has no dependency on the repository's scrapers, converters or
+history code. Python 3.11+ is required, matching DuckDB 2's minimum. Metadata and downloads use the standard
+library; SQL access additionally requires DuckDB 2 or newer.
+Rich is an optional dependency for CLI tables.
+
+## Installation
+
+Until the first PyPI release, install from this directory:
+
+```sh
+pip install .                 # Metadata API, download API and their CLI commands
+pip install '.[duckdb]'       # Also include DuckDB querying
+pip install '.[cli,duckdb]'    # All APIs and Rich table output
+```
+
+The distribution name is `lgd-query`; the Python import is `lgd_query`.
+After publication, the corresponding installs are `pip install lgd-query` and
+`pip install 'lgd-query[duckdb]'`.
+
+## 1. Metadata API
+
+```python
+from lgd_query import load_release, load_manifest
+
+release = load_release()
+villages = release.dataset("villages")
+
+print(release.generation, release.published_at, release.immutable_url)
+print(villages.observed_on, villages.rows, villages.status)
+print(villages.parquet.url, villages.parquet.size, villages.parquet.sha256)
+
+# Optional extra request for canonical schema and original-source provenance.
+manifest = load_manifest(villages)
+print(manifest["schema"])
+```
+
+`load_release()` fetches `metadata.json` once. Its `Release` object retains that
+generation, and datasets refer directly to immutable files even if the latest
+index changes. Accessing `release.datasets` or `release.dataset()` makes no
+requests. New datasets can be discovered without updating a hardcoded registry.
+`load_manifest()` verifies the manifest's size/hash and agreement with the index.
+
+`release.datasets` is a read-only mapping including unavailable datasets. Each
+`Dataset` exposes `name`, `available`, `status`, `reason`, `representation`,
+`observed_on`, `uploaded_at`, `rows`, `parquet`, `manifest`, `identity`,
+`coverage` and `metadata`. `parquet` and `manifest` are `Asset` references with
+`url`, `size` and `sha256`. An unavailable dataset has no file references;
+`release.dataset(name)` raises `DatasetUnavailableError` for it. An unknown name
+raises `KeyError`.
+
+“Latest” means the files advertised in one release index. Datasets can have
+different source dates. A failed update can retain a usable older file, with its
+actual date and failure status. Require an exact date when freshness matters:
+
+```python
+villages = release.dataset("villages", observed_on="2026-10-23")
+```
+
+A mismatch raises `ObservationDateError`; this is a freshness check, not a
+historical lookup. Cumulative `changes` and `transactions` reports have
+`observed_on=None`; their `coverage` describes event dates and collection coverage.
+Publication/upload timestamps do not establish an observation date. These files
+contain supplied report rows; omissions do not prove deletions.
+
+To reopen a generation, pass its immutable URL to `load_release(url)`. For an
+offline metadata lookup, save `release.metadata` as JSON and reopen it with
+`Release.from_dict(document)`. `metadata` and `coverage` return detached copies.
+Metadata requests accept `timeout=30` and `retries=2`. Transient HTTP responses,
+including a brief 404 during index replacement, get bounded retries. The reader
+does not silently fall back to `metadata.previous.json`; that URL can be supplied
+explicitly if the preceding generation is wanted.
+
+## 2. Download API
+
+```python
+from lgd_query import download
+
+path = download(villages, directory="./data")
+```
+
+The returned `pathlib.Path` names a complete Parquet file whose size and SHA-256
+match the resolved metadata. Files are named `<dataset>.<sha256>.parquet`, so
+different contents coexist in the cache. Every reuse verifies the cached file.
+A corrupt cached file is replaced only after its replacement verifies. Downloads
+stream to unique temporary files and rename after verification; failures clean
+up temporary files and never expose partial final files. No cache pruning occurs.
+
+`download()` accepts `timeout=30` and `retries=2` for HTTP connection/response
+retries. A failed transfer or checksum check raises an error; a subsequent call
+can retry the download. It requires neither DuckDB nor another metadata request.
+
+## 3. DuckDB API
+
+```python
+from lgd_query import connect
+
+states = release.dataset("states")
+districts = release.dataset("districts")
+
+with connect([states, districts]) as db:
+    rows = db.sql("""
+        SELECT s.state_name, count(*) AS district_count
+        FROM districts d JOIN states s USING (state_code)
+        GROUP BY s.state_name ORDER BY s.state_name
+    """).fetchall()
+```
+
+`connect()` returns a native `duckdb.DuckDBPyConnection`, with one view per
+selected dataset using its canonical name. Close it or use a context manager.
+Only the requested datasets are opened. Resolve them from one `Release` to pin
+one publication generation; their actual observation dates can still differ.
+
+By default DuckDB queries the resolved remote Parquet URLs with HTTP range
+requests. Its `httpfs` extension may be installed automatically on first use.
+Range queries do not verify the checksum of the entire file. For fully verified
+files and reusable downloads:
+
+```python
+with connect([states, districts], cache_dir="./data") as db:
+    rows = db.sql("SELECT * FROM districts WHERE state_code = ?", params=["33"]).fetchall()
+```
+
+With valid cached files and a saved index, this path works offline. DuckDB
+settings can be supplied with `config={"threads": 2, "memory_limit": "512MB"}`.
+`timeout` and `retries` affect cache downloads; remote range-request settings
+are controlled by DuckDB. No domain-specific query helpers are included yet.
+
+## CLI
+
+```sh
+lgd-query list
+lgd-query list --available-only
+lgd-query list --format json
+lgd-query resolve villages
+lgd-query resolve villages --manifest --observed-on 2026-10-23
+lgd-query download states districts --directory ./data
+lgd-query query 'SELECT state_code, count(*) FROM districts GROUP BY state_code' --datasets districts
+lgd-query query 'SELECT * FROM districts WHERE state_code = ?' --datasets districts --params '["33"]' --cache-dir ./data
+lgd-query query 'SELECT count(*) FROM districts' --datasets districts --format json
+```
+
+All commands default to Rich tables. Use `--format json` for scriptable output;
+JSON mode does not require Rich. Resolved metadata tables summarize the dataset
+and optional schema; JSON retains all manifest/source details. SQL JSON results
+have `columns` and `rows` arrays, preserving duplicate column names. Both formats
+use batches of at most 1,000 query rows; tables repeat their headers per batch.
+Dates and other non-JSON scalar types are represented as strings. Non-finite
+SQL floats use the JSON strings `"NaN"`, `"Infinity"` and `"-Infinity"`, including
+inside lists and structs; SQL NULL remains JSON null. Errors exit
+nonzero with a diagnostic.
+Maps with non-string keys use parallel `key` and `value` arrays, matching
+DuckDB's representation for complex map keys, so JSON object keys cannot
+silently change their meaning. Maps with string keys remain JSON objects.
+`python -m lgd_query` is equivalent to the installed `lgd-query` command.
+
+Pass `--metadata-url URL` before the command to select an immutable generation
+or mirror. `resolve`, `download` and `query` accept `--observed-on YYYY-MM-DD` to
+require the exact report date. Every command fetches discovery metadata only once.
+
+## Development and packaging
+
+```sh
+pip install -e '.[cli,duckdb]'
+python -m unittest discover -s tests -v
+uv build --out-dir dist
+```
+
+Build from this directory. Package discovery is restricted to `src/lgd_query`;
+the rest of the repository is outside the distribution. Tests use synthetic
+release metadata and small generated Parquet files. Tests that require DuckDB
+are skipped when the optional dependency is absent.
+
+## Version releases
+
+The repository workflow is `.github/workflows/lgd-query-publish.yml`, matching
+the PyPI trusted-publisher configuration. It uses the `pypi` GitHub environment
+and [OIDC trusted publishing](https://docs.pypi.org/trusted-publishers/using-a-publisher/).
+
+A pushed `lgd-query-v<version>` tag builds this directory, checks that the tag
+matches `pyproject.toml`, validates the distribution contents and PyPI metadata,
+and tests the built wheel on Python 3.11 and 3.14. Only after these checks pass
+does it upload the same wheel/source archive to PyPI and create a corresponding
+GitHub release with those files attached. No scraper or history files are bundled.
+
+For example, after committing version `0.1.0`:
+
+```sh
+git tag lgd-query-v0.1.0
+git push origin lgd-query-v0.1.0
+```
+
+Manual workflow runs on a branch only build/test. A manual run on a matching
+version tag also publishes. Failed publication jobs can be retried using the
+same built artifacts. GitHub release titles and tags use the library prefix to
+keep package versions separate from the data releases.
+
+Before a retry uploads files, the workflow compares any existing PyPI files with
+the checked local sizes and SHA-256 hashes. Different bytes under the same version
+stop publication. Before creating/updating the GitHub release, it verifies that
+both files match PyPI. If PyPI's metadata has not caught up yet, retry that job;
+it will not attach unconfirmed files. For a partial upload, rerun failed jobs
+with the original build artifacts instead of rebuilding the same version.

lgd_query-0.1.0.dist-info/RECORD

@@ -0,0 +1,14 @@
+lgd_query/__init__.py,sha256=BUuJVmgvI9NAoBZv6sJe_4zffiHkduyctMnFumBhQFo,556
+lgd_query/__main__.py,sha256=k1ocEWawweo1qCJWNFAAvyxz3tcY13dzvCenHszij30,48
+lgd_query/_http.py,sha256=4NvzmNU_FJETVG3yd0v-dc6UxNlsTSFaE7wy0cojNvA,2606
+lgd_query/cli.py,sha256=T3c0Gv8qIbzUIVMCdRcZ4An0hEPDajUHMNAjTquIRLQ,7964
+lgd_query/errors.py,sha256=gxldJyzdPw2Ivq43pzCr54nX-puM43jEN-obMtQQ_T8,486
+lgd_query/files.py,sha256=_3rtK8k8auOuWqSXnAPkRBvw4rV5VmSSa7dug-JJYhA,2378
+lgd_query/metadata.py,sha256=S-PJF83CbfF9NHWx8mVHImCPOd8_s-hbDGh1G0PZOgc,8644
+lgd_query/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+lgd_query/query.py,sha256=OQgk_kS5VAtGUgC9LVogAeY4ZAunPbNxfjr2-NMU2aw,1861
+lgd_query-0.1.0.dist-info/METADATA,sha256=lCUhp_W6xFtwvnciQmK6TK8r1mfTLc44pEUm8U3T5uA,10500
+lgd_query-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+lgd_query-0.1.0.dist-info/entry_points.txt,sha256=ixtw2oTwmENJLdp0OU729SA-HYuUH3bstnFoUVVJNP4,49
+lgd_query-0.1.0.dist-info/top_level.txt,sha256=HLr6qVbOD7ik5OIykqClTdCGTXgjI8XnqqVQ1P8_J3Y,10
+lgd_query-0.1.0.dist-info/RECORD,,

lgd_query-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.2.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

lgd_query-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+lgd-query = lgd_query.cli:main

lgd_query-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+lgd_query