quantilica-core 0.3.1__py3-none-any.whl

quantilica/core/__init__.py

@@ -0,0 +1,10 @@
+"""Common foundation utilities for Quantilica data projects."""
+
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("quantilica-core")
+except PackageNotFoundError:
+    __version__ = "0.0.0"
+
+__all__ = ["__version__"]

quantilica/core/cache.py

@@ -0,0 +1,124 @@
+"""File-based cache helpers for deterministic data downloads."""
+
+from __future__ import annotations
+
+import json
+import os
+from collections.abc import Mapping
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import Any
+from urllib.parse import urlencode
+
+from .dates import isoformat_utc, parse_iso_datetime, utc_now
+from .exceptions import StorageError
+from .files import ensure_dir, sha256_bytes, write_bytes_atomic, write_text_atomic
+
+
+@dataclass(frozen=True)
+class CacheEntry:
+    """Metadata for one cached object."""
+
+    key: str
+    path: str
+    created_at: str
+    size_bytes: int
+    sha256: str
+    metadata: dict[str, Any]
+
+
+class FileCache:
+    """Simple content cache stored on the local filesystem."""
+
+    def __init__(self, root: str | os.PathLike[str]) -> None:
+        self.root = ensure_dir(root)
+
+    def key_for_url(
+        self,
+        url: str,
+        *,
+        params: Mapping[str, Any] | None = None,
+        headers: Mapping[str, str] | None = None,
+    ) -> str:
+        """Return a deterministic key for a URL request."""
+        parts = [url]
+        if params:
+            parts.append(urlencode(sorted(params.items()), doseq=True))
+        if headers:
+            normalized_headers = {
+                key.lower(): value for key, value in sorted(headers.items())
+            }
+            parts.append(json.dumps(normalized_headers, sort_keys=True))
+        return sha256_bytes("\n".join(parts).encode("utf-8"))
+
+    def content_path(self, key: str) -> Path:
+        """Return the content path for a cache key."""
+        return self.root / key[:2] / key[2:]
+
+    def metadata_path(self, key: str) -> Path:
+        """Return the metadata path for a cache key."""
+        return self.root / key[:2] / f"{key[2:]}.json"
+
+    def exists(self, key: str) -> bool:
+        """Return True when content and metadata exist for a key."""
+        return self.content_path(key).exists() and self.metadata_path(key).exists()
+
+    def read_bytes(self, key: str) -> bytes:
+        """Read cached bytes."""
+        path = self.content_path(key)
+        try:
+            return path.read_bytes()
+        except OSError as exc:
+            raise StorageError(f"Could not read cache entry: {key}") from exc
+
+    def read_metadata(self, key: str) -> CacheEntry:
+        """Read cache metadata."""
+        path = self.metadata_path(key)
+        try:
+            payload = json.loads(path.read_text(encoding="utf-8"))
+        except (OSError, json.JSONDecodeError) as exc:
+            raise StorageError(f"Could not read cache metadata: {key}") from exc
+        return CacheEntry(**payload)
+
+    def write_bytes(
+        self,
+        key: str,
+        content: bytes,
+        *,
+        metadata: Mapping[str, Any] | None = None,
+    ) -> CacheEntry:
+        """Write content and metadata for a cache key."""
+        content_path = self.content_path(key)
+        metadata_path = self.metadata_path(key)
+        write_bytes_atomic(content_path, content)
+
+        entry = CacheEntry(
+            key=key,
+            path=str(content_path.relative_to(self.root)),
+            created_at=isoformat_utc(),
+            size_bytes=len(content),
+            sha256=sha256_bytes(content),
+            metadata=dict(metadata or {}),
+        )
+        write_text_atomic(
+            metadata_path,
+            json.dumps(asdict(entry), ensure_ascii=False, indent=2, sort_keys=True),
+        )
+        return entry
+
+    def is_fresh(self, key: str, *, ttl_seconds: int | None = None) -> bool:
+        """Return True if a cache entry exists and is within TTL."""
+        if not self.exists(key):
+            return False
+        if ttl_seconds is None:
+            return True
+        entry = self.read_metadata(key)
+        created_at = parse_iso_datetime(entry.created_at)
+        age = (utc_now() - created_at).total_seconds()
+        return age <= ttl_seconds
+
+    def get_bytes(self, key: str, *, ttl_seconds: int | None = None) -> bytes | None:
+        """Return cached bytes when present and fresh, otherwise None."""
+        if not self.is_fresh(key, ttl_seconds=ttl_seconds):
+            return None
+        return self.read_bytes(key)

quantilica/core/cli.py

@@ -0,0 +1,109 @@
+"""Shared Rich-based helpers for quantilica fetcher CLI plugins.
+
+These helpers are host-only: ``plugin.py`` modules run inside the
+``quantilica-cli`` host, which pulls in ``rich`` via the ``cli`` extra
+(``quantilica-core[cli]``). Standalone argparse CLIs should keep using
+:func:`quantilica.core.logging.configure_cli_logging` instead.
+"""
+
+from __future__ import annotations
+
+import logging
+
+try:
+    from rich.console import Console
+    from rich.logging import RichHandler
+    from rich.progress import (
+        BarColumn,
+        DownloadColumn,
+        MofNCompleteColumn,
+        Progress,
+        SpinnerColumn,
+        TextColumn,
+        TimeElapsedColumn,
+        TimeRemainingColumn,
+        TransferSpeedColumn,
+    )
+except ModuleNotFoundError as exc:  # pragma: no cover
+    raise ModuleNotFoundError(
+        "quantilica.core.cli requires the 'cli' extra; install quantilica-core[cli]"
+    ) from exc
+
+
+_console: Console | None = None
+
+
+def get_console() -> Console:
+    """Return a process-wide shared Rich console."""
+    global _console
+    if _console is None:
+        _console = Console()
+    return _console
+
+
+def setup_rich_logging(
+    verbose: bool,
+    *,
+    console: Console | None = None,
+) -> None:
+    """Configure logging via ``RichHandler`` without breaking progress bars.
+
+    ``verbose=False`` → WARNING only; ``verbose=True`` → DEBUG.
+    """
+    level = logging.DEBUG if verbose else logging.WARNING
+    logging.basicConfig(
+        level=level,
+        format="%(message)s",
+        datefmt="[%X]",
+        handlers=[RichHandler(console=console or get_console(), show_path=False)],
+        force=True,
+    )
+
+
+def make_batch_progress(console: Console | None = None) -> Progress:
+    """Build a Progress for overall/batch tracking (file counts)."""
+    return Progress(
+        SpinnerColumn(),
+        TextColumn("[progress.description]{task.description}"),
+        BarColumn(),
+        MofNCompleteColumn(),
+        TimeElapsedColumn(),
+        TimeRemainingColumn(),
+        console=console or get_console(),
+    )
+
+
+def make_download_progress(console: Console | None = None) -> Progress:
+    """Build a Progress for individual file downloads (bytes/speed)."""
+    return Progress(
+        SpinnerColumn(),
+        TextColumn("[dim]{task.description}[/dim]"),
+        BarColumn(),
+        DownloadColumn(),
+        TransferSpeedColumn(),
+        TimeRemainingColumn(),
+        console=console or get_console(),
+    )
+
+
+def expand_years_cli(
+    years: list[str] | None,
+    default_range: str | None = None,
+    console: Console | None = None,
+) -> list[int]:
+    """Expand CLI year/range arguments (e.g. ``["2020:2022", "2024"]``).
+
+    If ``years`` is empty and ``default_range`` is provided, it expands the
+    default range. Prints a warning to the console/stderr for any invalid specs.
+    """
+    from quantilica.core.dates import expand_year_range
+
+    con = console or get_console()
+    specs = years if years else ([default_range] if default_range else [])
+    result: list[int] = []
+    for arg in specs:
+        try:
+            result.extend(expand_year_range(arg))
+        except ValueError:
+            con.print(f"[yellow]Aviso:[/yellow] ano/intervalo inválido '{arg}'")
+    return result

quantilica/core/config.py

@@ -0,0 +1,97 @@
+"""Minimal configuration helpers for Quantilica projects."""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Mapping
+from dataclasses import dataclass
+from pathlib import Path
+
+from .exceptions import ConfigError
+
+
+@dataclass(frozen=True)
+class EnvSettings:
+    """Read settings from an environment mapping using an optional prefix."""
+
+    prefix: str = ""
+    environ: Mapping[str, str] | None = None
+
+    def _source(self) -> Mapping[str, str]:
+        return os.environ if self.environ is None else self.environ
+
+    def _key(self, name: str) -> str:
+        return f"{self.prefix}{name}".upper()
+
+    def get(self, name: str, default: str | None = None) -> str | None:
+        """Return a setting value or a default."""
+        return self._source().get(self._key(name), default)
+
+    def require(self, name: str) -> str:
+        """Return a setting value or raise ConfigError."""
+        value = self.get(name)
+        if value is None or value == "":
+            raise ConfigError(f"Missing required setting: {self._key(name)}")
+        return value
+
+    def get_bool(self, name: str, default: bool = False) -> bool:
+        """Return a boolean setting."""
+        value = self.get(name)
+        if value is None:
+            return default
+        normalized = value.strip().lower()
+        if normalized in {"1", "true", "yes", "y", "on"}:
+            return True
+        if normalized in {"0", "false", "no", "n", "off"}:
+            return False
+        raise ConfigError(f"Invalid boolean setting: {self._key(name)}={value!r}")
+
+    def get_int(self, name: str, default: int | None = None) -> int | None:
+        """Return an integer setting."""
+        value = self.get(name)
+        if value is None:
+            return default
+        try:
+            return int(value)
+        except ValueError as exc:
+            message = f"Invalid integer setting: {self._key(name)}={value!r}"
+            raise ConfigError(message) from exc
+
+    def path(self, name: str, default: str | Path | None = None) -> Path | None:
+        """Return a setting as an expanded Path."""
+        value = self.get(name)
+        if value is None:
+            if default is None:
+                return None
+            return Path(default).expanduser()
+        return Path(value).expanduser()
+
+
+def load_dotenv(
+    path: str | os.PathLike[str] = ".env",
+    *,
+    override: bool = False,
+) -> int:
+    """Load a simple dotenv file without adding a runtime dependency.
+
+    This parser intentionally supports only ``KEY=VALUE`` lines, comments, and
+    optional single or double quotes. It returns the number of variables loaded.
+    """
+    dotenv_path = Path(path).expanduser()
+    if not dotenv_path.exists():
+        return 0
+
+    loaded = 0
+    for raw_line in dotenv_path.read_text(encoding="utf-8").splitlines():
+        line = raw_line.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        key, value = line.split("=", 1)
+        key = key.strip()
+        value = value.strip().strip("'\"")
+        if not key:
+            continue
+        if override or key not in os.environ:
+            os.environ[key] = value
+            loaded += 1
+    return loaded

quantilica/core/dates.py

@@ -0,0 +1,61 @@
+"""Datetime helpers with explicit UTC behavior."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+
+
+def utc_now() -> datetime:
+    """Return the current timezone-aware UTC datetime."""
+    return datetime.now(UTC)
+
+
+def to_utc(value: datetime) -> datetime:
+    """Convert a datetime to timezone-aware UTC.
+
+    Naive datetimes are treated as UTC to avoid silently applying a local
+    machine timezone during ingestion jobs.
+    """
+    if value.tzinfo is None:
+        return value.replace(tzinfo=UTC)
+    return value.astimezone(UTC)
+
+
+def isoformat_utc(value: datetime | None = None) -> str:
+    """Return an ISO 8601 UTC timestamp using a trailing ``Z``."""
+    current = utc_now() if value is None else to_utc(value)
+    return current.isoformat(timespec="seconds").replace("+00:00", "Z")
+
+
+def parse_iso_datetime(value: str) -> datetime:
+    """Parse an ISO 8601 datetime and normalize it to UTC."""
+    normalized = value.replace("Z", "+00:00")
+    return to_utc(datetime.fromisoformat(normalized))
+
+
+def expand_year_range(*specs: str | int) -> list[int]:
+    """Expand year specs into a list of ints, preserving order.
+
+    Each spec is either a single year (``"2020"``) or an inclusive range
+    (``"2020:2025"``). Ranges may be descending (``"2025:2020"``). Raises
+    ``ValueError`` on malformed specs.
+    """
+    years: list[int] = []
+    for spec in specs:
+        text = str(spec).strip()
+        if ":" in text:
+            start_str, end_str = text.split(":", 1)
+            start, end = int(start_str), int(end_str)
+            step = 1 if start <= end else -1
+            years.extend(range(start, end + step, step))
+        else:
+            years.append(int(text))
+    return years
+
+
+def year_month_partition(year: int | str, month: int | str | None = None) -> str:
+    """Return a partition string: ``"YYYY"`` or ``"YYYYMM"``."""
+    y = int(year)
+    if month is None:
+        return f"{y:04d}"
+    return f"{y:04d}{int(month):02d}"

quantilica/core/exceptions.py

@@ -0,0 +1,29 @@
+"""Exception hierarchy shared by Quantilica projects."""
+
+
+class QuantilicaError(Exception):
+    """Base class for all domain-neutral Quantilica errors."""
+
+
+class ConfigError(QuantilicaError):
+    """Raised when configuration is missing or invalid."""
+
+
+class FetchError(QuantilicaError):
+    """Raised when remote data cannot be fetched."""
+
+
+class ParseError(QuantilicaError):
+    """Raised when input data cannot be parsed."""
+
+
+class StorageError(QuantilicaError):
+    """Raised when object or file storage operations fail."""
+
+
+class MetadataError(QuantilicaError):
+    """Raised when generic metadata is invalid or incomplete."""
+
+
+class ValidationError(QuantilicaError):
+    """Raised when a value fails validation."""

quantilica/core/fetcher.py

@@ -0,0 +1,157 @@
+"""Concurrent download orchestration for data fetcher packages."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from typing import Any
+
+from tqdm import tqdm
+
+from .http import AsyncHttpClient
+from .logging import get_logger
+from .storage import StampedDataRepository
+
+_logger = get_logger(__name__)
+
+
+@dataclass
+class RemoteResource:
+    """Description of a downloadable remote resource.
+
+    ``filename`` must be pre-computed by the caller (e.g. via ``slugify`` +
+    ``stamp_filename``) so the orchestrator can determine the local destination
+    and skip-check slug without knowing the source-specific naming rules.
+    """
+
+    name: str
+    url: str
+    filename: str
+    size: int = 0
+    format: str = ""
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+async def _download_one(
+    resource: RemoteResource,
+    repo: StampedDataRepository,
+    dataset_id: str,
+    client: AsyncHttpClient,
+    semaphore: asyncio.Semaphore,
+    *,
+    source_id: str,
+    producer: str,
+    ext: str,
+    logger: logging.Logger,
+    show_progress: bool,
+    on_file_done: Callable[[dict | None], None] | None,
+) -> dict | None:
+    dest = repo.dataset_path(dataset_id, resource.filename)
+
+    if resource.size > 0:
+        slug = resource.filename.partition("@")[0]
+        latest = repo.get_latest_stamped_file(dataset_id, slug, ext)
+        if latest is not None and latest.stat().st_size == resource.size:
+            logger.debug(
+                f"Skipping {resource.filename}: matching local copy {latest.name}"
+            )
+            if on_file_done:
+                on_file_done(None)
+            return None
+
+    pbar = None
+    _on_progress = None
+    if show_progress:
+        pbar = tqdm(
+            total=resource.size or None,
+            unit="B",
+            unit_scale=True,
+            desc=f"Downloading {resource.filename[:30]}...",
+            leave=False,
+        )
+        last_seen = 0
+
+        def _on_progress(downloaded: int, total: int) -> None:
+            nonlocal last_seen
+            if total and pbar.total != total:
+                pbar.total = total
+            pbar.update(downloaded - last_seen)
+            last_seen = downloaded
+
+    try:
+        async with semaphore:
+            await client.download_with_manifest(
+                resource.url,
+                dest,
+                source_id=source_id,
+                dataset_id=dataset_id,
+                producer=producer,
+                params=None,
+                progress=_on_progress,
+            )
+    except Exception as exc:
+        logger.error(f"Failed to download {resource.url}: {exc}")
+        if dest.exists():
+            try:
+                dest.unlink()
+            except OSError:
+                pass
+        if on_file_done:
+            on_file_done(None)
+        return None
+    finally:
+        if pbar is not None:
+            pbar.close()
+
+    result = {
+        "url": resource.url,
+        "filename": resource.filename,
+        "destination": dest,
+        "file_size": dest.stat().st_size,
+    }
+    if on_file_done:
+        on_file_done(result)
+    return result
+
+
+async def download_resources(
+    resources: list[RemoteResource],
+    repo: StampedDataRepository,
+    dataset_id: str,
+    client: AsyncHttpClient,
+    *,
+    source_id: str,
+    producer: str = "",
+    ext: str = "csv",
+    max_concurrency: int = 3,
+    logger: logging.Logger | None = None,
+    show_progress: bool = False,
+    on_file_done: Callable[[dict | None], None] | None = None,
+) -> list[dict]:
+    """Download resources concurrently; skip when local file size matches remote.
+
+    Returns a list of dicts with keys ``url``, ``filename``, ``destination``,
+    and ``file_size`` for each successfully downloaded resource.
+    """
+    _log = logger or _logger
+    semaphore = asyncio.Semaphore(max_concurrency)
+    tasks = [
+        _download_one(
+            resource,
+            repo,
+            dataset_id,
+            client,
+            semaphore,
+            source_id=source_id,
+            producer=producer,
+            ext=ext,
+            logger=_log,
+            show_progress=show_progress,
+            on_file_done=on_file_done,
+        )
+        for resource in resources
+    ]
+    results = await asyncio.gather(*tasks)
+    return [r for r in results if r is not None]