arcobi-sdk 0.1.0b1__py3-none-any.whl

arcobi_sdk/__init__.py

@@ -0,0 +1,88 @@
+"""arcobi_sdk: fetch, align and export energy-market time-series data.
+
+The default install contains the Datahub client and extraction helpers (fetch
+streams, align them on a time grid), time-zone conversion, export helpers
+(DataFrame/CSV/Parquet/JSON) and the credential-pool *client*. Heavier pieces are
+extras: ``arcobi-sdk[parquet]`` and ``[pool-server]`` (the credential-pool service).
+"""
+
+from arcobi_sdk.config import load_env
+from arcobi_sdk.datahub import (
+    DatahubClient,
+    DatasetSpec,
+    ResolutionPlan,
+    StreamSpec,
+    build_time_grid,
+    fetch_and_align,
+    fetch_streams,
+    interpolate_gaps,
+    plan_extraction,
+)
+from arcobi_sdk.errors import (
+    ArcobiError,
+    ConfigurationError,
+    DatahubAuthError,
+    DatahubError,
+    ExtractionError,
+    LeaseConflictError,
+    LeaseReclaimedError,
+    PoolError,
+    PoolExhaustedError,
+    PoolUnavailableError,
+    RateLimitedError,
+    StreamNotFoundError,
+    TokenExpiredError,
+)
+from arcobi_sdk.io import (
+    create_timestamp_directory,
+    find_latest_data_file,
+    save_json,
+    to_csv,
+    to_json,
+    to_parquet,
+    track_runtime,
+)
+from arcobi_sdk.pool import PoolClient, TokenLease, lease_token
+from arcobi_sdk.timeutils import convert_local_to_utc, format_timestamp_folder, timestamp_name
+
+__version__ = "0.1.0b1"
+
+__all__ = [
+    "ArcobiError",
+    "ConfigurationError",
+    "DatahubAuthError",
+    "DatahubClient",
+    "DatahubError",
+    "DatasetSpec",
+    "ExtractionError",
+    "LeaseConflictError",
+    "LeaseReclaimedError",
+    "PoolClient",
+    "PoolError",
+    "PoolExhaustedError",
+    "PoolUnavailableError",
+    "RateLimitedError",
+    "ResolutionPlan",
+    "StreamNotFoundError",
+    "StreamSpec",
+    "TokenExpiredError",
+    "TokenLease",
+    "__version__",
+    "build_time_grid",
+    "convert_local_to_utc",
+    "create_timestamp_directory",
+    "fetch_and_align",
+    "fetch_streams",
+    "find_latest_data_file",
+    "format_timestamp_folder",
+    "interpolate_gaps",
+    "lease_token",
+    "load_env",
+    "plan_extraction",
+    "save_json",
+    "timestamp_name",
+    "to_csv",
+    "to_json",
+    "to_parquet",
+    "track_runtime",
+]

arcobi_sdk/cli.py

@@ -0,0 +1,395 @@
+"""Command-line interface: ``arcobi-sdk fetch | check | version``.
+
+The CLI exists so that tools without a Python runtime of their own (Excel/VBA macros,
+PowerShell, schedulers) can pull Datahub data through the SDK. Its contract is
+designed for ``Shell``/``WScript.Shell`` consumers:
+
+* exit codes: ``0`` success, ``2`` bad arguments or configuration, ``3`` vendor or
+  authentication error, ``4`` the request succeeded but returned no data;
+* every failure is reported on **stderr** as exactly one line starting with
+  ``error:``; warnings from the SDK's loggers may precede it;
+* ``--out -`` streams the result to **stdout** (CSV by default, or
+  ``--format json-lines``), otherwise the format follows the ``--out`` extension and
+  stdout carries one ``ok:`` summary line (suppressed by ``--quiet``).
+
+Credentials are never taken on the command line: set ``DATAHUB_USERNAME`` /
+``DATAHUB_PASSWORD`` in the environment or in a ``.env`` file (``--env-file``,
+default ``./.env``). Only stdlib modules are used besides the SDK itself.
+"""
+
+from __future__ import annotations
+
+import argparse
+import logging
+import re
+import sys
+from collections.abc import Callable, Sequence
+from datetime import datetime, timedelta
+from pathlib import Path
+
+import pandas as pd
+
+from arcobi_sdk import __version__
+from arcobi_sdk.config import load_env
+from arcobi_sdk.datahub import DatahubClient, fetch_streams
+from arcobi_sdk.errors import ArcobiError, ConfigurationError, DatahubError, ExtractionError
+from arcobi_sdk.io import to_csv, to_json, to_parquet
+
+logger = logging.getLogger(__name__)
+
+EXIT_OK = 0
+EXIT_USAGE = 2
+EXIT_VENDOR = 3
+EXIT_NO_DATA = 4
+
+PROG = "arcobi-sdk"
+INDEX_LABEL = "timestamp"
+DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
+FORMATS: tuple[str, ...] = ("csv", "json", "json-lines", "parquet")
+_SUFFIX_FORMATS: dict[str, str] = {
+    ".csv": "csv",
+    ".txt": "csv",
+    ".json": "json",
+    ".jsonl": "json-lines",
+    ".ndjson": "json-lines",
+    ".parquet": "parquet",
+    ".pq": "parquet",
+}
+_LIST_SEPARATOR = re.compile(r"[,;\s]+")
+_NO_ROWS_MARKER = "returned no rows"
+
+
+class UsageError(Exception):
+    """Invalid command-line arguments; mapped to exit code 2."""
+
+
+class _Parser(argparse.ArgumentParser):
+    """ArgumentParser that raises :class:`UsageError` instead of exiting."""
+
+    def error(self, message: str) -> None:  # type: ignore[override]
+        raise UsageError(message)
+
+
+def parse_timestamp(text: str) -> datetime:
+    """Parse a CLI timestamp into a naive :class:`datetime`.
+
+    Accepts ISO 8601 (``2024-01-08``, ``2024-01-08 06:30``, ``2024-01-08T06:30:00``)
+    and the keywords ``now``, ``today``, ``yesterday`` and ``tomorrow``. Time-zone
+    offsets are dropped: Datahub interprets timestamps as market-local wall time.
+
+    Args:
+        text: The raw argument.
+
+    Returns:
+        A naive datetime.
+
+    Raises:
+        UsageError: If the text is not a recognised timestamp.
+    """
+    value = text.strip()
+    keyword = value.lower()
+    now = datetime.now()
+    midnight = now.replace(hour=0, minute=0, second=0, microsecond=0)
+    if keyword == "now":
+        return now.replace(microsecond=0)
+    if keyword == "today":
+        return midnight
+    if keyword == "yesterday":
+        return midnight - timedelta(days=1)
+    if keyword == "tomorrow":
+        return midnight + timedelta(days=1)
+    try:
+        parsed = datetime.fromisoformat(value)
+    except ValueError:
+        raise UsageError(
+            f"invalid timestamp {text!r}; use YYYY-MM-DD[ HH:MM[:SS]], "
+            "today, yesterday, tomorrow or now"
+        ) from None
+    return parsed.replace(tzinfo=None)
+
+
+def parse_stream_ids(text: str) -> list[int]:
+    """Parse ``"1001,1002"`` (commas, semicolons or spaces) into positive ints.
+
+    Raises:
+        UsageError: On an empty list or a non-integer item.
+    """
+    items = [item for item in _LIST_SEPARATOR.split(text.strip()) if item]
+    if not items:
+        raise UsageError("--streams needs at least one stream id")
+    ids: list[int] = []
+    for item in items:
+        try:
+            value = int(item)
+        except ValueError:
+            raise UsageError(f"invalid stream id {item!r}; expected an integer") from None
+        if value <= 0:
+            raise UsageError(f"invalid stream id {value}; expected a positive integer")
+        ids.append(value)
+    return ids
+
+
+def parse_names(text: str | None, count: int) -> list[str] | None:
+    """Parse ``--names a,b`` and check it matches the number of streams.
+
+    Raises:
+        UsageError: If the counts differ or a name is blank.
+    """
+    if text is None or not text.strip():
+        return None
+    names = [name.strip() for name in text.split(",")]
+    if any(not name for name in names):
+        raise UsageError("--names contains an empty name")
+    if len(names) != count:
+        raise UsageError(f"--names has {len(names)} items but --streams has {count}")
+    return names
+
+
+def resolve_format(out: str, explicit: str | None) -> str:
+    """Pick the output format from ``--format`` or the ``--out`` extension.
+
+    Args:
+        out: The ``--out`` argument (``-`` means stdout).
+        explicit: The ``--format`` argument, if given.
+
+    Returns:
+        One of :data:`FORMATS`.
+
+    Raises:
+        UsageError: Unknown extension, or Parquet requested on stdout.
+    """
+    if explicit is None:
+        if out == "-":
+            fmt = "csv"
+        else:
+            suffix = Path(out).suffix.lower()
+            fmt = _SUFFIX_FORMATS.get(suffix, "")
+            if not fmt:
+                raise UsageError(
+                    f"cannot infer the format of {out!r}; use one of "
+                    f"{sorted(_SUFFIX_FORMATS)} or pass --format"
+                )
+    else:
+        fmt = explicit
+    if out == "-" and fmt == "parquet":
+        raise UsageError("parquet cannot be written to stdout; use --out <file>.parquet")
+    return fmt
+
+
+def write_frame(df: pd.DataFrame, out: str, fmt: str) -> None:
+    """Write ``df`` to ``out`` (a path or ``-``) in ``fmt``.
+
+    The index is labelled ``timestamp`` and rendered as ``YYYY-MM-DD HH:MM:SS`` (CSV)
+    or ISO 8601 to the second (JSON), so VBA, PowerShell and Excel parse it directly.
+    """
+    frame = df.copy()
+    frame.index.name = INDEX_LABEL
+    if fmt == "csv":
+        if out == "-":
+            frame.to_csv(sys.stdout, date_format=DATE_FORMAT, lineterminator="\n")
+        else:
+            to_csv(frame, out, date_format=DATE_FORMAT, lineterminator="\n")
+        return
+    if fmt == "parquet":
+        to_parquet(frame, out)
+        return
+    lines = fmt == "json-lines"
+    if out == "-":
+        text = frame.reset_index().to_json(
+            orient="records", date_format="iso", date_unit="s", lines=lines
+        )
+        sys.stdout.write(text)
+        if not text.endswith("\n"):
+            sys.stdout.write("\n")
+        return
+    to_json(frame, out, date_unit="s", lines=lines)
+
+
+def _fail(message: str) -> None:
+    """Print one ``error:`` line on stderr (newlines collapsed)."""
+    flat = " ".join(str(message).split())
+    print(f"error: {flat}", file=sys.stderr)
+
+
+def cmd_fetch(args: argparse.Namespace) -> int:
+    """Run ``arcobi-sdk fetch``."""
+    stream_ids = parse_stream_ids(args.streams)
+    names = parse_names(args.names, len(stream_ids))
+    start = parse_timestamp(args.start)
+    end = parse_timestamp(args.end)
+    if end <= start:
+        raise UsageError(f"--end ({end}) must be after --start ({start})")
+    if args.resolution <= 0:
+        raise UsageError("--resolution must be a positive number of minutes")
+    fmt = resolve_format(args.out, args.format)
+    load_env(args.env_file)
+
+    with DatahubClient() as client:
+        try:
+            df = fetch_streams(
+                client,
+                stream_ids,
+                start,
+                end,
+                names=names,
+                data_option=args.data_option,
+                resolution=args.resolution,
+            )
+        except ExtractionError as exc:
+            if _NO_ROWS_MARKER in str(exc):
+                _fail(str(exc))
+                return EXIT_NO_DATA
+            raise
+
+    if df.empty or df.isna().all().all():
+        _fail(f"streams {args.streams} returned no data between {start} and {end}")
+        return EXIT_NO_DATA
+
+    df = df.sort_index()
+    write_frame(df, args.out, fmt)
+    if args.out != "-" and not args.quiet:
+        print(f"ok: {len(df)} rows, {len(df.columns)} columns -> {args.out}")
+    return EXIT_OK
+
+
+def cmd_check(args: argparse.Namespace) -> int:
+    """Run ``arcobi-sdk check``: authenticate, report the expiry, release the session."""
+    load_env(args.env_file)
+    with DatahubClient() as client:
+        client.get_token()
+        expires = client.expires_at
+    when = expires.strftime("%Y-%m-%dT%H:%M:%SZ") if expires is not None else "unknown"
+    print(f"ok: authenticated; token expires {when}; session released")
+    return EXIT_OK
+
+
+def cmd_version(_args: argparse.Namespace) -> int:
+    """Run ``arcobi-sdk version``."""
+    print(f"{PROG} {__version__}")
+    return EXIT_OK
+
+
+def _add_common_arguments(parser: argparse.ArgumentParser) -> None:
+    """Add the ``--env-file`` and verbosity options shared by the vendor-facing commands."""
+    parser.add_argument(
+        "--env-file",
+        default=".env",
+        metavar="PATH",
+        help="KEY=VALUE file with DATAHUB_USERNAME/DATAHUB_PASSWORD (default ./.env; "
+        "a missing file is ignored and never overrides the environment)",
+    )
+    verbosity = parser.add_mutually_exclusive_group()
+    verbosity.add_argument(
+        "--quiet", action="store_true", help="no 'ok:' summary and only error logs"
+    )
+    verbosity.add_argument("--verbose", action="store_true", help="info-level logs on stderr")
+
+
+def _add_fetch_arguments(parser: argparse.ArgumentParser) -> None:
+    """Add the stream, window and output options of the ``fetch`` command."""
+    parser.add_argument(
+        "--streams", required=True, metavar="IDS", help="comma-separated stream ids, e.g. 1001,1002"
+    )
+    parser.add_argument(
+        "--names", metavar="NAMES", help="comma-separated column names, one per stream"
+    )
+    parser.add_argument("--start", required=True, metavar="WHEN", help="window start (inclusive)")
+    parser.add_argument("--end", required=True, metavar="WHEN", help="window end")
+    parser.add_argument(
+        "--resolution", type=float, default=60, metavar="MIN", help="expected step in minutes"
+    )
+    parser.add_argument(
+        "--data-option", dest="data_option", metavar="X", help="Datahub dataOption value"
+    )
+    parser.add_argument(
+        "--out",
+        required=True,
+        metavar="PATH",
+        help="output file (.csv, .parquet, .json, .jsonl) or '-' for stdout",
+    )
+    parser.add_argument(
+        "--format",
+        choices=FORMATS,
+        help="output format (default: from the --out extension; csv for stdout)",
+    )
+
+
+def build_parser() -> argparse.ArgumentParser:
+    """Build the top-level parser with the ``fetch``, ``check`` and ``version`` commands."""
+    parser = _Parser(
+        prog=PROG,
+        description="Fetch Datahub time series into CSV/Parquet/JSON files or stdout.",
+        epilog=(
+            "exit codes: 0 ok, 2 bad arguments/configuration, 3 vendor/auth error, "
+            "4 no data. Errors are one 'error: ...' line on stderr."
+        ),
+    )
+    parser.add_argument("--version", action="version", version=f"{PROG} {__version__}")
+    sub = parser.add_subparsers(dest="command", metavar="command")
+    sub.required = True
+
+    fetch = sub.add_parser("fetch", help="fetch one or more streams and write them")
+    _add_fetch_arguments(fetch)
+    _add_common_arguments(fetch)
+    fetch.set_defaults(func=cmd_fetch)
+
+    check = sub.add_parser("check", help="authenticate, print the token expiry and release")
+    _add_common_arguments(check)
+    check.set_defaults(func=cmd_check)
+
+    version = sub.add_parser("version", help="print the SDK version")
+    version.set_defaults(func=cmd_version, quiet=False, verbose=False)
+    return parser
+
+
+def _configure_logging(quiet: bool, verbose: bool) -> None:
+    level = logging.ERROR if quiet else logging.INFO if verbose else logging.WARNING
+    logging.basicConfig(
+        level=level, stream=sys.stderr, format="%(levelname)s %(name)s: %(message)s", force=True
+    )
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    """Entry point for ``arcobi-sdk``.
+
+    Args:
+        argv: Arguments without the program name; defaults to ``sys.argv[1:]``.
+
+    Returns:
+        The process exit code (see the module docstring).
+    """
+    parser = build_parser()
+    try:
+        args = parser.parse_args(argv)
+    except UsageError as exc:
+        _fail(str(exc))
+        return EXIT_USAGE
+    except SystemExit as exc:  # --help / --version
+        return int(exc.code or 0)
+
+    _configure_logging(getattr(args, "quiet", False), getattr(args, "verbose", False))
+    command: Callable[[argparse.Namespace], int] = args.func
+    try:
+        return command(args)
+    except (UsageError, ConfigurationError) as exc:
+        _fail(str(exc))
+        return EXIT_USAGE
+    except DatahubError as exc:
+        _fail(str(exc))
+        return EXIT_VENDOR
+    except ArcobiError as exc:
+        _fail(str(exc))
+        return EXIT_VENDOR
+    except OSError as exc:
+        _fail(f"cannot write output: {exc}")
+        return EXIT_USAGE
+    except ValueError as exc:
+        _fail(str(exc))
+        return EXIT_USAGE
+    except KeyboardInterrupt:
+        _fail("interrupted")
+        return 130
+
+
+if __name__ == "__main__":  # pragma: no cover
+    sys.exit(main())

arcobi_sdk/config.py

@@ -0,0 +1,78 @@
+"""Credential and settings helpers.
+
+The SDK reads vendor credentials from constructor arguments first and from
+environment variables second. :func:`load_env` is a tiny, dependency-free
+``.env`` reader for local development so those variables do not have to be
+exported by hand. It never overrides variables that are already set, so CI
+secrets always win over a developer's file.
+"""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Mapping
+from pathlib import Path
+
+DATAHUB_ENV_VARS: tuple[str, ...] = (
+    "DATAHUB_USERNAME",
+    "DATAHUB_PASSWORD",
+    "DATAHUB_BEARER_TOKEN",
+)
+POOL_ENV_VARS: tuple[str, ...] = ("ARCOBI_POOL_URL", "ARCOBI_POOL_API_KEY")
+
+
+def parse_env_text(text: str) -> dict[str, str]:
+    """Parse ``KEY=VALUE`` lines (``export`` prefix, quotes and ``#`` comments allowed)."""
+    values: dict[str, str] = {}
+    for raw in text.splitlines():
+        line = raw.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        if line.startswith("export "):
+            line = line[len("export ") :].lstrip()
+        key, _, value = line.partition("=")
+        key = key.strip()
+        value = value.strip()
+        if value[:1] in {'"', "'"} and value[-1:] == value[:1] and len(value) >= 2:
+            value = value[1:-1]
+        elif " #" in value:
+            value = value.split(" #", 1)[0].rstrip()
+        if key:
+            values[key] = value
+    return values
+
+
+def load_env(
+    path: str | os.PathLike[str] = ".env",
+    *,
+    override: bool = False,
+    environ: Mapping[str, str] | None = None,
+) -> dict[str, str]:
+    """Load ``KEY=VALUE`` pairs from ``path`` into the process environment.
+
+    Args:
+        path: The file to read. A missing file is not an error (returns ``{}``).
+        override: Replace variables that are already set. Default keeps existing values.
+        environ: Mapping to update; defaults to :data:`os.environ`. Mainly for tests.
+
+    Returns:
+        The variables that were actually set by this call.
+    """
+    target: Mapping[str, str] = os.environ if environ is None else environ
+    file = Path(path)
+    if not file.is_file():
+        return {}
+    applied: dict[str, str] = {}
+    for key, value in parse_env_text(file.read_text(encoding="utf-8")).items():
+        if override or key not in target:
+            target[key] = value  # type: ignore[index]
+            applied[key] = value
+    return applied
+
+
+def datahub_credentials_present(environ: Mapping[str, str] | None = None) -> bool:
+    """True when the environment holds a bearer token or a username+password pair."""
+    env = os.environ if environ is None else environ
+    if env.get("DATAHUB_BEARER_TOKEN"):
+        return True
+    return bool(env.get("DATAHUB_USERNAME") and env.get("DATAHUB_PASSWORD"))

arcobi_sdk/datahub/__init__.py

@@ -0,0 +1,63 @@
+"""Datahub integration: authenticated client, multi-stream extraction and specs.
+
+Import :class:`DatahubClient` to talk to the API, describe what to pull with
+:class:`StreamSpec` / :class:`DatasetSpec`, and use :func:`fetch_streams` or the
+planner trio (:func:`plan_extraction`, :func:`build_time_grid`, :func:`fetch_and_align`)
+to get aligned DataFrames.
+"""
+
+from __future__ import annotations
+
+from arcobi_sdk.datahub.client import (
+    DEFAULT_BASE_URL,
+    INDEX_CANDIDATES,
+    DatahubClient,
+    TokenProvider,
+    format_api_datetime,
+    frame_from_payload,
+    select_value_column,
+)
+from arcobi_sdk.datahub.extract import (
+    DEFAULT_INTERPOLATE_LIMIT,
+    DEFAULT_PADDING_HOURS,
+    ResolutionPlan,
+    build_time_grid,
+    datahub_extract,
+    extract_merge_data_settings,
+    fetch_and_align,
+    fetch_streams,
+    interpolate_gaps,
+    plan_extraction,
+)
+from arcobi_sdk.datahub.specs import (
+    NAME_COLUMN_SEPARATOR,
+    DatasetSpec,
+    Role,
+    StreamSpec,
+    split_name_column,
+)
+
+__all__ = [
+    "DEFAULT_BASE_URL",
+    "DEFAULT_INTERPOLATE_LIMIT",
+    "DEFAULT_PADDING_HOURS",
+    "INDEX_CANDIDATES",
+    "NAME_COLUMN_SEPARATOR",
+    "DatahubClient",
+    "DatasetSpec",
+    "ResolutionPlan",
+    "Role",
+    "StreamSpec",
+    "TokenProvider",
+    "build_time_grid",
+    "datahub_extract",
+    "extract_merge_data_settings",
+    "fetch_and_align",
+    "fetch_streams",
+    "format_api_datetime",
+    "frame_from_payload",
+    "interpolate_gaps",
+    "plan_extraction",
+    "select_value_column",
+    "split_name_column",
+]