pymycorr 0.2.0__tar.gz

pymycorr-0.2.0/.gitignore

@@ -0,0 +1,40 @@
+# Byte-compiled / optimized
+__pycache__/
+*.py[cod]
+*$py.class
+
+# Distribution / packaging
+dist/
+build/
+*.egg-info/
+*.egg
+
+# Virtual environments
+.venv/
+venv/
+ENV/
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+.mypy_cache/
+.ruff_cache/
+
+# Jupyter
+.ipynb_checkpoints/
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Environment
+.env
+.env.local
+*.pem

pymycorr-0.2.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Recons Ltd
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pymycorr-0.2.0/PKG-INFO

@@ -0,0 +1,159 @@
+Metadata-Version: 2.4
+Name: pymycorr
+Version: 0.2.0
+Summary: Python client for fetching table data from the MyCorr API via Apache Arrow
+Project-URL: Homepage, https://github.com/recons-ltd/pymycorr
+Project-URL: Documentation, https://github.com/recons-ltd/pymycorr#readme
+Project-URL: Repository, https://github.com/recons-ltd/pymycorr
+Project-URL: Issues, https://github.com/recons-ltd/pymycorr/issues
+Project-URL: Changelog, https://github.com/recons-ltd/pymycorr/blob/main/CHANGELOG.md
+Author-email: Maximilian Schuberth <maxs@recons-ltd.com>
+License: MIT
+License-File: LICENSE
+Keywords: api-client,arrow,dataframe,mycorr,pandas,polars
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: pandas>=2.0.0
+Requires-Dist: pyarrow>=14.0.0
+Requires-Dist: python-dotenv>=1.0.0
+Provides-Extra: all
+Requires-Dist: nest-asyncio>=1.5.0; extra == 'all'
+Requires-Dist: polars>=0.20.0; extra == 'all'
+Provides-Extra: jupyter
+Requires-Dist: nest-asyncio>=1.5.0; extra == 'jupyter'
+Provides-Extra: polars
+Requires-Dist: polars>=0.20.0; extra == 'polars'
+Description-Content-Type: text/markdown
+
+# pymycorr
+
+[![PyPI version](https://img.shields.io/pypi/v/pymycorr)](https://pypi.org/project/pymycorr/)
+[![CI](https://img.shields.io/github/actions/workflow/status/sambaclab/mycorr-python/ci.yml?branch=main&label=CI)](https://github.com/sambaclab/mycorr-python/actions/workflows/ci.yml)
+[![Python](https://img.shields.io/badge/python-%E2%89%A53.10-blue)](https://github.com/sambaclab/mycorr-python/blob/main/pyproject.toml)
+[![License](https://img.shields.io/github/license/sambaclab/mycorr-python)](https://github.com/sambaclab/mycorr-python/blob/main/LICENSE)
+
+Python client for fetching table data from the [MyCorr](https://alpha.mycorr.app) API using Apache Arrow for efficient data transfer.
+
+> **Note:** The production API is coming soon. Set `MYCORR_API_URL` to your endpoint if you have early access.
+
+## Installation
+
+```bash
+pip install pymycorr
+```
+
+## Configuration
+
+Set your API token and (optionally) the API URL as environment variables:
+
+```bash
+export MYCORR_API_TOKEN="your-api-token"
+```
+
+Or create a `.env` file in your project root:
+
+```
+MYCORR_API_TOKEN=your-api-token
+```
+
+The client automatically loads from environment variables and `.env` files.
+
+## Quick Start
+
+```python
+from pymycorr import MyCorr
+
+# Initialize client (loads token from MYCORR_API_TOKEN env var or .env file)
+client = MyCorr()
+
+# Fetch as pandas DataFrame
+df = client.get_table("table-id")
+
+# Fetch a specific version
+df = client.get_table("table-id", version=2)
+
+# Fetch using version alias
+df = client.get_table("table-id", version="stable")
+
+# Get table metadata without fetching data
+info = client.get_table_info("table-id")
+print(info["schema"])
+```
+
+### Progress Display
+
+Progress is shown automatically in interactive environments (terminals and notebooks). You can control this behavior:
+
+```python
+# Always show progress
+client = MyCorr(progress=True)
+
+# Never show progress
+client = MyCorr(progress=False)
+
+# Override per-request
+df = client.get_table("table-id", progress=False)
+```
+
+## API Reference
+
+### MyCorr
+
+#### `__init__(url=None, token=None, env_file=None, progress="auto")`
+
+Initialize the client.
+
+- `url`: API base URL (optional). Falls back to `MYCORR_API_URL` env var, then default.
+- `token`: Authentication token (Bearer token from MyCorr UI). Falls back to `MYCORR_API_TOKEN` env var or `.env` file.
+- `env_file`: Path to custom `.env` file (optional).
+- `progress`: Show download progress. `True` always shows, `False` never shows, `"auto"` (default) shows in interactive environments.
+
+#### `get_table(table_id, version=None, engine="pandas", progress=None)`
+
+Fetch table data as a DataFrame.
+
+- `table_id`: Unique identifier for the table.
+- `version`: Version number (int) or alias (str like `"latest"`, `"stable"`).
+- `engine`: `"pandas"` (default) or `"polars"`.
+- `progress`: Override client's progress setting for this request.
+- Returns: pandas or polars DataFrame.
+
+#### `get_table_info(table_id, version=None)`
+
+Get table schema and metadata.
+
+- `table_id`: Unique identifier for the table.
+- `version`: Version number (int) or alias (str).
+- Returns: Dictionary with table metadata including schema.
+
+## Exceptions
+
+- `TableAPIError`: Base exception for API errors.
+- `TableNotFoundError`: Table not found (404).
+- `QuotaExceededError`: Egress quota exceeded (429).
+- `TableConversionError`: Failed to convert Arrow data to DataFrame.
+- `StreamingError`: Error during data streaming or IPC parsing.
+
+## Development
+
+See [.env.example](.env.example) for environment variable configuration. For local development:
+
+1. Copy `.env.example` to `.env`
+2. Set `MYCORR_API_URL` to your local/dev base URL (no path segments)
+3. SSL verification is automatically disabled for `localhost` and `127.0.0.1` URLs
+
+## License
+
+MIT License - see [LICENSE](LICENSE) for details.

pymycorr-0.2.0/README.md

@@ -0,0 +1,120 @@
+# pymycorr
+
+[![PyPI version](https://img.shields.io/pypi/v/pymycorr)](https://pypi.org/project/pymycorr/)
+[![CI](https://img.shields.io/github/actions/workflow/status/sambaclab/mycorr-python/ci.yml?branch=main&label=CI)](https://github.com/sambaclab/mycorr-python/actions/workflows/ci.yml)
+[![Python](https://img.shields.io/badge/python-%E2%89%A53.10-blue)](https://github.com/sambaclab/mycorr-python/blob/main/pyproject.toml)
+[![License](https://img.shields.io/github/license/sambaclab/mycorr-python)](https://github.com/sambaclab/mycorr-python/blob/main/LICENSE)
+
+Python client for fetching table data from the [MyCorr](https://alpha.mycorr.app) API using Apache Arrow for efficient data transfer.
+
+> **Note:** The production API is coming soon. Set `MYCORR_API_URL` to your endpoint if you have early access.
+
+## Installation
+
+```bash
+pip install pymycorr
+```
+
+## Configuration
+
+Set your API token and (optionally) the API URL as environment variables:
+
+```bash
+export MYCORR_API_TOKEN="your-api-token"
+```
+
+Or create a `.env` file in your project root:
+
+```
+MYCORR_API_TOKEN=your-api-token
+```
+
+The client automatically loads from environment variables and `.env` files.
+
+## Quick Start
+
+```python
+from pymycorr import MyCorr
+
+# Initialize client (loads token from MYCORR_API_TOKEN env var or .env file)
+client = MyCorr()
+
+# Fetch as pandas DataFrame
+df = client.get_table("table-id")
+
+# Fetch a specific version
+df = client.get_table("table-id", version=2)
+
+# Fetch using version alias
+df = client.get_table("table-id", version="stable")
+
+# Get table metadata without fetching data
+info = client.get_table_info("table-id")
+print(info["schema"])
+```
+
+### Progress Display
+
+Progress is shown automatically in interactive environments (terminals and notebooks). You can control this behavior:
+
+```python
+# Always show progress
+client = MyCorr(progress=True)
+
+# Never show progress
+client = MyCorr(progress=False)
+
+# Override per-request
+df = client.get_table("table-id", progress=False)
+```
+
+## API Reference
+
+### MyCorr
+
+#### `__init__(url=None, token=None, env_file=None, progress="auto")`
+
+Initialize the client.
+
+- `url`: API base URL (optional). Falls back to `MYCORR_API_URL` env var, then default.
+- `token`: Authentication token (Bearer token from MyCorr UI). Falls back to `MYCORR_API_TOKEN` env var or `.env` file.
+- `env_file`: Path to custom `.env` file (optional).
+- `progress`: Show download progress. `True` always shows, `False` never shows, `"auto"` (default) shows in interactive environments.
+
+#### `get_table(table_id, version=None, engine="pandas", progress=None)`
+
+Fetch table data as a DataFrame.
+
+- `table_id`: Unique identifier for the table.
+- `version`: Version number (int) or alias (str like `"latest"`, `"stable"`).
+- `engine`: `"pandas"` (default) or `"polars"`.
+- `progress`: Override client's progress setting for this request.
+- Returns: pandas or polars DataFrame.
+
+#### `get_table_info(table_id, version=None)`
+
+Get table schema and metadata.
+
+- `table_id`: Unique identifier for the table.
+- `version`: Version number (int) or alias (str).
+- Returns: Dictionary with table metadata including schema.
+
+## Exceptions
+
+- `TableAPIError`: Base exception for API errors.
+- `TableNotFoundError`: Table not found (404).
+- `QuotaExceededError`: Egress quota exceeded (429).
+- `TableConversionError`: Failed to convert Arrow data to DataFrame.
+- `StreamingError`: Error during data streaming or IPC parsing.
+
+## Development
+
+See [.env.example](.env.example) for environment variable configuration. For local development:
+
+1. Copy `.env.example` to `.env`
+2. Set `MYCORR_API_URL` to your local/dev base URL (no path segments)
+3. SSL verification is automatically disabled for `localhost` and `127.0.0.1` URLs
+
+## License
+
+MIT License - see [LICENSE](LICENSE) for details.

pymycorr-0.2.0/pyproject.toml

@@ -0,0 +1,95 @@
+[project]
+name = "pymycorr"
+version = "0.2.0"
+description = "Python client for fetching table data from the MyCorr API via Apache Arrow"
+readme = "README.md"
+license = { text = "MIT" }
+authors = [{ name = "Maximilian Schuberth", email = "maxs@recons-ltd.com" }]
+requires-python = ">=3.10"
+keywords = ["mycorr", "arrow", "dataframe", "api-client", "polars", "pandas"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Scientific/Engineering",
+    "Typing :: Typed",
+]
+dependencies = ["httpx>=0.27.0", "pyarrow>=14.0.0", "python-dotenv>=1.0.0", "pandas>=2.0.0"]
+
+[project.optional-dependencies]
+polars = ["polars>=0.20.0"]
+jupyter = ["nest-asyncio>=1.5.0"]
+all = ["polars>=0.20.0", "nest-asyncio>=1.5.0"]
+
+[project.urls]
+Homepage = "https://github.com/recons-ltd/pymycorr"
+Documentation = "https://github.com/recons-ltd/pymycorr#readme"
+Repository = "https://github.com/recons-ltd/pymycorr"
+Issues = "https://github.com/recons-ltd/pymycorr/issues"
+Changelog = "https://github.com/recons-ltd/pymycorr/blob/main/CHANGELOG.md"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.sdist]
+include = ["/src"]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/pymycorr"]
+
+[dependency-groups]
+dev = [
+    "pytest>=8.0",
+    "pytest-asyncio>=0.23",
+    "pytest-cov>=4.1",
+    "respx>=0.21.0",
+    "mypy>=1.8",
+    "ruff>=0.2",
+    "pre-commit>=3.6",
+    "ipykernel>=6.29",
+    "notebook>=7.0",
+    "build>=1.0",
+]
+
+[tool.ruff]
+target-version = "py310"
+line-length = 100
+src = ["src", "tests"]
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP", "B", "SIM"]
+
+[tool.mypy]
+python_version = "3.10"
+strict = true
+warn_return_any = true
+warn_unused_configs = true
+warn_unused_ignores = false
+
+[[tool.mypy.overrides]]
+module = ["pyarrow.*", "httpx.*", "polars.*", "pandas.*", "nest_asyncio.*", "IPython.*"]
+ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = ["tests.*"]
+disallow_untyped_decorators = false
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+asyncio_mode = "auto"
+asyncio_default_fixture_loop_scope = "function"
+
+[tool.coverage.run]
+source = ["src/pymycorr"]
+branch = true
+
+[tool.coverage.report]
+exclude_lines = ["pragma: no cover", "if TYPE_CHECKING:"]

pymycorr-0.2.0/src/pymycorr/__init__.py

@@ -0,0 +1,25 @@
+"""PyMyCorr - Python client for fetching table data from the MyCorr API."""
+
+from importlib.metadata import version
+
+from pymycorr.client import MyCorr
+from pymycorr.exceptions import (
+    QuotaExceededError,
+    RateLimitError,
+    StreamingError,
+    TableAPIError,
+    TableConversionError,
+    TableNotFoundError,
+)
+
+__version__ = version("pymycorr")
+__all__ = [
+    "MyCorr",
+    "QuotaExceededError",
+    "RateLimitError",
+    "StreamingError",
+    "TableAPIError",
+    "TableNotFoundError",
+    "TableConversionError",
+    "__version__",
+]

pymycorr-0.2.0/src/pymycorr/_progress.py

@@ -0,0 +1,174 @@
+"""Progress display utilities for streaming operations."""
+
+from __future__ import annotations
+
+import sys
+import time
+from typing import Any, Literal
+
+
+def detect_environment() -> Literal["notebook", "terminal", "non_interactive"]:
+    """Detect execution environment for appropriate progress display.
+
+    Returns:
+        'notebook' if running in Jupyter/IPython notebook,
+        'terminal' if running in interactive terminal,
+        'non_interactive' if output is piped or in CI.
+    """
+    try:
+        from IPython.core.getipython import get_ipython
+
+        ipython = get_ipython()  # type: ignore[no-untyped-call]
+        if ipython is not None and "IPKernelApp" in ipython.config:
+            return "notebook"
+    except (ImportError, AttributeError):
+        pass
+
+    if hasattr(sys.stdout, "isatty") and sys.stdout.isatty():
+        return "terminal"
+
+    return "non_interactive"
+
+
+def format_bytes(num_bytes: int | float) -> str:
+    """Format bytes as human-readable string.
+
+    Args:
+        num_bytes: Number of bytes.
+
+    Returns:
+        Formatted string like '12.5 MB'.
+    """
+    value = float(num_bytes)
+    for unit in ("B", "KB", "MB", "GB"):
+        if abs(value) < 1024:
+            return f"{value:.1f} {unit}"
+        value /= 1024
+    return f"{value:.1f} TB"
+
+
+def format_elapsed(seconds: float) -> str:
+    """Format elapsed time as MM:SS.
+
+    Args:
+        seconds: Elapsed time in seconds.
+
+    Returns:
+        Formatted string like '01:23'.
+    """
+    mins, secs = divmod(int(seconds), 60)
+    return f"{mins:02d}:{secs:02d}"
+
+
+def format_transfer_summary(
+    total_bytes: int,
+    elapsed_seconds: float,
+    uncompressed_bytes: int | None = None,
+) -> str:
+    """Format completion summary message.
+
+    Args:
+        total_bytes: Total compressed bytes transferred over the wire.
+        elapsed_seconds: Time taken in seconds.
+        uncompressed_bytes: Optional uncompressed (in-memory) size of the data.
+
+    Returns:
+        Formatted summary like 'Downloaded 4.2 MB → 14.7 MB in 2.9s (1.4 MB/s)'.
+    """
+    speed = total_bytes / elapsed_seconds if elapsed_seconds > 0 else 0
+    speed_str = format_bytes(speed)
+    size_str = format_bytes(total_bytes)
+    if uncompressed_bytes is not None:
+        size_str = f"{size_str} → {format_bytes(uncompressed_bytes)}"
+    return f"Downloaded {size_str} in {elapsed_seconds:.1f}s ({speed_str}/s)"
+
+
+class ProgressTracker:
+    """Context manager for tracking streaming progress with text output.
+
+    Prints live progress updates that overwrite the same line,
+    then prints a final summary on completion.
+    """
+
+    def __init__(
+        self,
+        enabled: bool | Literal["auto"],
+        desc: str = "Downloading",
+    ) -> None:
+        """Initialize the progress tracker.
+
+        Args:
+            enabled: True to always show, False to never show, 'auto' to detect.
+            desc: Description label for the progress output.
+        """
+        self._enabled = enabled
+        self._desc = desc
+        self._start_time: float = 0
+        self._elapsed: float = 0
+        self._total_bytes: int = 0
+        self._show_progress: bool = False
+        self._last_line_len: int = 0
+
+    def __enter__(self) -> ProgressTracker:
+        """Start tracking progress."""
+        self._start_time = time.monotonic()
+        self._show_progress = self._should_show()
+        return self
+
+    def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
+        """Stop tracking and clear progress line. Does not print summary."""
+        self._elapsed = time.monotonic() - self._start_time
+        if self._show_progress:
+            self._clear_line()
+
+    @property
+    def total_bytes(self) -> int:
+        """Total compressed bytes received so far."""
+        return self._total_bytes
+
+    def print_summary(self, uncompressed_bytes: int | None = None) -> None:
+        """Print the completion summary line.
+
+        Args:
+            uncompressed_bytes: Optional uncompressed (in-memory) size of the data.
+        """
+        if self._show_progress and self._total_bytes > 0:
+            summary = format_transfer_summary(self._total_bytes, self._elapsed, uncompressed_bytes)
+            sys.stdout.write(summary + "\n")
+            sys.stdout.flush()
+
+    def _should_show(self) -> bool:
+        """Determine if progress should be shown."""
+        if self._enabled is False:
+            return False
+        if self._enabled == "auto":
+            return detect_environment() != "non_interactive"
+        return True
+
+    def _clear_line(self) -> None:
+        """Clear the current line by overwriting with spaces."""
+        if self._last_line_len > 0:
+            sys.stdout.write("\r" + " " * self._last_line_len + "\r")
+
+    def _write_line(self, line: str) -> None:
+        """Write a line, tracking length for later clearing."""
+        self._clear_line()
+        sys.stdout.write(line)
+        sys.stdout.flush()
+        self._last_line_len = len(line)
+
+    def update(self, chunk_size: int) -> None:
+        """Update progress with received chunk.
+
+        Args:
+            chunk_size: Number of bytes received.
+        """
+        self._total_bytes += chunk_size
+        if self._show_progress:
+            elapsed = time.monotonic() - self._start_time
+            speed = self._total_bytes / elapsed if elapsed > 0 else 0
+            line = (
+                f"\r{self._desc}: {format_bytes(self._total_bytes)} "
+                f"[{format_elapsed(elapsed)}, {format_bytes(speed)}/s]"
+            )
+            self._write_line(line)

pymycorr-0.2.0/src/pymycorr/client.py

@@ -0,0 +1,593 @@
+"""MyCorr API client for fetching table data with Arrow format support."""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import queue
+import threading
+from collections.abc import AsyncIterator, Coroutine, Iterator
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Literal, TypeVar, cast
+from urllib.parse import urlparse
+
+import httpx
+import pyarrow as pa
+import pyarrow.ipc as ipc
+from dotenv import load_dotenv
+
+from pymycorr._progress import ProgressTracker
+from pymycorr.exceptions import (
+    QuotaExceededError,
+    RateLimitError,
+    StreamingError,
+    TableAPIError,
+    TableConversionError,
+    TableNotFoundError,
+)
+
+if TYPE_CHECKING:
+    import pandas as pd
+    import polars as pl
+
+T = TypeVar("T")
+
+
+class _StreamingBuffer:
+    """Bridges async HTTP chunks to PyArrow's synchronous read() interface."""
+
+    def __init__(self) -> None:
+        self._buffer = bytearray()
+        self._chunks: queue.Queue[bytes | None] = queue.Queue()
+        self._eof = False
+
+    def feed(self, chunk: bytes) -> None:
+        """Called by producer with incoming chunks."""
+        self._chunks.put(chunk)
+
+    def close(self) -> None:
+        """Signal end of stream."""
+        self._chunks.put(None)
+
+    def read(self, n: int = -1) -> bytes:
+        """Called by PyArrow - blocks until data available."""
+        while not self._eof and (n == -1 or len(self._buffer) < n):
+            try:
+                chunk = self._chunks.get(timeout=300)
+            except queue.Empty:
+                break
+            if chunk is None:
+                self._eof = True
+                break
+            self._buffer.extend(chunk)
+
+        if n == -1 or n >= len(self._buffer):
+            result = bytes(self._buffer)
+            self._buffer.clear()
+        else:
+            result = bytes(self._buffer[:n])
+            del self._buffer[:n]
+        return result
+
+
+class MyCorr:
+    """Client for fetching table data from API with Arrow format support."""
+
+    DEFAULT_URL = "https://mycorr.app"
+    _default_progress: bool | Literal["auto"]
+
+    def __init__(
+        self,
+        url: str | None = None,
+        token: str | None = None,
+        env_file: str | Path | None = None,
+        progress: bool | Literal["auto"] = "auto",
+    ) -> None:
+        """Initialize the client with authentication token and API URL.
+
+        Configuration is resolved in the following order:
+        1. Explicit parameters (highest priority)
+        2. Environment variables (MYCORR_API_URL, MYCORR_API_TOKEN)
+        3. Default values (URL only)
+
+        Args:
+            url: API base URL. Falls back to MYCORR_API_URL env var, then default.
+            token: Authentication token. Falls back to MYCORR_API_TOKEN env var.
+            env_file: Optional path to .env file. If None, auto-discovers .env.
+            progress: Show download progress. True always shows, False never shows,
+                'auto' (default) shows in notebooks/terminals but not in non-interactive.
+
+        Raises:
+            ValueError: If token cannot be resolved.
+        """
+        load_dotenv(dotenv_path=env_file)
+
+        self.url = (url or os.getenv("MYCORR_API_URL") or self.DEFAULT_URL).rstrip("/")
+        self.token = token or os.getenv("MYCORR_API_TOKEN")
+        hostname = urlparse(self.url).hostname or ""
+        self._verify_ssl = hostname not in ("localhost", "127.0.0.1")
+        self._default_progress = progress
+
+        if not self.token:
+            raise ValueError(
+                "token is required (pass explicitly or set MYCORR_API_TOKEN environment variable)"
+            )
+
+    def _run_sync(self, coro: Coroutine[Any, Any, T]) -> T:
+        """Run an async coroutine from a sync context.
+
+        Handles detection of running event loops and applies nest_asyncio
+        when needed (e.g., in Jupyter notebooks).
+        """
+        try:
+            loop = asyncio.get_running_loop()
+        except RuntimeError:
+            return asyncio.run(coro)
+
+        if loop.is_running():
+            try:
+                import nest_asyncio
+
+                nest_asyncio.apply()
+                return loop.run_until_complete(coro)
+            except ImportError as e:
+                raise RuntimeError(
+                    "nest_asyncio is required when calling from within an async context "
+                    "(e.g., Jupyter notebooks). Install with: pip install nest-asyncio"
+                ) from e
+        return loop.run_until_complete(coro)
+
+    @staticmethod
+    def _handle_error_response(status_code: int, body: bytes | str, context: str = "") -> None:
+        """Raise the appropriate exception for an HTTP error response.
+
+        Args:
+            status_code: The HTTP status code.
+            body: The raw response body (bytes or str).
+            context: Description of the operation for error messages.
+
+        Raises:
+            TableNotFoundError: If the status code is 404.
+            QuotaExceededError: If the status code is 429.
+            TableAPIError: For all other non-200 status codes.
+        """
+        try:
+            import json
+
+            error_body = json.loads(body) if isinstance(body, bytes | str) else {}
+        except Exception:
+            error_body = {}
+
+        if status_code == 404:
+            msg = error_body.get("message", "Not found")
+            raise TableNotFoundError(f"{context}{msg}")
+        if status_code == 429:
+            if error_body.get("error") == "rate_limit_exceeded":
+                raise RateLimitError(error_body)
+            raise QuotaExceededError(error_body)
+
+        msg = error_body.get("message", f"HTTP {status_code}")
+        raise TableAPIError(f"{context}{msg}")
+
+    @staticmethod
+    def _build_version_params(
+        table_id: str,
+        version: int | str | None,
+        *,
+        include_scope: bool = True,
+    ) -> dict[str, Any]:
+        """Build query params dict for version specification.
+
+        Args:
+            table_id: The table identifier.
+            version: Version number (int), alias (str), or None for 'latest'.
+            include_scope: Whether to include scope='read' in params.
+
+        Returns:
+            Dict with table_id, version/version_alias, and optionally scope.
+
+        Raises:
+            TypeError: If version has an invalid type.
+        """
+        if version is None:
+            version = "latest"
+
+        params: dict[str, Any] = {"table_id": table_id}
+        if include_scope:
+            params["scope"] = "read"
+
+        if isinstance(version, int):
+            params["version"] = version
+        elif isinstance(version, str):
+            params["version_alias"] = version
+        else:
+            raise TypeError(
+                f"Expected 'version' to be int, str, or None, got {type(version).__name__}"
+            )
+
+        return params
+
+    async def _get_data_stream(
+        self,
+        table_id: str,
+        version: int | str | None = None,
+        progress: bool | Literal["auto"] | None = None,
+    ) -> pa.Table:
+        """Fetch Arrow stream from API asynchronously.
+
+        Args:
+            table_id: Unique identifier for the table.
+            version: Version number (int) or version alias (str, e.g., 'latest', 'stable').
+            progress: Show download progress. None uses client default.
+
+        Returns:
+            PyArrow Table containing the data.
+
+        Raises:
+            ValueError: If table_id is empty.
+            TableNotFoundError: If the table is not found (404).
+            QuotaExceededError: If the egress quota is exceeded (429).
+            TableAPIError: For other API errors.
+            StreamingError: If IPC stream parsing fails.
+        """
+        batches = []
+        async for batch in self._stream_record_batches(table_id, version, progress=progress):
+            batches.append(batch)
+
+        if not batches:
+            return pa.table({})
+
+        return pa.Table.from_batches(batches)
+
+    async def _stream_record_batches(
+        self,
+        table_id: str,
+        version: int | str | None = None,
+        progress: bool | Literal["auto"] | None = None,
+    ) -> AsyncIterator[pa.RecordBatch]:
+        """Stream Arrow record batches from API asynchronously.
+
+        Yields individual RecordBatch objects as they arrive over the network,
+        allowing processing of large tables without loading everything into memory.
+
+        Args:
+            table_id: Unique identifier for the table.
+            version: Version number (int) or version alias (str, e.g., 'latest', 'stable').
+            progress: Show download progress. None uses client default.
+
+        Yields:
+            PyArrow RecordBatch objects.
+
+        Raises:
+            ValueError: If table_id is empty.
+            TableNotFoundError: If the table is not found (404).
+            QuotaExceededError: If the egress quota is exceeded (429).
+            TableAPIError: For other API errors.
+            StreamingError: If IPC stream parsing fails.
+        """
+        if not table_id:
+            raise ValueError("Table ID is required")
+
+        params = self._build_version_params(table_id, version)
+        effective_progress: bool | Literal["auto"] = (
+            progress if progress is not None else self._default_progress
+        )
+        timeout = httpx.Timeout(timeout=300.0, connect=30.0)
+
+        try:
+            async with (
+                httpx.AsyncClient(timeout=timeout, verify=self._verify_ssl) as client,
+                client.stream(
+                    "GET",
+                    f"{self.url}/data/table/stream",
+                    headers={
+                        "Authorization": f"Bearer {self.token}",
+                        "Accept": "application/vnd.apache.arrow.stream",
+                    },
+                    params=params,
+                ) as response,
+            ):
+                if response.status_code != 200:
+                    await response.aread()
+                    self._handle_error_response(
+                        response.status_code, response.content, "Error fetching table: "
+                    )
+
+                # Collect all chunks (server now sends single IPC stream)
+                chunks: list[bytes] = []
+                tracker = ProgressTracker(effective_progress, desc=f"Fetching {table_id}")
+                prev_downloaded = 0
+                with tracker:
+                    async for chunk in response.aiter_bytes():
+                        wire_bytes = response.num_bytes_downloaded
+                        tracker.update(wire_bytes - prev_downloaded)
+                        prev_downloaded = wire_bytes
+                        chunks.append(chunk)
+
+                # Parse single IPC stream with PyArrow's native reader
+                data = b"".join(chunks)
+                try:
+                    reader = ipc.open_stream(data)
+                    uncompressed = 0
+                    for batch in reader:
+                        uncompressed += batch.nbytes
+                        yield batch
+                except (pa.ArrowInvalid, pa.ArrowIOError) as e:
+                    raise StreamingError(f"Failed to parse IPC stream: {e}") from e
+                else:
+                    tracker.print_summary(uncompressed_bytes=uncompressed)
+
+        except httpx.HTTPError as e:
+            raise StreamingError(f"Connection error: {e!s}") from e
+
+    def _iter_record_batches(
+        self,
+        table_id: str,
+        version: int | str | None = None,
+        progress: bool | Literal["auto"] | None = None,
+    ) -> Iterator[pa.RecordBatch]:
+        """Synchronous iterator over record batches.
+
+        Uses a background thread with queue for true streaming in sync contexts.
+        In Jupyter notebooks or running event loops, falls back to collecting
+        all batches first (requires nest-asyncio).
+
+        Args:
+            table_id: Unique identifier for the table.
+            version: Version number (int) or version alias (str, e.g., 'latest', 'stable').
+            progress: Show download progress. None uses client default.
+
+        Yields:
+            PyArrow RecordBatch objects.
+
+        Raises:
+            ValueError: If table_id is empty.
+            TableNotFoundError: If the table is not found (404).
+            QuotaExceededError: If the egress quota is exceeded (429).
+            TableAPIError: For other API errors.
+            StreamingError: If IPC stream parsing fails.
+            RuntimeError: If nest_asyncio required but not installed.
+        """
+        try:
+            asyncio.get_running_loop()
+            has_running_loop = True
+        except RuntimeError:
+            has_running_loop = False
+
+        if has_running_loop:
+            # In Jupyter or async context - collect all batches via _run_sync
+            async def collect() -> list[pa.RecordBatch]:
+                return [
+                    batch
+                    async for batch in self._stream_record_batches(
+                        table_id, version, progress=progress
+                    )
+                ]
+
+            yield from self._run_sync(collect())
+        else:
+            # No running loop - use thread + queue for true streaming
+            yield from self._sync_stream_batches(table_id, version, progress=progress)
+
+    def _sync_stream_batches(
+        self,
+        table_id: str,
+        version: int | str | None,
+        progress: bool | Literal["auto"] | None = None,
+    ) -> Iterator[pa.RecordBatch]:
+        """True streaming sync iteration using thread + PyArrow native reader.
+
+        Uses _StreamingBuffer to bridge async HTTP chunks to PyArrow's
+        synchronous read() interface, enabling true streaming where batches
+        are yielded as data arrives over the network.
+        """
+        buffer = _StreamingBuffer()
+        error_holder: list[Exception] = []
+        tracker_holder: list[ProgressTracker] = []
+
+        def producer() -> None:
+            async def fetch() -> None:
+                params = self._build_version_params(table_id, version)
+                effective_progress: bool | Literal["auto"] = (
+                    progress if progress is not None else self._default_progress
+                )
+                timeout = httpx.Timeout(timeout=300.0, connect=30.0)
+
+                try:
+                    async with (
+                        httpx.AsyncClient(timeout=timeout, verify=self._verify_ssl) as client,
+                        client.stream(
+                            "GET",
+                            f"{self.url}/data/table/stream",
+                            headers={
+                                "Authorization": f"Bearer {self.token}",
+                                "Accept": "application/vnd.apache.arrow.stream",
+                            },
+                            params=params,
+                        ) as response,
+                    ):
+                        if response.status_code != 200:
+                            await response.aread()
+                            self._handle_error_response(
+                                response.status_code,
+                                response.content,
+                                "Error fetching table: ",
+                            )
+
+                        tracker = ProgressTracker(effective_progress, desc=f"Fetching {table_id}")
+                        tracker_holder.append(tracker)
+                        prev_downloaded = 0
+                        with tracker:
+                            async for chunk in response.aiter_bytes():
+                                wire_bytes = response.num_bytes_downloaded
+                                tracker.update(wire_bytes - prev_downloaded)
+                                prev_downloaded = wire_bytes
+                                buffer.feed(chunk)
+                except Exception as e:
+                    error_holder.append(e)
+                finally:
+                    buffer.close()
+
+            asyncio.run(fetch())
+
+        thread = threading.Thread(target=producer, daemon=True)
+        thread.start()
+
+        # PyArrow reads from buffer as data arrives
+        uncompressed = 0
+        try:
+            reader = ipc.open_stream(buffer)
+            for batch in reader:
+                uncompressed += batch.nbytes
+                yield batch
+        except (pa.ArrowInvalid, pa.ArrowIOError) as e:
+            thread.join()
+            if error_holder:
+                raise error_holder[0] from e
+            raise StreamingError(f"Failed to parse IPC stream: {e}") from e
+
+        thread.join()
+
+        # Re-raise any error from the producer thread
+        if error_holder:
+            raise error_holder[0]
+
+        # Print summary with uncompressed size from main thread
+        if tracker_holder:
+            tracker_holder[0].print_summary(uncompressed_bytes=uncompressed)
+
+    def _stream_to_dataframes(
+        self,
+        table_id: str,
+        version: int | str | None = None,
+        engine: Literal["pandas", "polars"] = "pandas",
+        progress: bool | Literal["auto"] | None = None,
+    ) -> Iterator[Any]:
+        """Stream data as individual DataFrames per batch.
+
+        Useful for processing large tables in chunks without loading
+        everything into memory.
+
+        Args:
+            table_id: Unique identifier for the table.
+            version: Version number (int) or version alias (str, e.g., 'latest', 'stable').
+            engine: Data processing engine ('pandas' or 'polars').
+            progress: Show download progress. None uses client default.
+
+        Yields:
+            DataFrame in the specified format (pandas or polars).
+
+        Raises:
+            ValueError: If engine is not supported.
+            TableNotFoundError: If the table is not found (404).
+            QuotaExceededError: If the egress quota is exceeded (429).
+            TableAPIError: For other API errors.
+            StreamingError: If IPC stream parsing fails.
+            TableConversionError: If conversion to DataFrame fails.
+        """
+        if engine not in ("pandas", "polars"):
+            raise ValueError(f"Engine must be 'pandas' or 'polars', got '{engine}'")
+
+        for batch in self._iter_record_batches(table_id, version, progress=progress):
+            try:
+                if engine == "pandas":
+                    # Convert batch to table first to ensure DataFrame output
+                    table = pa.Table.from_batches([batch])
+                    yield table.to_pandas()
+                else:
+                    import polars as pl_module
+
+                    yield pl_module.from_arrow(batch)
+            except Exception as e:
+                raise TableConversionError(
+                    f"Failed to convert batch to {engine} format: {e!s}"
+                ) from e
+
+    def get_table(
+        self,
+        table_id: str,
+        version: int | str | None = None,
+        engine: Literal["pandas", "polars"] = "pandas",
+        progress: bool | Literal["auto"] | None = None,
+    ) -> pd.DataFrame | pl.DataFrame:
+        """Fetch table data synchronously and convert to DataFrame.
+
+        Args:
+            table_id: Unique identifier for the table.
+            version: Version number (int) or version alias (str, e.g., 'latest', 'stable').
+            engine: Data processing engine ('pandas' or 'polars').
+            progress: Show download progress. None uses client default.
+
+        Returns:
+            DataFrame in the specified format (pandas or polars).
+
+        Raises:
+            TypeError: If version has wrong type.
+            ValueError: If engine is not supported.
+            TableNotFoundError: If the table is not found (404).
+            QuotaExceededError: If the egress quota is exceeded (429).
+            TableAPIError: For other API-related errors.
+            TableConversionError: If conversion to DataFrame fails.
+            StreamingError: If IPC stream parsing fails.
+        """
+        if engine not in ("pandas", "polars"):
+            raise ValueError(f"Engine must be 'pandas' or 'polars', got '{engine}'")
+
+        async def get_dataframe_async() -> pd.DataFrame | pl.DataFrame:
+            """Internal async function to fetch and convert data."""
+            data_stream = await self._get_data_stream(table_id, version, progress=progress)
+
+            try:
+                if engine == "pandas":
+                    return cast("pd.DataFrame", data_stream.to_pandas())
+                else:
+                    import polars as pl_module
+
+                    return cast("pl.DataFrame", pl_module.from_arrow(data_stream))
+            except Exception as e:
+                raise TableConversionError(
+                    f"Failed to convert table to {engine} format: {e!s}"
+                ) from e
+
+        return self._run_sync(get_dataframe_async())
+
+    def get_table_info(self, table_id: str, version: int | str | None = None) -> dict[str, Any]:
+        """Get table schema and metadata without fetching full data.
+
+        Args:
+            table_id: Unique identifier for the table.
+            version: Version number (int) or version alias (str, e.g., 'latest', 'stable').
+
+        Returns:
+            Dictionary containing table snapshot (version, meta, schema).
+
+        Raises:
+            ValueError: If table_id is empty.
+            TableNotFoundError: If the table is not found (404).
+            QuotaExceededError: If the egress quota is exceeded (429).
+            TableAPIError: For other API-related errors.
+        """
+        if not table_id:
+            raise ValueError("Table ID is required")
+
+        params: dict[str, Any] = {
+            "table_id": table_id,
+            "scope": "read",
+            "version_alias": version if isinstance(version, str) else "latest",
+        }
+        if isinstance(version, int):
+            params["version"] = version
+
+        response = httpx.get(
+            f"{self.url}/data/tableinfo",
+            headers={"Authorization": f"Bearer {self.token}"},
+            params=params,
+            verify=self._verify_ssl,
+        )
+
+        if response.status_code != 200:
+            self._handle_error_response(
+                response.status_code, response.text, "Error fetching table info: "
+            )
+
+        return cast(dict[str, Any], response.json())

pymycorr-0.2.0/src/pymycorr/exceptions.py

@@ -0,0 +1,56 @@
+"""Custom exceptions for the PyMyCorr client."""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+class TableAPIError(Exception):
+    """Base exception for table API errors."""
+
+
+class TableNotFoundError(TableAPIError):
+    """Raised when table is not found."""
+
+
+class TableConversionError(TableAPIError):
+    """Raised when table conversion fails."""
+
+
+class StreamingError(TableAPIError):
+    """Raised when an error occurs during data streaming."""
+
+    def __init__(self, message: str, batches_received: int = 0):
+        super().__init__(message)
+        self.batches_received = batches_received
+
+
+class RateLimitError(TableAPIError):
+    """Raised when the API rate limit is exceeded (HTTP 429, governor).
+
+    The ``retry_after`` attribute contains the number of seconds to wait
+    before retrying (parsed from the server's ``retry_after_secs`` field),
+    or ``None`` if the server did not provide it.
+    """
+
+    def __init__(self, detail: dict[str, Any]) -> None:
+        message = detail.get("message") or detail.get("error", "rate limit exceeded")
+        super().__init__(message)
+        self.detail = detail
+        try:
+            self.retry_after: int | None = int(detail["retry_after_secs"])
+        except (KeyError, ValueError, TypeError):
+            self.retry_after = None
+
+
+class QuotaExceededError(TableAPIError):
+    """Raised when the daily egress quota is exceeded (HTTP 429).
+
+    The ``detail`` attribute contains the raw error payload from the server
+    (e.g. error code and reset time when the server provides it).
+    """
+
+    def __init__(self, detail: dict[str, Any]) -> None:
+        message = detail.get("message") or detail.get("error", "egress quota exceeded")
+        super().__init__(message)
+        self.detail = detail