reflective-data-catalog 0.0.1__tar.gz

reflective_data_catalog-0.0.1/.github/workflows/build-and-push.yml

@@ -0,0 +1,79 @@
+name: Publish Python 🐍 distribution 📦 to PyPI and TestPyPI
+
+on: push
+
+jobs:
+  build:
+    name: Build distribution 📦
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        persist-credentials: false
+        fetch-depth: 0  # full history so setuptools-scm can compute the version
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version: "3.x"
+
+    - name: Install pypa/build
+      run: >-
+        python3 -m
+        pip install
+        build
+        --user
+    - name: Build a binary wheel and a source tarball
+      run: python3 -m build
+    - name: Store the distribution packages
+      uses: actions/upload-artifact@v5
+      with:
+        name: python-package-distributions
+        path: dist/
+
+  publish-to-pypi:
+    name: >-
+      Publish Python 🐍 distribution 📦 to PyPI
+    if: startsWith(github.ref, 'refs/tags/')  # only publish to PyPI on tag pushes
+    needs:
+    - build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/reflective-data-catalog
+    permissions:
+      id-token: write  # IMPORTANT: mandatory for trusted publishing
+
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v6
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Publish distribution 📦 to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+  
+  publish-to-testpypi:
+    name: Publish Python 🐍 distribution 📦 to TestPyPI
+    needs:
+    - build
+    runs-on: ubuntu-latest
+
+    environment:
+      name: testpypi
+      url: https://test.pypi.org/p/reflective-data-catalog
+
+    permissions:
+      id-token: write  # IMPORTANT: mandatory for trusted publishing
+
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v6
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Publish distribution 📦 to TestPyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        repository-url: https://test.pypi.org/legacy/
+        verbose: true

reflective_data_catalog-0.0.1/.github/workflows/tests.yml

@@ -0,0 +1,46 @@
+name: Tests
+
+on:
+  push:
+    branches: [main]
+    tags: ["*"]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    name: Tests (Python ${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v6
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+
+      - name: Lint with Ruff
+        run: |
+          ruff check .
+          ruff format --check .
+
+      - name: Run tests with coverage
+        run: |
+          pytest --cov=reflective_data_catalog --cov-report=term-missing --cov-report=xml
+
+      - name: Upload coverage to Codecov
+        if: matrix.python-version == '3.12'
+        uses: codecov/codecov-action@v5
+        with:
+          files: ./coverage.xml
+          fail_ci_if_error: false

reflective_data_catalog-0.0.1/.gitignore

@@ -0,0 +1,56 @@
+# Python bytecode
+**/__pycache__/
+*.py[cod]
+
+# Virtual environments
+venv/
+env/
+.venv/
+
+# Environment variables
+.env
+
+# IDE settings
+.vscode/
+.idea/
+
+# OS generated files
+.DS_Store
+Thumbs.db
+
+# Data files
+*.joblib
+*.nc
+dump.rdb
+*.npy
+*.pkl
+*.csv
+
+# Python packages
+*.egg-info/
+*.egg
+*.eggs
+*.eggs.zip
+*.eggs.tar.gz
+*.eggs.tar.bz2
+*.eggs.tar.xz
+*.eggs.tar.lzma
+*.eggs.tar.lz
+*.eggs.tar.zst
+*.eggs.tar.zstd
+*.eggs.tar.zstd.tar.gz
+
+# Jupyter notebooks cruft
+**/.ipynb_checkpoints/
+
+# Ruff cache
+.ruff_cache/
+
+# Pytest cache
+.pytest_cache/
+
+# Pytest coverage
+.coverage
+coverage.xml
+htmlcov/
+.coverage.*

reflective_data_catalog-0.0.1/.pre-commit-config.yaml

@@ -0,0 +1,9 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.8.4
+    hooks:
+      # Run the linter
+      - id: ruff
+        args: [--fix]
+      # Run the formatter
+      - id: ruff-format

reflective_data_catalog-0.0.1/CLAUDE.md

@@ -0,0 +1,78 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with this codebase.
+
+## Project Overview
+
+Reflective Data Catalog is a pip-installable Python package that provides a unified interface for accessing SAI (Stratospheric Aerosol Injection) climate model data stored in cloud storage (S3, GCS, Azure, Cloudflare R2). It wraps multiple climate model sources (CESM2-WACCM, MIROC-ES2H, UKESM1, E3SMv3) behind a consistent API.
+
+## Project Structure
+
+- `src/reflective_data_catalog/` — Package source (src layout)
+  - `main.py` — `ReflectiveCatalog` class, the main entry point
+  - `flexibleSources.py` — `FlexibleSourceConfig`, `FlexibleSource`, `SourceDiscovery`, `FlexibleSourceRegistry` classes
+  - `reflective_data.py` — `DEFAULT_FLEXIBLE_SOURCES` list of all source configurations
+  - `storage.py` — `CloudFileSystem` class wrapping obstore for multi-cloud access
+  - `esgf.py` — `ESGFHelper` for ESGF data access
+  - `esm.py` — `ESMCatalog` and `GeoMIPCloudHelper` for Google Cloud CMIP6 catalog via intake-esm
+  - `help_text.py` — Help text utilities
+- `tests/` — Unit tests (pytest)
+  - `conftest.py` — Shared fixtures and mocks
+  - `test_flexible_sources.py` — FlexibleSourceConfig, Registry, Discovery
+  - `test_storage.py` — CloudFileSystem (including R2)
+  - `test_esgf.py` — ESGFHelper (mocked)
+  - `test_esm.py` — ESMCatalog, GeoMIPCloudHelper (mocked)
+  - `test_catalog.py` — ReflectiveCatalog integration
+- `pyproject.toml` — Package metadata, dependencies, and Ruff config
+- `.github/workflows/tests.yml` — CI: runs tests and coverage on push/PR
+
+## Key Architecture
+
+- **`FlexibleSourceConfig`** is a frozen dataclass that defines how to build cloud storage paths and filename patterns for each data source. It supports placeholders: `{base}`, `{ensemble}`, `{table_path}`, `{variable}`, `{variant}`, `{time}`, `{ensemble_id}`.
+- **`FlexibleSource`** wraps a config and provides `.to_dask()`, `.read()`, `.list_variables()`, `.list_ensembles()`, `.list_tables()`, and `.discover()`.
+- **`SourceDiscovery`** handles cloud storage scanning via `obstore` (through `CloudFileSystem`) to discover available data.
+- **`CloudFileSystem`** wraps `obstore` to provide a unified API (glob, ls, exists, open) across S3, GCS, Azure, Cloudflare R2, and other cloud providers. It auto-detects the provider from the URL scheme (`s3://`, `gs://`, `az://`, `r2://`). R2 support uses `S3Store` with the Cloudflare endpoint.
+- **`ESMCatalog`** provides intake-esm access to the Google Cloud CMIP6 catalog (search, load, list experiments/models/variables).
+- **`GeoMIPCloudHelper`** wraps ESMCatalog with GeoMIP-specific convenience methods (g6sulfur, g6solar, load_ensemble).
+- **`ReflectiveCatalog`** uses `__getattr__` to dynamically expose registered sources (e.g., `catalog.cesm2_waccm_ssp245()`). Also exposes `catalog.esm` and `catalog.geomip_cloud` for cloud-optimized Zarr access.
+- Sources are registered from `DEFAULT_FLEXIBLE_SOURCES` in `reflective_data.py`.
+
+## Common Commands
+
+```bash
+# Install in development mode
+pip install -e ".[dev]"
+
+# Lint
+ruff check .
+
+# Auto-fix lint issues
+ruff check --fix .
+
+# Format
+ruff format .
+
+# Run tests
+pytest
+
+# Run tests with coverage
+pytest --cov=reflective_data_catalog --cov-report=term-missing
+```
+
+## Code Style
+
+- Uses Ruff for linting and formatting (configured in `pyproject.toml`)
+- Line length: 88 characters
+- Double quotes for strings
+- All imports use relative paths within the package (e.g., `from .flexibleSources import ...`)
+
+## Important Notes
+
+- Cloud storage access uses `obstore` via the `CloudFileSystem` wrapper — auto-detects provider from URL scheme (`s3://`, `gs://`, `az://`, `r2://`).
+- Cloudflare R2 uses `r2://` URLs and requires an account ID via `r2_account_id` kwarg or `CLOUDFLARE_R2_ACCOUNT_ID` / `CLOUDFLARE_ACCOUNT_ID` env var.
+- AWS credentials are needed for S3 access; similarly for GCS and Azure.
+- Google Cloud CMIP6/GeoMIP data is accessible via `catalog.esm` and `catalog.geomip_cloud` (requires `intake-esm`).
+- Some sources use `ensemble_mapping` to convert ensemble names to IDs in filenames (e.g., `r1` → `001` for CESM2).
+- MIROC sources use a `variant` parameter to distinguish between file types (e.g., `baseline` vs `G6-1.5K-SAI`).
+- UKESM1 and E3SMv3 sources have `{variable}` in the directory pattern, not just the filename.
+- All external services are mocked in tests — no network or cloud credentials needed to run `pytest`.

reflective_data_catalog-0.0.1/CONTRIBUTING.md

@@ -0,0 +1,163 @@
+# Contributing to Reflective Data Catalog
+
+Thank you for your interest in contributing! This document provides guidelines for contributing to the project.
+
+## Getting Started
+
+1. Fork and clone the repository:
+
+```bash
+git clone https://github.com/ReflectiveCloud/reflective-data-catalog.git
+cd reflective-data-catalog
+```
+
+2. Install in development mode:
+
+```bash
+pip install -e ".[dev]"
+```
+
+## Development Workflow
+
+1. Create a branch for your work:
+
+```bash
+git checkout -b feature/your-feature-name
+```
+
+2. Make your changes and ensure they pass linting and tests.
+
+3. Commit your changes with a clear message:
+
+```bash
+git commit -m "Add brief description of change"
+```
+
+4. Push and open a pull request.
+
+## Code Style
+
+This project uses [Ruff](https://docs.astral.sh/ruff/) for linting and formatting. Run both before submitting:
+
+```bash
+ruff check .
+ruff format .
+```
+
+Auto-fix lint issues where possible:
+
+```bash
+ruff check --fix .
+```
+
+Key style rules:
+
+- Line length: 88 characters
+- Double quotes for strings
+- Space indentation (no tabs)
+- Imports sorted with isort rules
+
+## Running Tests
+
+Run the full test suite:
+
+```bash
+pytest
+```
+
+Run with coverage report:
+
+```bash
+pytest --cov=reflective_data_catalog --cov-report=term-missing
+```
+
+Run a specific test file or test:
+
+```bash
+pytest tests/test_flexible_sources.py
+pytest tests/test_catalog.py::TestSearch::test_search_by_term
+```
+
+All external services (S3, ESGF, intake-esm) are mocked in the test suite — no network access or cloud credentials are needed.
+
+### Writing Tests
+
+- Tests live in `tests/` and use [pytest](https://docs.pytest.org/).
+- Shared fixtures and mocks are in `tests/conftest.py`.
+- Mock any cloud or network calls; tests must run offline.
+- Aim for one test file per source module (e.g. `test_storage.py` for `storage.py`).
+
+## Adding a New Data Source
+
+To add a new flexible data source, create a `FlexibleSourceConfig` entry in `src/reflective_data_catalog/reflective_data.py`:
+
+```python
+FlexibleSourceConfig(
+    name='model_experiment',                    # Snake-case identifier
+    base='s3://bucket/path/to/data',            # Cloud storage base URL (s3://, gs://, az://, r2://)
+    pattern='{base}/{ensemble}/{table_path}',   # Directory pattern
+    filename_pattern='{variable}.*.nc',         # Filename pattern
+    table_mapping={'Amon': 'Amon'},             # Table name mapping
+    default_table='Amon',
+    default_variable='tas',
+    default_ensemble='r1i1p1f1',
+    driver='netcdf',
+    description='Model experiment description'
+)
+```
+
+The source will automatically be available as `catalog.model_experiment()`.
+
+### Configuration Fields
+
+| Field | Description |
+|-------|-------------|
+| `name` | Unique snake_case identifier |
+| `base` | Cloud storage base URL (`s3://`, `gs://`, `az://`, `r2://`) |
+| `pattern` | Directory path template with `{base}`, `{ensemble}`, `{table_path}`, `{variable}`, `{time}` placeholders |
+| `filename_pattern` | Filename template with `{variable}`, `{ensemble}`, `{variant}`, `{ensemble_id}`, `{time}`, and `*` wildcards |
+| `table_mapping` | Dict mapping table names to S3 directory names |
+| `ensemble_mapping` | Optional dict mapping ensemble names to filename IDs (e.g. `{'r1': '001'}`) |
+| `default_table` | Default table when none is specified |
+| `default_variable` | Default variable when none is specified |
+| `default_ensemble` | Default ensemble member |
+| `default_variant` | Optional default variant (e.g. `'baseline'`) |
+| `default_time` | Optional default time/frequency (e.g. `'AERmon'`) |
+| `driver` | `'netcdf'` or `'zarr'` |
+| `combine_files` | How to combine multi-file sources: `'by_coords'`, `'nested'`, or `'first'` |
+| `concat_dim` | Dimension to concatenate along (default: `'time'`) |
+| `description` | Human-readable description |
+
+## Project Structure
+
+```
+src/reflective_data_catalog/
+├── __init__.py          # Package exports
+├── main.py              # ReflectiveCatalog class
+├── flexible_sources.py   # FlexibleSourceConfig, FlexibleSource, SourceDiscovery
+├── reflective_data.py   # Default source configurations
+├── esgf.py              # ESGF data access helper
+├── esm.py               # intake-esm Google Cloud CMIP6/GeoMIP access
+├── storage.py           # CloudFileSystem (obstore wrapper)
+└── help_text.py         # Help text utilities
+tests/
+├── conftest.py              # Shared fixtures and mocks
+├── test_flexible_sources.py # FlexibleSourceConfig, Registry, Discovery
+├── test_storage.py          # CloudFileSystem
+├── test_esgf.py             # ESGFHelper (mocked)
+├── test_esm.py              # ESMCatalog, GeoMIPCloudHelper (mocked)
+└── test_catalog.py          # ReflectiveCatalog integration
+```
+
+## Reporting Issues
+
+When reporting a bug, please include:
+
+- Python version
+- Package version
+- Steps to reproduce the issue
+- Full error traceback
+
+## License
+
+By contributing, you agree that your contributions will be licensed under the Apache 2.0 license.