atdata 0.2.2b1__tar.gz → 0.3.0b1__tar.gz

atdata-0.3.0b1/.github/workflows/uv-publish-pypi.yml

@@ -0,0 +1,65 @@
+name: Build and upload package to PyPI
+
+on:
+  release:
+    types:
+      - published
+
+permissions:
+  contents: read
+  id-token: write
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    name: Build release distribution
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version-file: "pyproject.toml"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+
+      - name: Install project
+        run: uv sync --locked --all-extras --dev
+
+      - name: Build release distributions
+        run: uv build
+
+      - name: Upload dist artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    name: Publish to PyPI
+    runs-on: ubuntu-latest
+    needs: build
+    environment:
+      name: pypi
+      url: https://pypi.org/project/atdata/
+
+    steps:
+      - name: Download dist artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+
+      - name: Publish to PyPI
+        run: uv publish --trusted-publishing always dist/*

atdata-0.3.0b1/.github/workflows/uv-test.yml

@@ -0,0 +1,134 @@
+name: Run tests with `uv`
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+permissions:
+  contents: read
+  actions: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+
+      - name: Install the project
+        run: uv sync --locked --dev
+
+      - name: Run ruff check
+        run: uv run ruff check src/ tests/
+
+      - name: Run ruff format check
+        run: uv run ruff format --check src/ tests/
+
+  test:
+    name: Test (py${{ matrix.python-version }}, redis${{ matrix.redis-version }})
+    runs-on: ubuntu-latest
+    environment:
+      name: test
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
+        redis-version: [6, 7]
+
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+
+      - name: Install the project
+        run: uv sync --locked --all-extras --dev
+
+      - name: Start Redis
+        uses: supercharge/redis-github-action@1.8.1
+        with:
+          redis-version: ${{ matrix.redis-version }}
+
+      - name: Run tests with coverage
+        run: uv run pytest --cov=atdata --cov-report=xml --cov-report=term
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v5
+        with:
+          fail_ci_if_error: false
+          token: ${{ secrets.CODECOV_TOKEN }}
+
+  benchmark:
+    name: Benchmarks
+    runs-on: ubuntu-latest
+    needs: [lint]
+    permissions:
+      contents: write
+      actions: write
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.14"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+
+      - name: Install just
+        uses: extractions/setup-just@v2
+
+      - name: Install the project
+        run: uv sync --locked --all-extras --dev
+
+      - name: Start Redis
+        uses: supercharge/redis-github-action@1.8.1
+        with:
+          redis-version: 7
+
+      - name: Run benchmarks
+        run: just bench
+
+      - name: Copy report to docs
+        run: |
+          mkdir -p docs/benchmarks
+          cp .bench/report.html docs/benchmarks/index.html
+
+      - name: Commit updated benchmark docs
+        if: github.event_name == 'push'
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git add docs/benchmarks/index.html
+          git diff --cached --quiet || git commit -m "docs: update benchmark report [skip ci]"
+          git push
+
+      - name: Upload benchmark report
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: benchmark-report
+          path: |
+            .bench/report.html
+            .bench/*.json

{atdata-0.2.2b1 → atdata-0.3.0b1}/.gitignore

@@ -5,8 +5,10 @@
 # Don't commit any .env files
 **/*.env
 # Don't commit `uv` lockfiles
-**/uv.lock
+# (SIKE it turns out we actually do want to commit)
+# **/uv.lock
 # Development tooling (keep local, not in upstream)
+# (SIKE it turns out we actually do want to commit)
 # .chainlink/
 # .claude/
 
@@ -50,6 +52,9 @@ MANIFEST
 pip-log.txt
 pip-delete-this-directory.txt
 
+# Benchmark results
+.bench/
+
 # Unit test / coverage reports
 htmlcov/
 .tox/

{atdata-0.2.2b1 → atdata-0.3.0b1}/.vscode/settings.json

@@ -5,15 +5,19 @@
         "atproto",
         "creds",
         "dtype",
+        "fastparquet",
         "getattr",
         "hgetall",
         "hset",
+        "libipld",
         "maxcount",
         "minioadmin",
         "msgpack",
         "ndarray",
         "NSID",
         "ormsgpack",
+        "psycopg",
+        "pydantic",
         "pypi",
         "pyproject",
         "pytest",
@@ -24,6 +28,8 @@
         "schemamodels",
         "shardlists",
         "tariterators",
+        "tqdm",
+        "typer",
         "unpackb",
         "webdataset"
     ],  

{atdata-0.2.2b1 → atdata-0.3.0b1}/CHANGELOG.md

@@ -25,6 +25,116 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - **Comprehensive integration test suite**: 593 tests covering E2E flows, error handling, edge cases
 
 ### Changed
+- Investigate upload-artifact not finding benchmark output (#512)
+- Fix duplicate CI runs for push+PR overlap (#511)
+- Scope contents:write permission to benchmark job only (#510)
+- Add benchmark docs auto-commit to CI workflow (#509)
+- Submit PR for v0.3.0b1 release to upstream/main (#508)
+- Implement GH#39: Production hardening (observability, error handling, testing infra) (#504)
+- Add pluggable structured logging via atdata.configure_logging (#507)
+- Add PartialFailureError and shard-level error handling to Dataset.map (#506)
+- Add atdata.testing module with mock clients, fixtures, and helpers (#505)
+- Fix CI linting failures (20 ruff errors) (#503)
+- Adversarial review: Post-benchmark suite assessment (#494)
+- Remove redundant protocol docstrings that restate signatures (#500)
+- Add missing unit tests for _type_utils.py (#499)
+- Strengthen weak assertions (assert X is not None → value checks) (#498)
+- Trim verbose exception constructor docstrings (#501)
+- Analyze benchmark results for performance improvement opportunities (#502)
+- Consolidate remaining duplicate sample types in test files (#497)
+- Remove dead code: _repo_legacy.py legacy UUID field, unused imports (#496)
+- Trim verbose docstrings in dataset.py and _index.py (#495)
+- Benchmark report: replace mean/stddev with median/IQR, add per-sample columns (#492)
+- Add parameter descriptions to benchmark suite with automatic report introspection (#491)
+- HTML benchmark reports with CI integration (#487)
+- Add bench + render step to CI on highest Python version only (#490)
+- Update justfile bench commands to export JSON and render (#489)
+- Create render_report.py script to convert JSON to HTML (#488)
+- Increase test coverage for low-coverage modules (#480)
+- Add providers/_postgres.py tests (mock-based) (#485)
+- Add _stub_manager.py tests (#484)
+- Add manifest/_query.py tests (#483)
+- Add repository.py tests (#482)
+- Add CLI tests (cli/__init__, diagnose, local, preview, schema) (#481)
+- Check test coverage for CLI utils (#479)
+- Add performance benchmark suite for atdata (#471)
+- Verify benchmarks run (#478)
+- Update pyproject.toml and justfile (#477)
+- Create bench_atmosphere.py (#476)
+- Create bench_query.py (#475)
+- Create bench_dataset_io.py (#474)
+- Create bench_index_providers.py (#473)
+- Create benchmarks/conftest.py with shared fixtures (#472)
+- Add per-shard manifest and query system (GH #35) (#462)
+- Write unit and integration tests (#470)
+- Integrate manifest into write path and Dataset.query() (#469)
+- Implement QueryExecutor and SampleLocation (#468)
+- Implement ManifestWriter (JSON + parquet) (#467)
+- Implement ManifestBuilder (#465)
+- Implement ShardManifest data model (#466)
+- Implement aggregate collectors (categorical, numeric, set) (#464)
+- Implement ManifestField annotation and resolve_manifest_fields() (#463)
+- Migrate type annotations from PackableSample to Packable protocol (#461)
+- Remove LocalIndex factory — consolidate to Index (#460)
+- Split local.py monolith into local/ package (#452)
+- Verify tests and lint pass (#459)
+- Create __init__.py re-export facade and delete local.py (#458)
+- Create _repo_legacy.py with deprecated Repo class (#457)
+- Create _index.py with Index class and LocalIndex factory (#456)
+- Create _s3.py with S3DataStore and S3 helpers (#455)
+- Create _schema.py with schema models and helpers (#454)
+- Create _entry.py with LocalDatasetEntry and constants (#453)
+- Migrate CLI from argparse to typer (#449)
+- Investigate test failures (#450)
+- Fix ensure_stub receiving LocalSchemaRecord instead of dict (#451)
+- GH#38: Developer experience improvements (#437)
+- CLI: atdata preview command (#440)
+- CLI: atdata schema show/diff commands (#439)
+- CLI: atdata inspect command (#438)
+- Dataset.__len__ and Dataset.select() for sample count and indexed access (#447)
+- Dataset.to_pandas() and Dataset.to_dict() export methods (#446)
+- Dataset.filter() and Dataset.map() streaming transforms (#445)
+- Dataset.get(key) for keyed sample access (#442)
+- Dataset.describe() summary statistics (#444)
+- Dataset.schema property and column_names (#443)
+- Dataset.head(n) and Dataset.__iter__ convenience methods (#441)
+- Custom exception hierarchy with actionable error messages (#448)
+- Adversarial review: Post-Repository consolidation assessment (#430)
+- Remove backwards-compat dict-access methods from SchemaField and LocalSchemaRecord (#436)
+- Add missing test coverage for Repository prefix routing edge cases and error paths (#435)
+- Trim over-verbose docstrings in local.py module/class level (#434)
+- Fix formally incorrect test assertions (batch_size, CID, brace notation) (#433)
+- Consolidate duplicate test sample types across test files into conftest.py (#432)
+- Consolidate duplicate entry-creation logic in Index (add_entry vs _insert_dataset_to_provider) (#431)
+- Switch default Index provider from Redis to SQLite (#429)
+- Consolidated Index with Repository system (#424)
+- Phase 4: Deprecate AtmosphereIndex, update exports (#428)
+- Phase 3: Default Index singleton and load_dataset integration (#427)
+- Phase 2: Extend Index with repos/atmosphere params and prefix routing (#426)
+- Phase 1: Create Repository dataclass and _AtmosphereBackend in repository.py (#425)
+- Adversarial review: Post-IndexProvider pluggable storage assessment (#417)
+- Convert TODO comments to tracked issues or remove (#422)
+- Remove deprecated shard_list property references from docstrings (#421)
+- Replace bare except in _stub_manager.py and cli/local.py with specific exceptions (#423)
+- Tighten generic pytest.raises(Exception) to specific exception types in tests (#420)
+- Replace assert statements with ValueError in production code (#419)
+- Consolidate duplicated _parse_semver into _type_utils.py (#418)
+- feat: Add SQLite/PostgreSQL index providers (GH #42) (#409)
+- Update documentation and public API exports (#416)
+- Add tests for all providers (#415)
+- Refactor Index class to accept provider parameter (#414)
+- Implement PostgresIndexProvider (#413)
+- Implement SqliteIndexProvider (#412)
+- Implement RedisIndexProvider (extract from Index class) (#411)
+- Define IndexProvider protocol in _protocols.py (#410)
+- Add just lint command to justfile (#408)
+- Add SQLite/PostgreSQL providers for LocalIndex (in addition to Redis) (#407)
+- Fix type hints for @atdata.packable decorator to show PackableSample methods (#406)
+- Review GitHub workflows and recommend CI improvements (#405)
+- Fix type signatures for Dataset.ordered and Dataset.shuffled (GH#28) (#404)
+- Investigate quartodoc Example section rendering - missing CSS classes on pre/code tags (#401)
+- Update all docstrings from Example: to Examples: format (#403)
+- Create GitHub issues for v0.3 roadmap feature domains (#402)
 - Expand Quarto documentation with architectural narrative (#395)
 - Expand atmosphere tutorial with federation context (#400)
 - Expand local-workflow tutorial with system narrative (#399)

{atdata-0.2.2b1 → atdata-0.3.0b1}/CLAUDE.md

@@ -46,8 +46,10 @@ uv build
 Development tasks are managed with [just](https://github.com/casey/just), a command runner. Available commands:
 
 ```bash
-# Build documentation (runs quartodoc + quarto)
-just docs
+just test              # Run all tests with coverage
+just test tests/test_dataset.py  # Run specific test file
+just lint              # Run ruff check + format check
+just docs              # Build documentation (runs quartodoc + quarto)
 ```
 
 The `justfile` is in the project root. Add new dev tasks there rather than creating shell scripts.
@@ -209,11 +211,11 @@ def test_repo_insert_with_s3(mock_s3, clean_redis):
 
 ## Docstring Formatting
 
-This project uses **Google-style docstrings** with quartodoc for API documentation generation. The most important formatting requirement is for **Example sections**.
+This project uses **Google-style docstrings** with quartodoc for API documentation generation. The most important formatting requirement is for **Examples sections**.
 
-### Example Section Format
+### Examples Section Format
 
-Example sections must use reStructuredText literal block syntax (`::`) to render correctly in quartodoc-generated documentation:
+Use `Examples:` (plural) for code examples. This is recognized by griffe's Google docstring parser and rendered with proper syntax highlighting by quartodoc:
 
 ```python
 def my_function():
@@ -227,50 +229,44 @@ def my_function():
     Returns:
         Description of return value.
 
-    Example:
-        ::
-
-            >>> result = my_function()
-            >>> print(result)
-            'output'
+    Examples:
+        >>> result = my_function()
+        >>> print(result)
+        'output'
     """
 ```
 
 **Key formatting rules:**
 
-1. `Example:` with a colon, 4-space indented from the docstring margin
-2. `::` on its own line, 8-space indented (4 more than `Example:`)
-3. Blank line after `::`
-4. Code examples indented 12 spaces (4 more than `::`)
-5. Use `>>>` for Python prompts and `...` for continuation lines
+1. Use `Examples:` (plural, not `Example:` singular)
+2. Code examples are indented 8 spaces (4 more than `Examples:`)
+3. Use `>>>` for Python prompts and `...` for continuation lines
+4. No `::` marker needed - griffe handles the parsing automatically
 
-**Incorrect format (will not render properly):**
+**Incorrect format (will not render with syntax highlighting):**
 ```python
-    Example:
-        >>> code_here()  # Wrong - missing :: and extra indentation
+    Example:  # Wrong - singular form is treated as an admonition
+        ::    # Wrong - reST literal block marker not needed
+            >>> code_here()
 ```
 
 **Correct format:**
 ```python
-    Example:
-        ::
-
-            >>> code_here()  # Correct - has :: and proper indentation
+    Examples:
+        >>> code_here()  # Correct - plural form, proper indentation
 ```
 
 ### Multiple Examples
 
-For multiple examples, use the same pattern:
+For multiple examples, continue in the same section:
 
 ```python
-    Example:
-        ::
-
-            >>> # First example
-            >>> x = create_thing()
+    Examples:
+        >>> # First example
+        >>> x = create_thing()
 
-            >>> # Second example
-            >>> y = other_thing()
+        >>> # Second example
+        >>> y = other_thing()
 ```
 
 ### Class and Method Docstrings
@@ -281,20 +277,16 @@ Apply the same format to class docstrings and method docstrings:
 class MyClass:
     """Class description.
 
-    Example:
-        ::
-
-            >>> obj = MyClass()
-            >>> obj.do_something()
+    Examples:
+        >>> obj = MyClass()
+        >>> obj.do_something()
     """
 
     def method(self):
         """Method description.
 
-        Example:
-            ::
-
-                >>> self.method()
+        Examples:
+            >>> self.method()
         """
 ```
 

{atdata-0.2.2b1 → atdata-0.3.0b1}/PKG-INFO

@@ -1,11 +1,12 @@
 Metadata-Version: 2.4
 Name: atdata
-Version: 0.2.2b1
+Version: 0.3.0b1
 Summary: A loose federation of distributed, typed datasets
 Author-email: Maxine Levesque <hello@maxine.science>, "Maxine @ Forecast Bio" <maxine@forecast.bio>
 License-File: LICENSE
 Requires-Python: >=3.12
 Requires-Dist: atproto>=0.0.65
+Requires-Dist: boto3>=1.41.5
 Requires-Dist: fastparquet>=2024.11.0
 Requires-Dist: libipld>=3.3.2
 Requires-Dist: msgpack>=1.1.2
@@ -19,9 +20,12 @@ Requires-Dist: requests>=2.32.5
 Requires-Dist: s3fs>=2025.12.0
 Requires-Dist: schemamodels>=0.9.1
 Requires-Dist: tqdm>=4.67.1
+Requires-Dist: typer>=0.21.1
 Requires-Dist: webdataset>=1.0.2
 Provides-Extra: atmosphere
 Requires-Dist: atproto>=0.0.55; extra == 'atmosphere'
+Provides-Extra: postgres
+Requires-Dist: psycopg[binary]>=3.1; extra == 'postgres'
 Description-Content-Type: text/markdown
 
 # atdata