dr-wandb 0.1.0__tar.gz

dr_wandb-0.1.0/.claude/settings.local.json

@@ -0,0 +1,10 @@
+{
+  "permissions": {
+    "allow": [
+      "WebSearch",
+      "WebFetch(domain:docs.wandb.ai)"
+    ],
+    "deny": [],
+    "ask": []
+  }
+}

dr_wandb-0.1.0/.example.env

@@ -0,0 +1,12 @@
+# .example.env
+# Copy to .env and update with your values
+
+# Database connection (required for production)
+DR_WANDB_DATABASE_URL=postgresql+psycopg2://localhost/wandb
+
+# Output dir (optional)
+DR_WANDB_OUTPUT_DIR=data
+
+# Default W&B project (optional - can use cli instead)
+DR_WANDB_PROJECT=project
+DR_WANDB_ENTITY=entity

dr_wandb-0.1.0/.gitignore

@@ -0,0 +1,209 @@
+data/
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[codz]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+#poetry.toml
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
+#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
+#pdm.lock
+#pdm.toml
+.pdm-python
+.pdm-build/
+
+# pixi
+#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
+#pixi.lock
+#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
+#   in the .venv directory. It is recommended not to include this directory in version control.
+.pixi
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.envrc
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Abstra
+# Abstra is an AI-powered process automation framework.
+# Ignore directories containing user credentials, local state, and settings.
+# Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#  Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
+#  that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#  and can be added to the global gitignore or merged into this file. However, if you prefer, 
+#  you could uncomment the following to ignore the entire vscode folder
+# .vscode/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Cursor
+#  Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to
+#  exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
+#  refer to https://docs.cursor.com/context/ignore-files
+.cursorignore
+.cursorindexingignore
+
+# Marimo
+marimo/_static/
+marimo/_lsp/
+__marimo__/

dr_wandb-0.1.0/.python-version

@@ -0,0 +1 @@
+3.12

dr_wandb-0.1.0/CLAUDE.md

@@ -0,0 +1,116 @@
+# CLAUDE.md
+
+This file provides project level guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## 🚨 READ FIRST - DESIGN PHILOSOPHY
+**MANDATORY:** Before starting ANY work, read `docs/processes/design_philosophy.md` to understand the core principles and methodology that guide this project. All code changes must align with these principles.
+
+- **No Backward Compatibility**: This is a research library - breaking changes are acceptable for better design
+- **Fail Fast, Fail Loudly**: Use assertions, avoid defensive programming that hides bugs
+- **No Exception Handling**: Never use try-catch blocks - let errors surface immediately
+- **Assertions Over Exceptions**: Use `assert condition, "message"` instead of `raise ValueError()`
+- **Minimize Friction**: Every design choice should reduce friction between idea and visualization
+- **Embrace Change, Demand Consistency**: When making changes, update ALL affected parts
+
+Remember: The goal is code that *disappears* into the background, allowing researchers to focus on their work.
+
+## Essential Commands
+- `us` runs `uv sync` - Install all dependencies including dev, test, and test-ml groups
+- `lint` runs `uv run ruff check --fix .` - Lint code with ruff and apply autofixes where possible
+- `ft` runs `uv run ruff format .` - Format code with ruff  
+- `uv run pytest` - Run tests with pytest (supports parallel execution with xdist)
+- `lint_fix` - Run ruff format and then check with --fix
+
+**IMPORTANT**: Do NOT run tests, linting, type checking, or formatting unless explicitly requested by the user. Focus on the requested changes only.
+
+## 🎯 CODE STYLE REQUIREMENTS
+
+### Zero Comments Policy
+- **NEVER add ANY comments** - no docstrings, no inline comments, no block comments
+- Code must be self-documenting through clear naming and structure
+- Remove ALL existing comments when editing files (docstrings, # comments, etc.)
+
+### Comprehensive Typing
+- **ALL function signatures** must have complete type hints for parameters and return values
+- Use `from typing import Any, Optional` etc. as needed
+- Prefer `list`, `dict` etc over `List` and `Dict`
+- Add `from __future__ import annotations` and use modern type hints
+- Import types like `import pandas as pd` when using `pd.DataFrame` in hints
+- If a circular import exists, use `TYPE_CHECKING` to gate
+- All `__init__` methods must have `-> None` return type
+- All class methods need proper `self` typing context
+- Use specific types over `Any` when possible (e.g., `pd.DataFrame` not `Any`)
+- Create custom types for clarity: `type GroupKey = Tuple[Tuple[str, Any], ...]`
+- Example pattern:
+  ```python
+  def method_name(self, param: str, optional_param: Optional[int] = None) -> Dict[str, Any]:
+  ```
+
+### File Structure
+- **ALL imports at the very top** - no imports anywhere else in the file
+- Type aliases near top after imports
+- Magic values should NEVER be hardcoded throughout, all constants be semantically named at the top of the module
+- No module-level docstrings - remove entirely
+- Class definitions without docstrings
+- Methods without docstrings but with full type hints
+
+### Replace Comments with Structure
+- **Instead of comments** → Extract succinctly named helper functions
+- **Instead of complex types** → Create descriptive type aliases
+- Examples:
+  ```python
+  # BAD: Complex code with comments
+  def process_data(self, data):
+      # Convert categorical columns to numeric for ML processing
+      processed = data.copy()
+      # ... complex logic ...
+      
+  # GOOD: Self-documenting through function names and types
+  type CategoricalColumns = List[str]
+  type NumericData = pd.DataFrame
+  
+  def process_data(self, data: pd.DataFrame) -> NumericData:
+      return self._convert_categorical_to_numeric(data)
+      
+  def _convert_categorical_to_numeric(self, data: pd.DataFrame) -> NumericData:
+      # Clear, focused function that explains itself
+  ```
+
+### Fail Fast and Loud: Asserts Not Try-Except
+- **Always aim to check assumptions with asserts**
+- Avoid nested try-except blocks
+- Instead, identify assumptions and assert them at the top of the function
+
+## 🛠️ DEVELOPMENT WORKFLOW
+
+### When Editing Files
+1. **Read design philosophy first** - understand the core method principles
+2. **Strip ALL comments** - docstrings, inline comments, everything
+3. **Add comprehensive type hints** - every parameter, every return value
+4. **Extract helper functions** - instead of complex inline logic with comments
+5. **Import required typing modules** - add to imports as needed
+6. **Test functionality** - ensure no behavioral changes from refactoring
+
+### Code Quality Gates
+- **Use type hints** on all functions
+- **ALL imports at file top** - never mid-file, never in functions, never anywhere else
+- **Use assertions, not exceptions** - single line `assert condition, "message"` instead of try-catch or raising exceptions
+- **Never use try-catch blocks** - let errors bubble up; use assertions for validation
+- **Show full modified functions**, not just diffs
+- **Prefer explicit code** over clever code
+- **Follow "Leave No Trace"** - remove all legacy patterns when making changes
+
+### Git Shortcuts
+| Shortcut | Command | Use |
+|----------|---------|-----|
+| `gst` | `git status` | Check state |
+| `gd_agent` | `git --no-pager diff` | See changes |
+| `glo` | `git log --oneline -10` | Recent commits |
+| `ga .` | `git add .` | Stage files |
+| `gc -m "msg"` | `git commit -m "msg"` | Commit |
+
+### 📋 COMMIT STRATEGY
+- **Small, semantic commits**: 20-30 lines per commit with clear purpose
+- **Single line messages**: Succinct and clear, imperative mood
+- **Quality gates**: Run linting/formatting before commits only when explicitly requested
+- **Incremental building**: Each commit should be reviewable and complete

dr_wandb-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Danielle Rothermel
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

dr_wandb-0.1.0/PKG-INFO

@@ -0,0 +1,123 @@
+Metadata-Version: 2.4
+Name: dr-wandb
+Version: 0.1.0
+Summary: Interact with wandb from python
+Author-email: Danielle Rothermel <danielle.rothermel@gmail.com>
+License-File: LICENSE
+Requires-Python: >=3.12
+Requires-Dist: pandas>=2.3.2
+Requires-Dist: psycopg2>=2.9.10
+Requires-Dist: pyarrow>=21.0.0
+Requires-Dist: pydantic-settings>=2.10.1
+Requires-Dist: sqlalchemy>=2.0.43
+Requires-Dist: wandb>=0.21.4
+Description-Content-Type: text/markdown
+
+# dr_wandb
+
+A command-line utility for downloading and archiving Weights & Biases experiment data to local storage formats optimized for offline analysis. Stores to PostgreSQL db + Parquet files, supports incremental updates and selective data retrieval.
+
+## Installation
+
+```bash
+uv add dr_wandb
+```
+
+### Prerequisites
+
+- Python 3.12 or higher
+- PostgreSQL database server
+- Weights & Biases account with API access
+- PyArrow for Parquet file operations
+
+### Authentication
+
+Configure Weights & Biases authentication using one of these methods:
+
+```bash
+wandb login
+```
+
+Or set the API key as an environment variable:
+
+```bash
+export WANDB_API_KEY=your_api_key_here
+```
+
+## Basic Usage
+
+Download all runs from a Weights & Biases project:
+
+```bash
+wandb-download --entity your_entity --project your_project
+
+Options:
+  --entity TEXT        WandB entity (username or team name)
+  --project TEXT       WandB project name
+  --runs-only          Download only run metadata, skip training history
+  --force-refresh      Download all data, ignoring existing records
+  --db-url TEXT        PostgreSQL connection string
+  --output-dir TEXT    Directory for exported Parquet files
+  --help              Show help message and exit
+```
+
+The tool creates a PostgreSQL database, downloads experiment data, and exports Parquet files to the configured output directory. It tool tracks existing data and downloads only new or updated runs by default. A run is considered for update if:
+
+- It does not exist in the local database
+- Its state is "running" (indicating potential new data)
+
+Use `--force-refresh` to download all runs regardless of existing data.
+
+### Environment Variables
+
+The tool reads configuration from environment variables with the `DR_WANDB_` prefix and supports `.env` files:
+
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `DR_WANDB_ENTITY` | Weights & Biases entity name | None |
+| `DR_WANDB_PROJECT` | Weights & Biases project name | None |
+| `DR_WANDB_DATABASE_URL` | PostgreSQL connection string | `postgresql+psycopg2://localhost/wandb` |
+| `DR_WANDB_OUTPUT_DIR` | Directory for exported files | `./data` |
+
+### Database Configuration
+
+The PostgreSQL connection string follows the standard format:
+
+```
+postgresql+psycopg2://username:password@host:port/database_name
+```
+
+If the specified database does not exist, the tool will attempt to create it automatically.
+
+## Data Schema
+
+
+The tool generates the following files in the output directory:
+
+- `runs_metadata.parquet` - Complete run metadata including configurations, summaries, and system information
+- `runs_history.parquet` - Training metrics and logged values over time
+- `runs_metadata_{component}.parquet` - Component-specific files for config, summary, wandb_metadata, system_metrics, system_attrs, and sweep_info
+
+
+**Run Records**
+- **run_id**: Unique identifier for the experiment run
+- **run_name**: Human-readable name assigned to the run
+- **state**: Current state (finished, running, crashed, failed, killed)
+- **project**: Project name
+- **entity**: Entity name
+- **created_at**: Timestamp of run creation
+- **config**: Experiment configuration parameters (JSONB)
+- **summary**: Final metrics and outputs (JSONB)
+- **wandb_metadata**: Platform-specific metadata (JSONB)
+- **system_metrics**: Hardware and system information (JSONB)
+- **system_attrs**: Additional system attributes (JSONB)
+- **sweep_info**: Hyperparameter sweep information (JSONB)
+
+**Training History Records**
+- **run_id**: Reference to the parent run
+- **step**: Training step number
+- **timestamp**: Time of metric logging
+- **runtime**: Elapsed time since run start
+- **wandb_metadata**: Platform logging metadata (JSONB)
+- **metrics**: All logged metrics and values (JSONB, flattened in Parquet export)
+

dr_wandb-0.1.0/README.md

@@ -0,0 +1,108 @@
+# dr_wandb
+
+A command-line utility for downloading and archiving Weights & Biases experiment data to local storage formats optimized for offline analysis. Stores to PostgreSQL db + Parquet files, supports incremental updates and selective data retrieval.
+
+## Installation
+
+```bash
+uv add dr_wandb
+```
+
+### Prerequisites
+
+- Python 3.12 or higher
+- PostgreSQL database server
+- Weights & Biases account with API access
+- PyArrow for Parquet file operations
+
+### Authentication
+
+Configure Weights & Biases authentication using one of these methods:
+
+```bash
+wandb login
+```
+
+Or set the API key as an environment variable:
+
+```bash
+export WANDB_API_KEY=your_api_key_here
+```
+
+## Basic Usage
+
+Download all runs from a Weights & Biases project:
+
+```bash
+wandb-download --entity your_entity --project your_project
+
+Options:
+  --entity TEXT        WandB entity (username or team name)
+  --project TEXT       WandB project name
+  --runs-only          Download only run metadata, skip training history
+  --force-refresh      Download all data, ignoring existing records
+  --db-url TEXT        PostgreSQL connection string
+  --output-dir TEXT    Directory for exported Parquet files
+  --help              Show help message and exit
+```
+
+The tool creates a PostgreSQL database, downloads experiment data, and exports Parquet files to the configured output directory. It tool tracks existing data and downloads only new or updated runs by default. A run is considered for update if:
+
+- It does not exist in the local database
+- Its state is "running" (indicating potential new data)
+
+Use `--force-refresh` to download all runs regardless of existing data.
+
+### Environment Variables
+
+The tool reads configuration from environment variables with the `DR_WANDB_` prefix and supports `.env` files:
+
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `DR_WANDB_ENTITY` | Weights & Biases entity name | None |
+| `DR_WANDB_PROJECT` | Weights & Biases project name | None |
+| `DR_WANDB_DATABASE_URL` | PostgreSQL connection string | `postgresql+psycopg2://localhost/wandb` |
+| `DR_WANDB_OUTPUT_DIR` | Directory for exported files | `./data` |
+
+### Database Configuration
+
+The PostgreSQL connection string follows the standard format:
+
+```
+postgresql+psycopg2://username:password@host:port/database_name
+```
+
+If the specified database does not exist, the tool will attempt to create it automatically.
+
+## Data Schema
+
+
+The tool generates the following files in the output directory:
+
+- `runs_metadata.parquet` - Complete run metadata including configurations, summaries, and system information
+- `runs_history.parquet` - Training metrics and logged values over time
+- `runs_metadata_{component}.parquet` - Component-specific files for config, summary, wandb_metadata, system_metrics, system_attrs, and sweep_info
+
+
+**Run Records**
+- **run_id**: Unique identifier for the experiment run
+- **run_name**: Human-readable name assigned to the run
+- **state**: Current state (finished, running, crashed, failed, killed)
+- **project**: Project name
+- **entity**: Entity name
+- **created_at**: Timestamp of run creation
+- **config**: Experiment configuration parameters (JSONB)
+- **summary**: Final metrics and outputs (JSONB)
+- **wandb_metadata**: Platform-specific metadata (JSONB)
+- **system_metrics**: Hardware and system information (JSONB)
+- **system_attrs**: Additional system attributes (JSONB)
+- **sweep_info**: Hyperparameter sweep information (JSONB)
+
+**Training History Records**
+- **run_id**: Reference to the parent run
+- **step**: Training step number
+- **timestamp**: Time of metric logging
+- **runtime**: Elapsed time since run start
+- **wandb_metadata**: Platform logging metadata (JSONB)
+- **metrics**: All logged metrics and values (JSONB, flattened in Parquet export)
+

dr_wandb-0.1.0/docs/processes/CODING_PRINCIPLES.md

@@ -0,0 +1 @@
+/Users/daniellerothermel/drotherm/repos/dr_ref/docs/processes/CODING_PRINCIPLES.md

dr_wandb-0.1.0/docs/processes/README.md

@@ -0,0 +1 @@
+/Users/daniellerothermel/drotherm/repos/dr_ref/docs/processes/README.md

dr_wandb-0.1.0/docs/processes/audit_synthesis_pipeline.md

@@ -0,0 +1 @@
+/Users/daniellerothermel/drotherm/repos/dr_ref/docs/processes/audit_synthesis_pipeline.md

dr_wandb-0.1.0/docs/processes/design_philosophy.md

@@ -0,0 +1 @@
+/Users/daniellerothermel/drotherm/repos/dr_ref/docs/processes/design_philosophy.md

dr_wandb-0.1.0/docs/processes/documentation_organizer_guide.md

@@ -0,0 +1 @@
+/Users/daniellerothermel/drotherm/repos/dr_ref/docs/processes/documentation_organizer_guide.md

dr_wandb-0.1.0/docs/processes/fresh_eyes_review_guide.md

@@ -0,0 +1 @@
+/Users/daniellerothermel/drotherm/repos/dr_ref/docs/processes/fresh_eyes_review_guide.md

dr_wandb-0.1.0/docs/processes/general_project_extraction_prompt.md

@@ -0,0 +1 @@
+/Users/daniellerothermel/drotherm/repos/dr_ref/docs/processes/general_project_extraction_prompt.md

dr_wandb-0.1.0/docs/processes/project_consolidation_methodology.md

@@ -0,0 +1 @@
+/Users/daniellerothermel/drotherm/repos/dr_ref/docs/processes/project_consolidation_methodology.md

dr_wandb-0.1.0/docs/processes/reporting_guide.md

@@ -0,0 +1 @@
+/Users/daniellerothermel/drotherm/repos/dr_ref/docs/processes/reporting_guide.md

dr_wandb-0.1.0/docs/processes/strategic_collaboration_guide.md

@@ -0,0 +1 @@
+/Users/daniellerothermel/drotherm/repos/dr_ref/docs/processes/strategic_collaboration_guide.md

dr_wandb-0.1.0/docs/processes/tactical_execution_guide.md

@@ -0,0 +1 @@
+/Users/daniellerothermel/drotherm/repos/dr_ref/docs/processes/tactical_execution_guide.md