cryoflow 0.2.0__tar.gz

cryoflow-0.2.0/.github/actions/setup-nix/action.yaml

@@ -0,0 +1,19 @@
+name: 'Setup Nix'
+description: 'Install Nix and configure Cachix'
+inputs:
+  cachix-auth-token:
+    description: 'Cachix authentication token'
+    required: false
+runs:
+  using: 'composite'
+  steps:
+    - name: Install Nix
+      uses: cachix/install-nix-action@2126ae7fc54c9df00dd18f7f18754393182c73cd # v31.9.1
+      with:
+        github_access_token: ${{ github.token }}
+
+    - name: Setup Cachix (yasunori)
+      uses: cachix/cachix-action@3ba601ff5bbb07c7220846facfa2cd81eeee15a1 # v16
+      with:
+        name: yasunori0418
+        authToken: ${{ inputs.cachix-auth-token }}

cryoflow-0.2.0/.github/pull_request_template.md

@@ -0,0 +1,77 @@
+## Overview
+
+<!-- Briefly describe the purpose and context of this PR -->
+
+## Type of Change
+
+<!-- Check the items that apply -->
+
+- [ ] 🎉 New feature (non-breaking change which adds functionality)
+- [ ] 🐛 Bug fix (non-breaking change which fixes an issue)
+- [ ] 💥 Breaking change (fix or feature that would cause existing functionality to not work as expected)
+- [ ] 📝 Documentation update
+- [ ] ♻️ Refactoring (code improvement without changing functionality)
+- [ ] 🎨 Style change (formatting, missing semicolons, etc.)
+- [ ] ⚡ Performance improvement
+- [ ] ✅ Test addition/modification
+- [ ] 🔧 Configuration change
+- [ ] 🏗️ Build/CI/CD related
+
+## Changes
+
+<!-- List the details of the changes made -->
+
+-
+-
+-
+
+## Related Issues
+
+<!-- Link related issues if any -->
+
+Closes #
+Related to #
+
+## Testing
+
+<!-- Describe the testing status -->
+
+### Test Results
+
+```bash
+# Paste test execution commands and results
+```
+
+### Verification
+
+- [ ] Verified in local environment
+- [ ] All existing tests pass
+- [ ] Added new tests (if applicable)
+
+## Screenshots
+
+<!-- Attach Before/After screenshots if there are UI changes -->
+
+## Checklist
+
+<!-- Check items that should be verified before merging -->
+
+- [ ] Code follows the project's coding conventions
+- [ ] Self-reviewed the code
+- [ ] Added comments (especially for complex logic)
+- [ ] Updated documentation (if applicable)
+- [ ] No warnings generated
+- [ ] All existing tests pass
+- [ ] Added tests for new changes
+- [ ] Included migration guide (if breaking change)
+
+## Additional Information
+
+<!-- Add any additional information for reviewers -->
+
+## Review Focus
+
+<!-- Highlight specific areas that need reviewer attention -->
+
+-
+-

cryoflow-0.2.0/.github/workflows/cachix-push.yml

@@ -0,0 +1,23 @@
+name: Cachix Push
+
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Setup Nix
+        uses: ./.github/actions/setup-nix
+        with:
+          cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
+
+      - name: Build packages
+        run: |
+          nix build .#default
+          nix build .#test

cryoflow-0.2.0/.github/workflows/publish.yml

@@ -0,0 +1,26 @@
+name: Publish to PyPI
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Setup uv
+        uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # v7.3.1
+
+      - name: Build packages
+        run: uv build --all-packages
+
+      - name: Publish to PyPI
+        run: uv publish
+        env:
+          UV_PUBLISH_TOKEN: ${{ secrets.PYPI_TOKEN }}

cryoflow-0.2.0/.github/workflows/test.yml

@@ -0,0 +1,22 @@
+name: Test
+
+on:
+  workflow_dispatch:
+  push:
+  pull_request:
+
+jobs:
+  pytest:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Setup Nix
+        uses: ./.github/actions/setup-nix
+        with:
+          cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
+
+      - name: Run pytest
+        run: nix shell .#test -c pytest
+        env:
+          TERM: "dumb"

cryoflow-0.2.0/.gitignore

@@ -0,0 +1,227 @@
+# Python.gitignore
+# @see: https://github.com/github/gitignore/blob/main/Python.gitignore
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[codz]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#   Usually these files are written by a python script from a template
+#   before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+# Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+# uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+# poetry.lock
+# poetry.toml
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
+#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
+# pdm.lock
+# pdm.toml
+.pdm-python
+.pdm-build/
+
+# pixi
+#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
+# pixi.lock
+#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
+#   in the .venv directory. It is recommended not to include this directory in version control.
+.pixi
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# Redis
+*.rdb
+*.aof
+*.pid
+
+# RabbitMQ
+mnesia/
+rabbitmq/
+rabbitmq-data/
+
+# ActiveMQ
+activemq-data/
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.envrc
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#   JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#   be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#   and can be added to the global gitignore or merged into this file.  For a more nuclear
+#   option (not recommended) you can uncomment the following to ignore the entire idea folder.
+# .idea/
+
+# Abstra
+#   Abstra is an AI-powered process automation framework.
+#   Ignore directories containing user credentials, local state, and settings.
+#   Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#   Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
+#   that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#   and can be added to the global gitignore or merged into this file. However, if you prefer, 
+#   you could uncomment the following to ignore the entire vscode folder
+# .vscode/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Marimo
+marimo/_static/
+marimo/_lsp/
+__marimo__/
+
+# Streamlit
+.streamlit/secrets.toml
+
+# Nix.gitignore
+# @see: https://github.com/github/gitignore/blob/main/Nix.gitignore
+# Ignore build outputs from performing a nix-build or `nix build` command
+result
+result-*
+
+# Ignore automatically generated direnv output
+.direnv

cryoflow-0.2.0/.python-version

@@ -0,0 +1 @@
+3.11

cryoflow-0.2.0/CHANGELOG.md

@@ -0,0 +1,212 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [0.2.0] - 2026-02-22
+
+### Added
+
+- **`InputPlugin` ABC** (`cryoflow-core`):
+  - New `InputPlugin` base class for loading data from arbitrary sources
+  - `execute()` takes no arguments and returns `Result[FrameData, Exception]`
+  - `dry_run()` returns schema without loading actual data
+  - Registered via new `register_input_plugins` hookspec in `hookspecs.py`
+
+- **`label` attribute on `BasePlugin`** (`cryoflow-core`):
+  - All plugins now carry a `label` string (default: `'default'`)
+  - Enables label-based routing of multiple concurrent data streams
+
+- **Concrete `InputPlugin` implementations** (`cryoflow-plugin-collections`):
+  - `cryoflow_plugin_collections.input.parquet_scan` — reads Parquet files via `pl.scan_parquet()`
+  - `cryoflow_plugin_collections.input.ipc_scan` — reads Arrow IPC files via `pl.scan_ipc()`
+
+- **`libs/returns` submodule split** (`cryoflow-plugin-collections`):
+  - `libs/returns/result.py` — re-exports `Result`, `Success`, `Failure`, and related utilities
+  - `libs/returns/maybe.py` — re-exports `Maybe`, `Some`, `Nothing`, and related utilities
+  - Both are re-exported from `libs/returns/__init__.py` for backward compatibility
+
+- **Docs**: InputPlugin guide integrated into `docs/plugin_development.md` / `docs/plugin_development_ja.md`
+
+### Changed
+
+- **BREAKING**: `config.py` — `input_path: Path` field removed from `RunConfig`; replaced by `input_plugins: list[PluginConfig]`
+- **BREAKING**: `config.py` — `plugins` field renamed to `transform_plugins` and `output_plugins` (separate lists)
+- **BREAKING**: `config.py` — `label` field added to `PluginConfig`; required for multi-stream routing
+- **BREAKING**: `pipeline.py` — `load_data()` and `_detect_format()` removed; pipeline now built on `LabeledDataMap` keyed by label
+- **BREAKING**: `loader.py` — `InputPlugin` load/registration added; `label` passed to all plugin instances at instantiation
+
+### Tests
+
+- Reorganized all test files from flat layout into per-module directories (one class per file)
+- Added pipeline tests: label routing, `LabeledDataMap` construction, `execute_transform_chain`, `execute_output`, dry-run pipeline
+- Added `InputPlugin` unit tests, `label` attribute tests
+- Total: 232 tests passing
+
+---
+
+### Migration Guide
+
+#### Configuration File Migration
+
+The top-level `input_path` key is removed. Input data sources must now be declared as `[[input_plugins]]` entries.
+
+```toml
+# Before (0.1.x)
+input_path = "data/input.parquet"
+
+[[plugins]]
+name = "column-multiplier"
+module = "cryoflow_plugin_collections.transform.multiplier"
+
+[[plugins]]
+name = "parquet-writer"
+module = "cryoflow_plugin_collections.output.parquet_writer"
+
+# After (0.2.0)
+[[input_plugins]]
+name = "parquet-scan"
+module = "cryoflow_plugin_collections.input.parquet_scan"
+label = "default"
+[input_plugins.options]
+input_path = "data/input.parquet"
+
+[[transform_plugins]]
+name = "column-multiplier"
+module = "cryoflow_plugin_collections.transform.multiplier"
+label = "default"
+
+[[output_plugins]]
+name = "parquet-writer"
+module = "cryoflow_plugin_collections.output.parquet_writer"
+label = "default"
+```
+
+#### Plugin Development Migration
+
+If you have custom plugins that referenced the old `plugins` config key or `load_data()` pipeline API, update as follows:
+
+- Replace `plugins` with `transform_plugins` or `output_plugins` in your config files
+- Remove any code relying on `pipeline.load_data()` or `pipeline._detect_format()`
+- `InputPlugin.execute()` takes **no arguments** (unlike `TransformPlugin` / `OutputPlugin` which receive `df: FrameData`)
+- Assign a `label` to every plugin config entry; use `'default'` for single-stream pipelines
+
+#### `libs/returns` Import Path (unchanged)
+
+The `libs/returns` module was split internally into `result.py` and `maybe.py`, but all public symbols remain importable from `cryoflow_plugin_collections.libs.returns` — no import changes required.
+
+---
+
+## [0.1.3] - 2026-02-17
+
+### Added
+
+- **Complete polars API re-export** (`cryoflow-plugin-collections`):
+  - Re-export all 228+ public APIs from polars using wildcard import
+  - Support for 3 import patterns:
+    1. Module import: `from libs import polars as pl; pl.col()`
+    2. Object import: `from libs.polars import pl; pl.col()`
+    3. Individual imports: `from libs.polars import col, lit, when`
+  - Dynamic `__all__` generation for automatic tracking of polars updates
+  - Zero maintenance overhead and zero runtime overhead
+  - Full type safety and IDE autocomplete support
+
+- **Complete returns library re-export** (`cryoflow-plugin-collections`):
+  - Re-export 146+ unique public APIs from 13 major returns modules
+  - Newly available containers and utilities:
+    - `Maybe` monad for optional values (`Some`, `Nothing`)
+    - `IO` containers for side effect management (`IO`, `IOResult`)
+    - `Future` for async operations (`Future`, `FutureResult`)
+    - `Context` for dependency injection (`RequiresContext`)
+    - Pipeline utilities (`flow`, `pipe`)
+    - Pointfree operations (`bind`, `map`, `alt`, `lash`)
+    - Curry/partial application utilities
+    - Converters and methods for container transformations
+  - Support for individual imports: `from libs.returns import Maybe, IO, flow`
+  - Full functional programming toolkit for plugin developers
+
+### Changed
+
+- **Improved `__all__` construction** (`cryoflow-plugin-collections`):
+  - Refactored returns module to use single assignment pattern
+  - Replaced destructive extend/reassign with list comprehension and temporary variable
+  - Better static analysis compatibility (reduced Pyright warnings)
+
+### Tests
+
+- Added comprehensive tests for polars re-export (4 new test cases)
+- Added comprehensive tests for returns re-export (4 new test cases)
+- All 44 tests passing with full backward compatibility
+
+## [0.1.0] - 2025-02-12
+
+### Added
+
+- **Path resolution system**: All file paths in configuration are now resolved relative to the config file's directory
+  - `config.py`: Added `_resolve_path_relative_to_config()` helper function
+  - `BasePlugin`: Added `resolve_path()` method for consistent path resolution across plugins
+  - Comprehensive E2E tests for relative path resolution
+
+### Changed
+
+- **BREAKING CHANGE**: `BasePlugin.__init__()` now requires `config_dir` parameter (no longer optional)
+  - Remove backward compatibility fallback to eliminate technical debt
+  - All plugins must explicitly receive `config_dir` during instantiation
+- **BREAKING CHANGE**: Relative paths in `input_path` are now resolved relative to config file directory instead of current working directory
+- **BREAKING CHANGE**: Plugin option paths are resolved relative to config file directory
+- Updated `ParquetWriterPlugin` to use `resolve_path()` for output paths
+- Updated all example configurations to use relative paths
+
+### Documentation
+
+- Added comprehensive path resolution behavior section to `docs/spec.md` and `docs/spec_ja.md`
+- Updated `README.md` and `README_ja.md` with path resolution guidelines
+- Updated plugin development guides (`docs/plugin_development.md`, `docs/plugin_development_ja.md`) with new API
+
+### Migration Guide
+
+#### For Configuration Files
+
+Update relative paths to be relative to the config file location:
+
+```toml
+# Before (0.0.x): paths relative to current working directory
+input_path = "examples/data/input.parquet"
+
+# After (0.1.0): paths relative to config file directory
+# If config.toml is in project root:
+input_path = "examples/data/input.parquet"  # Same if running from project root
+
+# If config.toml is in config/:
+input_path = "../examples/data/input.parquet"  # OR use absolute path
+```
+
+#### For Plugin Development
+
+Update plugin constructors to accept `config_dir`:
+
+```python
+# Before (0.0.x)
+class MyPlugin(OutputPlugin):
+    def execute(self, df):
+        output_path = Path(self.options.get('output_path'))
+        # ...
+
+# After (0.1.0)
+class MyPlugin(OutputPlugin):
+    def execute(self, df):
+        output_path = self.resolve_path(self.options.get('output_path'))
+        # ...
+```
+
+## [0.0.4] - 2025-02-11
+
+### Changed
+
+- Remove unused `output_target` configuration field
+
+## [0.0.3] - 2025-02-11
+
+Initial working release with core functionality.