wt-compiler 0.1.0__tar.gz

wt_compiler-0.1.0/PKG-INFO

@@ -0,0 +1,301 @@
+Metadata-Version: 2.4
+Name: wt-compiler
+Version: 0.1.0
+Summary: Workflow compiler for generating DAG artifacts from workflow specifications
+Author: Ecoscope Team
+License: MIT
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+Requires-Dist: wt-contracts<1.0.0,>=0.1.0
+Requires-Dist: pydantic<3.0.0,>=2.0.0
+Requires-Dist: jinja2>=3.0.0
+Requires-Dist: ruamel.yaml>=0.17.0
+Requires-Dist: py-rattler<0.23.0,>=0.22.0
+Requires-Dist: datamodel-code-generator>=0.25.0
+Requires-Dist: pydot>=1.4.0
+Requires-Dist: ruff>=0.1.0
+Requires-Dist: tomli-w>=1.0.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.23.0; extra == "dev"
+Requires-Dist: mypy>=1.0.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+
+# wt-compiler
+
+Workflow compiler for generating DAG artifacts from workflow specifications.
+
+## Overview
+
+`wt-compiler` is a key component of the wt (workflow toolkit) ecosystem. It compiles workflow specifications (YAML files) into complete, executable workflow packages including:
+
+- DAG Python code (async, sequential, and Jupytext variants)
+- Pydantic parameter models with JSON schemas
+- CLI interfaces for workflow execution
+- Pixi configuration for dependency management
+- Dockerfiles for containerized deployment
+- Test suites
+
+## Key Innovation: Environment-Isolated Task Discovery
+
+Unlike legacy systems that require importing task libraries directly, `wt-compiler` uses **subprocess-based task discovery**:
+
+1. Creates ephemeral rattler/pixi environments with specified requirements
+2. Calls `wt-registry` CLI in that environment
+3. Parses JSON output (validated against `wt-contracts` schemas)
+4. Compiles workflows without Python import dependencies on task libraries
+
+This enables:
+- ✅ Cross-environment compilation (Python 3.10 compiler can target Python 3.12 tasks)
+- ✅ Isolation from task library dependency conflicts
+- ✅ Type-safe contracts via `wt-contracts` package
+- ✅ No circular dependencies between packages
+
+## Installation
+
+```bash
+# From source (development)
+cd wt/wt-compiler
+uv sync
+
+# Once published to PyPI
+uv add wt-compiler
+```
+
+## Usage
+
+### Basic Compilation
+
+```python
+from wt_compiler import compile_workflow, Spec
+from rattler import MatchSpec
+
+# Load a workflow specification
+spec = Spec.parse_file("workflow/spec.yaml")
+
+# Compile to artifacts
+artifacts = compile_workflow(
+    spec=spec,
+    spec_relpath="workflow/spec.yaml"
+)
+
+# Write artifacts to disk
+artifacts.dump(clobber=True)
+```
+
+### Task Discovery
+
+```python
+from wt_compiler.discovery import discover_tasks_from_requirements
+from rattler import MatchSpec
+
+# Discover tasks from requirements
+requirements = [
+    MatchSpec("my-task-library>=1.0.0"),
+    MatchSpec("another-library>=2.0.0"),
+]
+
+tasks = discover_tasks_from_requirements(requirements)
+# Returns: dict[task_name, dict[module_path, KnownTask]]
+```
+
+### Workflow Specification Format
+
+```yaml
+id: my-workflow
+requirements:
+  - name: my-task-library
+    version: ">=1.0.0"
+    channel: conda-forge
+
+workflow:
+  - id: task1
+    task: extract_data
+    partial:
+      source: "s3://my-bucket/data.csv"
+
+  - id: task2
+    task: transform_data
+    partial:
+      input_data: "${{ workflow.task1.return }}"
+    map:
+      argnames: param
+      argvalues: "${{ workflow.task1.return }}"
+```
+
+## Architecture
+
+### Package Structure
+
+```
+wt-compiler/
+├── src/wt_compiler/
+│   ├── __init__.py          # Public exports
+│   ├── spec.py              # Spec and TaskInstance models
+│   ├── compiler.py          # DagCompiler class
+│   ├── discovery.py         # Task discovery via rattler + CLI
+│   ├── artifacts.py         # Artifact generation models
+│   ├── jsonschema.py        # JSON schema utilities
+│   ├── requirements.py      # Rattler channel/matchspec handling
+│   ├── util.py              # Import validation utilities
+│   ├── formatting.py        # Ruff formatting decorator
+│   ├── _models.py           # Pydantic base classes
+│   └── templates/           # Jinja2 templates
+│       ├── pkg/
+│       │   ├── dags/
+│       │   │   ├── run_async.jinja2
+│       │   │   ├── run_sequential.jinja2
+│       │   │   └── jupytext.jinja2
+│       │   ├── cli.jinja2
+│       │   ├── dispatch.jinja2
+│       │   └── ...
+│       ├── tests/
+│       ├── Dockerfile.jinja2
+│       └── pixi.jinja2
+└── tests/
+    ├── test_spec.py
+    ├── test_compiler.py
+    ├── test_discovery.py
+    └── ...
+```
+
+### Dependencies
+
+- **wt-contracts** (>=0.1.0): Shared type contracts (RegistryOutput, TaskProtocol, etc.)
+- **pydantic** (>=2.0.0): Data validation and modeling
+- **jinja2**: Template rendering
+- **ruamel.yaml**: YAML parsing
+- **rattler** (>=0.8.0): Conda environment management
+- **datamodel-code-generator**: Generate Pydantic models from JSON schemas
+- **pydot**: DAG visualization
+
+## Implementation Status
+
+### ✅ Completed Components
+
+1. **Package Structure** - Full directory layout with src/ structure
+2. **pyproject.toml** - setuptools-scm configuration, dependencies, tool configs
+3. **spec.py** - Complete Spec, TaskInstance, and related models (~700 lines)
+4. **discovery.py** - Task discovery via rattler + wt-registry CLI
+5. **artifacts.py** - All artifact models (Dags, PixiToml, WorkflowArtifacts, etc.)
+6. **requirements.py** - Channel and MatchSpec handling
+7. **jsonschema.py** - JSON schema utilities with RJSF support
+8. **util.py** - Import reference validation
+9. **formatting.py** - Ruff formatting decorator
+10. **_models.py** - Pydantic base model classes
+11. **templates/** - All Jinja2 templates copied from legacy codebase
+12. **compiler.py** - Core DagCompiler class structure
+
+### ⚠️ Needs Expansion
+
+The following areas are implemented as simplified stubs and need full implementation:
+
+#### compiler.py TODOs
+
+1. **get_params_jsonschema()** - Currently returns empty schema
+   - Needs: Extract schemas from discovered task metadata
+   - Needs: Merge schemas for task groups
+   - Needs: Apply omit_args logic
+   - Needs: Generate proper UI schema
+   - Needs: Apply RJSF overrides
+
+2. **generate_params_model()** - Stub implementation
+   - Needs: Use datamodel-code-generator to create Pydantic model from JSON schema
+   - Needs: Proper imports and type hints
+
+3. **Graph visualization** - Not implemented
+   - Needs: Generate pydot graphs showing task dependencies
+   - Needs: Export to PNG
+
+4. **README generation** - Not implemented
+   - Needs: Generate README.md with fingerprint information
+   - Needs: Include workflow diagram, parameter documentation
+
+5. **Version management** - Basic implementation only
+   - Needs: Full VERSION.yaml bump logic
+   - Needs: Lockfile carryover for updates
+
+6. **get_per_taskinstance_params_notebook()** - Empty stub
+   - Needs: Generate parameter notebooks for Jupytext DAG
+
+#### discovery.py TODOs
+
+1. **rattler-py native API** - Currently uses subprocess fallback
+   - Needs: Update when rattler-py solve/install API is stable
+   - Needs: Better error handling
+
+2. **Schema validation** - Basic validation only
+   - Needs: Full wt-contracts schema validation
+   - Needs: Better error messages for malformed CLI output
+
+#### Testing
+
+- **Unit tests** - Not yet written
+  - Need tests for: spec parsing, validation, compilation
+  - Need tests for: task discovery with mock environments
+  - Need tests for: artifact generation
+  - Need tests for: template rendering
+
+## Development
+
+### Setup
+
+```bash
+cd wt/wt-compiler
+uv sync
+```
+
+### Run Tests
+
+```bash
+uv run pytest
+```
+
+### Type Checking
+
+```bash
+uv run mypy src/wt_compiler
+```
+
+### Linting
+
+```bash
+uv run ruff check src/wt_compiler
+uv run ruff format src/wt_compiler
+```
+
+## Relationship to Other Packages
+
+- **wt-contracts**: Depends on (provides type contracts)
+- **wt-registry**: Called via subprocess (no Python dependency)
+- **wt-task**: No dependency (generates code that uses it)
+- **wt-runner**: No dependency (runner may depend on compiler in future)
+- **wt-invokers**: No dependency
+
+## Migration from Legacy
+
+This package replaces `ecoscope_workflows_core.compiler`. Key differences:
+
+1. **No direct task imports** - Uses CLI-based discovery instead
+2. **wt-contracts integration** - Type-safe schemas for all interfaces
+3. **Modular dependencies** - Only depends on wt-contracts
+4. **Simplified models** - Spec models are now in spec.py instead of compiler.py
+
+## Future Work
+
+1. Complete all TODO areas in compiler.py
+2. Write comprehensive test suite
+3. Add CLI tool for standalone compilation
+4. Add workflow visualization tools
+5. Add workflow validation tools
+6. Performance optimization for large workflows
+7. Better error messages and debugging tools
+
+## Contributing
+
+See main wt repository CONTRIBUTING.md for guidelines.
+
+## License
+
+MIT

wt_compiler-0.1.0/README.md

@@ -0,0 +1,277 @@
+# wt-compiler
+
+Workflow compiler for generating DAG artifacts from workflow specifications.
+
+## Overview
+
+`wt-compiler` is a key component of the wt (workflow toolkit) ecosystem. It compiles workflow specifications (YAML files) into complete, executable workflow packages including:
+
+- DAG Python code (async, sequential, and Jupytext variants)
+- Pydantic parameter models with JSON schemas
+- CLI interfaces for workflow execution
+- Pixi configuration for dependency management
+- Dockerfiles for containerized deployment
+- Test suites
+
+## Key Innovation: Environment-Isolated Task Discovery
+
+Unlike legacy systems that require importing task libraries directly, `wt-compiler` uses **subprocess-based task discovery**:
+
+1. Creates ephemeral rattler/pixi environments with specified requirements
+2. Calls `wt-registry` CLI in that environment
+3. Parses JSON output (validated against `wt-contracts` schemas)
+4. Compiles workflows without Python import dependencies on task libraries
+
+This enables:
+- ✅ Cross-environment compilation (Python 3.10 compiler can target Python 3.12 tasks)
+- ✅ Isolation from task library dependency conflicts
+- ✅ Type-safe contracts via `wt-contracts` package
+- ✅ No circular dependencies between packages
+
+## Installation
+
+```bash
+# From source (development)
+cd wt/wt-compiler
+uv sync
+
+# Once published to PyPI
+uv add wt-compiler
+```
+
+## Usage
+
+### Basic Compilation
+
+```python
+from wt_compiler import compile_workflow, Spec
+from rattler import MatchSpec
+
+# Load a workflow specification
+spec = Spec.parse_file("workflow/spec.yaml")
+
+# Compile to artifacts
+artifacts = compile_workflow(
+    spec=spec,
+    spec_relpath="workflow/spec.yaml"
+)
+
+# Write artifacts to disk
+artifacts.dump(clobber=True)
+```
+
+### Task Discovery
+
+```python
+from wt_compiler.discovery import discover_tasks_from_requirements
+from rattler import MatchSpec
+
+# Discover tasks from requirements
+requirements = [
+    MatchSpec("my-task-library>=1.0.0"),
+    MatchSpec("another-library>=2.0.0"),
+]
+
+tasks = discover_tasks_from_requirements(requirements)
+# Returns: dict[task_name, dict[module_path, KnownTask]]
+```
+
+### Workflow Specification Format
+
+```yaml
+id: my-workflow
+requirements:
+  - name: my-task-library
+    version: ">=1.0.0"
+    channel: conda-forge
+
+workflow:
+  - id: task1
+    task: extract_data
+    partial:
+      source: "s3://my-bucket/data.csv"
+
+  - id: task2
+    task: transform_data
+    partial:
+      input_data: "${{ workflow.task1.return }}"
+    map:
+      argnames: param
+      argvalues: "${{ workflow.task1.return }}"
+```
+
+## Architecture
+
+### Package Structure
+
+```
+wt-compiler/
+├── src/wt_compiler/
+│   ├── __init__.py          # Public exports
+│   ├── spec.py              # Spec and TaskInstance models
+│   ├── compiler.py          # DagCompiler class
+│   ├── discovery.py         # Task discovery via rattler + CLI
+│   ├── artifacts.py         # Artifact generation models
+│   ├── jsonschema.py        # JSON schema utilities
+│   ├── requirements.py      # Rattler channel/matchspec handling
+│   ├── util.py              # Import validation utilities
+│   ├── formatting.py        # Ruff formatting decorator
+│   ├── _models.py           # Pydantic base classes
+│   └── templates/           # Jinja2 templates
+│       ├── pkg/
+│       │   ├── dags/
+│       │   │   ├── run_async.jinja2
+│       │   │   ├── run_sequential.jinja2
+│       │   │   └── jupytext.jinja2
+│       │   ├── cli.jinja2
+│       │   ├── dispatch.jinja2
+│       │   └── ...
+│       ├── tests/
+│       ├── Dockerfile.jinja2
+│       └── pixi.jinja2
+└── tests/
+    ├── test_spec.py
+    ├── test_compiler.py
+    ├── test_discovery.py
+    └── ...
+```
+
+### Dependencies
+
+- **wt-contracts** (>=0.1.0): Shared type contracts (RegistryOutput, TaskProtocol, etc.)
+- **pydantic** (>=2.0.0): Data validation and modeling
+- **jinja2**: Template rendering
+- **ruamel.yaml**: YAML parsing
+- **rattler** (>=0.8.0): Conda environment management
+- **datamodel-code-generator**: Generate Pydantic models from JSON schemas
+- **pydot**: DAG visualization
+
+## Implementation Status
+
+### ✅ Completed Components
+
+1. **Package Structure** - Full directory layout with src/ structure
+2. **pyproject.toml** - setuptools-scm configuration, dependencies, tool configs
+3. **spec.py** - Complete Spec, TaskInstance, and related models (~700 lines)
+4. **discovery.py** - Task discovery via rattler + wt-registry CLI
+5. **artifacts.py** - All artifact models (Dags, PixiToml, WorkflowArtifacts, etc.)
+6. **requirements.py** - Channel and MatchSpec handling
+7. **jsonschema.py** - JSON schema utilities with RJSF support
+8. **util.py** - Import reference validation
+9. **formatting.py** - Ruff formatting decorator
+10. **_models.py** - Pydantic base model classes
+11. **templates/** - All Jinja2 templates copied from legacy codebase
+12. **compiler.py** - Core DagCompiler class structure
+
+### ⚠️ Needs Expansion
+
+The following areas are implemented as simplified stubs and need full implementation:
+
+#### compiler.py TODOs
+
+1. **get_params_jsonschema()** - Currently returns empty schema
+   - Needs: Extract schemas from discovered task metadata
+   - Needs: Merge schemas for task groups
+   - Needs: Apply omit_args logic
+   - Needs: Generate proper UI schema
+   - Needs: Apply RJSF overrides
+
+2. **generate_params_model()** - Stub implementation
+   - Needs: Use datamodel-code-generator to create Pydantic model from JSON schema
+   - Needs: Proper imports and type hints
+
+3. **Graph visualization** - Not implemented
+   - Needs: Generate pydot graphs showing task dependencies
+   - Needs: Export to PNG
+
+4. **README generation** - Not implemented
+   - Needs: Generate README.md with fingerprint information
+   - Needs: Include workflow diagram, parameter documentation
+
+5. **Version management** - Basic implementation only
+   - Needs: Full VERSION.yaml bump logic
+   - Needs: Lockfile carryover for updates
+
+6. **get_per_taskinstance_params_notebook()** - Empty stub
+   - Needs: Generate parameter notebooks for Jupytext DAG
+
+#### discovery.py TODOs
+
+1. **rattler-py native API** - Currently uses subprocess fallback
+   - Needs: Update when rattler-py solve/install API is stable
+   - Needs: Better error handling
+
+2. **Schema validation** - Basic validation only
+   - Needs: Full wt-contracts schema validation
+   - Needs: Better error messages for malformed CLI output
+
+#### Testing
+
+- **Unit tests** - Not yet written
+  - Need tests for: spec parsing, validation, compilation
+  - Need tests for: task discovery with mock environments
+  - Need tests for: artifact generation
+  - Need tests for: template rendering
+
+## Development
+
+### Setup
+
+```bash
+cd wt/wt-compiler
+uv sync
+```
+
+### Run Tests
+
+```bash
+uv run pytest
+```
+
+### Type Checking
+
+```bash
+uv run mypy src/wt_compiler
+```
+
+### Linting
+
+```bash
+uv run ruff check src/wt_compiler
+uv run ruff format src/wt_compiler
+```
+
+## Relationship to Other Packages
+
+- **wt-contracts**: Depends on (provides type contracts)
+- **wt-registry**: Called via subprocess (no Python dependency)
+- **wt-task**: No dependency (generates code that uses it)
+- **wt-runner**: No dependency (runner may depend on compiler in future)
+- **wt-invokers**: No dependency
+
+## Migration from Legacy
+
+This package replaces `ecoscope_workflows_core.compiler`. Key differences:
+
+1. **No direct task imports** - Uses CLI-based discovery instead
+2. **wt-contracts integration** - Type-safe schemas for all interfaces
+3. **Modular dependencies** - Only depends on wt-contracts
+4. **Simplified models** - Spec models are now in spec.py instead of compiler.py
+
+## Future Work
+
+1. Complete all TODO areas in compiler.py
+2. Write comprehensive test suite
+3. Add CLI tool for standalone compilation
+4. Add workflow visualization tools
+5. Add workflow validation tools
+6. Performance optimization for large workflows
+7. Better error messages and debugging tools
+
+## Contributing
+
+See main wt repository CONTRIBUTING.md for guidelines.
+
+## License
+
+MIT

wt_compiler-0.1.0/pyproject.toml

@@ -0,0 +1,128 @@
+[build-system]
+requires = ["setuptools>=64", "setuptools-scm>=8"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "wt-compiler"
+dynamic = ["version"]
+description = "Workflow compiler for generating DAG artifacts from workflow specifications"
+requires-python = ">=3.12"
+readme = "README.md"
+license = { text = "MIT" }
+authors = [
+    { name = "Ecoscope Team" }
+]
+dependencies = [
+    "wt-contracts>=0.1.0,<1.0.0",
+    "pydantic>=2.0.0,<3.0.0",
+    "jinja2>=3.0.0",
+    "ruamel.yaml>=0.17.0",
+    "py-rattler>=0.22.0,<0.23.0",
+    "datamodel-code-generator>=0.25.0",
+    "pydot>=1.4.0",
+    "ruff>=0.1.0",
+    "tomli-w>=1.0.0",
+]
+
+[project.scripts]
+wt-compiler = "wt_compiler.cli:main"
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0.0",
+    "pytest-cov>=4.0.0",
+    "pytest-asyncio>=0.23.0",
+    "mypy>=1.0.0",
+    "ruff>=0.1.0",
+]
+
+[dependency-groups]
+dev = [
+    "pytest>=7.0.0",
+    "pytest-cov>=4.0.0",
+    "pytest-asyncio>=0.23.0",
+    "mypy>=1.0.0",
+    "ruff>=0.1.0",
+]
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools_scm]
+# Version comes from git tags matching this pattern
+tag_regex = "^wt-compiler/v(?P<version>[0-9.]+)$"
+# Root is the git repository root (parent of wt/)
+root = ".."
+# Generate version file
+version_file = "src/wt_compiler/_version.py"
+git_describe_command = ["git", "describe", "--dirty", "--tags", "--long", "--match", "wt-compiler/v*"]
+
+[tool.uv.sources]
+# Development override - uses local editable wt-contracts
+# Published packages will reference PyPI version normally
+wt-contracts = { path = "../wt-contracts", editable = true }
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["tests"]
+python_files = ["test_*.py"]
+addopts = "--cov=wt_compiler --cov-report=term-missing"
+asyncio_mode = "auto"
+
+[tool.mypy]
+strict = true
+python_version = "3.12"
+warn_return_any = true
+warn_unused_configs = true
+disallow_untyped_defs = true
+
+[tool.ruff]
+line-length = 100
+target-version = "py312"
+
+[tool.ruff.lint]
+select = [
+    "E",  # pycodestyle errors
+    "W",  # pycodestyle warnings
+    "F",  # pyflakes
+    "I",  # isort
+    "B",  # flake8-bugbear
+    "C4",  # flake8-comprehensions
+    "UP",  # pyupgrade
+]
+ignore = []
+
+[tool.ruff.lint.per-file-ignores]
+"__init__.py" = ["F401"]  # Allow unused imports in __init__.py
+
+# ============================================================================
+# Pixi Build Configuration (conda package building)
+# ============================================================================
+
+[tool.pixi.package]
+name = "wt-compiler"
+version = "0.1.0"
+
+[tool.pixi.package.build]
+backend = { name = "pixi-build-python", version = "*", channels = ["conda-forge", "https://prefix.dev/pixi-build-backends"] }
+
+[tool.pixi.package.build.config]
+noarch = true
+
+[tool.pixi.package.host-dependencies]
+python = ">=3.12,<3.16"
+setuptools = ">=64"
+setuptools-scm = ">=8"
+pip = "*"
+
+[tool.pixi.package.run-dependencies]
+python = ">=3.12,<3.16"
+pydantic = ">=2.0.0,<3.0.0"
+jinja2 = ">=3.0.0"
+"ruamel.yaml" = ">=0.17.0"
+py-rattler = ">=0.22.0,<0.23.0"
+datamodel-code-generator = ">=0.25.0"
+pydot = ">=1.4.0"
+ruff = ">=0.1.0"
+tomli-w = ">=1.0.0"
+wt-contracts = ">=0.1.0,<1.0.0"

wt_compiler-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+