just-dna-compiler 0.1.0__tar.gz

just_dna_compiler-0.1.0/PKG-INFO

@@ -0,0 +1,35 @@
+Metadata-Version: 2.3
+Name: just-dna-compiler
+Version: 0.1.0
+Summary: Reference compiler for just-dna annotation modules: spec directory → parquet artifact + manifest
+Requires-Dist: just-dna-format
+Requires-Dist: polars>=1.42.0
+Requires-Dist: duckdb>=1.1.0
+Requires-Dist: pyyaml>=6.0.2
+Requires-Dist: platformdirs>=4.0.0
+Requires-Dist: python-dotenv>=1.0.1
+Requires-Python: >=3.13
+Description-Content-Type: text/markdown
+
+# just-dna-compiler
+
+The reference **compiler** for just-dna annotation modules: it turns an authored spec directory
+(`module_spec.yaml` + `variants.csv` + `studies.csv`) into the deployable three-parquet artifact
+(`weights` / `annotations` / `studies`) **plus a `manifest.json`** with integrity digests.
+
+It consumes the schema/contract from [`just-dna-format`](../schema) and is the shared transform
+called by both `just-dna-pipelines` (local compile) and `just-dna-marketplace` (server-side
+recompile on publish).
+
+```python
+from just_dna_compiler.compiler import validate_spec, compile_module
+
+validate_spec(spec_dir)                       # -> ValidationResult (genes/categories lists)
+compile_module(spec_dir, out_dir,             # -> CompilationResult (+ manifest.json written)
+               resolve_with_ensembl=False,    # inject an Ensembl DuckDB/parquet dir to resolve
+               compiled_by="marketplace-server")
+```
+
+**Dependencies:** `just-dna-format`, `polars`, `duckdb`, `pyyaml` — deliberately no Dagster / LLM
+SDKs. The Ensembl reference is **injected** by the caller (a `.duckdb` file or a parquet dir); this
+package never downloads it.

just_dna_compiler-0.1.0/README.md

@@ -0,0 +1,22 @@
+# just-dna-compiler
+
+The reference **compiler** for just-dna annotation modules: it turns an authored spec directory
+(`module_spec.yaml` + `variants.csv` + `studies.csv`) into the deployable three-parquet artifact
+(`weights` / `annotations` / `studies`) **plus a `manifest.json`** with integrity digests.
+
+It consumes the schema/contract from [`just-dna-format`](../schema) and is the shared transform
+called by both `just-dna-pipelines` (local compile) and `just-dna-marketplace` (server-side
+recompile on publish).
+
+```python
+from just_dna_compiler.compiler import validate_spec, compile_module
+
+validate_spec(spec_dir)                       # -> ValidationResult (genes/categories lists)
+compile_module(spec_dir, out_dir,             # -> CompilationResult (+ manifest.json written)
+               resolve_with_ensembl=False,    # inject an Ensembl DuckDB/parquet dir to resolve
+               compiled_by="marketplace-server")
+```
+
+**Dependencies:** `just-dna-format`, `polars`, `duckdb`, `pyyaml` — deliberately no Dagster / LLM
+SDKs. The Ensembl reference is **injected** by the caller (a `.duckdb` file or a parquet dir); this
+package never downloads it.

just_dna_compiler-0.1.0/pyproject.toml

@@ -0,0 +1,26 @@
+[project]
+name = "just-dna-compiler"
+version = "0.1.0"
+description = "Reference compiler for just-dna annotation modules: spec directory → parquet artifact + manifest"
+readme = "README.md"
+requires-python = ">=3.13"
+dependencies = [
+    "just-dna-format",
+    "polars>=1.42.0",
+    "duckdb>=1.1.0",
+    "pyyaml>=6.0.2",
+    "platformdirs>=4.0.0",
+    "python-dotenv>=1.0.1",
+]
+
+[build-system]
+requires = ["uv-build"]
+build-backend = "uv_build"
+
+[tool.uv.sources]
+just-dna-format = { workspace = true }
+
+[dependency-groups]
+dev = [
+    "pytest>=9.0.3",
+]

just_dna_compiler-0.1.0/src/just_dna_compiler/__init__.py

@@ -0,0 +1,9 @@
+"""
+just-dna-compiler — the reference transform from an authored module spec directory to the
+deployable three-parquet artifact plus its `manifest.json`.
+
+Consumes the schema/contract from `just-dna-format`; both `just-dna-pipelines` (local compile) and
+`just-dna-marketplace` (server-side recompile on publish) call the same `compile_module`.
+
+    from just_dna_compiler.compiler import validate_spec, compile_module, reverse_module
+"""

just_dna_compiler-0.1.0/src/just_dna_compiler/cache.py

@@ -0,0 +1,78 @@
+"""
+Ensembl reference cache resolution — mirrors just-dna-lite's on-disk layout so a marketplace or
+standalone compile can **reuse an existing just-dna-lite deployment's cache** (disk economy, no
+re-download), pointed via `.env`.
+
+Layout (identical to `just-dna-pipelines`)::
+
+    <base>/ensembl_variations/data/*.parquet
+    <base>/ensembl_variations/ensembl_variations.duckdb    # optional prebuilt view
+
+where ``<base>`` is ``$JUST_DNA_PIPELINES_CACHE_DIR`` (the same var just-dna-lite uses), or the
+platformdirs user cache for ``"just-dna-pipelines"``. ``$JUST_DNA_ENSEMBL_CACHE`` (a ``.duckdb``
+file or a directory) overrides everything for explicit pointing.
+
+This module **never downloads**: if no cache is present, resolution returns ``None`` and the
+resolver skips with a warning. Provisioning the reference is the deployment's job.
+"""
+
+import os
+from pathlib import Path
+from typing import Optional
+
+from dotenv import find_dotenv, load_dotenv
+from platformdirs import user_cache_dir
+
+APPNAME: str = "just-dna-pipelines"
+ENSEMBL_SUBDIR: str = "ensembl_variations"
+DUCKDB_NAME: str = "ensembl_variations.duckdb"
+
+
+def load_env(override: bool = False) -> Optional[str]:
+    """Load the nearest `.env` (walking up from CWD), so cache paths can be set there.
+    Returns the loaded path, or None."""
+    env_path = find_dotenv(usecwd=True)
+    if env_path:
+        load_dotenv(env_path, override=override)
+        return env_path
+    return None
+
+
+def default_ensembl_cache_dir() -> Path:
+    """The `<base>/ensembl_variations` directory, matching just-dna-lite's convention."""
+    base = os.getenv("JUST_DNA_PIPELINES_CACHE_DIR")
+    root = Path(base) if base else Path(user_cache_dir(appname=APPNAME))
+    return root / ENSEMBL_SUBDIR
+
+
+def resolve_ensembl_reference(
+    ensembl_cache: Optional[Path] = None, *, load_dotenv_file: bool = True
+) -> Optional[Path]:
+    """Locate a usable Ensembl reference without downloading.
+
+    Precedence: explicit `ensembl_cache` → ``$JUST_DNA_ENSEMBL_CACHE`` → the just-dna-lite layout
+    under ``$JUST_DNA_PIPELINES_CACHE_DIR`` / platformdirs. Prefers a prebuilt
+    ``ensembl_variations.duckdb``; otherwise the directory of parquet files. Returns the resolved
+    path (a ``.duckdb`` file or a directory), or ``None`` if nothing is present.
+    """
+    if load_dotenv_file:
+        load_env()
+
+    candidate = ensembl_cache or os.getenv("JUST_DNA_ENSEMBL_CACHE")
+    search_dir = Path(candidate) if candidate else default_ensembl_cache_dir()
+
+    # Explicit pointing at a specific DuckDB file.
+    if search_dir.is_file() and search_dir.suffix == ".duckdb":
+        return search_dir
+
+    # Otherwise return the cache directory if it holds a prebuilt db or parquet data; the
+    # connection layer decides whether the db is usable and falls back to parquet if not.
+    if search_dir.is_dir():
+        data_dir = search_dir / "data"
+        has_db = (search_dir / DUCKDB_NAME).is_file()
+        has_parquet = (data_dir.is_dir() and any(data_dir.glob("*.parquet"))) or any(
+            search_dir.glob("*.parquet")
+        )
+        if has_db or has_parquet:
+            return search_dir
+    return None

just_dna_compiler-0.1.0/src/just_dna_compiler/compiler.py

@@ -0,0 +1,676 @@
+"""
+Module spec compiler: validates a spec directory and compiles it to the three-parquet artifact
+(weights, annotations, studies) plus a `manifest.json`.
+
+Public API:
+    validate_spec(spec_dir) -> ValidationResult
+    compile_module(spec_dir, output_dir, ...) -> CompilationResult   (emits manifest.json)
+    reverse_module(parquet_dir, output_dir, ...) -> Path
+
+The DSL/manifest schema comes from `just-dna-format`; this package is the transform between them.
+"""
+
+import csv
+import shutil
+from datetime import datetime, timezone
+from importlib.metadata import PackageNotFoundError, version
+from pathlib import Path
+from typing import Any, Optional
+
+import polars as pl
+import yaml
+from just_dna_format.integrity import build_artifact, file_entries
+from just_dna_format.manifest import (
+    Compilation,
+    Display,
+    FileEntry,
+    Identity,
+    ModuleManifest,
+    Stats,
+    write_manifest,
+)
+from just_dna_format.spec import ModuleSpecConfig, StudyRow, VariantRow
+from pydantic import ValidationError
+
+from just_dna_compiler.models import CompilationResult, ValidationResult
+
+_INPUT_FILES: tuple[str, ...] = ("module_spec.yaml", "variants.csv", "studies.csv")
+_OUTPUT_FILES: tuple[str, ...] = ("weights.parquet", "annotations.parquet", "studies.parquet")
+
+
+def _compiler_version() -> str:
+    try:
+        return f"just-dna-compiler {version('just-dna-compiler')}"
+    except PackageNotFoundError:
+        return "just-dna-compiler unknown"
+
+
+def _now_iso() -> str:
+    return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+def _collect_logs(
+    spec_dir: Path, output_dir: Path, explicit: Optional[list[Path]]
+) -> list[FileEntry]:
+    """Gather optional run/provenance logs into the module dir and hash them.
+
+    Auto-discovers a top-level aggregate log (`*.log` in `spec_dir`) plus per-role files under a
+    `spec_dir/logs/` folder, preserving each file's path relative to the module. An explicit
+    `log_files` list overrides discovery. Files are copied into `output_dir` (so they ship with the
+    module) and returned as hashed `FileEntry` rows. No logs → empty list (a valid module).
+    """
+    pairs: list[tuple[str, Path]] = []  # (relative name in module dir, source file)
+    if explicit is not None:
+        for path in map(Path, explicit):
+            try:
+                rel = path.relative_to(spec_dir).as_posix()
+            except ValueError:
+                rel = path.name
+            pairs.append((rel, path))
+    else:
+        for path in sorted(spec_dir.glob("*.log")):
+            pairs.append((path.name, path))
+        logs_dir = spec_dir / "logs"
+        if logs_dir.is_dir():
+            for path in sorted(logs_dir.rglob("*.log")):
+                pairs.append((path.relative_to(spec_dir).as_posix(), path))
+
+    seen: set[str] = set()
+    names: list[str] = []
+    for rel, src in pairs:
+        if rel in seen or not src.is_file():
+            continue
+        seen.add(rel)
+        dest = output_dir / rel
+        if dest.resolve() != src.resolve():
+            dest.parent.mkdir(parents=True, exist_ok=True)
+            shutil.copyfile(src, dest)
+        names.append(rel)
+    return file_entries(output_dir, names)
+
+
+# ── File loading helpers ───────────────────────────────────────────────────────
+
+
+def _load_yaml(path: Path) -> tuple[Optional[ModuleSpecConfig], list[str]]:
+    """Load and validate module_spec.yaml. Returns (config, errors)."""
+    if not path.exists():
+        return None, [f"module_spec.yaml not found at {path}"]
+    raw = yaml.safe_load(path.read_text(encoding="utf-8"))
+    if raw is None:
+        return None, ["module_spec.yaml is empty"]
+    try:
+        return ModuleSpecConfig.model_validate(raw), []
+    except ValidationError as exc:
+        errors = []
+        for err in exc.errors():
+            loc = " → ".join(str(x) for x in err["loc"])
+            errors.append(f"module_spec.yaml [{loc}]: {err['msg']}")
+        return None, errors
+
+
+def _load_csv_rows(
+    path: Path, row_model: type, file_label: str
+) -> tuple[list[Any], list[str], list[str]]:
+    """Load a CSV and validate each row against a Pydantic model. Returns (rows, errors, warnings)."""
+    errors: list[str] = []
+    rows: list[Any] = []
+    if not path.exists():
+        return [], [f"{file_label} not found at {path}"], []
+
+    with open(path, encoding="utf-8", newline="") as f:
+        reader = csv.DictReader(f)
+        if reader.fieldnames is None:
+            return [], [f"{file_label} has no header row"], []
+        for line_num, raw_row in enumerate(reader, start=2):
+            cleaned = {
+                k.strip(): (v.strip() if isinstance(v, str) and v.strip() != "" else None)
+                for k, v in raw_row.items()
+                if k is not None
+            }
+            try:
+                rows.append(row_model.model_validate(cleaned))
+            except ValidationError as exc:
+                for err in exc.errors():
+                    loc = " → ".join(str(x) for x in err["loc"])
+                    errors.append(f"{file_label} line {line_num} [{loc}]: {err['msg']}")
+    return rows, errors, []
+
+
+# ── Cross-row validation ───────────────────────────────────────────────────────
+
+
+def _cross_validate_variants(variants: list[VariantRow]) -> tuple[list[str], list[str]]:
+    """Validate consistency across variant rows. Returns (errors, warnings)."""
+    errors: list[str] = []
+    warnings: list[str] = []
+
+    key_positions: dict[str, tuple[Optional[str], Optional[int]]] = {}
+    for row in variants:
+        key = row.variant_key
+        pos = (row.chrom, row.start)
+        if key in key_positions:
+            if key_positions[key] != pos:
+                errors.append(f"Inconsistent positions for {key}: {key_positions[key]} vs {pos}")
+        else:
+            key_positions[key] = pos
+
+    seen_keys: set[tuple[str, str]] = set()
+    for row in variants:
+        key = (row.variant_key, row.genotype)
+        if key in seen_keys:
+            errors.append(f"Duplicate (variant, genotype): ({row.variant_key}, {row.genotype})")
+        seen_keys.add(key)
+
+    for row in variants:
+        if row.weight is None:
+            continue
+        if row.state == "risk" and row.weight > 0:
+            warnings.append(
+                f"{row.variant_key} genotype {row.genotype}: state='risk' but weight={row.weight} > 0"
+            )
+        if row.state == "protective" and row.weight < 0:
+            warnings.append(
+                f"{row.variant_key} genotype {row.genotype}: state='protective' but weight={row.weight} < 0"
+            )
+    return errors, warnings
+
+
+def _cross_validate_studies(
+    studies: list[StudyRow], variant_keys: set[str]
+) -> tuple[list[str], list[str]]:
+    """Validate study rows against variant keys. Returns (errors, warnings)."""
+    warnings: list[str] = []
+    orphan_keys = {row.variant_key for row in studies} - variant_keys
+    if orphan_keys:
+        warnings.append(f"Studies reference variants not in variants.csv: {sorted(orphan_keys)}")
+    seen: set[tuple[str, str]] = set()
+    for row in studies:
+        key = (row.variant_key, row.pmid)
+        if key in seen:
+            warnings.append(f"Duplicate (variant, pmid): ({row.variant_key}, {row.pmid})")
+        seen.add(key)
+    return [], warnings
+
+
+# ── Public API ─────────────────────────────────────────────────────────────────
+
+
+def validate_spec(spec_dir: Path) -> ValidationResult:
+    """Validate a module spec directory without producing output.
+
+    Stats include `genes`/`categories` as lists (filtering None) plus `variant_count`,
+    `gene_count`, and `study_count` — the fields the manifest needs.
+    """
+    spec_dir = Path(spec_dir)
+    all_errors: list[str] = []
+    all_warnings: list[str] = []
+
+    if not spec_dir.is_dir():
+        return ValidationResult(valid=False, errors=[f"Spec directory does not exist: {spec_dir}"])
+
+    config, yaml_errors = _load_yaml(spec_dir / "module_spec.yaml")
+    all_errors.extend(yaml_errors)
+
+    variants, var_errors, var_warnings = _load_csv_rows(
+        spec_dir / "variants.csv", VariantRow, "variants.csv"
+    )
+    all_errors.extend(var_errors)
+    all_warnings.extend(var_warnings)
+
+    studies_path = spec_dir / "studies.csv"
+    studies: list[StudyRow] = []
+    if studies_path.exists():
+        studies, study_errors, study_warnings = _load_csv_rows(
+            studies_path, StudyRow, "studies.csv"
+        )
+        all_errors.extend(study_errors)
+        all_warnings.extend(study_warnings)
+        if not studies and not study_errors:
+            all_errors.append(
+                "studies.csv is present but has no study rows. Grounding evidence is mandatory."
+            )
+    else:
+        all_errors.append(
+            "studies.csv is missing. Grounding evidence is mandatory; add study rows with PMIDs."
+        )
+
+    if variants:
+        cross_errors, cross_warnings = _cross_validate_variants(variants)
+        all_errors.extend(cross_errors)
+        all_warnings.extend(cross_warnings)
+        variant_keys = {v.variant_key for v in variants}
+        if studies:
+            _, study_warnings = _cross_validate_studies(studies, variant_keys)
+            all_warnings.extend(study_warnings)
+
+    stats: dict[str, Any] = {}
+    if variants:
+        variant_keys_set = {v.variant_key for v in variants}
+        genes = sorted({v.gene for v in variants if v.gene})
+        categories = sorted({v.category for v in variants if v.category})
+        stats = {
+            "variant_count": len(variant_keys_set),
+            "unique_rsids": len({v.rsid for v in variants if v.rsid is not None}),
+            "gene_count": len(genes),
+            "genes": genes,
+            "categories": categories,
+            "study_count": len(studies),
+        }
+        if config:
+            stats["module_name"] = config.module.name
+
+    return ValidationResult(
+        valid=len(all_errors) == 0, errors=all_errors, warnings=all_warnings, stats=stats
+    )
+
+
+def compile_module(
+    spec_dir: Path,
+    output_dir: Path,
+    compression: str = "zstd",
+    resolve_with_ensembl: bool = True,
+    ensembl_cache: Optional[Path] = None,
+    compiled_by: Optional[str] = None,
+    ensembl_reference: Optional[str] = None,
+    log_files: Optional[list[Path]] = None,
+) -> CompilationResult:
+    """Compile a module spec directory into parquet files plus a `manifest.json`.
+
+    Args:
+        spec_dir: Path to the module spec directory.
+        output_dir: Directory for output parquet files + manifest.json.
+        compression: Parquet compression codec.
+        resolve_with_ensembl: Resolve missing rsid/position via an injected Ensembl DuckDB.
+        ensembl_cache: Path to a prebuilt Ensembl DuckDB or a parquet cache dir (required to
+            resolve; the standalone compiler does not download it — the caller injects it).
+        compiled_by: Provenance tag for the manifest (the marketplace passes "marketplace-server";
+            a local compile leaves it None, so downloaders treat it as untrusted).
+        ensembl_reference: Pinned reference id recorded in the manifest for reproducibility.
+        log_files: Explicit run/provenance log files to record. If None, auto-discovers a top-level
+            `*.log` plus per-role files under `spec_dir/logs/`. Logs are optional.
+    """
+    spec_dir = Path(spec_dir)
+    output_dir = Path(output_dir)
+
+    validation = validate_spec(spec_dir)
+    if not validation.valid:
+        return CompilationResult(
+            success=False, errors=validation.errors, warnings=validation.warnings
+        )
+
+    config, _ = _load_yaml(spec_dir / "module_spec.yaml")
+    assert config is not None
+    variants, _, _ = _load_csv_rows(spec_dir / "variants.csv", VariantRow, "variants.csv")
+    studies: list[StudyRow] = []
+    if (spec_dir / "studies.csv").exists():
+        studies, _, _ = _load_csv_rows(spec_dir / "studies.csv", StudyRow, "studies.csv")
+
+    all_warnings = list(validation.warnings)
+    if resolve_with_ensembl:
+        from just_dna_compiler.resolver import resolve_variants
+
+        variants, resolve_warnings = resolve_variants(variants, ensembl_cache)
+        all_warnings.extend(resolve_warnings)
+
+    module_name = config.module.name
+    weights_df = _build_weights(variants, config)
+    annotations_df = _build_annotations(variants, module_name)
+    studies_df = _build_studies(studies, module_name) if studies else None
+
+    output_dir.mkdir(parents=True, exist_ok=True)
+    weights_df.write_parquet(output_dir / "weights.parquet", compression=compression)
+    annotations_df.write_parquet(output_dir / "annotations.parquet", compression=compression)
+    if studies_df is not None:
+        studies_df.write_parquet(output_dir / "studies.parquet", compression=compression)
+
+    logs = _collect_logs(spec_dir, output_dir, log_files)
+    manifest = _build_manifest(
+        config=config,
+        spec_dir=spec_dir,
+        output_dir=output_dir,
+        validation=validation,
+        weights_rows=weights_df.height,
+        warnings=all_warnings,
+        compiled_by=compiled_by,
+        ensembl_reference=ensembl_reference,
+        logs=logs,
+    )
+    write_manifest(manifest, output_dir / "manifest.json")
+
+    stats: dict[str, Any] = {
+        "module_name": module_name,
+        "weights_rows": weights_df.height,
+        "annotations_rows": annotations_df.height,
+        "studies_rows": studies_df.height if studies_df is not None else 0,
+    }
+    return CompilationResult(
+        success=True,
+        output_dir=output_dir,
+        errors=[],
+        warnings=all_warnings,
+        stats=stats,
+        manifest=manifest,
+    )
+
+
+def _build_manifest(
+    *,
+    config: ModuleSpecConfig,
+    spec_dir: Path,
+    output_dir: Path,
+    validation: ValidationResult,
+    weights_rows: int,
+    warnings: list[str],
+    compiled_by: Optional[str],
+    ensembl_reference: Optional[str],
+    logs: list[FileEntry],
+) -> ModuleManifest:
+    """Assemble the manifest from the spec, validation stats, and hashed input/output/log files."""
+    module = config.module
+    vstats = validation.stats
+    return ModuleManifest(
+        identity=Identity(name=module.name),
+        display=Display(
+            title=module.title,
+            description=module.description,
+            report_title=module.report_title,
+            icon=module.icon,
+            color=module.color,
+        ),
+        genome_build=config.genome_build,
+        curator=config.defaults.curator,
+        method=config.defaults.method,
+        stats=Stats(
+            variant_count=vstats.get("variant_count", 0),
+            weights_rows=weights_rows,
+            study_count=vstats.get("study_count", 0),
+            gene_count=vstats.get("gene_count", 0),
+            genes=vstats.get("genes", []),
+            categories=vstats.get("categories", []),
+        ),
+        compilation=Compilation(
+            compile_success=True,
+            compiled_by=compiled_by,
+            compiler_version=_compiler_version(),
+            ensembl_reference=ensembl_reference,
+            compiled_at=_now_iso(),
+            warnings=warnings,
+        ),
+        inputs=file_entries(spec_dir, list(_INPUT_FILES)),
+        artifact=build_artifact(output_dir, list(_OUTPUT_FILES)),
+        logs=logs,
+    )
+
+
+# ── Parquet builders ───────────────────────────────────────────────────────────
+
+
+def _build_weights(variants: list[VariantRow], config: ModuleSpecConfig) -> pl.DataFrame:
+    """Build the weights.parquet DataFrame from validated variant rows."""
+    defaults = config.defaults
+    module_name = config.module.name
+    records: list[dict[str, Any]] = []
+    for v in variants:
+        priority = v.priority if v.priority is not None else defaults.priority
+        records.append(
+            {
+                "rsid": v.rsid,
+                "genotype": v.genotype.split("/"),
+                "module": module_name,
+                "weight": v.weight,
+                "state": v.state,
+                "priority": priority,
+                "conclusion": v.conclusion,
+                "curator": v.curator or defaults.curator,
+                "method": v.method or defaults.method,
+                "chrom": v.chrom,
+                "start": v.start,
+                "end": v.start,
+                "ref": v.ref,
+                "alts": v.alts.split(",") if v.alts else None,
+                "clinvar": v.clinvar if v.clinvar is not None else False,
+                "pathogenic": v.pathogenic if v.pathogenic is not None else False,
+                "benign": v.benign if v.benign is not None else False,
+                "likely_pathogenic": False,
+                "likely_benign": False,
+            }
+        )
+    schema = {
+        "rsid": pl.Utf8,
+        "genotype": pl.List(pl.Utf8),
+        "module": pl.Utf8,
+        "weight": pl.Float64,
+        "state": pl.Utf8,
+        "priority": pl.Utf8,
+        "conclusion": pl.Utf8,
+        "curator": pl.Utf8,
+        "method": pl.Utf8,
+        "chrom": pl.Utf8,
+        "start": pl.UInt32,
+        "end": pl.UInt32,
+        "ref": pl.Utf8,
+        "alts": pl.List(pl.Utf8),
+        "clinvar": pl.Boolean,
+        "pathogenic": pl.Boolean,
+        "benign": pl.Boolean,
+        "likely_pathogenic": pl.Boolean,
+        "likely_benign": pl.Boolean,
+    }
+    return pl.DataFrame(records, schema=schema)
+
+
+def _build_annotations(variants: list[VariantRow], module_name: str) -> pl.DataFrame:
+    """Build annotations.parquet, deduplicated by variant_key (first occurrence wins)."""
+    seen_keys: set[str] = set()
+    records: list[dict[str, Optional[str]]] = []
+    for v in variants:
+        key = v.variant_key
+        if key not in seen_keys:
+            records.append(
+                {
+                    "rsid": v.rsid,
+                    "module": module_name,
+                    "gene": v.gene or "",
+                    "phenotype": v.phenotype or "",
+                    "category": v.category or "",
+                }
+            )
+            seen_keys.add(key)
+    schema = {
+        "rsid": pl.Utf8,
+        "module": pl.Utf8,
+        "gene": pl.Utf8,
+        "phenotype": pl.Utf8,
+        "category": pl.Utf8,
+    }
+    return pl.DataFrame(records, schema=schema)
+
+
+def _build_studies(studies: list[StudyRow], module_name: str) -> pl.DataFrame:
+    """Build the studies.parquet DataFrame from validated study rows."""
+    records: list[dict[str, Any]] = []
+    for s in studies:
+        records.append(
+            {
+                "rsid": s.rsid,
+                "module": module_name,
+                "pmid": s.pmid,
+                "population": s.population,
+                "p_value": s.p_value,
+                "conclusion": s.conclusion,
+                "study_design": s.study_design,
+            }
+        )
+    schema = {
+        "rsid": pl.Utf8,
+        "module": pl.Utf8,
+        "pmid": pl.Utf8,
+        "population": pl.Utf8,
+        "p_value": pl.Utf8,
+        "conclusion": pl.Utf8,
+        "study_design": pl.Utf8,
+    }
+    return pl.DataFrame(records, schema=schema)
+
+
+# ── Reverse engineering ────────────────────────────────────────────────────────
+
+
+def reverse_module(
+    parquet_dir: Path,
+    output_dir: Path,
+    module_name: Optional[str] = None,
+    title: Optional[str] = None,
+    description: Optional[str] = None,
+    report_title: Optional[str] = None,
+    icon: str = "database",
+    color: str = "#6435c9",
+) -> Path:
+    """Reverse-engineer a parquet module back into the spec DSL (yaml + csv). Returns output_dir."""
+    parquet_dir = Path(parquet_dir)
+    output_dir = Path(output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    weights_df = pl.read_parquet(parquet_dir / "weights.parquet")
+    if module_name is None:
+        if "module" in weights_df.columns:
+            module_name = weights_df["module"].drop_nulls().unique().to_list()[0]
+        else:
+            module_name = parquet_dir.name
+
+    default_curator = _most_common(weights_df, "curator") or "unknown"
+    default_method = _most_common(weights_df, "method") or "unknown"
+    default_priority: Optional[str] = None
+    if "priority" in weights_df.columns:
+        non_null = weights_df["priority"].drop_nulls()
+        if non_null.len() > 0:
+            default_priority = non_null.mode().to_list()[0]
+
+    annotations_df: Optional[pl.DataFrame] = None
+    ann_path = parquet_dir / "annotations.parquet"
+    if ann_path.exists():
+        annotations_df = pl.read_parquet(ann_path)
+
+    defaults_dict: dict[str, Any] = {"curator": default_curator, "method": default_method}
+    if default_priority is not None:
+        defaults_dict["priority"] = default_priority
+
+    spec = {
+        "schema_version": "1.0",
+        "module": {
+            "name": module_name,
+            "title": title or module_name.replace("_", " ").title(),
+            "description": description or f"Annotation module: {module_name}",
+            "report_title": report_title or module_name.replace("_", " ").title(),
+            "icon": icon,
+            "color": color,
+        },
+        "defaults": defaults_dict,
+        "genome_build": "GRCh38",
+    }
+    (output_dir / "module_spec.yaml").write_text(
+        yaml.dump(spec, default_flow_style=False, sort_keys=False), encoding="utf-8"
+    )
+
+    ann_lookup: dict[str, dict[str, str]] = {}
+    if annotations_df is not None:
+        for row in annotations_df.iter_rows(named=True):
+            ann_lookup[row["rsid"]] = {
+                "gene": row.get("gene", ""),
+                "phenotype": row.get("phenotype", ""),
+                "category": row.get("category", ""),
+            }
+
+    _write_variants_csv(
+        weights_df, ann_lookup, default_curator, default_method, default_priority,
+        output_dir / "variants.csv",
+    )
+    studies_path = parquet_dir / "studies.parquet"
+    if studies_path.exists():
+        _write_studies_csv(pl.read_parquet(studies_path), output_dir / "studies.csv")
+    return output_dir
+
+
+def _most_common(df: pl.DataFrame, col: str) -> Optional[str]:
+    """Return the most common non-null value in a column, or None."""
+    if col not in df.columns:
+        return None
+    non_null = df[col].drop_nulls()
+    if non_null.len() == 0:
+        return None
+    return non_null.mode().to_list()[0]
+
+
+def _write_variants_csv(
+    weights_df: pl.DataFrame,
+    ann_lookup: dict[str, dict[str, str]],
+    default_curator: str,
+    default_method: str,
+    default_priority: Optional[str],
+    output_path: Path,
+) -> None:
+    """Write variants.csv from weights parquet + annotations lookup."""
+    fieldnames = [
+        "rsid", "chrom", "start", "ref", "alts", "genotype", "weight", "state", "conclusion",
+        "priority", "gene", "phenotype", "category", "clinvar", "pathogenic", "benign",
+        "curator", "method",
+    ]
+    with open(output_path, "w", encoding="utf-8", newline="") as f:
+        writer = csv.DictWriter(f, fieldnames=fieldnames)
+        writer.writeheader()
+        for row in weights_df.iter_rows(named=True):
+            rsid = row.get("rsid") or ""
+            ann = ann_lookup.get(rsid, {})
+            genotype_list = row.get("genotype", [])
+            alts_list = row.get("alts")
+            curator = row.get("curator", "")
+            method = row.get("method", "")
+            priority = row.get("priority")
+            clinvar = row.get("clinvar", False)
+            pathogenic = row.get("pathogenic", False)
+            benign = row.get("benign", False)
+            writer.writerow(
+                {
+                    "rsid": rsid,
+                    "chrom": row.get("chrom", ""),
+                    "start": row.get("start", ""),
+                    "ref": row.get("ref", ""),
+                    "alts": ",".join(alts_list) if alts_list else "",
+                    "genotype": "/".join(genotype_list) if genotype_list else "",
+                    "weight": row.get("weight") if row.get("weight") is not None else "",
+                    "state": row.get("state", ""),
+                    "conclusion": row.get("conclusion", ""),
+                    "priority": priority if priority != default_priority else "",
+                    "gene": ann.get("gene", ""),
+                    "phenotype": ann.get("phenotype", ""),
+                    "category": ann.get("category", ""),
+                    "clinvar": str(clinvar).lower() if clinvar else "",
+                    "pathogenic": str(pathogenic).lower() if pathogenic else "",
+                    "benign": str(benign).lower() if benign else "",
+                    "curator": curator if curator != default_curator else "",
+                    "method": method if method != default_method else "",
+                }
+            )
+
+
+def _write_studies_csv(studies_df: pl.DataFrame, output_path: Path) -> None:
+    """Write studies.csv from studies parquet."""
+    fieldnames = ["rsid", "pmid", "population", "p_value", "conclusion", "study_design"]
+    with open(output_path, "w", encoding="utf-8", newline="") as f:
+        writer = csv.DictWriter(f, fieldnames=fieldnames)
+        writer.writeheader()
+        for row in studies_df.iter_rows(named=True):
+            pmid = row.get("pmid")
+            if pmid is None or str(pmid).strip() == "":
+                continue
+            writer.writerow(
+                {
+                    "rsid": row["rsid"],
+                    "pmid": str(pmid).strip(),
+                    "population": row.get("population") or "",
+                    "p_value": row.get("p_value") or "",
+                    "conclusion": row.get("conclusion") or "",
+                    "study_design": row.get("study_design") or "",
+                }
+            )

just_dna_compiler-0.1.0/src/just_dna_compiler/models.py

@@ -0,0 +1,33 @@
+"""Result types returned by the compiler. The compiled *manifest* itself is the
+`just_dna_format.manifest.ModuleManifest` — these wrap validation/compilation outcomes."""
+
+from pathlib import Path
+from typing import Any, Optional
+
+from just_dna_format.manifest import ModuleManifest
+from pydantic import BaseModel, Field
+
+
+class ValidationResult(BaseModel):
+    """Result of spec validation."""
+
+    valid: bool = Field(description="Whether the spec is valid")
+    errors: list[str] = Field(default_factory=list, description="Validation errors")
+    warnings: list[str] = Field(default_factory=list, description="Non-fatal warnings")
+    stats: dict[str, Any] = Field(
+        default_factory=dict,
+        description="Summary stats: variant_count, gene_count, genes, categories, study_count",
+    )
+
+
+class CompilationResult(BaseModel):
+    """Result of spec compilation, including the emitted manifest."""
+
+    success: bool = Field(description="Whether compilation succeeded")
+    output_dir: Optional[Path] = Field(default=None, description="Directory with output parquets")
+    errors: list[str] = Field(default_factory=list)
+    warnings: list[str] = Field(default_factory=list)
+    stats: dict[str, Any] = Field(default_factory=dict)
+    manifest: Optional[ModuleManifest] = Field(
+        default=None, description="The manifest written next to the parquets (None on failure)"
+    )

just_dna_compiler-0.1.0/src/just_dna_compiler/resolver.py

@@ -0,0 +1,192 @@
+"""
+Bidirectional rsid <-> position resolver using an Ensembl DuckDB (GRCh38).
+
+Unlike the original pipelines resolver, this standalone version **injects** the Ensembl
+reference: the caller passes either a prebuilt `.duckdb` file or a directory of Ensembl parquet
+files, and the resolver builds an ephemeral `ensembl_variations` view over it. It never reaches
+into `just-dna-pipelines` and never downloads anything — provisioning the reference is the
+caller's responsibility (the marketplace pins one reference for the whole ecosystem).
+"""
+
+import logging
+from pathlib import Path
+from typing import Optional
+
+import duckdb
+
+from just_dna_format.spec import VariantRow
+
+from just_dna_compiler.cache import DUCKDB_NAME, resolve_ensembl_reference
+
+logger = logging.getLogger(__name__)
+
+
+class EnsemblReferenceError(FileNotFoundError):
+    """Raised when a provided Ensembl reference is neither a usable .duckdb nor a parquet dir."""
+
+
+def _has_ensembl_table(con: duckdb.DuckDBPyConnection) -> bool:
+    return any(r[0] == "ensembl_variations" for r in con.execute("SHOW TABLES").fetchall())
+
+
+def _view_over_parquet(reference: Path) -> Optional[duckdb.DuckDBPyConnection]:
+    """Build an in-memory `ensembl_variations` view over the cache's parquet files, or None."""
+    parquet_glob: Optional[Path] = None
+    if (reference / "data").is_dir() and any((reference / "data").glob("*.parquet")):
+        parquet_glob = reference / "data"
+    elif reference.is_dir() and any(reference.glob("*.parquet")):
+        parquet_glob = reference
+    if parquet_glob is None:
+        return None
+    # DuckDB can't bind a parameter inside CREATE VIEW ... read_parquet(). The pattern is a local
+    # path from our own cache resolution, not user input; single-quote-escape it defensively.
+    pattern = f"{parquet_glob}/*.parquet".replace("'", "''")
+    con = duckdb.connect(":memory:")
+    con.execute(f"CREATE VIEW ensembl_variations AS SELECT * FROM read_parquet('{pattern}')")
+    return con
+
+
+def _connect(reference: Path) -> duckdb.DuckDBPyConnection:
+    """Open a DuckDB connection exposing an `ensembl_variations` relation.
+
+    `reference` (from `resolve_ensembl_reference`) is a `.duckdb` file or a cache directory. A
+    prebuilt DuckDB is used only if it actually contains the `ensembl_variations` table (a stale/
+    empty db, as just-dna-lite may leave behind, is ignored in favor of the parquet `data/`).
+    """
+    reference = Path(reference)
+
+    if reference.is_file() and reference.suffix == ".duckdb":
+        con = duckdb.connect(str(reference), read_only=True)
+        if _has_ensembl_table(con):
+            return con
+        con.close()
+        raise EnsemblReferenceError(f"{reference} has no ensembl_variations table")
+
+    db = reference / DUCKDB_NAME
+    if db.is_file():
+        con = duckdb.connect(str(db), read_only=True)
+        if _has_ensembl_table(con):
+            return con
+        con.close()  # empty/stale prebuilt db — fall back to parquet
+
+    con = _view_over_parquet(reference)
+    if con is None:
+        raise EnsemblReferenceError(f"no usable Ensembl .duckdb or parquet files at {reference}")
+    return con
+
+
+def resolve_variants(
+    variants: list[VariantRow],
+    ensembl_cache: Optional[Path] = None,
+) -> tuple[list[VariantRow], list[str]]:
+    """Fill in missing rsid or position from the injected Ensembl reference (GRCh38).
+
+    Variants that already carry both identifiers are returned unchanged. If no reference is
+    available, resolution is skipped with a warning rather than raising.
+    """
+    need_pos = [v for v in variants if v.rsid is not None and v.chrom is None]
+    need_rsid = [v for v in variants if v.rsid is None and v.chrom is not None]
+    if not need_pos and not need_rsid:
+        return variants, []
+
+    reference = resolve_ensembl_reference(ensembl_cache)
+    if reference is None:
+        msg = (
+            "Ensembl resolution skipped: no reference cache found "
+            "(set JUST_DNA_PIPELINES_CACHE_DIR or JUST_DNA_ENSEMBL_CACHE, or pass ensembl_cache)"
+        )
+        logger.warning(msg)
+        return variants, [msg]
+
+    try:
+        con = _connect(reference)
+    except EnsemblReferenceError as exc:
+        msg = f"Ensembl resolution skipped: {exc}"
+        logger.warning(msg)
+        return variants, [msg]
+
+    warnings: list[str] = []
+    rsid_to_pos: dict[str, dict] = {}
+    if need_pos:
+        unique_rsids = list({v.rsid for v in need_pos if v.rsid is not None})
+        rsid_to_pos = _lookup_positions_by_rsid(con, unique_rsids, warnings)
+    pos_to_rsid: dict[str, str] = {}
+    if need_rsid:
+        unique_positions = list(
+            {(v.chrom, v.start, v.ref) for v in need_rsid if v.chrom is not None and v.start is not None}
+        )
+        pos_to_rsid = _lookup_rsids_by_position(con, unique_positions, warnings)
+    con.close()
+
+    patched: list[VariantRow] = []
+    for v in variants:
+        if v.rsid is not None and v.chrom is None and v.rsid in rsid_to_pos:
+            patched.append(v.model_copy(update=rsid_to_pos[v.rsid]))
+        elif v.rsid is None and v.chrom is not None:
+            key = f"{v.chrom}:{v.start}:{v.ref}"
+            if key in pos_to_rsid:
+                patched.append(v.model_copy(update={"rsid": pos_to_rsid[key]}))
+            else:
+                warnings.append(f"Position {key}: no rsid found in Ensembl")
+                patched.append(v)
+        else:
+            patched.append(v)
+    return patched, warnings
+
+
+def _lookup_positions_by_rsid(
+    con: duckdb.DuckDBPyConnection, rsids: list[str], warnings: list[str]
+) -> dict[str, dict]:
+    """Batch lookup: rsid -> {chrom, start, ref, alts}."""
+    if not rsids:
+        return {}
+    placeholders = ", ".join("?" for _ in rsids)
+    rows = con.execute(
+        f"""
+        SELECT id, chrom, start, ref, string_agg(DISTINCT alt, ',' ORDER BY alt) AS alts
+        FROM ensembl_variations
+        WHERE id IN ({placeholders})
+        GROUP BY id, chrom, start, ref
+        """,
+        rsids,
+    ).fetchall()
+    result: dict[str, dict] = {}
+    for row_id, chrom, start, ref, alts in rows:
+        if row_id in result:
+            continue
+        result[row_id] = {"chrom": str(chrom), "start": int(start), "ref": str(ref), "alts": str(alts)}
+    for rsid in rsids:
+        if rsid not in result:
+            warnings.append(f"{rsid}: not found in Ensembl, position remains unset")
+    return result
+
+
+def _lookup_rsids_by_position(
+    con: duckdb.DuckDBPyConnection,
+    positions: list[tuple[Optional[str], Optional[int], Optional[str]]],
+    warnings: list[str],
+) -> dict[str, str]:
+    """Batch lookup: (chrom, start, ref) -> rsid."""
+    if not positions:
+        return {}
+    conditions = []
+    params: list[object] = []
+    for chrom, start, ref in positions:
+        if ref is not None:
+            conditions.append("(chrom = ? AND start = ? AND ref = ?)")
+            params.extend([chrom, start, ref])
+        else:
+            conditions.append("(chrom = ? AND start = ?)")
+            params.extend([chrom, start])
+    where = " OR ".join(conditions)
+    rows = con.execute(
+        f"SELECT DISTINCT chrom, start, ref, id FROM ensembl_variations "
+        f"WHERE ({where}) AND id LIKE 'rs%'",
+        params,
+    ).fetchall()
+    result: dict[str, str] = {}
+    for chrom, start, ref, row_id in rows:
+        key = f"{chrom}:{start}:{ref}"
+        if key not in result:
+            result[key] = str(row_id)
+    return result