crochet-migration 0.1.0__py3-none-any.whl

crochet/__init__.py

@@ -0,0 +1,3 @@
+"""Crochet — Versioned schema & data migrations for neomodel Neo4j graphs."""
+
+__version__ = "0.1.0"

crochet/cli.py

@@ -0,0 +1,327 @@
+"""Crochet CLI — command-line interface for managing neomodel migrations."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import click
+
+from crochet import __version__
+from crochet.config import CrochetConfig, load_config, find_project_root
+from crochet.errors import CrochetError, ProjectNotInitializedError
+from crochet.ledger.sqlite import Ledger
+
+
+def _get_config(ctx: click.Context) -> CrochetConfig:
+    """Load config, attaching it to the Click context."""
+    if "config" not in ctx.obj:
+        ctx.obj["config"] = load_config()
+    return ctx.obj["config"]
+
+
+def _get_ledger(ctx: click.Context) -> Ledger:
+    """Open the ledger, attaching it to the Click context for cleanup."""
+    if "ledger" not in ctx.obj:
+        config = _get_config(ctx)
+        ctx.obj["ledger"] = Ledger(config.ledger_file)
+    return ctx.obj["ledger"]
+
+
+# ======================================================================
+# Root group
+# ======================================================================
+
+
+@click.group()
+@click.version_option(__version__, prog_name="crochet")
+@click.pass_context
+def main(ctx: click.Context) -> None:
+    """Crochet — versioned schema & data migrations for neomodel graphs."""
+    ctx.ensure_object(dict)
+
+
+# ======================================================================
+# new-project
+# ======================================================================
+
+
+@main.command("new-project")
+@click.option("--name", default="my-graph", help="Project name.")
+@click.option(
+    "--path",
+    type=click.Path(),
+    default=".",
+    help="Directory to initialize (default: current directory).",
+)
+@click.pass_context
+def new_project(ctx: click.Context, name: str, path: str) -> None:
+    """Initialize a new Crochet project."""
+    root = Path(path).resolve()
+
+    config = CrochetConfig(project_name=name, project_root=root)
+    config.save()
+
+    # Create directories
+    config.models_dir.mkdir(parents=True, exist_ok=True)
+    (config.models_dir / "__init__.py").touch()
+    config.migrations_dir.mkdir(parents=True, exist_ok=True)
+    (config.migrations_dir / "__init__.py").touch()
+
+    # Initialize ledger
+    with Ledger(config.ledger_file):
+        pass
+
+    click.echo(f"Initialized crochet project '{name}' at {root}")
+    click.echo(f"  config:     {root / 'crochet.toml'}")
+    click.echo(f"  models:     {config.models_dir}")
+    click.echo(f"  migrations: {config.migrations_dir}")
+    click.echo(f"  ledger:     {config.ledger_file}")
+
+
+# ======================================================================
+# create-node
+# ======================================================================
+
+
+@main.command("create-node")
+@click.argument("class_name")
+@click.option("--kgid", default=None, help="Explicit __kgid__ (auto-generated if omitted).")
+@click.pass_context
+def create_node(ctx: click.Context, class_name: str, kgid: str | None) -> None:
+    """Scaffold a new StructuredNode model file."""
+    from crochet.scaffold.node import scaffold_node
+
+    config = _get_config(ctx)
+    path = scaffold_node(config.models_dir, class_name, kgid=kgid)
+    click.echo(f"Created node model: {path}")
+
+
+# ======================================================================
+# create-relationship
+# ======================================================================
+
+
+@main.command("create-relationship")
+@click.argument("class_name")
+@click.option("--rel-type", default=None, help="Neo4j relationship type (default: CLASS_NAME).")
+@click.option("--kgid", default=None, help="Explicit __kgid__ (auto-generated if omitted).")
+@click.pass_context
+def create_relationship(
+    ctx: click.Context, class_name: str, rel_type: str | None, kgid: str | None
+) -> None:
+    """Scaffold a new StructuredRel model file."""
+    from crochet.scaffold.relationship import scaffold_relationship
+
+    config = _get_config(ctx)
+    path = scaffold_relationship(
+        config.models_dir, class_name, rel_type=rel_type, kgid=kgid
+    )
+    click.echo(f"Created relationship model: {path}")
+
+
+# ======================================================================
+# create-migration
+# ======================================================================
+
+
+@main.command("create-migration")
+@click.argument("description")
+@click.option("--no-snapshot", is_flag=True, help="Skip schema snapshot.")
+@click.option("--unsafe", is_flag=True, help="Mark migration as rollback-unsafe.")
+@click.pass_context
+def create_migration(
+    ctx: click.Context, description: str, no_snapshot: bool, unsafe: bool
+) -> None:
+    """Create a new migration file."""
+    from crochet.ir.parser import parse_models_directory
+    from crochet.migrations.engine import MigrationEngine
+
+    config = _get_config(ctx)
+    ledger = _get_ledger(ctx)
+    engine = MigrationEngine(config, ledger)
+
+    snapshot = None
+    if not no_snapshot:
+        try:
+            snapshot = parse_models_directory(config.models_dir)
+        except CrochetError as exc:
+            click.echo(f"Warning: could not parse models: {exc}", err=True)
+            snapshot = None
+
+    path = engine.create_migration(
+        description=description,
+        current_snapshot=snapshot,
+        rollback_safe=not unsafe,
+    )
+    click.echo(f"Created migration: {path}")
+    if snapshot:
+        click.echo(f"  schema hash: {snapshot.schema_hash[:16]}…")
+
+
+# ======================================================================
+# upgrade
+# ======================================================================
+
+
+@main.command()
+@click.option("--target", default=None, help="Migrate up to this revision.")
+@click.option("--dry-run", is_flag=True, help="Show what would be applied without executing.")
+@click.pass_context
+def upgrade(ctx: click.Context, target: str | None, dry_run: bool) -> None:
+    """Apply pending migrations."""
+    from crochet.migrations.engine import MigrationEngine
+
+    config = _get_config(ctx)
+    ledger = _get_ledger(ctx)
+    engine = MigrationEngine(config, ledger)
+
+    driver = _try_connect_neo4j(config) if not dry_run else None
+
+    try:
+        applied = engine.upgrade(target=target, driver=driver, dry_run=dry_run)
+    finally:
+        if driver:
+            driver.close()
+
+    if not applied:
+        click.echo("Nothing to apply — already up to date.")
+    else:
+        prefix = "[dry-run] " if dry_run else ""
+        for rev in applied:
+            click.echo(f"{prefix}Applied: {rev}")
+        click.echo(f"{prefix}{len(applied)} migration(s) applied.")
+
+
+# ======================================================================
+# downgrade
+# ======================================================================
+
+
+@main.command()
+@click.option("--target", default=None, help="Revert down to (but not including) this revision.")
+@click.option("--dry-run", is_flag=True, help="Show what would be reverted without executing.")
+@click.pass_context
+def downgrade(ctx: click.Context, target: str | None, dry_run: bool) -> None:
+    """Revert the most recent migration (or down to --target)."""
+    from crochet.migrations.engine import MigrationEngine
+
+    config = _get_config(ctx)
+    ledger = _get_ledger(ctx)
+    engine = MigrationEngine(config, ledger)
+
+    driver = _try_connect_neo4j(config) if not dry_run else None
+
+    try:
+        reverted = engine.downgrade(target=target, driver=driver, dry_run=dry_run)
+    except CrochetError as exc:
+        raise click.ClickException(str(exc)) from exc
+    finally:
+        if driver:
+            driver.close()
+
+    if not reverted:
+        click.echo("Nothing to revert.")
+    else:
+        prefix = "[dry-run] " if dry_run else ""
+        for rev in reverted:
+            click.echo(f"{prefix}Reverted: {rev}")
+        click.echo(f"{prefix}{len(reverted)} migration(s) reverted.")
+
+
+# ======================================================================
+# status
+# ======================================================================
+
+
+@main.command()
+@click.pass_context
+def status(ctx: click.Context) -> None:
+    """Show migration status."""
+    from crochet.migrations.engine import MigrationEngine
+
+    config = _get_config(ctx)
+    ledger = _get_ledger(ctx)
+    engine = MigrationEngine(config, ledger)
+
+    all_migrations = engine.discover_migrations()
+    applied = engine.applied_migrations()
+    pending = engine.pending_migrations()
+
+    click.echo(f"Project: {config.project_name}")
+    click.echo(f"Total migrations:   {len(all_migrations)}")
+    click.echo(f"Applied:            {len(applied)}")
+    click.echo(f"Pending:            {len(pending)}")
+
+    head = ledger.get_head()
+    if head:
+        click.echo(f"Head:               {head.revision_id}")
+        click.echo(f"  applied at:       {head.applied_at}")
+        click.echo(f"  schema hash:      {head.schema_hash[:16]}…" if head.schema_hash else "")
+    else:
+        click.echo("Head:               (none)")
+
+    if pending:
+        click.echo("\nPending migrations:")
+        for m in pending:
+            safe = "safe" if m.rollback_safe else "UNSAFE"
+            click.echo(f"  - {m.revision_id} [{safe}]")
+
+    # Dataset batches
+    batches = ledger.get_batches()
+    if batches:
+        click.echo(f"\nDataset batches:    {len(batches)}")
+        for b in batches[-5:]:  # show last 5
+            click.echo(f"  - {b.batch_id} ({b.source_file or 'no file'})")
+
+
+# ======================================================================
+# verify
+# ======================================================================
+
+
+@main.command()
+@click.option("--with-neo4j", is_flag=True, help="Also verify Neo4j connectivity.")
+@click.pass_context
+def verify(ctx: click.Context, with_neo4j: bool) -> None:
+    """Run verification checks."""
+    from crochet.verify import verify_project
+
+    config = _get_config(ctx)
+    ledger = _get_ledger(ctx)
+
+    driver = None
+    if with_neo4j:
+        driver = _try_connect_neo4j(config)
+
+    try:
+        report = verify_project(config, ledger, driver=driver)
+    finally:
+        if driver:
+            driver.close()
+
+    click.echo(report.summary())
+    if not report.passed:
+        raise SystemExit(1)
+
+
+# ======================================================================
+# Helpers
+# ======================================================================
+
+
+def _try_connect_neo4j(config: CrochetConfig) -> object | None:
+    """Try to create a Neo4j driver. Returns None on failure."""
+    try:
+        from neo4j import GraphDatabase
+
+        return GraphDatabase.driver(
+            config.neo4j.uri,
+            auth=(config.neo4j.username, config.neo4j.password),
+        )
+    except Exception:
+        return None
+
+
+if __name__ == "__main__":
+    main()

crochet/config.py

@@ -0,0 +1,116 @@
+"""Project configuration for Crochet."""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import toml
+
+from crochet.errors import ConfigError, ProjectNotInitializedError
+
+CONFIG_FILENAME = "crochet.toml"
+DEFAULT_MODELS_PATH = "models"
+DEFAULT_MIGRATIONS_PATH = "migrations"
+DEFAULT_LEDGER_PATH = ".crochet/ledger.db"
+
+
+@dataclass
+class Neo4jConfig:
+    uri: str = "bolt://localhost:7687"
+    username: str = "neo4j"
+    password: str = ""
+
+    def __post_init__(self) -> None:
+        self.uri = os.environ.get("CROCHET_NEO4J_URI", self.uri)
+        self.username = os.environ.get("CROCHET_NEO4J_USERNAME", self.username)
+        self.password = os.environ.get("CROCHET_NEO4J_PASSWORD", self.password)
+
+
+@dataclass
+class CrochetConfig:
+    project_name: str = "my-graph"
+    models_path: str = DEFAULT_MODELS_PATH
+    migrations_path: str = DEFAULT_MIGRATIONS_PATH
+    ledger_path: str = DEFAULT_LEDGER_PATH
+    neo4j: Neo4jConfig = field(default_factory=Neo4jConfig)
+    project_root: Path = field(default_factory=lambda: Path.cwd())
+
+    @property
+    def models_dir(self) -> Path:
+        return self.project_root / self.models_path
+
+    @property
+    def migrations_dir(self) -> Path:
+        return self.project_root / self.migrations_path
+
+    @property
+    def ledger_file(self) -> Path:
+        return self.project_root / self.ledger_path
+
+    def to_dict(self) -> dict:
+        return {
+            "project": {
+                "name": self.project_name,
+                "models_path": self.models_path,
+                "migrations_path": self.migrations_path,
+            },
+            "neo4j": {
+                "uri": self.neo4j.uri,
+                "username": self.neo4j.username,
+            },
+            "ledger": {
+                "path": self.ledger_path,
+            },
+        }
+
+    def save(self, path: Path | None = None) -> None:
+        target = path or (self.project_root / CONFIG_FILENAME)
+        target.parent.mkdir(parents=True, exist_ok=True)
+        with open(target, "w") as f:
+            toml.dump(self.to_dict(), f)
+
+
+def find_project_root(start: Path | None = None) -> Path:
+    """Walk up from *start* looking for crochet.toml."""
+    current = (start or Path.cwd()).resolve()
+    while True:
+        if (current / CONFIG_FILENAME).exists():
+            return current
+        parent = current.parent
+        if parent == current:
+            raise ProjectNotInitializedError(str(start or Path.cwd()))
+        current = parent
+
+
+def load_config(project_root: Path | None = None) -> CrochetConfig:
+    """Load and return the project configuration."""
+    root = project_root or find_project_root()
+    config_path = root / CONFIG_FILENAME
+    if not config_path.exists():
+        raise ProjectNotInitializedError(str(root))
+
+    try:
+        data = toml.load(config_path)
+    except Exception as exc:
+        raise ConfigError(f"Failed to parse {config_path}: {exc}") from exc
+
+    proj = data.get("project", {})
+    neo = data.get("neo4j", {})
+    ledger = data.get("ledger", {})
+
+    neo4j_config = Neo4jConfig(
+        uri=neo.get("uri", "bolt://localhost:7687"),
+        username=neo.get("username", "neo4j"),
+        password=neo.get("password", ""),
+    )
+
+    return CrochetConfig(
+        project_name=proj.get("name", "my-graph"),
+        models_path=proj.get("models_path", DEFAULT_MODELS_PATH),
+        migrations_path=proj.get("migrations_path", DEFAULT_MIGRATIONS_PATH),
+        ledger_path=ledger.get("path", DEFAULT_LEDGER_PATH),
+        neo4j=neo4j_config,
+        project_root=root,
+    )

crochet/errors.py

@@ -0,0 +1,75 @@
+"""Custom exceptions for the Crochet framework."""
+
+
+class CrochetError(Exception):
+    """Base exception for all Crochet errors."""
+
+
+class ProjectNotInitializedError(CrochetError):
+    """Raised when a crochet command is run outside an initialized project."""
+
+    def __init__(self, path: str = "."):
+        super().__init__(
+            f"No crochet project found at '{path}'. Run 'crochet new-project' first."
+        )
+
+
+class ConfigError(CrochetError):
+    """Raised for configuration file issues."""
+
+
+class SchemaError(CrochetError):
+    """Raised for schema parsing or validation issues."""
+
+
+class MissingKGIDError(SchemaError):
+    """Raised when a neomodel class is missing a __kgid__."""
+
+    def __init__(self, class_name: str):
+        super().__init__(
+            f"Class '{class_name}' is missing a __kgid__ attribute. "
+            "Every node and relationship model must declare an immutable __kgid__."
+        )
+
+
+class DuplicateKGIDError(SchemaError):
+    """Raised when two classes share the same __kgid__."""
+
+    def __init__(self, kgid: str, class1: str, class2: str):
+        super().__init__(
+            f"Duplicate __kgid__ '{kgid}' found on classes '{class1}' and '{class2}'."
+        )
+
+
+class MigrationError(CrochetError):
+    """Raised for migration execution issues."""
+
+
+class MigrationChainError(MigrationError):
+    """Raised when the migration chain is broken or inconsistent."""
+
+
+class RollbackUnsafeError(MigrationError):
+    """Raised when attempting to downgrade a non-rollback-safe migration."""
+
+    def __init__(self, revision_id: str):
+        super().__init__(
+            f"Migration '{revision_id}' is marked as rollback-unsafe. "
+            "Downgrade is not permitted."
+        )
+
+
+class LedgerError(CrochetError):
+    """Raised for SQLite ledger issues."""
+
+
+class LedgerIntegrityError(LedgerError):
+    """Raised when the ledger state is inconsistent."""
+
+
+class IngestError(CrochetError):
+    """Raised for data ingest issues."""
+
+
+class VerificationError(CrochetError):
+    """Raised when verification checks fail."""

crochet/ingest/__init__.py

@@ -0,0 +1,5 @@
+"""Data ingest and batch tracking."""
+
+from crochet.ingest.batch import compute_file_checksum, IngestTracker
+
+__all__ = ["compute_file_checksum", "IngestTracker"]

crochet/ingest/batch.py

@@ -0,0 +1,61 @@
+"""Deterministic data-ingest tracking: checksums, provenance, batch IDs."""
+
+from __future__ import annotations
+
+import hashlib
+import uuid
+from pathlib import Path
+
+from crochet.errors import IngestError
+from crochet.ledger.sqlite import DatasetBatch, Ledger
+
+
+def compute_file_checksum(path: Path, algorithm: str = "sha256") -> str:
+    """Return the hex digest of a file."""
+    h = hashlib.new(algorithm)
+    with open(path, "rb") as f:
+        for chunk in iter(lambda: f.read(8192), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+class IngestTracker:
+    """High-level helper that ties data loading to the ledger."""
+
+    def __init__(self, ledger: Ledger, loader_version: str = "1.0") -> None:
+        self._ledger = ledger
+        self._loader_version = loader_version
+
+    def register_batch(
+        self,
+        source_file: Path | None = None,
+        migration_id: str | None = None,
+        record_count: int | None = None,
+        batch_id: str | None = None,
+    ) -> DatasetBatch:
+        bid = batch_id or uuid.uuid4().hex[:12]
+        checksum = None
+        fname = None
+        if source_file is not None:
+            if not source_file.exists():
+                raise IngestError(f"Source file not found: {source_file}")
+            checksum = compute_file_checksum(source_file)
+            fname = str(source_file)
+
+        return self._ledger.record_batch(
+            batch_id=bid,
+            migration_id=migration_id,
+            source_file=fname,
+            file_checksum=checksum,
+            loader_version=self._loader_version,
+            record_count=record_count,
+        )
+
+    def verify_file(self, batch: DatasetBatch) -> bool:
+        """Check that the source file still matches the recorded checksum."""
+        if batch.source_file is None or batch.file_checksum is None:
+            return True
+        path = Path(batch.source_file)
+        if not path.exists():
+            return False
+        return compute_file_checksum(path) == batch.file_checksum

crochet/ir/__init__.py

@@ -0,0 +1,23 @@
+"""Intermediate Representation for neomodel schemas."""
+
+from crochet.ir.schema import (
+    PropertyIR,
+    NodeIR,
+    RelationshipIR,
+    SchemaSnapshot,
+)
+from crochet.ir.parser import parse_models_directory, parse_module
+from crochet.ir.diff import SchemaDiff, diff_snapshots
+from crochet.ir.hash import hash_snapshot
+
+__all__ = [
+    "PropertyIR",
+    "NodeIR",
+    "RelationshipIR",
+    "SchemaSnapshot",
+    "parse_models_directory",
+    "parse_module",
+    "SchemaDiff",
+    "diff_snapshots",
+    "hash_snapshot",
+]