lsst-felis 26.2024.900__py3-none-any.whl → 29.2025.4500__py3-none-any.whl

felis/cli.py

@@ -1,3 +1,5 @@
+"""Click command line interface."""
+
 # This file is part of felis.
 #
 # Developed for the LSST Data Management System.
@@ -21,313 +23,384 @@
 
 from __future__ import annotations
 
-import io
-import json
 import logging
-import sys
-from collections.abc import Iterable, Mapping, MutableMapping
-from typing import Any
+from collections.abc import Iterable
+from typing import IO
 
 import click
-import yaml
 from pydantic import ValidationError
-from pyld import jsonld
-from sqlalchemy.engine import Engine, create_engine, create_mock_engine, make_url
-from sqlalchemy.engine.mock import MockConnection
+from sqlalchemy.engine import Engine, create_engine, make_url
+from sqlalchemy.engine.mock import MockConnection, create_mock_engine
 
-from . import DEFAULT_CONTEXT, DEFAULT_FRAME, __version__
-from .check import CheckingVisitor
+from . import __version__
 from .datamodel import Schema
-from .sql import SQLVisitor
-from .tap import Tap11Base, TapLoadingVisitor, init_tables
-from .utils import ReorderingVisitor
-from .validation import get_schema
+from .db.schema import create_database
+from .db.utils import DatabaseContext, is_mock_url
+from .diff import DatabaseDiff, FormattedSchemaDiff, SchemaDiff
+from .metadata import MetaDataBuilder
+from .tap_schema import DataLoader, MetadataInserter, TableManager
+
+__all__ = ["cli"]
 
 logger = logging.getLogger("felis")
 
+loglevel_choices = ["CRITICAL", "FATAL", "ERROR", "WARNING", "INFO", "DEBUG"]
+
 
 @click.group()
 @click.version_option(__version__)
-def cli() -> None:
-    """Felis Command Line Tools."""
-    logging.basicConfig(level=logging.INFO)
+@click.option(
+    "--log-level",
+    type=click.Choice(loglevel_choices),
+    envvar="FELIS_LOGLEVEL",
+    help="Felis log level",
+    default=logging.getLevelName(logging.INFO),
+)
+@click.option(
+    "--log-file",
+    type=click.Path(),
+    envvar="FELIS_LOGFILE",
+    help="Felis log file path",
+)
+@click.option(
+    "--id-generation/--no-id-generation",
+    is_flag=True,
+    help="Generate IDs for all objects that do not have them",
+    default=True,
+)
+@click.pass_context
+def cli(ctx: click.Context, log_level: str, log_file: str | None, id_generation: bool) -> None:
+    """Felis command line tools"""
+    ctx.ensure_object(dict)
+    ctx.obj["id_generation"] = id_generation
+    if ctx.obj["id_generation"]:
+        logger.info("ID generation is enabled")
+    else:
+        logger.info("ID generation is disabled")
+    if log_file:
+        logging.basicConfig(filename=log_file, level=log_level)
+    else:
+        logging.basicConfig(level=log_level)
 
 
-@cli.command("create-all")
-@click.option("--engine-url", envvar="ENGINE_URL", help="SQLAlchemy Engine URL")
-@click.option("--schema-name", help="Alternate Schema Name for Felis File")
-@click.option("--dry-run", is_flag=True, help="Dry Run Only. Prints out the DDL that would be executed")
+@cli.command("create", help="Create database objects from the Felis file")
+@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL", default="sqlite://")
+@click.option("--schema-name", help="Alternate schema name to override Felis file")
+@click.option(
+    "--initialize",
+    is_flag=True,
+    help="Create the schema in the database if it does not exist (error if already exists)",
+)
+@click.option(
+    "--drop", is_flag=True, help="Drop schema if it already exists in the database (implies --initialize)"
+)
+@click.option("--echo", is_flag=True, help="Echo database commands as they are executed")
+@click.option("--dry-run", is_flag=True, help="Dry run only to print out commands instead of executing")
+@click.option(
+    "--output-file", "-o", type=click.File(mode="w"), help="Write SQL commands to a file instead of executing"
+)
+@click.option("--ignore-constraints", is_flag=True, help="Ignore constraints when creating tables")
 @click.argument("file", type=click.File())
-def create_all(engine_url: str, schema_name: str, dry_run: bool, file: io.TextIOBase) -> None:
-    """Create schema objects from the Felis FILE."""
-    schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
-    visitor = SQLVisitor(schema_name=schema_name)
-    schema = visitor.visit_schema(schema_obj)
-
-    metadata = schema.metadata
-
-    engine: Engine | MockConnection
-    if not dry_run:
-        engine = create_engine(engine_url)
-    else:
-        _insert_dump = InsertDump()
-        engine = create_mock_engine(make_url(engine_url), executor=_insert_dump.dump)
-        _insert_dump.dialect = engine.dialect
-    metadata.create_all(engine)
-
-
-@cli.command("init-tap")
-@click.option("--tap-schema-name", help="Alt Schema Name for TAP_SCHEMA")
-@click.option("--tap-schemas-table", help="Alt Table Name for TAP_SCHEMA.schemas")
-@click.option("--tap-tables-table", help="Alt Table Name for TAP_SCHEMA.tables")
-@click.option("--tap-columns-table", help="Alt Table Name for TAP_SCHEMA.columns")
-@click.option("--tap-keys-table", help="Alt Table Name for TAP_SCHEMA.keys")
-@click.option("--tap-key-columns-table", help="Alt Table Name for TAP_SCHEMA.key_columns")
-@click.argument("engine-url")
-def init_tap(
+@click.pass_context
+def create(
+    ctx: click.Context,
     engine_url: str,
-    tap_schema_name: str,
-    tap_schemas_table: str,
-    tap_tables_table: str,
-    tap_columns_table: str,
-    tap_keys_table: str,
-    tap_key_columns_table: str,
+    schema_name: str | None,
+    initialize: bool,
+    drop: bool,
+    echo: bool,
+    dry_run: bool,
+    output_file: IO[str] | None,
+    ignore_constraints: bool,
+    file: IO[str],
 ) -> None:
-    """Initialize TAP 1.1 TAP_SCHEMA objects.
-
-    Please verify the schema/catalog you are executing this in in your
-    engine URL.
+    """Create database objects from the Felis file.
+
+    Parameters
+    ----------
+    engine_url
+        SQLAlchemy Engine URL.
+    schema_name
+        Alternate schema name to override Felis file.
+    initialize
+        Create the schema in the database if it does not exist.
+    drop
+        Drop schema if it already exists in the database.
+    echo
+        Echo database commands as they are executed.
+    dry_run
+        Dry run only to print out commands instead of executing.
+    output_file
+        Write SQL commands to a file instead of executing.
+    ignore_constraints
+        Ignore constraints when creating tables.
+    file
+        Felis file to read.
     """
-    engine = create_engine(engine_url, echo=True)
-    init_tables(
-        tap_schema_name,
-        tap_schemas_table,
-        tap_tables_table,
-        tap_columns_table,
-        tap_keys_table,
-        tap_key_columns_table,
-    )
-    Tap11Base.metadata.create_all(engine)
-
-
-@cli.command("load-tap")
-@click.option("--engine-url", envvar="ENGINE_URL", help="SQLAlchemy Engine URL to catalog")
-@click.option("--schema-name", help="Alternate Schema Name for Felis file")
-@click.option("--catalog-name", help="Catalog Name for Schema")
-@click.option("--dry-run", is_flag=True, help="Dry Run Only. Prints out the DDL that would be executed")
-@click.option("--tap-schema-name", help="Alt Schema Name for TAP_SCHEMA")
-@click.option("--tap-tables-postfix", help="Postfix for TAP table names")
-@click.option("--tap-schemas-table", help="Alt Table Name for TAP_SCHEMA.schemas")
-@click.option("--tap-tables-table", help="Alt Table Name for TAP_SCHEMA.tables")
-@click.option("--tap-columns-table", help="Alt Table Name for TAP_SCHEMA.columns")
-@click.option("--tap-keys-table", help="Alt Table Name for TAP_SCHEMA.keys")
-@click.option("--tap-key-columns-table", help="Alt Table Name for TAP_SCHEMA.key_columns")
+    try:
+        schema = Schema.from_stream(file, context={"id_generation": ctx.obj["id_generation"]})
+        url = make_url(engine_url)
+        if schema_name:
+            logger.info(f"Overriding schema name with: {schema_name}")
+            schema.name = schema_name
+        elif url.drivername == "sqlite":
+            logger.info("Overriding schema name for sqlite with: main")
+            schema.name = "main"
+        if not url.host and not url.drivername == "sqlite":
+            dry_run = True
+            logger.info("Forcing dry run for non-sqlite engine URL with no host")
+
+        metadata = MetaDataBuilder(schema, ignore_constraints=ignore_constraints).build()
+        logger.debug(f"Created metadata with schema name: {metadata.schema}")
+
+        engine: Engine | MockConnection
+        if not dry_run and not output_file:
+            engine = create_engine(url, echo=echo)
+        else:
+            if dry_run:
+                logger.info("Dry run will be executed")
+            engine = DatabaseContext.create_mock_engine(url, output_file)
+            if output_file:
+                logger.info("Writing SQL output to: " + output_file.name)
+
+        context = DatabaseContext(metadata, engine)
+
+        if drop and initialize:
+            raise ValueError("Cannot drop and initialize schema at the same time")
+
+        if drop:
+            logger.debug("Dropping schema if it exists")
+            context.drop()
+            initialize = True  # If schema is dropped, it needs to be recreated.
+
+        if initialize:
+            logger.debug("Creating schema if not exists")
+            context.initialize()
+
+        context.create_all()
+    except Exception as e:
+        logger.exception(e)
+        raise click.ClickException(str(e))
+
+
+@cli.command("load-tap-schema", help="Load metadata from a Felis file into a TAP_SCHEMA database")
+@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL")
+@click.option(
+    "--tap-schema-name", "-n", help="Name of the TAP_SCHEMA schema in the database (default: TAP_SCHEMA)"
+)
+@click.option(
+    "--tap-tables-postfix",
+    "-p",
+    help="Postfix which is applied to standard TAP_SCHEMA table names",
+    default="",
+)
+@click.option("--tap-schema-index", "-i", type=int, help="TAP_SCHEMA index of the schema in this environment")
+@click.option("--dry-run", "-D", is_flag=True, help="Execute dry run only. Does not insert any data.")
+@click.option("--echo", "-e", is_flag=True, help="Print out the generated insert statements to stdout")
+@click.option("--output-file", "-o", type=click.Path(), help="Write SQL commands to a file")
+@click.option(
+    "--force-unbounded-arraysize",
+    is_flag=True,
+    help="Use unbounded arraysize by default for all variable length string columns"
+    ", e.g., ``votable:arraysize: *`` (workaround for astropy bug #18099)",
+)  # DM-50899: Variable-length bounded strings are not handled correctly in astropy
+@click.option(
+    "--unique-keys",
+    "-u",
+    is_flag=True,
+    help="Generate unique key_id values for keys and key_columns tables by prepending the schema name",
+    default=False,
+)
 @click.argument("file", type=click.File())
-def load_tap(
+@click.pass_context
+def load_tap_schema(
+    ctx: click.Context,
     engine_url: str,
-    schema_name: str,
-    catalog_name: str,
-    dry_run: bool,
     tap_schema_name: str,
     tap_tables_postfix: str,
-    tap_schemas_table: str,
-    tap_tables_table: str,
-    tap_columns_table: str,
-    tap_keys_table: str,
-    tap_key_columns_table: str,
-    file: io.TextIOBase,
+    tap_schema_index: int,
+    dry_run: bool,
+    echo: bool,
+    output_file: str | None,
+    force_unbounded_arraysize: bool,
+    unique_keys: bool,
+    file: IO[str],
 ) -> None:
-    """Load TAP metadata from a Felis FILE.
-
-    This command loads the associated TAP metadata from a Felis FILE
-    to the TAP_SCHEMA tables.
+    """Load TAP metadata from a Felis file.
+
+    Parameters
+    ----------
+    engine_url
+        SQLAlchemy Engine URL.
+    tap_tables_postfix
+        Postfix which is applied to standard TAP_SCHEMA table names.
+    tap_schema_index
+        TAP_SCHEMA index of the schema in this environment.
+    dry_run
+        Execute dry run only. Does not insert any data.
+    echo
+        Print out the generated insert statements to stdout.
+    output_file
+        Output file for writing generated SQL.
+    file
+        Felis file to read.
+
+    Notes
+    -----
+    The TAP_SCHEMA database must already exist or the command will fail. This
+    command will not initialize the TAP_SCHEMA tables.
     """
-    top_level_object = yaml.load(file, Loader=yaml.SafeLoader)
-    schema_obj: dict
-    if isinstance(top_level_object, dict):
-        schema_obj = top_level_object
-        if "@graph" not in schema_obj:
-            schema_obj["@type"] = "felis:Schema"
-        schema_obj["@context"] = DEFAULT_CONTEXT
-    elif isinstance(top_level_object, list):
-        schema_obj = {"@context": DEFAULT_CONTEXT, "@graph": top_level_object}
+    url = make_url(engine_url)
+    engine: Engine | MockConnection
+    if dry_run or is_mock_url(url):
+        engine = create_mock_engine(url, executor=None)
     else:
-        logger.error("Schema object not of recognizable type")
-        raise click.exceptions.Exit(1)
-
-    normalized = _normalize(schema_obj, embed="@always")
-    if len(normalized["@graph"]) > 1 and (schema_name or catalog_name):
-        logger.error("--schema-name and --catalog-name incompatible with multiple schemas")
-        raise click.exceptions.Exit(1)
-
-    # Force normalized["@graph"] to a list, which is what happens when there's
-    # multiple schemas
-    if isinstance(normalized["@graph"], dict):
-        normalized["@graph"] = [normalized["@graph"]]
-
-    tap_tables = init_tables(
-        tap_schema_name,
-        tap_tables_postfix,
-        tap_schemas_table,
-        tap_tables_table,
-        tap_columns_table,
-        tap_keys_table,
-        tap_key_columns_table,
-    )
-
-    if not dry_run:
         engine = create_engine(engine_url)
+    mgr = TableManager(
+        engine=engine,
+        apply_schema_to_metadata=False if engine.dialect.name == "sqlite" else True,
+        schema_name=tap_schema_name,
+        table_name_postfix=tap_tables_postfix,
+    )
 
-        if engine_url == "sqlite://" and not dry_run:
-            # In Memory SQLite - Mostly used to test
-            Tap11Base.metadata.create_all(engine)
-
-        for schema in normalized["@graph"]:
-            tap_visitor = TapLoadingVisitor(
-                engine,
-                catalog_name=catalog_name,
-                schema_name=schema_name,
-                tap_tables=tap_tables,
-            )
-            tap_visitor.visit_schema(schema)
-    else:
-        _insert_dump = InsertDump()
-        conn = create_mock_engine(make_url(engine_url), executor=_insert_dump.dump, paramstyle="pyformat")
-        # After the engine is created, update the executor with the dialect
-        _insert_dump.dialect = conn.dialect
-
-        for schema in normalized["@graph"]:
-            tap_visitor = TapLoadingVisitor.from_mock_connection(
-                conn,
-                catalog_name=catalog_name,
-                schema_name=schema_name,
-                tap_tables=tap_tables,
-            )
-            tap_visitor.visit_schema(schema)
-
-
-@cli.command("modify-tap")
-@click.option("--start-schema-at", type=int, help="Rewrite index for tap:schema_index", default=0)
-@click.argument("files", nargs=-1, type=click.File())
-def modify_tap(start_schema_at: int, files: Iterable[io.TextIOBase]) -> None:
-    """Modify TAP information in Felis schema FILES.
-
-    This command has some utilities to aid in rewriting felis FILES
-    in specific ways. It will write out a merged version of these files.
-    """
-    count = 0
-    graph = []
-    for file in files:
-        schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
-        if "@graph" not in schema_obj:
-            schema_obj["@type"] = "felis:Schema"
-        schema_obj["@context"] = DEFAULT_CONTEXT
-        schema_index = schema_obj.get("tap:schema_index")
-        if not schema_index or (schema_index and schema_index > start_schema_at):
-            schema_index = start_schema_at + count
-            count += 1
-        schema_obj["tap:schema_index"] = schema_index
-        graph.extend(jsonld.flatten(schema_obj))
-    merged = {"@context": DEFAULT_CONTEXT, "@graph": graph}
-    normalized = _normalize(merged, embed="@always")
-    _dump(normalized)
-
-
-@cli.command("basic-check")
-@click.argument("file", type=click.File())
-def basic_check(file: io.TextIOBase) -> None:
-    """Perform a basic check on a felis FILE.
-
-    This performs a very check to ensure required fields are
-    populated and basic semantics are okay. It does not ensure semantics
-    are valid for other commands like create-all or load-tap.
-    """
-    schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
-    schema_obj["@type"] = "felis:Schema"
-    # Force Context and Schema Type
-    schema_obj["@context"] = DEFAULT_CONTEXT
-    check_visitor = CheckingVisitor()
-    check_visitor.visit_schema(schema_obj)
-
-
-@cli.command("normalize")
-@click.argument("file", type=click.File())
-def normalize(file: io.TextIOBase) -> None:
-    """Normalize a Felis FILE.
-
-    Takes a felis schema FILE, expands it (resolving the full URLs),
-    then compacts it, and finally produces output in the canonical
-    format.
-
-    (This is most useful in some debugging scenarios)
-
-    See Also :
+    schema = Schema.from_stream(
+        file,
+        context={
+            "id_generation": ctx.obj["id_generation"],
+            "force_unbounded_arraysize": force_unbounded_arraysize,
+        },
+    )
 
-        https://json-ld.org/spec/latest/json-ld/#expanded-document-form
-        https://json-ld.org/spec/latest/json-ld/#compacted-document-form
+    DataLoader(
+        schema,
+        mgr,
+        engine,
+        tap_schema_index=tap_schema_index,
+        dry_run=dry_run,
+        print_sql=echo,
+        output_path=output_file,
+        unique_keys=unique_keys,
+    ).load()
+
+
+@cli.command("init-tap-schema", help="Initialize a standard TAP_SCHEMA database")
+@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL", required=True)
+@click.option("--tap-schema-name", help="Name of the TAP_SCHEMA schema in the database")
+@click.option(
+    "--tap-tables-postfix", help="Postfix which is applied to standard TAP_SCHEMA table names", default=""
+)
+@click.option(
+    "--insert-metadata/--no-insert-metadata",
+    is_flag=True,
+    help="Insert metadata describing TAP_SCHEMA itself",
+    default=True,
+)
+@click.pass_context
+def init_tap_schema(
+    ctx: click.Context, engine_url: str, tap_schema_name: str, tap_tables_postfix: str, insert_metadata: bool
+) -> None:
+    """Initialize a standard TAP_SCHEMA database.
+
+    Parameters
+    ----------
+    engine_url
+        SQLAlchemy Engine URL.
+    tap_schema_name
+        Name of the TAP_SCHEMA schema in the database.
+    tap_tables_postfix
+        Postfix which is applied to standard TAP_SCHEMA table names.
+    insert_metadata
+        Insert metadata describing TAP_SCHEMA itself.
+        If set to False, only the TAP_SCHEMA tables will be created, but no
+        metadata will be inserted.
     """
-    schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
-    schema_obj["@type"] = "felis:Schema"
-    # Force Context and Schema Type
-    schema_obj["@context"] = DEFAULT_CONTEXT
-    expanded = jsonld.expand(schema_obj)
-    normalized = _normalize(expanded, embed="@always")
-    _dump(normalized)
-
+    url = make_url(engine_url)
+    engine: Engine | MockConnection
+    if is_mock_url(url):
+        raise click.ClickException("Mock engine URL is not supported for this command")
+    engine = create_engine(engine_url)
+    mgr = TableManager(
+        apply_schema_to_metadata=False if engine.dialect.name == "sqlite" else True,
+        schema_name=tap_schema_name,
+        table_name_postfix=tap_tables_postfix,
+    )
+    mgr.initialize_database(engine)
+    if insert_metadata:
+        inserter = MetadataInserter(mgr, engine)
+        inserter.insert_metadata()
 
-@cli.command("merge")
-@click.argument("files", nargs=-1, type=click.File())
-def merge(files: Iterable[io.TextIOBase]) -> None:
-    """Merge a set of Felis FILES.
 
-    This will expand out the felis FILES so that it is easy to
-    override values (using @Id), then normalize to a single
-    output.
-    """
-    graph = []
-    for file in files:
-        schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
-        if "@graph" not in schema_obj:
-            schema_obj["@type"] = "felis:Schema"
-        schema_obj["@context"] = DEFAULT_CONTEXT
-        graph.extend(jsonld.flatten(schema_obj))
-    updated_map: MutableMapping[str, Any] = {}
-    for item in graph:
-        _id = item["@id"]
-        item_to_update = updated_map.get(_id, item)
-        if item_to_update and item_to_update != item:
-            logger.debug(f"Overwriting {_id}")
-        item_to_update.update(item)
-        updated_map[_id] = item_to_update
-    merged = {"@context": DEFAULT_CONTEXT, "@graph": list(updated_map.values())}
-    normalized = _normalize(merged, embed="@always")
-    _dump(normalized)
-
-
-@cli.command("validate")
+@cli.command("validate", help="Validate one or more Felis YAML files")
+@click.option(
+    "--check-description", is_flag=True, help="Check that all objects have a description", default=False
+)
 @click.option(
-    "-s",
-    "--schema-name",
-    help="Schema name for validation",
-    type=click.Choice(["RSP", "default"]),
-    default="default",
+    "--check-redundant-datatypes", is_flag=True, help="Check for redundant datatype overrides", default=False
+)
+@click.option(
+    "--check-tap-table-indexes",
+    is_flag=True,
+    help="Check that every table has a unique TAP table index",
+    default=False,
+)
+@click.option(
+    "--check-tap-principal",
+    is_flag=True,
+    help="Check that at least one column per table is flagged as TAP principal",
+    default=False,
 )
-@click.option("-d", "--require-description", is_flag=True, help="Require description for all objects")
 @click.argument("files", nargs=-1, type=click.File())
-def validate(schema_name: str, require_description: bool, files: Iterable[io.TextIOBase]) -> None:
-    """Validate one or more felis YAML files."""
-    schema_class = get_schema(schema_name)
-    logger.info(f"Using schema '{schema_class.__name__}'")
-
-    if require_description:
-        Schema.require_description(True)
-
+@click.pass_context
+def validate(
+    ctx: click.Context,
+    check_description: bool,
+    check_redundant_datatypes: bool,
+    check_tap_table_indexes: bool,
+    check_tap_principal: bool,
+    files: Iterable[IO[str]],
+) -> None:
+    """Validate one or more felis YAML files.
+
+    Parameters
+    ----------
+    check_description
+        Check that all objects have a valid description.
+    check_redundant_datatypes
+        Check for redundant type overrides.
+    check_tap_table_indexes
+        Check that every table has a unique TAP table index.
+    check_tap_principal
+        Check that at least one column per table is flagged as TAP principal.
+    files
+        The Felis YAML files to validate.
+
+    Raises
+    ------
+    click.exceptions.Exit
+        Raised if any validation errors are found. The ``ValidationError``
+        which is thrown when a schema fails to validate will be logged as an
+        error message.
+
+    Notes
+    -----
+    All of the ``check`` flags are turned off by default and represent
+    optional validations controlled by the Pydantic context.
+    """
     rc = 0
     for file in files:
         file_name = getattr(file, "name", None)
         logger.info(f"Validating {file_name}")
         try:
-            schema_class.model_validate(yaml.load(file, Loader=yaml.SafeLoader))
+            Schema.from_stream(
+                file,
+                context={
+                    "check_description": check_description,
+                    "check_redundant_datatypes": check_redundant_datatypes,
+                    "check_tap_table_indexes": check_tap_table_indexes,
+                    "check_tap_principal": check_tap_principal,
+                    "id_generation": ctx.obj["id_generation"],
+                },
+            )
         except ValidationError as e:
             logger.error(e)
             rc = 1
@@ -335,80 +408,103 @@ def validate(schema_name: str, require_description: bool, files: Iterable[io.Tex
         raise click.exceptions.Exit(rc)
 
 
-@cli.command("dump-json")
-@click.option("-x", "--expanded", is_flag=True, help="Extended schema before dumping.")
-@click.option("-f", "--framed", is_flag=True, help="Frame schema before dumping.")
-@click.option("-c", "--compacted", is_flag=True, help="Compact schema before dumping.")
-@click.option("-g", "--graph", is_flag=True, help="Pass graph option to compact.")
-@click.argument("file", type=click.File())
-def dump_json(
-    file: io.TextIOBase,
-    expanded: bool = False,
-    compacted: bool = False,
-    framed: bool = False,
-    graph: bool = False,
+@cli.command(
+    "diff",
+    help="""
+    Compare two schemas or a schema and a database for changes
+
+    Examples:
+
+      felis diff schema1.yaml schema2.yaml
+
+      felis diff -c alembic schema1.yaml schema2.yaml
+
+      felis diff --engine-url sqlite:///test.db schema.yaml
+    """,
+)
+@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL")
+@click.option(
+    "-c",
+    "--comparator",
+    type=click.Choice(["alembic", "deepdiff"], case_sensitive=False),
+    help="Comparator to use for schema comparison",
+    default="deepdiff",
+)
+@click.option("-E", "--error-on-change", is_flag=True, help="Exit with error code if schemas are different")
+@click.argument("files", nargs=-1, type=click.File())
+@click.pass_context
+def diff(
+    ctx: click.Context,
+    engine_url: str | None,
+    comparator: str,
+    error_on_change: bool,
+    files: Iterable[IO[str]],
+) -> None:
+    schemas = [
+        Schema.from_stream(file, context={"id_generation": ctx.obj["id_generation"]}) for file in files
+    ]
+
+    diff: SchemaDiff
+    if len(schemas) == 2 and engine_url is None:
+        if comparator == "alembic":
+            db_context = create_database(schemas[0])
+            assert isinstance(db_context.engine, Engine)
+            diff = DatabaseDiff(schemas[1], db_context.engine)
+        else:
+            diff = FormattedSchemaDiff(schemas[0], schemas[1])
+    elif len(schemas) == 1 and engine_url is not None:
+        engine = create_engine(engine_url)
+        diff = DatabaseDiff(schemas[0], engine)
+    else:
+        raise click.ClickException(
+            "Invalid arguments - provide two schemas or a schema and a database engine URL"
+        )
+
+    diff.print()
+
+    if diff.has_changes and error_on_change:
+        raise click.ClickException("Schema was changed")
+
+
+@cli.command(
+    "dump",
+    help="""
+    Dump a schema file to YAML or JSON format
+
+    Example:
+
+      felis dump schema.yaml schema.json
+
+      felis dump schema.yaml schema_dump.yaml
+    """,
+)
+@click.option(
+    "--strip-ids/--no-strip-ids",
+    is_flag=True,
+    help="Strip IDs from the output schema",
+    default=False,
+)
+@click.argument("files", nargs=2, type=click.Path())
+@click.pass_context
+def dump(
+    ctx: click.Context,
+    strip_ids: bool,
+    files: list[str],
 ) -> None:
-    """Dump JSON representation using various JSON-LD options."""
-    schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
-    schema_obj["@type"] = "felis:Schema"
-    # Force Context and Schema Type
-    schema_obj["@context"] = DEFAULT_CONTEXT
-
-    if expanded:
-        schema_obj = jsonld.expand(schema_obj)
-    if framed:
-        schema_obj = jsonld.frame(schema_obj, DEFAULT_FRAME)
-    if compacted:
-        options = {}
-        if graph:
-            options["graph"] = True
-        schema_obj = jsonld.compact(schema_obj, DEFAULT_CONTEXT, options=options)
-    json.dump(schema_obj, sys.stdout, indent=4)
-
-
-def _dump(obj: Mapping[str, Any]) -> None:
-    class OrderedDumper(yaml.Dumper):
-        pass
-
-    def _dict_representer(dumper: yaml.Dumper, data: Any) -> Any:
-        return dumper.represent_mapping(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, data.items())
-
-    OrderedDumper.add_representer(dict, _dict_representer)
-    print(yaml.dump(obj, Dumper=OrderedDumper, default_flow_style=False))
-
-
-def _normalize(schema_obj: Mapping[str, Any], embed: str = "@last") -> MutableMapping[str, Any]:
-    framed = jsonld.frame(schema_obj, DEFAULT_FRAME, options=dict(embed=embed))
-    compacted = jsonld.compact(framed, DEFAULT_CONTEXT, options=dict(graph=True))
-    graph = compacted["@graph"]
-    graph = [ReorderingVisitor(add_type=True).visit_schema(schema_obj) for schema_obj in graph]
-    compacted["@graph"] = graph if len(graph) > 1 else graph[0]
-    return compacted
-
-
-class InsertDump:
-    """An Insert Dumper for SQL statements."""
-
-    dialect: Any = None
-
-    def dump(self, sql: Any, *multiparams: Any, **params: Any) -> None:
-        compiled = sql.compile(dialect=self.dialect)
-        sql_str = str(compiled) + ";"
-        params_list = [compiled.params]
-        for params in params_list:
-            if not params:
-                print(sql_str)
-                continue
-            new_params = {}
-            for key, value in params.items():
-                if isinstance(value, str):
-                    new_params[key] = f"'{value}'"
-                elif value is None:
-                    new_params[key] = "null"
-                else:
-                    new_params[key] = value
-
-            print(sql_str % new_params)
+    if strip_ids:
+        logger.info("Stripping IDs from the output schema")
+    if files[1].endswith(".json"):
+        format = "json"
+    elif files[1].endswith(".yaml"):
+        format = "yaml"
+    else:
+        raise click.ClickException("Output file must have a .json or .yaml extension")
+    schema = Schema.from_uri(files[0], context={"id_generation": ctx.obj["id_generation"]})
+    with open(files[1], "w") as f:
+        if format == "yaml":
+            schema.dump_yaml(f, strip_ids=strip_ids)
+        elif format == "json":
+            schema.dump_json(f, strip_ids=strip_ids)
 
 
 if __name__ == "__main__":