lsst-felis 28.2024.4500__py3-none-any.whl → 29.2025.4500__py3-none-any.whl

felis/__init__.py

@@ -19,4 +19,15 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-from .version import *
+from .datamodel import Schema
+from .db.schema import create_database
+from .diff import DatabaseDiff, FormattedSchemaDiff, SchemaDiff
+from .metadata import MetaDataBuilder
+
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("lsst-felis")
+except PackageNotFoundError:
+    # Package not installed or scons not run.
+    __version__ = "0.0.0"

felis/cli.py

@@ -34,10 +34,11 @@ from sqlalchemy.engine.mock import MockConnection, create_mock_engine
 
 from . import __version__
 from .datamodel import Schema
+from .db.schema import create_database
 from .db.utils import DatabaseContext, is_mock_url
+from .diff import DatabaseDiff, FormattedSchemaDiff, SchemaDiff
 from .metadata import MetaDataBuilder
-from .tap import Tap11Base, TapLoadingVisitor, init_tables
-from .tap_schema import DataLoader, TableManager
+from .tap_schema import DataLoader, MetadataInserter, TableManager
 
 __all__ = ["cli"]
 
@@ -62,7 +63,10 @@ loglevel_choices = ["CRITICAL", "FATAL", "ERROR", "WARNING", "INFO", "DEBUG"]
     help="Felis log file path",
 )
 @click.option(
-    "--id-generation", is_flag=True, help="Generate IDs for all objects that do not have them", default=False
+    "--id-generation/--no-id-generation",
+    is_flag=True,
+    help="Generate IDs for all objects that do not have them",
+    default=True,
 )
 @click.pass_context
 def cli(ctx: click.Context, log_level: str, log_file: str | None, id_generation: bool) -> None:
@@ -71,6 +75,8 @@ def cli(ctx: click.Context, log_level: str, log_file: str | None, id_generation:
     ctx.obj["id_generation"] = id_generation
     if ctx.obj["id_generation"]:
         logger.info("ID generation is enabled")
+    else:
+        logger.info("ID generation is disabled")
     if log_file:
         logging.basicConfig(filename=log_file, level=log_level)
     else:
@@ -177,181 +183,34 @@ def create(
         raise click.ClickException(str(e))
 
 
-@cli.command("init-tap", help="Initialize TAP_SCHEMA objects in the database")
-@click.option("--tap-schema-name", help="Alternate database schema name for 'TAP_SCHEMA'")
-@click.option("--tap-schemas-table", help="Alternate table name for 'schemas'")
-@click.option("--tap-tables-table", help="Alternate table name for 'tables'")
-@click.option("--tap-columns-table", help="Alternate table name for 'columns'")
-@click.option("--tap-keys-table", help="Alternate table name for 'keys'")
-@click.option("--tap-key-columns-table", help="Alternate table name for 'key_columns'")
-@click.argument("engine-url")
-def init_tap(
-    engine_url: str,
-    tap_schema_name: str,
-    tap_schemas_table: str,
-    tap_tables_table: str,
-    tap_columns_table: str,
-    tap_keys_table: str,
-    tap_key_columns_table: str,
-) -> None:
-    """Initialize TAP_SCHEMA objects in the database.
-
-    Parameters
-    ----------
-    engine_url
-        SQLAlchemy Engine URL. The target PostgreSQL schema or MySQL database
-        must already exist and be referenced in the URL.
-    tap_schema_name
-        Alterate name for the database schema ``TAP_SCHEMA``.
-    tap_schemas_table
-        Alterate table name for ``schemas``.
-    tap_tables_table
-        Alterate table name for ``tables``.
-    tap_columns_table
-        Alterate table name for ``columns``.
-    tap_keys_table
-        Alterate table name for ``keys``.
-    tap_key_columns_table
-        Alterate table name for ``key_columns``.
-
-    Notes
-    -----
-    The supported version of TAP_SCHEMA in the SQLAlchemy metadata is 1.1. The
-    tables are created in the database schema specified by the engine URL,
-    which must be a PostgreSQL schema or MySQL database that already exists.
-    """
-    engine = create_engine(engine_url)
-    init_tables(
-        tap_schema_name,
-        tap_schemas_table,
-        tap_tables_table,
-        tap_columns_table,
-        tap_keys_table,
-        tap_key_columns_table,
-    )
-    Tap11Base.metadata.create_all(engine)
-
-
-@cli.command("load-tap", help="Load metadata from a Felis file into a TAP_SCHEMA database")
-@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL")
-@click.option("--schema-name", help="Alternate Schema Name for Felis file")
-@click.option("--catalog-name", help="Catalog Name for Schema")
-@click.option("--dry-run", is_flag=True, help="Dry Run Only. Prints out the DDL that would be executed")
-@click.option("--tap-schema-name", help="Alternate schema name for 'TAP_SCHEMA'")
-@click.option("--tap-tables-postfix", help="Postfix for TAP_SCHEMA table names")
-@click.option("--tap-schemas-table", help="Alternate table name for 'schemas'")
-@click.option("--tap-tables-table", help="Alternate table name for 'tables'")
-@click.option("--tap-columns-table", help="Alternate table name for 'columns'")
-@click.option("--tap-keys-table", help="Alternate table name for 'keys'")
-@click.option("--tap-key-columns-table", help="Alternate table name for 'key_columns'")
-@click.option("--tap-schema-index", type=int, help="TAP_SCHEMA index of the schema in this environment")
-@click.argument("file", type=click.File())
-def load_tap(
-    engine_url: str,
-    schema_name: str,
-    catalog_name: str,
-    dry_run: bool,
-    tap_schema_name: str,
-    tap_tables_postfix: str,
-    tap_schemas_table: str,
-    tap_tables_table: str,
-    tap_columns_table: str,
-    tap_keys_table: str,
-    tap_key_columns_table: str,
-    tap_schema_index: int,
-    file: IO[str],
-) -> None:
-    """Load TAP metadata from a Felis file.
-
-    This command loads the associated TAP metadata from a Felis YAML file
-    into the TAP_SCHEMA tables.
-
-    Parameters
-    ----------
-    engine_url
-        SQLAlchemy Engine URL to catalog.
-    schema_name
-        Alternate schema name. This overrides the schema name in the
-        ``catalog`` field of the Felis file.
-    catalog_name
-        Catalog name for the schema. This possibly duplicates the
-        ``tap_schema_name`` argument (DM-44870).
-    dry_run
-        Dry run only to print out commands instead of executing.
-    tap_schema_name
-        Alternate name for the schema of TAP_SCHEMA in the database.
-    tap_tables_postfix
-        Postfix for TAP table names that will be automatically appended.
-    tap_schemas_table
-        Alternate table name for ``schemas``.
-    tap_tables_table
-        Alternate table name for ``tables``.
-    tap_columns_table
-        Alternate table name for ``columns``.
-    tap_keys_table
-        Alternate table name for ``keys``.
-    tap_key_columns_table
-        Alternate table name for ``key_columns``.
-    tap_schema_index
-        TAP_SCHEMA index of the schema in this TAP environment.
-    file
-        Felis file to read.
-
-    Notes
-    -----
-    The data will be loaded into the TAP_SCHEMA from the engine URL. The
-    tables must have already been initialized or an error will occur.
-    """
-    schema = Schema.from_stream(file)
-
-    tap_tables = init_tables(
-        tap_schema_name,
-        tap_tables_postfix,
-        tap_schemas_table,
-        tap_tables_table,
-        tap_columns_table,
-        tap_keys_table,
-        tap_key_columns_table,
-    )
-
-    if not dry_run:
-        engine = create_engine(engine_url)
-
-        if engine_url == "sqlite://" and not dry_run:
-            # In Memory SQLite - Mostly used to test
-            Tap11Base.metadata.create_all(engine)
-
-        tap_visitor = TapLoadingVisitor(
-            engine,
-            catalog_name=catalog_name,
-            schema_name=schema_name,
-            tap_tables=tap_tables,
-            tap_schema_index=tap_schema_index,
-        )
-        tap_visitor.visit_schema(schema)
-    else:
-        conn = DatabaseContext.create_mock_engine(engine_url)
-
-        tap_visitor = TapLoadingVisitor.from_mock_connection(
-            conn,
-            catalog_name=catalog_name,
-            schema_name=schema_name,
-            tap_tables=tap_tables,
-            tap_schema_index=tap_schema_index,
-        )
-        tap_visitor.visit_schema(schema)
-
-
 @cli.command("load-tap-schema", help="Load metadata from a Felis file into a TAP_SCHEMA database")
 @click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL")
-@click.option("--tap-schema-name", help="Name of the TAP_SCHEMA schema in the database")
 @click.option(
-    "--tap-tables-postfix", help="Postfix which is applied to standard TAP_SCHEMA table names", default=""
+    "--tap-schema-name", "-n", help="Name of the TAP_SCHEMA schema in the database (default: TAP_SCHEMA)"
+)
+@click.option(
+    "--tap-tables-postfix",
+    "-p",
+    help="Postfix which is applied to standard TAP_SCHEMA table names",
+    default="",
+)
+@click.option("--tap-schema-index", "-i", type=int, help="TAP_SCHEMA index of the schema in this environment")
+@click.option("--dry-run", "-D", is_flag=True, help="Execute dry run only. Does not insert any data.")
+@click.option("--echo", "-e", is_flag=True, help="Print out the generated insert statements to stdout")
+@click.option("--output-file", "-o", type=click.Path(), help="Write SQL commands to a file")
+@click.option(
+    "--force-unbounded-arraysize",
+    is_flag=True,
+    help="Use unbounded arraysize by default for all variable length string columns"
+    ", e.g., ``votable:arraysize: *`` (workaround for astropy bug #18099)",
+)  # DM-50899: Variable-length bounded strings are not handled correctly in astropy
+@click.option(
+    "--unique-keys",
+    "-u",
+    is_flag=True,
+    help="Generate unique key_id values for keys and key_columns tables by prepending the schema name",
+    default=False,
 )
-@click.option("--tap-schema-index", type=int, help="TAP_SCHEMA index of the schema in this environment")
-@click.option("--dry-run", is_flag=True, help="Execute dry run only. Does not insert any data.")
-@click.option("--echo", is_flag=True, help="Print out the generated insert statements to stdout")
-@click.option("--output-file", type=click.Path(), help="Write SQL commands to a file")
 @click.argument("file", type=click.File())
 @click.pass_context
 def load_tap_schema(
@@ -363,6 +222,8 @@ def load_tap_schema(
     dry_run: bool,
     echo: bool,
     output_file: str | None,
+    force_unbounded_arraysize: bool,
+    unique_keys: bool,
     file: IO[str],
 ) -> None:
     """Load TAP metadata from a Felis file.
@@ -402,7 +263,13 @@ def load_tap_schema(
         table_name_postfix=tap_tables_postfix,
     )
 
-    schema = Schema.from_stream(file, context={"id_generation": ctx.obj["id_generation"]})
+    schema = Schema.from_stream(
+        file,
+        context={
+            "id_generation": ctx.obj["id_generation"],
+            "force_unbounded_arraysize": force_unbounded_arraysize,
+        },
+    )
 
     DataLoader(
         schema,
@@ -412,9 +279,57 @@ def load_tap_schema(
         dry_run=dry_run,
         print_sql=echo,
         output_path=output_file,
+        unique_keys=unique_keys,
     ).load()
 
 
+@cli.command("init-tap-schema", help="Initialize a standard TAP_SCHEMA database")
+@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL", required=True)
+@click.option("--tap-schema-name", help="Name of the TAP_SCHEMA schema in the database")
+@click.option(
+    "--tap-tables-postfix", help="Postfix which is applied to standard TAP_SCHEMA table names", default=""
+)
+@click.option(
+    "--insert-metadata/--no-insert-metadata",
+    is_flag=True,
+    help="Insert metadata describing TAP_SCHEMA itself",
+    default=True,
+)
+@click.pass_context
+def init_tap_schema(
+    ctx: click.Context, engine_url: str, tap_schema_name: str, tap_tables_postfix: str, insert_metadata: bool
+) -> None:
+    """Initialize a standard TAP_SCHEMA database.
+
+    Parameters
+    ----------
+    engine_url
+        SQLAlchemy Engine URL.
+    tap_schema_name
+        Name of the TAP_SCHEMA schema in the database.
+    tap_tables_postfix
+        Postfix which is applied to standard TAP_SCHEMA table names.
+    insert_metadata
+        Insert metadata describing TAP_SCHEMA itself.
+        If set to False, only the TAP_SCHEMA tables will be created, but no
+        metadata will be inserted.
+    """
+    url = make_url(engine_url)
+    engine: Engine | MockConnection
+    if is_mock_url(url):
+        raise click.ClickException("Mock engine URL is not supported for this command")
+    engine = create_engine(engine_url)
+    mgr = TableManager(
+        apply_schema_to_metadata=False if engine.dialect.name == "sqlite" else True,
+        schema_name=tap_schema_name,
+        table_name_postfix=tap_tables_postfix,
+    )
+    mgr.initialize_database(engine)
+    if insert_metadata:
+        inserter = MetadataInserter(mgr, engine)
+        inserter.insert_metadata()
+
+
 @cli.command("validate", help="Validate one or more Felis YAML files")
 @click.option(
     "--check-description", is_flag=True, help="Check that all objects have a description", default=False
@@ -493,5 +408,104 @@ def validate(
         raise click.exceptions.Exit(rc)
 
 
+@cli.command(
+    "diff",
+    help="""
+    Compare two schemas or a schema and a database for changes
+
+    Examples:
+
+      felis diff schema1.yaml schema2.yaml
+
+      felis diff -c alembic schema1.yaml schema2.yaml
+
+      felis diff --engine-url sqlite:///test.db schema.yaml
+    """,
+)
+@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL")
+@click.option(
+    "-c",
+    "--comparator",
+    type=click.Choice(["alembic", "deepdiff"], case_sensitive=False),
+    help="Comparator to use for schema comparison",
+    default="deepdiff",
+)
+@click.option("-E", "--error-on-change", is_flag=True, help="Exit with error code if schemas are different")
+@click.argument("files", nargs=-1, type=click.File())
+@click.pass_context
+def diff(
+    ctx: click.Context,
+    engine_url: str | None,
+    comparator: str,
+    error_on_change: bool,
+    files: Iterable[IO[str]],
+) -> None:
+    schemas = [
+        Schema.from_stream(file, context={"id_generation": ctx.obj["id_generation"]}) for file in files
+    ]
+
+    diff: SchemaDiff
+    if len(schemas) == 2 and engine_url is None:
+        if comparator == "alembic":
+            db_context = create_database(schemas[0])
+            assert isinstance(db_context.engine, Engine)
+            diff = DatabaseDiff(schemas[1], db_context.engine)
+        else:
+            diff = FormattedSchemaDiff(schemas[0], schemas[1])
+    elif len(schemas) == 1 and engine_url is not None:
+        engine = create_engine(engine_url)
+        diff = DatabaseDiff(schemas[0], engine)
+    else:
+        raise click.ClickException(
+            "Invalid arguments - provide two schemas or a schema and a database engine URL"
+        )
+
+    diff.print()
+
+    if diff.has_changes and error_on_change:
+        raise click.ClickException("Schema was changed")
+
+
+@cli.command(
+    "dump",
+    help="""
+    Dump a schema file to YAML or JSON format
+
+    Example:
+
+      felis dump schema.yaml schema.json
+
+      felis dump schema.yaml schema_dump.yaml
+    """,
+)
+@click.option(
+    "--strip-ids/--no-strip-ids",
+    is_flag=True,
+    help="Strip IDs from the output schema",
+    default=False,
+)
+@click.argument("files", nargs=2, type=click.Path())
+@click.pass_context
+def dump(
+    ctx: click.Context,
+    strip_ids: bool,
+    files: list[str],
+) -> None:
+    if strip_ids:
+        logger.info("Stripping IDs from the output schema")
+    if files[1].endswith(".json"):
+        format = "json"
+    elif files[1].endswith(".yaml"):
+        format = "yaml"
+    else:
+        raise click.ClickException("Output file must have a .json or .yaml extension")
+    schema = Schema.from_uri(files[0], context={"id_generation": ctx.obj["id_generation"]})
+    with open(files[1], "w") as f:
+        if format == "yaml":
+            schema.dump_yaml(f, strip_ids=strip_ids)
+        elif format == "json":
+            schema.dump_json(f, strip_ids=strip_ids)
+
+
 if __name__ == "__main__":
     cli()

felis/config/tap_schema/columns.csv

@@ -0,0 +1,33 @@
+table_name,column_name,utype,ucd,unit,description,datatype,arraysize,xtype,size,principal,indexed,std,column_index
+tap_schema.columns,"""size""",\N,\N,\N,deprecated: use arraysize,int,\N,\N,\N,1,0,1,9
+tap_schema.columns,arraysize,\N,\N,\N,lists the size of variable-length columns in the tableset,char,16*,\N,16,1,0,1,8
+tap_schema.columns,column_index,\N,\N,\N,recommended sort order when listing columns of a table,int,\N,\N,\N,1,0,1,13
+tap_schema.columns,column_name,\N,\N,\N,the column name,char,64*,\N,64,1,0,1,2
+tap_schema.columns,datatype,\N,\N,\N,lists the ADQL datatype of columns in the tableset,char,64*,\N,64,1,0,1,7
+tap_schema.columns,description,\N,\N,\N,describes the columns in the tableset,char,512*,\N,512,1,0,1,6
+tap_schema.columns,indexed,\N,\N,\N,"an indexed column; 1 means 1, 0 means 0",int,\N,\N,\N,1,0,1,11
+tap_schema.columns,principal,\N,\N,\N,"a principal column; 1 means 1, 0 means 0",int,\N,\N,\N,1,0,1,10
+tap_schema.columns,std,\N,\N,\N,"a standard column; 1 means 1, 0 means 0",int,\N,\N,\N,1,0,1,12
+tap_schema.columns,table_name,\N,\N,\N,the table this column belongs to,char,64*,\N,64,1,0,1,1
+tap_schema.columns,ucd,\N,\N,\N,lists the UCDs of columns in the tableset,char,64*,\N,64,1,0,1,4
+tap_schema.columns,unit,\N,\N,\N,lists the unit used for column values in the tableset,char,64*,\N,64,1,0,1,5
+tap_schema.columns,utype,\N,\N,\N,lists the utypes of columns in the tableset,char,512*,\N,512,1,0,1,3
+tap_schema.columns,xtype,\N,\N,\N,a DALI or custom extended type annotation,char,64*,\N,64,1,0,1,7
+tap_schema.key_columns,from_column,\N,\N,\N,column in the from_table,char,64*,\N,64,1,0,1,2
+tap_schema.key_columns,key_id,\N,\N,\N,key to join to tap_schema.keys,char,64*,\N,64,1,0,1,1
+tap_schema.key_columns,target_column,\N,\N,\N,column in the target_table,char,64*,\N,64,1,0,1,3
+tap_schema.keys,description,\N,\N,\N,describes keys in the tableset,char,512*,\N,512,1,0,1,5
+tap_schema.keys,from_table,\N,\N,\N,the table with the foreign key,char,64*,\N,64,1,0,1,2
+tap_schema.keys,key_id,\N,\N,\N,unique key to join to tap_schema.key_columns,char,64*,\N,64,1,0,1,1
+tap_schema.keys,target_table,\N,\N,\N,the table with the primary key,char,64*,\N,64,1,0,1,3
+tap_schema.keys,utype,\N,\N,\N,lists the utype of keys in the tableset,char,512*,\N,512,1,0,1,4
+tap_schema.schemas,description,\N,\N,\N,describes schemas in the tableset,char,512*,\N,512,1,0,1,3
+tap_schema.schemas,schema_index,\N,\N,\N,recommended sort order when listing schemas,int,\N,\N,\N,1,0,1,4
+tap_schema.schemas,schema_name,\N,\N,\N,schema name for reference to tap_schema.schemas,char,64*,\N,64,1,0,1,1
+tap_schema.schemas,utype,\N,\N,\N,lists the utypes of schemas in the tableset,char,512*,\N,512,1,0,1,2
+tap_schema.tables,description,\N,\N,\N,describes tables in the tableset,char,512*,\N,512,1,0,1,5
+tap_schema.tables,schema_name,\N,\N,\N,the schema this table belongs to,char,512*,\N,512,1,0,1,1
+tap_schema.tables,table_index,\N,\N,\N,recommended sort order when listing tables,int,\N,\N,\N,1,0,1,6
+tap_schema.tables,table_name,\N,\N,\N,the fully qualified table name,char,64*,\N,64,1,0,1,2
+tap_schema.tables,table_type,\N,\N,\N,one of: table view,char,8*,\N,8,1,0,1,3
+tap_schema.tables,utype,\N,\N,\N,lists the utype of tables in the tableset,char,512*,\N,512,1,0,1,4

felis/config/tap_schema/key_columns.csv

@@ -0,0 +1,8 @@
+key_id,from_column,target_column
+k1,schema_name,schema_name
+k2,table_name,table_name
+k3,from_table,table_name
+k4,target_table,table_name
+k5,key_id,key_id
+k6,from_column,column_name
+k7,target_column,column_name

felis/config/tap_schema/keys.csv

@@ -0,0 +1,8 @@
+key_id,from_table,target_table,utype,description
+k1,tap_schema.tables,tap_schema.schemas,\N,\N
+k2,tap_schema.columns,tap_schema.tables,\N,\N
+k3,tap_schema.keys,tap_schema.tables,\N,\N
+k4,tap_schema.keys,tap_schema.tables,\N,\N
+k5,tap_schema.key_columns,tap_schema.keys,\N,\N
+k6,tap_schema.key_columns,tap_schema.columns,\N,\N
+k7,tap_schema.key_columns,tap_schema.columns,\N,\N

felis/config/tap_schema/schemas.csv

@@ -0,0 +1,2 @@
+schema_name,utype,description,schema_index
+tap_schema,\N,A TAP-standard-mandated schema to describe tablesets in a TAP 1.1 service,100000

felis/config/tap_schema/tables.csv

@@ -0,0 +1,6 @@
+schema_name,table_name,table_type,utype,description,table_index
+tap_schema,tap_schema.columns,table,\N,description of columns in this tableset,102000
+tap_schema,tap_schema.key_columns,table,\N,description of foreign key columns in this tableset,104000
+tap_schema,tap_schema.keys,table,\N,description of foreign keys in this tableset,103000
+tap_schema,tap_schema.schemas,table,\N,description of schemas in this tableset,100000
+tap_schema,tap_schema.tables,table,\N,description of tables in this tableset,101000