lsst-felis 29.2025.4500__py3-none-any.whl → 30.0.0rc3__py3-none-any.whl

felis/__init__.py

@@ -19,10 +19,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-from .datamodel import Schema
-from .db.schema import create_database
-from .diff import DatabaseDiff, FormattedSchemaDiff, SchemaDiff
-from .metadata import MetaDataBuilder
+from .datamodel import *
 
 from importlib.metadata import PackageNotFoundError, version
 

felis/cli.py

@@ -29,15 +29,12 @@ from typing import IO
 
 import click
 from pydantic import ValidationError
-from sqlalchemy.engine import Engine, create_engine, make_url
-from sqlalchemy.engine.mock import MockConnection, create_mock_engine
 
 from . import __version__
 from .datamodel import Schema
-from .db.schema import create_database
-from .db.utils import DatabaseContext, is_mock_url
+from .db.database_context import create_database_context
 from .diff import DatabaseDiff, FormattedSchemaDiff, SchemaDiff
-from .metadata import MetaDataBuilder
+from .metadata import create_metadata
 from .tap_schema import DataLoader, MetadataInserter, TableManager
 
 __all__ = ["cli"]
@@ -100,6 +97,7 @@ def cli(ctx: click.Context, log_level: str, log_file: str | None, id_generation:
     "--output-file", "-o", type=click.File(mode="w"), help="Write SQL commands to a file instead of executing"
 )
 @click.option("--ignore-constraints", is_flag=True, help="Ignore constraints when creating tables")
+@click.option("--skip-indexes", is_flag=True, help="Skip creating indexes when building metadata")
 @click.argument("file", type=click.File())
 @click.pass_context
 def create(
@@ -112,6 +110,7 @@ def create(
     dry_run: bool,
     output_file: IO[str] | None,
     ignore_constraints: bool,
+    skip_indexes: bool,
     file: IO[str],
 ) -> None:
     """Create database objects from the Felis file.
@@ -134,55 +133,111 @@ def create(
         Write SQL commands to a file instead of executing.
     ignore_constraints
         Ignore constraints when creating tables.
+    skip_indexes
+        Skip creating indexes when building metadata.
     file
         Felis file to read.
     """
     try:
-        schema = Schema.from_stream(file, context={"id_generation": ctx.obj["id_generation"]})
-        url = make_url(engine_url)
-        if schema_name:
-            logger.info(f"Overriding schema name with: {schema_name}")
-            schema.name = schema_name
-        elif url.drivername == "sqlite":
-            logger.info("Overriding schema name for sqlite with: main")
-            schema.name = "main"
-        if not url.host and not url.drivername == "sqlite":
-            dry_run = True
-            logger.info("Forcing dry run for non-sqlite engine URL with no host")
-
-        metadata = MetaDataBuilder(schema, ignore_constraints=ignore_constraints).build()
-        logger.debug(f"Created metadata with schema name: {metadata.schema}")
-
-        engine: Engine | MockConnection
-        if not dry_run and not output_file:
-            engine = create_engine(url, echo=echo)
-        else:
-            if dry_run:
-                logger.info("Dry run will be executed")
-            engine = DatabaseContext.create_mock_engine(url, output_file)
-            if output_file:
-                logger.info("Writing SQL output to: " + output_file.name)
+        metadata = create_metadata(
+            file,
+            id_generation=ctx.obj["id_generation"],
+            schema_name=schema_name,
+            ignore_constraints=ignore_constraints,
+            skip_indexes=skip_indexes,
+            engine_url=engine_url,
+        )
 
-        context = DatabaseContext(metadata, engine)
+        with create_database_context(
+            engine_url,
+            metadata,
+            echo=echo,
+            dry_run=dry_run,
+            output_file=output_file,
+        ) as db_ctx:
+            if drop and initialize:
+                raise ValueError("Cannot drop and initialize schema at the same time")
 
-        if drop and initialize:
-            raise ValueError("Cannot drop and initialize schema at the same time")
+            if drop:
+                logger.debug("Dropping schema if it exists")
+                db_ctx.drop()
+                initialize = True  # If schema is dropped, it needs to be recreated.
 
-        if drop:
-            logger.debug("Dropping schema if it exists")
-            context.drop()
-            initialize = True  # If schema is dropped, it needs to be recreated.
+            if initialize:
+                logger.debug("Creating schema if not exists")
+                db_ctx.initialize()
 
-        if initialize:
-            logger.debug("Creating schema if not exists")
-            context.initialize()
+            db_ctx.create_all()
 
-        context.create_all()
     except Exception as e:
         logger.exception(e)
         raise click.ClickException(str(e))
 
 
+@cli.command("create-indexes", help="Create database indexes defined in the Felis file")
+@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL", default="sqlite://")
+@click.option("--schema-name", help="Alternate schema name to override Felis file")
+@click.argument("file", type=click.File())
+@click.pass_context
+def create_indexes(
+    ctx: click.Context,
+    engine_url: str,
+    schema_name: str | None,
+    file: IO[str],
+) -> None:
+    """Create indexes from a Felis YAML file in a target database.
+
+    Parameters
+    ----------
+    engine_url
+        SQLAlchemy Engine URL.
+    file
+        Felis file to read.
+    """
+    try:
+        metadata = create_metadata(
+            file, id_generation=ctx.obj["id_generation"], schema_name=schema_name, engine_url=engine_url
+        )
+        with create_database_context(engine_url, metadata) as db_ctx:
+            db_ctx.create_indexes()
+    except Exception as e:
+        logger.exception(e)
+        raise click.ClickException("Error creating indexes: " + str(e))
+
+
+@cli.command("drop-indexes", help="Drop database indexes defined in the Felis file")
+@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL", default="sqlite://")
+@click.option("--schema-name", help="Alternate schema name to override Felis file")
+@click.argument("file", type=click.File())
+@click.pass_context
+def drop_indexes(
+    ctx: click.Context,
+    engine_url: str,
+    schema_name: str | None,
+    file: IO[str],
+) -> None:
+    """Drop indexes from a Felis YAML file in a target database.
+
+    Parameters
+    ----------
+    engine_url
+        SQLAlchemy Engine URL.
+    schema-name
+        Alternate schema name to override Felis file.
+    file
+        Felis file to read.
+    """
+    try:
+        metadata = create_metadata(
+            file, id_generation=ctx.obj["id_generation"], schema_name=schema_name, engine_url=engine_url
+        )
+        with create_database_context(engine_url, metadata) as db:
+            db.drop_indexes()
+    except Exception as e:
+        logger.exception(e)
+        raise click.ClickException("Error dropping indexes: " + str(e))
+
+
 @cli.command("load-tap-schema", help="Load metadata from a Felis file into a TAP_SCHEMA database")
 @click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL")
 @click.option(
@@ -197,7 +252,9 @@ def create(
 @click.option("--tap-schema-index", "-i", type=int, help="TAP_SCHEMA index of the schema in this environment")
 @click.option("--dry-run", "-D", is_flag=True, help="Execute dry run only. Does not insert any data.")
 @click.option("--echo", "-e", is_flag=True, help="Print out the generated insert statements to stdout")
-@click.option("--output-file", "-o", type=click.Path(), help="Write SQL commands to a file")
+@click.option(
+    "--output-file", "-o", type=click.File(mode="w"), help="Write SQL commands to a file instead of executing"
+)
 @click.option(
     "--force-unbounded-arraysize",
     is_flag=True,
@@ -221,7 +278,7 @@ def load_tap_schema(
     tap_schema_index: int,
     dry_run: bool,
     echo: bool,
-    output_file: str | None,
+    output_file: IO[str] | None,
     force_unbounded_arraysize: bool,
     unique_keys: bool,
     file: IO[str],
@@ -250,42 +307,51 @@ def load_tap_schema(
     The TAP_SCHEMA database must already exist or the command will fail. This
     command will not initialize the TAP_SCHEMA tables.
     """
-    url = make_url(engine_url)
-    engine: Engine | MockConnection
-    if dry_run or is_mock_url(url):
-        engine = create_mock_engine(url, executor=None)
-    else:
-        engine = create_engine(engine_url)
+    # Create TableManager with automatic dialect detection
     mgr = TableManager(
-        engine=engine,
-        apply_schema_to_metadata=False if engine.dialect.name == "sqlite" else True,
+        engine_url=engine_url,
         schema_name=tap_schema_name,
         table_name_postfix=tap_tables_postfix,
     )
 
-    schema = Schema.from_stream(
-        file,
-        context={
-            "id_generation": ctx.obj["id_generation"],
-            "force_unbounded_arraysize": force_unbounded_arraysize,
-        },
-    )
+    # Create DatabaseContext using TableManager's metadata
+    with create_database_context(
+        engine_url, mgr.metadata, echo=echo, dry_run=dry_run, output_file=output_file
+    ) as db_ctx:
+        schema = Schema.from_stream(
+            file,
+            context={
+                "id_generation": ctx.obj["id_generation"],
+                "force_unbounded_arraysize": force_unbounded_arraysize,
+            },
+        )
 
-    DataLoader(
-        schema,
-        mgr,
-        engine,
-        tap_schema_index=tap_schema_index,
-        dry_run=dry_run,
-        print_sql=echo,
-        output_path=output_file,
-        unique_keys=unique_keys,
-    ).load()
+        DataLoader(
+            schema,
+            mgr,
+            db_context=db_ctx,
+            tap_schema_index=tap_schema_index,
+            dry_run=dry_run,
+            print_sql=echo,
+            output_file=output_file,
+            unique_keys=unique_keys,
+        ).load()
 
 
 @cli.command("init-tap-schema", help="Initialize a standard TAP_SCHEMA database")
 @click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL", required=True)
 @click.option("--tap-schema-name", help="Name of the TAP_SCHEMA schema in the database")
+@click.option(
+    "--extensions",
+    type=str,
+    default=None,
+    help=(
+        "Optional path to extensions YAML file (system path or resource:// URI). "
+        "If not provided, no extensions will be applied. "
+        "Example (default packaged extensions): "
+        "--extensions resource://felis/config/tap_schema/tap_schema_extensions.yaml"
+    ),
+)
 @click.option(
     "--tap-tables-postfix", help="Postfix which is applied to standard TAP_SCHEMA table names", default=""
 )
@@ -297,7 +363,12 @@ def load_tap_schema(
 )
 @click.pass_context
 def init_tap_schema(
-    ctx: click.Context, engine_url: str, tap_schema_name: str, tap_tables_postfix: str, insert_metadata: bool
+    ctx: click.Context,
+    engine_url: str,
+    tap_schema_name: str,
+    extensions: str | None,
+    tap_tables_postfix: str,
+    insert_metadata: bool,
 ) -> None:
     """Initialize a standard TAP_SCHEMA database.
 
@@ -307,6 +378,8 @@ def init_tap_schema(
         SQLAlchemy Engine URL.
     tap_schema_name
         Name of the TAP_SCHEMA schema in the database.
+    extensions
+        Extensions YAML file.
     tap_tables_postfix
         Postfix which is applied to standard TAP_SCHEMA table names.
     insert_metadata
@@ -314,20 +387,19 @@ def init_tap_schema(
         If set to False, only the TAP_SCHEMA tables will be created, but no
         metadata will be inserted.
     """
-    url = make_url(engine_url)
-    engine: Engine | MockConnection
-    if is_mock_url(url):
-        raise click.ClickException("Mock engine URL is not supported for this command")
-    engine = create_engine(engine_url)
+    # Create TableManager with automatic dialect detection
     mgr = TableManager(
-        apply_schema_to_metadata=False if engine.dialect.name == "sqlite" else True,
+        engine_url=engine_url,
         schema_name=tap_schema_name,
         table_name_postfix=tap_tables_postfix,
+        extensions_path=extensions,
     )
-    mgr.initialize_database(engine)
-    if insert_metadata:
-        inserter = MetadataInserter(mgr, engine)
-        inserter.insert_metadata()
+
+    # Create DatabaseContext using TableManager's metadata
+    with create_database_context(engine_url, mgr.metadata) as db_ctx:
+        mgr.initialize_database(db_context=db_ctx)
+        if insert_metadata:
+            MetadataInserter(mgr, db_context=db_ctx).insert_metadata()
 
 
 @cli.command("validate", help="Validate one or more Felis YAML files")
@@ -440,24 +512,37 @@ def diff(
     error_on_change: bool,
     files: Iterable[IO[str]],
 ) -> None:
+    files_list = list(files)
     schemas = [
-        Schema.from_stream(file, context={"id_generation": ctx.obj["id_generation"]}) for file in files
+        Schema.from_stream(file, context={"id_generation": ctx.obj["id_generation"]}) for file in files_list
     ]
-
     diff: SchemaDiff
-    if len(schemas) == 2 and engine_url is None:
+    if len(schemas) == 2:
         if comparator == "alembic":
-            db_context = create_database(schemas[0])
-            assert isinstance(db_context.engine, Engine)
-            diff = DatabaseDiff(schemas[1], db_context.engine)
+            # Reset file stream to beginning before re-reading
+            files_list[0].seek(0)
+            metadata = create_metadata(
+                files_list[0], id_generation=ctx.obj["id_generation"], engine_url=engine_url
+            )
+            with create_database_context(
+                engine_url if engine_url else "sqlite:///:memory:", metadata
+            ) as db_ctx:
+                db_ctx.initialize()
+                db_ctx.create_all()
+                diff = DatabaseDiff(schemas[1], db_ctx.engine)
         else:
             diff = FormattedSchemaDiff(schemas[0], schemas[1])
     elif len(schemas) == 1 and engine_url is not None:
-        engine = create_engine(engine_url)
-        diff = DatabaseDiff(schemas[0], engine)
+        # Create minimal metadata for the context manager
+        from sqlalchemy import MetaData
+
+        metadata = MetaData()
+
+        with create_database_context(engine_url, metadata) as db_ctx:
+            diff = DatabaseDiff(schemas[0], db_ctx.engine)
     else:
         raise click.ClickException(
-            "Invalid arguments - provide two schemas or a schema and a database engine URL"
+            "Invalid arguments - provide two schemas or a single schema and a database engine URL"
         )
 
     diff.print()

felis/config/tap_schema/tap_schema_extensions.yaml

@@ -0,0 +1,73 @@
+# TAP_SCHEMA Extensions
+# This file defines additional columns to be added to the standard TAP_SCHEMA tables
+# These are extensions beyond the IVOA TAP 1.1 specification and needed for the CADC TAP service
+
+# Extension columns for each TAP_SCHEMA table
+name: tap_schema_extensions
+description: Extensions to the standard TAP_SCHEMA tables
+
+tables:
+  - name: schemas
+    description: "Extensions to TAP_SCHEMA.schemas table"
+    columns:
+      - name: owner_id
+        datatype: char
+        length: 32
+        description: "Owner identifier for user-created content"
+
+      - name: read_anon
+        datatype: int
+        description: "Anonymous read permission flag (0 or 1)"
+
+      - name: read_only_group
+        datatype: char
+        length: 128
+        description: "Read-only group identifier"
+
+      - name: read_write_group
+        datatype: char
+        length: 128
+        description: "Read-write group identifier"
+
+      - name: api_created
+        datatype: int
+        description: "Flag indicating if schema was created via TAP service API (0 or 1)"
+
+  - name: tables
+    description: "Extensions to TAP_SCHEMA.tables table"
+    columns:
+      - name: owner_id
+        datatype: char
+        length: 32
+        description: "Owner identifier for user-created content"
+
+      - name: read_anon
+        datatype: int
+        description: "Anonymous read permission flag (0 or 1)"
+
+      - name: read_only_group
+        datatype: char
+        length: 128
+        description: "Read-only group identifier"
+
+      - name: read_write_group
+        datatype: char
+        length: 128
+        description: "Read-write group identifier"
+
+      - name: api_created
+        datatype: int
+        description: "Flag indicating if table was created via TAP service API (0 or 1)"
+
+      - name: view_target
+        datatype: char
+        length: 128
+        description: "View target identifier"
+
+  - name: columns
+    description: "Extensions to TAP_SCHEMA.columns table"
+    columns:
+      - name: column_id
+        datatype: char
+        length: 32
+        description: "Globally unique columnID for use as an XML ID attribute on the FIELD in VOTable output"

felis/datamodel.py

@@ -47,9 +47,8 @@ from pydantic import (
 )
 from pydantic_core import InitErrorDetails
 
-from .db.dialects import get_supported_dialects
-from .db.sqltypes import get_type_func
-from .db.utils import string_to_typeengine
+from .db._dialects import get_supported_dialects, string_to_typeengine
+from .db._sqltypes import get_type_func
 from .types import Boolean, Byte, Char, Double, FelisType, Float, Int, Long, Short, String, Text, Unicode
 
 logger = logging.getLogger(__name__)

felis/db/{dialects.py → _dialects.py}

@@ -1,4 +1,4 @@
-"""Get SQLAlchemy dialects and their type modules."""
+"""Utilities for accessing SQLAlchemy dialects and their type modules."""
 
 # This file is part of felis.
 #
@@ -23,16 +23,18 @@
 
 from __future__ import annotations
 
+import re
 from collections.abc import Mapping
 from types import MappingProxyType, ModuleType
 
-from sqlalchemy import dialects
+from sqlalchemy import dialects, types
 from sqlalchemy.engine import Dialect
 from sqlalchemy.engine.mock import create_mock_engine
+from sqlalchemy.types import TypeEngine
 
-from .sqltypes import MYSQL, POSTGRES, SQLITE
+from ._sqltypes import MYSQL, POSTGRES, SQLITE
 
-__all__ = ["get_dialect_module", "get_supported_dialects"]
+__all__ = ["get_dialect_module", "get_supported_dialects", "string_to_typeengine"]
 
 _DIALECT_NAMES = (MYSQL, POSTGRES, SQLITE)
 """List of supported dialect names.
@@ -40,6 +42,9 @@ _DIALECT_NAMES = (MYSQL, POSTGRES, SQLITE)
 This list is used to create the dialect and module dictionaries.
 """
 
+_DATATYPE_REGEXP = re.compile(r"(\w+)(\((.*)\))?")
+"""Regular expression to match data types with parameters in parentheses."""
+
 
 def _dialect(dialect_name: str) -> Dialect:
     """Create the SQLAlchemy dialect for the given name using a mock engine.
@@ -114,3 +119,63 @@ def get_dialect_module(dialect_name: str) -> ModuleType:
     if dialect_name not in _DIALECT_MODULES:
         raise ValueError(f"Unsupported dialect: {dialect_name}")
     return _DIALECT_MODULES[dialect_name]
+
+
+def string_to_typeengine(
+    type_string: str, dialect: Dialect | None = None, length: int | None = None
+) -> TypeEngine:
+    """Convert a string representation of a datatype to a SQLAlchemy type.
+
+    Parameters
+    ----------
+    type_string
+        The string representation of the data type.
+    dialect
+        The SQLAlchemy dialect to use. If None, the default dialect will be
+        used.
+    length
+        The length of the data type. If the data type does not have a length
+        attribute, this parameter will be ignored.
+
+    Returns
+    -------
+    `sqlalchemy.types.TypeEngine`
+        The SQLAlchemy type engine object.
+
+    Raises
+    ------
+    ValueError
+        Raised if the type string is invalid or the type is not supported.
+
+    Notes
+    -----
+    This function is used when converting type override strings defined in
+    fields such as ``mysql:datatype`` in the schema data.
+    """
+    match = _DATATYPE_REGEXP.search(type_string)
+    if not match:
+        raise ValueError(f"Invalid type string: {type_string}")
+
+    type_name, _, params = match.groups()
+    if dialect is None:
+        type_class = getattr(types, type_name.upper(), None)
+    else:
+        try:
+            dialect_module = get_dialect_module(dialect.name)
+        except KeyError:
+            raise ValueError(f"Unsupported dialect: {dialect}")
+        type_class = getattr(dialect_module, type_name.upper(), None)
+
+    if not type_class:
+        raise ValueError(f"Unsupported type: {type_name.upper()}")
+
+    if params:
+        params = [int(param) if param.isdigit() else param for param in params.split(",")]
+        type_obj = type_class(*params)
+    else:
+        type_obj = type_class()
+
+    if hasattr(type_obj, "length") and getattr(type_obj, "length") is None and length is not None:
+        type_obj.length = length
+
+    return type_obj

felis/db/{variants.py → _variants.py}

@@ -32,7 +32,7 @@ from sqlalchemy import types
 from sqlalchemy.types import TypeEngine
 
 from ..datamodel import Column
-from .dialects import get_dialect_module, get_supported_dialects
+from ._dialects import get_dialect_module, get_supported_dialects
 
 __all__ = ["make_variant_dict"]