lsst-felis 26.2024.1200__tar.gz → 26.2024.1500__tar.gz

{lsst-felis-26.2024.1200/python/lsst_felis.egg-info → lsst-felis-26.2024.1500}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lsst-felis
-Version: 26.2024.1200
+Version: 26.2024.1500
 Summary: A vocabulary for describing catalogs and acting on those descriptions
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: GNU General Public License v3 or later (GPLv3+)
@@ -23,5 +23,6 @@ Requires-Dist: click>=7
 Requires-Dist: pyyaml>=6
 Requires-Dist: pyld>=2
 Requires-Dist: pydantic<3,>=2
+Requires-Dist: lsst-utils
 Provides-Extra: test
 Requires-Dist: pytest>=3.2; extra == "test"

{lsst-felis-26.2024.1200 → lsst-felis-26.2024.1500}/pyproject.toml

@@ -26,7 +26,8 @@ dependencies = [
     "click >= 7",
     "pyyaml >= 6",
     "pyld >= 2",
-    "pydantic >= 2, < 3"
+    "pydantic >= 2, < 3",
+    "lsst-utils"
 ]
 requires-python = ">=3.11.0"
 dynamic = ["version"]
@@ -147,3 +148,13 @@ target-version = "py311"
 extend-select = [
     "RUF100", # Warn about unused noqa
 ]
+
+[tool.pydeps]
+max_bacon = 2
+no_show = true
+verbose = 0
+pylib = false
+format = "png"
+exclude = [
+    "sqlalchemy"
+]

{lsst-felis-26.2024.1200 → lsst-felis-26.2024.1500}/python/felis/cli.py

@@ -26,7 +26,7 @@ import json
 import logging
 import sys
 from collections.abc import Iterable, Mapping, MutableMapping
-from typing import Any
+from typing import IO, Any
 
 import click
 import yaml
@@ -38,7 +38,7 @@ from sqlalchemy.engine.mock import MockConnection
 from . import DEFAULT_CONTEXT, DEFAULT_FRAME, __version__
 from .check import CheckingVisitor
 from .datamodel import Schema
-from .sql import SQLVisitor
+from .metadata import DatabaseContext, InsertDump, MetaDataBuilder
 from .tap import Tap11Base, TapLoadingVisitor, init_tables
 from .utils import ReorderingVisitor
 from .validation import get_schema
@@ -71,27 +71,70 @@ def cli(log_level: str, log_file: str | None) -> None:
         logging.basicConfig(level=log_level)
 
 
-@cli.command("create-all")
-@click.option("--engine-url", envvar="ENGINE_URL", help="SQLAlchemy Engine URL")
-@click.option("--schema-name", help="Alternate Schema Name for Felis File")
-@click.option("--dry-run", is_flag=True, help="Dry Run Only. Prints out the DDL that would be executed")
+@cli.command("create")
+@click.option("--engine-url", envvar="ENGINE_URL", help="SQLAlchemy Engine URL", default="sqlite://")
+@click.option("--schema-name", help="Alternate schema name to override Felis file")
+@click.option(
+    "--create-if-not-exists", is_flag=True, help="Create the schema in the database if it does not exist"
+)
+@click.option("--drop-if-exists", is_flag=True, help="Drop schema if it already exists in the database")
+@click.option("--echo", is_flag=True, help="Echo database commands as they are executed")
+@click.option("--dry-run", is_flag=True, help="Dry run only to print out commands instead of executing")
+@click.option(
+    "--output-file", "-o", type=click.File(mode="w"), help="Write SQL commands to a file instead of executing"
+)
 @click.argument("file", type=click.File())
-def create_all(engine_url: str, schema_name: str, dry_run: bool, file: io.TextIOBase) -> None:
-    """Create schema objects from the Felis FILE."""
-    schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
-    visitor = SQLVisitor(schema_name=schema_name)
-    schema = visitor.visit_schema(schema_obj)
-
-    metadata = schema.metadata
+def create(
+    engine_url: str,
+    schema_name: str | None,
+    create_if_not_exists: bool,
+    drop_if_exists: bool,
+    echo: bool,
+    dry_run: bool,
+    output_file: IO[str] | None,
+    file: IO,
+) -> None:
+    """Create database objects from the Felis file."""
+    yaml_data = yaml.safe_load(file)
+    schema = Schema.model_validate(yaml_data)
+    url_obj = make_url(engine_url)
+    if schema_name:
+        logger.info(f"Overriding schema name with: {schema_name}")
+        schema.name = schema_name
+    elif url_obj.drivername == "sqlite":
+        logger.info("Overriding schema name for sqlite with: main")
+        schema.name = "main"
+    if not url_obj.host and not url_obj.drivername == "sqlite":
+        dry_run = True
+        logger.info("Forcing dry run for non-sqlite engine URL with no host")
+
+    builder = MetaDataBuilder(schema)
+    builder.build()
+    metadata = builder.metadata
+    logger.debug(f"Created metadata with schema name: {metadata.schema}")
 
     engine: Engine | MockConnection
-    if not dry_run:
-        engine = create_engine(engine_url)
+    if not dry_run and not output_file:
+        engine = create_engine(engine_url, echo=echo)
     else:
-        _insert_dump = InsertDump()
-        engine = create_mock_engine(make_url(engine_url), executor=_insert_dump.dump)
-        _insert_dump.dialect = engine.dialect
-    metadata.create_all(engine)
+        if dry_run:
+            logger.info("Dry run will be executed")
+        engine = DatabaseContext.create_mock_engine(url_obj, output_file)
+        if output_file:
+            logger.info("Writing SQL output to: " + output_file.name)
+
+    context = DatabaseContext(metadata, engine)
+
+    if drop_if_exists:
+        logger.debug("Dropping schema if it exists")
+        context.drop_if_exists()
+        create_if_not_exists = True  # If schema is dropped, it needs to be recreated.
+
+    if create_if_not_exists:
+        logger.debug("Creating schema if not exists")
+        context.create_if_not_exists()
+
+    context.create_all()
 
 
 @cli.command("init-tap")
@@ -404,30 +447,5 @@ def _normalize(schema_obj: Mapping[str, Any], embed: str = "@last") -> MutableMa
     return compacted
 
 
-class InsertDump:
-    """An Insert Dumper for SQL statements."""
-
-    dialect: Any = None
-
-    def dump(self, sql: Any, *multiparams: Any, **params: Any) -> None:
-        compiled = sql.compile(dialect=self.dialect)
-        sql_str = str(compiled) + ";"
-        params_list = [compiled.params]
-        for params in params_list:
-            if not params:
-                print(sql_str)
-                continue
-            new_params = {}
-            for key, value in params.items():
-                if isinstance(value, str):
-                    new_params[key] = f"'{value}'"
-                elif value is None:
-                    new_params[key] = "null"
-                else:
-                    new_params[key] = value
-
-            print(sql_str % new_params)
-
-
 if __name__ == "__main__":
     cli()

{lsst-felis-26.2024.1200 → lsst-felis-26.2024.1500}/python/felis/datamodel.py

@@ -31,7 +31,6 @@ from astropy.io.votable import ucd  # type: ignore
 from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
 
 logger = logging.getLogger(__name__)
-# logger.setLevel(logging.DEBUG)
 
 __all__ = (
     "BaseObject",
@@ -50,7 +49,6 @@ __all__ = (
 CONFIG = ConfigDict(
     populate_by_name=True,  # Populate attributes by name.
     extra="forbid",  # Do not allow extra fields.
-    use_enum_values=True,  # Use enum values instead of names.
     validate_assignment=True,  # Validate assignments after model is created.
     str_strip_whitespace=True,  # Strip whitespace from string fields.
 )
@@ -130,8 +128,13 @@ class Column(BaseObject):
     length: int | None = None
     """The length of the column."""
 
-    nullable: bool = True
-    """Whether the column can be `NULL`."""
+    nullable: bool | None = None
+    """Whether the column can be ``NULL``.
+
+    If `None`, this value was not set explicitly in the YAML data. In this
+    case, it will be set to `False` for columns with numeric types and `True`
+    otherwise.
+    """
 
     value: Any = None
     """The default value of the column."""
@@ -142,6 +145,9 @@ class Column(BaseObject):
     mysql_datatype: str | None = Field(None, alias="mysql:datatype")
     """The MySQL datatype of the column."""
 
+    postgresql_datatype: str | None = Field(None, alias="postgresql:datatype")
+    """The PostgreSQL datatype of the column."""
+
     ivoa_ucd: str | None = Field(None, alias="ivoa:ucd")
     """The IVOA UCD of the column."""
 
@@ -280,7 +286,7 @@ class Table(BaseObject):
     indexes: list[Index] = Field(default_factory=list)
     """The indexes on the table."""
 
-    primaryKey: str | list[str] | None = None
+    primary_key: str | list[str] | None = Field(None, alias="primaryKey")
     """The primary key of the table."""
 
     tap_table_index: int | None = Field(None, alias="tap:table_index")
@@ -427,6 +433,9 @@ class Schema(BaseObject):
     @model_validator(mode="after")
     def create_id_map(self: Schema) -> Schema:
         """Create a map of IDs to objects."""
+        if len(self.id_map):
+            logger.debug("ID map was already populated")
+            return self
         visitor: SchemaIdVisitor = SchemaIdVisitor()
         visitor.visit_schema(self)
         logger.debug(f"ID map contains {len(self.id_map.keys())} objects")

lsst-felis-26.2024.1500/python/felis/db/_variants.py

@@ -0,0 +1,94 @@
+# This file is part of felis.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import re
+from typing import Any
+
+from sqlalchemy import types
+from sqlalchemy.dialects import mysql, oracle, postgresql, sqlite
+from sqlalchemy.types import TypeEngine
+
+from ..datamodel import Column
+
+MYSQL = "mysql"
+ORACLE = "oracle"
+POSTGRES = "postgresql"
+SQLITE = "sqlite"
+
+TABLE_OPTS = {
+    "mysql:engine": "mysql_engine",
+    "mysql:charset": "mysql_charset",
+    "oracle:compress": "oracle_compress",
+}
+
+COLUMN_VARIANT_OVERRIDE = {
+    "mysql:datatype": "mysql",
+    "oracle:datatype": "oracle",
+    "postgresql:datatype": "postgresql",
+    "sqlite:datatype": "sqlite",
+}
+
+DIALECT_MODULES = {MYSQL: mysql, ORACLE: oracle, SQLITE: sqlite, POSTGRES: postgresql}
+
+_length_regex = re.compile(r"\((\d+)\)")
+"""A regular expression that is looking for numbers within parentheses."""
+
+
+def process_variant_override(dialect_name: str, variant_override_str: str) -> types.TypeEngine:
+    """Return variant type for given dialect."""
+    dialect = DIALECT_MODULES[dialect_name]
+    variant_type_name = variant_override_str.split("(")[0]
+
+    # Process Variant Type
+    if variant_type_name not in dir(dialect):
+        raise ValueError(f"Type {variant_type_name} not found in dialect {dialect_name}")
+    variant_type = getattr(dialect, variant_type_name)
+    length_params = []
+    if match := _length_regex.search(variant_override_str):
+        length_params.extend([int(i) for i in match.group(1).split(",")])
+    return variant_type(*length_params)
+
+
+def make_variant_dict(column_obj: Column) -> dict[str, TypeEngine[Any]]:
+    """Handle variant overrides for a `felis.datamodel.Column`.
+
+    This function will return a dictionary of `str` to
+    `sqlalchemy.types.TypeEngine` containing variant datatype information
+    (e.g., for mysql, postgresql, etc).
+
+    Parameters
+    ----------
+    column_obj : `felis.datamodel.Column`
+        The column object from which to build the variant dictionary.
+
+    Returns
+    -------
+    variant_dict : `dict`
+        The dictionary of `str` to `sqlalchemy.types.TypeEngine` containing
+        variant datatype information (e.g., for mysql, postgresql, etc).
+    """
+    variant_dict = {}
+    for field_name, value in iter(column_obj):
+        if field_name in COLUMN_VARIANT_OVERRIDE:
+            dialect = COLUMN_VARIANT_OVERRIDE[field_name]
+            variant: TypeEngine = process_variant_override(dialect, value)
+            variant_dict[dialect] = variant
+    return variant_dict