lsst-felis 27.0.0rc3__py3-none-any.whl → 27.2024.1900__py3-none-any.whl

felis/__init__.py

@@ -19,29 +19,4 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-from . import types
-from .check import *
 from .version import *
-from .visitor import *
-
-DEFAULT_CONTEXT = {
-    "@vocab": "http://lsst.org/felis/",
-    "mysql": "http://mysql.com/",
-    "postgres": "http://posgresql.org/",
-    "oracle": "http://oracle.com/database/",
-    "sqlite": "http://sqlite.org/",
-    "fits": "http://fits.gsfc.nasa.gov/FITS/4.0/",
-    "ivoa": "http://ivoa.net/rdf/",
-    "votable": "http://ivoa.net/rdf/VOTable/",
-    "tap": "http://ivoa.net/documents/TAP/",
-    "tables": {"@container": "@list", "@type": "@id", "@id": "felis:Table"},
-    "columns": {"@container": "@list", "@type": "@id", "@id": "felis:Column"},
-    "constraints": {"@container": "@list", "@type": "@id"},
-    "indexes": {"@container": "@list", "@type": "@id", "@id": "felis:Index"},
-    "referencedColumns": {"@container": "@list", "@type": "@id"},
-}
-
-DEFAULT_FRAME = {
-    "@context": DEFAULT_CONTEXT,
-    "@type": "felis:Schema",
-}

felis/cli.py

@@ -22,25 +22,20 @@
 from __future__ import annotations
 
 import io
-import json
 import logging
-import sys
-from collections.abc import Iterable, Mapping, MutableMapping
-from typing import IO, Any
+from collections.abc import Iterable
+from typing import IO
 
 import click
 import yaml
 from pydantic import ValidationError
-from pyld import jsonld
 from sqlalchemy.engine import Engine, create_engine, create_mock_engine, make_url
 from sqlalchemy.engine.mock import MockConnection
 
-from . import DEFAULT_CONTEXT, DEFAULT_FRAME, __version__
-from .check import CheckingVisitor
+from . import __version__
 from .datamodel import Schema
 from .metadata import DatabaseContext, InsertDump, MetaDataBuilder
 from .tap import Tap11Base, TapLoadingVisitor, init_tables
-from .utils import ReorderingVisitor
 from .validation import get_schema
 
 logger = logging.getLogger("felis")
@@ -183,6 +178,7 @@ def init_tap(
 @click.option("--tap-columns-table", help="Alt Table Name for TAP_SCHEMA.columns")
 @click.option("--tap-keys-table", help="Alt Table Name for TAP_SCHEMA.keys")
 @click.option("--tap-key-columns-table", help="Alt Table Name for TAP_SCHEMA.key_columns")
+@click.option("--tap-schema-index", type=int, help="TAP_SCHEMA index of the schema")
 @click.argument("file", type=click.File())
 def load_tap(
     engine_url: str,
@@ -196,6 +192,7 @@ def load_tap(
     tap_columns_table: str,
     tap_keys_table: str,
     tap_key_columns_table: str,
+    tap_schema_index: int,
     file: io.TextIOBase,
 ) -> None:
     """Load TAP metadata from a Felis FILE.
@@ -203,28 +200,8 @@ def load_tap(
     This command loads the associated TAP metadata from a Felis FILE
     to the TAP_SCHEMA tables.
     """
-    top_level_object = yaml.load(file, Loader=yaml.SafeLoader)
-    schema_obj: dict
-    if isinstance(top_level_object, dict):
-        schema_obj = top_level_object
-        if "@graph" not in schema_obj:
-            schema_obj["@type"] = "felis:Schema"
-        schema_obj["@context"] = DEFAULT_CONTEXT
-    elif isinstance(top_level_object, list):
-        schema_obj = {"@context": DEFAULT_CONTEXT, "@graph": top_level_object}
-    else:
-        logger.error("Schema object not of recognizable type")
-        raise click.exceptions.Exit(1)
-
-    normalized = _normalize(schema_obj, embed="@always")
-    if len(normalized["@graph"]) > 1 and (schema_name or catalog_name):
-        logger.error("--schema-name and --catalog-name incompatible with multiple schemas")
-        raise click.exceptions.Exit(1)
-
-    # Force normalized["@graph"] to a list, which is what happens when there's
-    # multiple schemas
-    if isinstance(normalized["@graph"], dict):
-        normalized["@graph"] = [normalized["@graph"]]
+    yaml_data = yaml.load(file, Loader=yaml.SafeLoader)
+    schema = Schema.model_validate(yaml_data)
 
     tap_tables = init_tables(
         tap_schema_name,
@@ -243,126 +220,28 @@ def load_tap(
             # In Memory SQLite - Mostly used to test
             Tap11Base.metadata.create_all(engine)
 
-        for schema in normalized["@graph"]:
-            tap_visitor = TapLoadingVisitor(
-                engine,
-                catalog_name=catalog_name,
-                schema_name=schema_name,
-                tap_tables=tap_tables,
-            )
-            tap_visitor.visit_schema(schema)
+        tap_visitor = TapLoadingVisitor(
+            engine,
+            catalog_name=catalog_name,
+            schema_name=schema_name,
+            tap_tables=tap_tables,
+            tap_schema_index=tap_schema_index,
+        )
+        tap_visitor.visit_schema(schema)
     else:
         _insert_dump = InsertDump()
         conn = create_mock_engine(make_url(engine_url), executor=_insert_dump.dump, paramstyle="pyformat")
         # After the engine is created, update the executor with the dialect
         _insert_dump.dialect = conn.dialect
 
-        for schema in normalized["@graph"]:
-            tap_visitor = TapLoadingVisitor.from_mock_connection(
-                conn,
-                catalog_name=catalog_name,
-                schema_name=schema_name,
-                tap_tables=tap_tables,
-            )
-            tap_visitor.visit_schema(schema)
-
-
-@cli.command("modify-tap")
-@click.option("--start-schema-at", type=int, help="Rewrite index for tap:schema_index", default=0)
-@click.argument("files", nargs=-1, type=click.File())
-def modify_tap(start_schema_at: int, files: Iterable[io.TextIOBase]) -> None:
-    """Modify TAP information in Felis schema FILES.
-
-    This command has some utilities to aid in rewriting felis FILES
-    in specific ways. It will write out a merged version of these files.
-    """
-    count = 0
-    graph = []
-    for file in files:
-        schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
-        if "@graph" not in schema_obj:
-            schema_obj["@type"] = "felis:Schema"
-        schema_obj["@context"] = DEFAULT_CONTEXT
-        schema_index = schema_obj.get("tap:schema_index")
-        if not schema_index or (schema_index and schema_index > start_schema_at):
-            schema_index = start_schema_at + count
-            count += 1
-        schema_obj["tap:schema_index"] = schema_index
-        graph.extend(jsonld.flatten(schema_obj))
-    merged = {"@context": DEFAULT_CONTEXT, "@graph": graph}
-    normalized = _normalize(merged, embed="@always")
-    _dump(normalized)
-
-
-@cli.command("basic-check")
-@click.argument("file", type=click.File())
-def basic_check(file: io.TextIOBase) -> None:
-    """Perform a basic check on a felis FILE.
-
-    This performs a very check to ensure required fields are
-    populated and basic semantics are okay. It does not ensure semantics
-    are valid for other commands like create-all or load-tap.
-    """
-    schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
-    schema_obj["@type"] = "felis:Schema"
-    # Force Context and Schema Type
-    schema_obj["@context"] = DEFAULT_CONTEXT
-    check_visitor = CheckingVisitor()
-    check_visitor.visit_schema(schema_obj)
-
-
-@cli.command("normalize")
-@click.argument("file", type=click.File())
-def normalize(file: io.TextIOBase) -> None:
-    """Normalize a Felis FILE.
-
-    Takes a felis schema FILE, expands it (resolving the full URLs),
-    then compacts it, and finally produces output in the canonical
-    format.
-
-    (This is most useful in some debugging scenarios)
-
-    See Also :
-
-        https://json-ld.org/spec/latest/json-ld/#expanded-document-form
-        https://json-ld.org/spec/latest/json-ld/#compacted-document-form
-    """
-    schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
-    schema_obj["@type"] = "felis:Schema"
-    # Force Context and Schema Type
-    schema_obj["@context"] = DEFAULT_CONTEXT
-    expanded = jsonld.expand(schema_obj)
-    normalized = _normalize(expanded, embed="@always")
-    _dump(normalized)
-
-
-@cli.command("merge")
-@click.argument("files", nargs=-1, type=click.File())
-def merge(files: Iterable[io.TextIOBase]) -> None:
-    """Merge a set of Felis FILES.
-
-    This will expand out the felis FILES so that it is easy to
-    override values (using @Id), then normalize to a single
-    output.
-    """
-    graph = []
-    for file in files:
-        schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
-        if "@graph" not in schema_obj:
-            schema_obj["@type"] = "felis:Schema"
-        schema_obj["@context"] = DEFAULT_CONTEXT
-        graph.extend(jsonld.flatten(schema_obj))
-    updated_map: MutableMapping[str, Any] = {}
-    for item in graph:
-        _id = item["@id"]
-        item_to_update = updated_map.get(_id, item)
-        if item_to_update and item_to_update != item:
-            logger.debug(f"Overwriting {_id}")
-        item_to_update.update(item)
-        updated_map[_id] = item_to_update
-    merged = {"@context": DEFAULT_CONTEXT, "@graph": list(updated_map.values())}
-    normalized = _normalize(merged, embed="@always")
-    _dump(normalized)
+        tap_visitor = TapLoadingVisitor.from_mock_connection(
+            conn,
+            catalog_name=catalog_name,
+            schema_name=schema_name,
+            tap_tables=tap_tables,
+            tap_schema_index=tap_schema_index,
+        )
+        tap_visitor.visit_schema(schema)
 
 
 @cli.command("validate")
@@ -411,56 +290,5 @@ def validate(
         raise click.exceptions.Exit(rc)
 
 
-@cli.command("dump-json")
-@click.option("-x", "--expanded", is_flag=True, help="Extended schema before dumping.")
-@click.option("-f", "--framed", is_flag=True, help="Frame schema before dumping.")
-@click.option("-c", "--compacted", is_flag=True, help="Compact schema before dumping.")
-@click.option("-g", "--graph", is_flag=True, help="Pass graph option to compact.")
-@click.argument("file", type=click.File())
-def dump_json(
-    file: io.TextIOBase,
-    expanded: bool = False,
-    compacted: bool = False,
-    framed: bool = False,
-    graph: bool = False,
-) -> None:
-    """Dump JSON representation using various JSON-LD options."""
-    schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
-    schema_obj["@type"] = "felis:Schema"
-    # Force Context and Schema Type
-    schema_obj["@context"] = DEFAULT_CONTEXT
-
-    if expanded:
-        schema_obj = jsonld.expand(schema_obj)
-    if framed:
-        schema_obj = jsonld.frame(schema_obj, DEFAULT_FRAME)
-    if compacted:
-        options = {}
-        if graph:
-            options["graph"] = True
-        schema_obj = jsonld.compact(schema_obj, DEFAULT_CONTEXT, options=options)
-    json.dump(schema_obj, sys.stdout, indent=4)
-
-
-def _dump(obj: Mapping[str, Any]) -> None:
-    class OrderedDumper(yaml.Dumper):
-        pass
-
-    def _dict_representer(dumper: yaml.Dumper, data: Any) -> Any:
-        return dumper.represent_mapping(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, data.items())
-
-    OrderedDumper.add_representer(dict, _dict_representer)
-    print(yaml.dump(obj, Dumper=OrderedDumper, default_flow_style=False))
-
-
-def _normalize(schema_obj: Mapping[str, Any], embed: str = "@last") -> MutableMapping[str, Any]:
-    framed = jsonld.frame(schema_obj, DEFAULT_FRAME, options=dict(embed=embed))
-    compacted = jsonld.compact(framed, DEFAULT_CONTEXT, options=dict(graph=True))
-    graph = compacted["@graph"]
-    graph = [ReorderingVisitor(add_type=True).visit_schema(schema_obj) for schema_obj in graph]
-    compacted["@graph"] = graph if len(graph) > 1 else graph[0]
-    return compacted
-
-
 if __name__ == "__main__":
     cli()

felis/datamodel.py

@@ -37,7 +37,7 @@ from sqlalchemy.engine.interfaces import Dialect
 from sqlalchemy.types import TypeEngine
 
 from .db.sqltypes import get_type_func
-from .types import FelisType
+from .types import Boolean, Byte, Char, Double, FelisType, Float, Int, Long, Short, String, Text, Unicode
 
 logger = logging.getLogger(__name__)
 
@@ -93,18 +93,20 @@ class BaseObject(BaseModel):
     description: DescriptionStr | None = None
     """A description of the database object."""
 
-    @model_validator(mode="after")  # type: ignore[arg-type]
-    @classmethod
-    def check_description(cls, object: BaseObject, info: ValidationInfo) -> BaseObject:
+    votable_utype: str | None = Field(None, alias="votable:utype")
+    """The VOTable utype (usage-specific or unique type) of the object."""
+
+    @model_validator(mode="after")
+    def check_description(self, info: ValidationInfo) -> BaseObject:
         """Check that the description is present if required."""
         context = info.context
         if not context or not context.get("require_description", False):
-            return object
-        if object.description is None or object.description == "":
+            return self
+        if self.description is None or self.description == "":
             raise ValueError("Description is required and must be non-empty")
-        if len(object.description) < DESCR_MIN_LENGTH:
+        if len(self.description) < DESCR_MIN_LENGTH:
             raise ValueError(f"Description must be at least {DESCR_MIN_LENGTH} characters long")
-        return object
+        return self
 
 
 class DataType(StrEnum):
@@ -176,18 +178,13 @@ class Column(BaseObject):
     datatype: DataType
     """The datatype of the column."""
 
-    length: int | None = None
+    length: int | None = Field(None, gt=0)
     """The length of the column."""
 
-    nullable: bool | None = None
-    """Whether the column can be ``NULL``.
-
-    If `None`, this value was not set explicitly in the YAML data. In this
-    case, it will be set to `False` for columns with numeric types and `True`
-    otherwise.
-    """
+    nullable: bool = True
+    """Whether the column can be ``NULL``."""
 
-    value: Any = None
+    value: str | int | float | bool | None = None
     """The default value of the column."""
 
     autoincrement: bool | None = None
@@ -222,12 +219,33 @@ class Column(BaseObject):
     """TAP_SCHEMA indication that this column is defined by an IVOA standard.
     """
 
-    votable_utype: str | None = Field(None, alias="votable:utype")
-    """The VOTable utype (usage-specific or unique type) of the column."""
-
     votable_xtype: str | None = Field(None, alias="votable:xtype")
     """The VOTable xtype (extended type) of the column."""
 
+    votable_datatype: str | None = Field(None, alias="votable:datatype")
+    """The VOTable datatype of the column."""
+
+    @model_validator(mode="after")
+    def check_value(self) -> Column:
+        """Check that the default value is valid."""
+        if (value := self.value) is not None:
+            if value is not None and self.autoincrement is True:
+                raise ValueError("Column cannot have both a default value and be autoincremented")
+            felis_type = FelisType.felis_type(self.datatype)
+            if felis_type.is_numeric:
+                if felis_type in (Byte, Short, Int, Long) and not isinstance(value, int):
+                    raise ValueError("Default value must be an int for integer type columns")
+                elif felis_type in (Float, Double) and not isinstance(value, float):
+                    raise ValueError("Default value must be a decimal number for float and double columns")
+            elif felis_type in (String, Char, Unicode, Text):
+                if not isinstance(value, str):
+                    raise ValueError("Default value must be a string for string columns")
+                if not len(value):
+                    raise ValueError("Default value must be a non-empty string for string columns")
+            elif felis_type is Boolean and not isinstance(value, bool):
+                raise ValueError("Default value must be a boolean for boolean columns")
+        return self
+
     @field_validator("ivoa_ucd")
     @classmethod
     def check_ivoa_ucd(cls, ivoa_ucd: str) -> str:
@@ -258,52 +276,69 @@ class Column(BaseObject):
 
         return values
 
-    @model_validator(mode="after")  # type: ignore[arg-type]
+    @model_validator(mode="before")
     @classmethod
-    def validate_datatypes(cls, col: Column, info: ValidationInfo) -> Column:
+    def check_length(cls, values: dict[str, Any]) -> dict[str, Any]:
+        """Check that a valid length is provided for sized types."""
+        datatype = values.get("datatype")
+        if datatype is None:
+            # Skip this validation if datatype is not provided
+            return values
+        length = values.get("length")
+        felis_type = FelisType.felis_type(datatype)
+        if felis_type.is_sized and length is None:
+            raise ValueError(
+                f"Length must be provided for type '{datatype}'"
+                + (f" in column '{values['@id']}'" if "@id" in values else "")
+            )
+        elif not felis_type.is_sized and length is not None:
+            logger.warning(
+                f"The datatype '{datatype}' does not support a specified length"
+                + (f" in column '{values['@id']}'" if "@id" in values else "")
+            )
+        return values
+
+    @model_validator(mode="after")
+    def check_datatypes(self, info: ValidationInfo) -> Column:
         """Check for redundant datatypes on columns."""
         context = info.context
         if not context or not context.get("check_redundant_datatypes", False):
-            return col
-        if all(getattr(col, f"{dialect}:datatype", None) is not None for dialect in _DIALECTS.keys()):
-            return col
+            return self
+        if all(getattr(self, f"{dialect}:datatype", None) is not None for dialect in _DIALECTS.keys()):
+            return self
 
-        datatype = col.datatype
-        length: int | None = col.length or None
+        datatype = self.datatype
+        length: int | None = self.length or None
 
         datatype_func = get_type_func(datatype)
         felis_type = FelisType.felis_type(datatype)
         if felis_type.is_sized:
-            if length is not None:
-                datatype_obj = datatype_func(length)
-            else:
-                raise ValueError(f"Length must be provided for sized type '{datatype}' in column '{col.id}'")
+            datatype_obj = datatype_func(length)
         else:
             datatype_obj = datatype_func()
 
         for dialect_name, dialect in _DIALECTS.items():
             db_annotation = f"{dialect_name}_datatype"
-            if datatype_string := col.model_dump().get(db_annotation):
+            if datatype_string := self.model_dump().get(db_annotation):
                 db_datatype_obj = string_to_typeengine(datatype_string, dialect, length)
                 if datatype_obj.compile(dialect) == db_datatype_obj.compile(dialect):
                     raise ValueError(
-                        "'{}: {}' is the same as 'datatype: {}' in column '{}'".format(
-                            db_annotation, datatype_string, col.datatype, col.id
+                        "'{}: {}' is a redundant override of 'datatype: {}' in column '{}'{}".format(
+                            db_annotation,
+                            datatype_string,
+                            self.datatype,
+                            self.id,
+                            "" if length is None else f" with length {length}",
                         )
                     )
                 else:
                     logger.debug(
-                        "Valid type override of 'datatype: {}' with '{}: {}' in column '{}'".format(
-                            col.datatype, db_annotation, datatype_string, col.id
-                        )
-                    )
-                    logger.debug(
-                        "Compiled datatype '{}' with {} compiled override '{}'".format(
-                            datatype_obj.compile(dialect), dialect_name, db_datatype_obj.compile(dialect)
-                        )
+                        f"Type override of 'datatype: {self.datatype}' "
+                        f"with '{db_annotation}: {datatype_string}' in column '{self.id}' "
+                        f"compiled to '{datatype_obj.compile(dialect)}' and "
+                        f"'{db_datatype_obj.compile(dialect)}'"
                     )
-
-        return col
+        return self
 
 
 class Constraint(BaseObject):
@@ -458,7 +493,7 @@ class SchemaIdVisitor:
 
     def __init__(self) -> None:
         """Create a new SchemaVisitor."""
-        self.schema: "Schema" | None = None
+        self.schema: Schema | None = None
         self.duplicates: set[str] = set()
 
     def add(self, obj: BaseObject) -> None:
@@ -471,7 +506,7 @@ class SchemaIdVisitor:
                 else:
                     self.schema.id_map[obj_id] = obj
 
-    def visit_schema(self, schema: "Schema") -> None:
+    def visit_schema(self, schema: Schema) -> None:
         """Visit the schema object that was added during initialization.
 
         This will set an internal variable pointing to the schema object.

felis/db/sqltypes.py

@@ -20,8 +20,8 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 import builtins
-from collections.abc import Mapping
-from typing import Any, Callable
+from collections.abc import Callable, Mapping
+from typing import Any
 
 from sqlalchemy import SmallInteger, types
 from sqlalchemy.dialects import mysql, oracle, postgresql
@@ -47,7 +47,7 @@ def compile_tinyint(type_: Any, compiler: Any, **kw: Any) -> str:
 
 _TypeMap = Mapping[str, types.TypeEngine | type[types.TypeEngine]]
 
-boolean_map: _TypeMap = {MYSQL: mysql.BIT(1), ORACLE: oracle.NUMBER(1), POSTGRES: postgresql.BOOLEAN()}
+boolean_map: _TypeMap = {MYSQL: mysql.BOOLEAN, ORACLE: oracle.NUMBER(1), POSTGRES: postgresql.BOOLEAN()}
 
 byte_map: _TypeMap = {
     MYSQL: mysql.TINYINT(),

felis/metadata.py

@@ -34,10 +34,10 @@ from sqlalchemy import (
     ForeignKeyConstraint,
     Index,
     MetaData,
-    Numeric,
     PrimaryKeyConstraint,
     ResultProxy,
     Table,
+    TextClause,
     UniqueConstraint,
     create_mock_engine,
     make_url,
@@ -135,6 +135,9 @@ def get_datatype_with_variants(column_obj: datamodel.Column) -> TypeEngine:
     return datatype
 
 
+_VALID_SERVER_DEFAULTS = ("CURRENT_TIMESTAMP", "NOW()", "LOCALTIMESTAMP", "NULL")
+
+
 class MetaDataBuilder:
     """A class for building a `MetaData` object from a Felis `Schema`."""
 
@@ -264,29 +267,35 @@ class MetaDataBuilder:
         name = column_obj.name
         id = column_obj.id
         description = column_obj.description
-        default = column_obj.value
+        value = column_obj.value
+        nullable = column_obj.nullable
 
-        # Handle variant overrides for the column (e.g., "mysql:datatype").
+        # Get datatype, handling variant overrides such as "mysql:datatype".
         datatype = get_datatype_with_variants(column_obj)
 
-        # Set default value of nullable based on column type and then whether
-        # it was explicitly provided in the schema data.
-        nullable = column_obj.nullable
-        if nullable is None:
-            nullable = False if isinstance(datatype, Numeric) else True
-
-        # Set autoincrement depending on if it was provided explicitly.
+        # Set autoincrement, depending on if it was provided explicitly.
         autoincrement: Literal["auto"] | bool = (
             column_obj.autoincrement if column_obj.autoincrement is not None else "auto"
         )
 
+        server_default: str | TextClause | None = None
+        if value is not None:
+            server_default = str(value)
+            if server_default in _VALID_SERVER_DEFAULTS or not isinstance(value, str):
+                # If the server default is a valid keyword or not a string,
+                # use it as is.
+                server_default = text(server_default)
+
+        if server_default is not None:
+            logger.debug(f"Column '{id}' has default value: {server_default}")
+
         column: Column = Column(
             name,
             datatype,
             comment=description,
             autoincrement=autoincrement,
             nullable=nullable,
-            server_default=default,
+            server_default=server_default,
         )
 
         self._objects[id] = column
@@ -475,7 +484,7 @@ class DatabaseContext:
                 self.connection.execute(text(f"DROP DATABASE IF EXISTS {schema_name}"))
             elif db_type == "postgresql":
                 logger.info(f"Dropping PostgreSQL schema if exists: {schema_name}")
-                self.connection.execute(sqa_schema.DropSchema(schema_name, if_exists=True))
+                self.connection.execute(sqa_schema.DropSchema(schema_name, if_exists=True, cascade=True))
             else:
                 raise ValueError(f"Unsupported database type: {db_type}")
         except SQLAlchemyError as e: