lsst-felis 26.2024.1400__py3-none-any.whl → 26.2024.1600__py3-none-any.whl

felis/cli.py

@@ -373,22 +373,37 @@ def merge(files: Iterable[io.TextIOBase]) -> None:
     type=click.Choice(["RSP", "default"]),
     default="default",
 )
-@click.option("-d", "--require-description", is_flag=True, help="Require description for all objects")
+@click.option(
+    "-d", "--require-description", is_flag=True, help="Require description for all objects", default=False
+)
+@click.option(
+    "-t", "--check-redundant-datatypes", is_flag=True, help="Check for redundant datatypes", default=False
+)
 @click.argument("files", nargs=-1, type=click.File())
-def validate(schema_name: str, require_description: bool, files: Iterable[io.TextIOBase]) -> None:
+def validate(
+    schema_name: str,
+    require_description: bool,
+    check_redundant_datatypes: bool,
+    files: Iterable[io.TextIOBase],
+) -> None:
     """Validate one or more felis YAML files."""
     schema_class = get_schema(schema_name)
-    logger.info(f"Using schema '{schema_class.__name__}'")
-
-    if require_description:
-        Schema.require_description(True)
+    if schema_name != "default":
+        logger.info(f"Using schema '{schema_class.__name__}'")
 
     rc = 0
     for file in files:
         file_name = getattr(file, "name", None)
         logger.info(f"Validating {file_name}")
         try:
-            schema_class.model_validate(yaml.load(file, Loader=yaml.SafeLoader))
+            data = yaml.load(file, Loader=yaml.SafeLoader)
+            schema_class.model_validate(
+                data,
+                context={
+                    "check_redundant_datatypes": check_redundant_datatypes,
+                    "require_description": require_description,
+                },
+            )
         except ValidationError as e:
             logger.error(e)
             rc = 1

felis/datamodel.py

@@ -22,13 +22,22 @@
 from __future__ import annotations
 
 import logging
+import re
 from collections.abc import Mapping, Sequence
-from enum import Enum
+from enum import StrEnum, auto
 from typing import Annotated, Any, Literal, TypeAlias
 
 from astropy import units as units  # type: ignore
 from astropy.io.votable import ucd  # type: ignore
-from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validator, model_validator
+from sqlalchemy import dialects
+from sqlalchemy import types as sqa_types
+from sqlalchemy.engine import create_mock_engine
+from sqlalchemy.engine.interfaces import Dialect
+from sqlalchemy.types import TypeEngine
+
+from .db.sqltypes import get_type_func
+from .types import FelisType
 
 logger = logging.getLogger(__name__)
 
@@ -49,7 +58,6 @@ __all__ = (
 CONFIG = ConfigDict(
     populate_by_name=True,  # Populate attributes by name.
     extra="forbid",  # Do not allow extra fields.
-    validate_assignment=True,  # Validate assignments after model is created.
     str_strip_whitespace=True,  # Strip whitespace from string fields.
 )
 """Pydantic model configuration as described in:
@@ -83,40 +91,83 @@ class BaseObject(BaseModel):
     """
 
     description: DescriptionStr | None = None
-    """A description of the database object.
-
-    By default, the description is optional but will be required if
-    `BaseObject.Config.require_description` is set to `True` by the user.
-    """
+    """A description of the database object."""
 
-    @model_validator(mode="before")
+    @model_validator(mode="after")  # type: ignore[arg-type]
     @classmethod
-    def check_description(cls, values: dict[str, Any]) -> dict[str, Any]:
+    def check_description(cls, object: BaseObject, info: ValidationInfo) -> BaseObject:
         """Check that the description is present if required."""
-        if Schema.is_description_required():
-            if "description" not in values or not values["description"]:
-                raise ValueError("Description is required and must be non-empty")
-            if len(values["description"].strip()) < DESCR_MIN_LENGTH:
-                raise ValueError(f"Description must be at least {DESCR_MIN_LENGTH} characters long")
-        return values
+        context = info.context
+        if not context or not context.get("require_description", False):
+            return object
+        if object.description is None or object.description == "":
+            raise ValueError("Description is required and must be non-empty")
+        if len(object.description) < DESCR_MIN_LENGTH:
+            raise ValueError(f"Description must be at least {DESCR_MIN_LENGTH} characters long")
+        return object
 
 
-class DataType(Enum):
+class DataType(StrEnum):
     """`Enum` representing the data types supported by Felis."""
 
-    BOOLEAN = "boolean"
-    BYTE = "byte"
-    SHORT = "short"
-    INT = "int"
-    LONG = "long"
-    FLOAT = "float"
-    DOUBLE = "double"
-    CHAR = "char"
-    STRING = "string"
-    UNICODE = "unicode"
-    TEXT = "text"
-    BINARY = "binary"
-    TIMESTAMP = "timestamp"
+    boolean = auto()
+    byte = auto()
+    short = auto()
+    int = auto()
+    long = auto()
+    float = auto()
+    double = auto()
+    char = auto()
+    string = auto()
+    unicode = auto()
+    text = auto()
+    binary = auto()
+    timestamp = auto()
+
+
+_DIALECTS = {
+    "mysql": create_mock_engine("mysql://", executor=None).dialect,
+    "postgresql": create_mock_engine("postgresql://", executor=None).dialect,
+}
+"""Dictionary of dialect names to SQLAlchemy dialects."""
+
+_DIALECT_MODULES = {"mysql": getattr(dialects, "mysql"), "postgresql": getattr(dialects, "postgresql")}
+"""Dictionary of dialect names to SQLAlchemy dialect modules."""
+
+_DATATYPE_REGEXP = re.compile(r"(\w+)(\((.*)\))?")
+"""Regular expression to match data types in the form "type(length)"""
+
+
+def string_to_typeengine(
+    type_string: str, dialect: Dialect | None = None, length: int | None = None
+) -> TypeEngine:
+    match = _DATATYPE_REGEXP.search(type_string)
+    if not match:
+        raise ValueError(f"Invalid type string: {type_string}")
+
+    type_name, _, params = match.groups()
+    if dialect is None:
+        type_class = getattr(sqa_types, type_name.upper(), None)
+    else:
+        try:
+            dialect_module = _DIALECT_MODULES[dialect.name]
+        except KeyError:
+            raise ValueError(f"Unsupported dialect: {dialect}")
+        type_class = getattr(dialect_module, type_name.upper(), None)
+
+    if not type_class:
+        raise ValueError(f"Unsupported type: {type_class}")
+
+    if params:
+        params = [int(param) if param.isdigit() else param for param in params.split(",")]
+        type_obj = type_class(*params)
+    else:
+        type_obj = type_class()
+
+    if hasattr(type_obj, "length") and getattr(type_obj, "length") is None and length is not None:
+        type_obj.length = length
+
+    return type_obj
 
 
 class Column(BaseObject):
@@ -207,6 +258,53 @@ class Column(BaseObject):
 
         return values
 
+    @model_validator(mode="after")  # type: ignore[arg-type]
+    @classmethod
+    def validate_datatypes(cls, col: Column, info: ValidationInfo) -> Column:
+        """Check for redundant datatypes on columns."""
+        context = info.context
+        if not context or not context.get("check_redundant_datatypes", False):
+            return col
+        if all(getattr(col, f"{dialect}:datatype", None) is not None for dialect in _DIALECTS.keys()):
+            return col
+
+        datatype = col.datatype
+        length: int | None = col.length or None
+
+        datatype_func = get_type_func(datatype)
+        felis_type = FelisType.felis_type(datatype)
+        if felis_type.is_sized:
+            if length is not None:
+                datatype_obj = datatype_func(length)
+            else:
+                raise ValueError(f"Length must be provided for sized type '{datatype}' in column '{col.id}'")
+        else:
+            datatype_obj = datatype_func()
+
+        for dialect_name, dialect in _DIALECTS.items():
+            db_annotation = f"{dialect_name}_datatype"
+            if datatype_string := col.model_dump().get(db_annotation):
+                db_datatype_obj = string_to_typeengine(datatype_string, dialect, length)
+                if datatype_obj.compile(dialect) == db_datatype_obj.compile(dialect):
+                    raise ValueError(
+                        "'{}: {}' is the same as 'datatype: {}' in column '{}'".format(
+                            db_annotation, datatype_string, col.datatype, col.id
+                        )
+                    )
+                else:
+                    logger.debug(
+                        "Valid type override of 'datatype: {}' with '{}: {}' in column '{}'".format(
+                            col.datatype, db_annotation, datatype_string, col.id
+                        )
+                    )
+                    logger.debug(
+                        "Compiled datatype '{}' with {} compiled override '{}'".format(
+                            datatype_obj.compile(dialect), dialect_name, db_datatype_obj.compile(dialect)
+                        )
+                    )
+
+        return col
+
 
 class Constraint(BaseObject):
     """A database table constraint."""
@@ -404,15 +502,6 @@ class SchemaIdVisitor:
 class Schema(BaseObject):
     """The database schema containing the tables."""
 
-    class ValidationConfig:
-        """Validation configuration which is specific to Felis."""
-
-        _require_description = False
-        """Flag to require a description for all objects.
-
-        This is set by the `require_description` class method.
-        """
-
     version: SchemaVersion | str | None = None
     """The version of the schema."""
 
@@ -430,21 +519,29 @@ class Schema(BaseObject):
             raise ValueError("Table names must be unique")
         return tables
 
-    @model_validator(mode="after")
-    def create_id_map(self: Schema) -> Schema:
-        """Create a map of IDs to objects."""
+    def _create_id_map(self: Schema) -> Schema:
+        """Create a map of IDs to objects.
+
+        This method should not be called by users. It is called automatically
+        by the ``model_post_init()`` method. If the ID map is already
+        populated, this method will return immediately.
+        """
         if len(self.id_map):
-            logger.debug("ID map was already populated")
+            logger.debug("Ignoring call to create_id_map() - ID map was already populated")
             return self
         visitor: SchemaIdVisitor = SchemaIdVisitor()
         visitor.visit_schema(self)
-        logger.debug(f"ID map contains {len(self.id_map.keys())} objects")
+        logger.debug(f"Created schema ID map with {len(self.id_map.keys())} objects")
         if len(visitor.duplicates):
             raise ValueError(
                 "Duplicate IDs found in schema:\n    " + "\n    ".join(visitor.duplicates) + "\n"
             )
         return self
 
+    def model_post_init(self, ctx: Any) -> None:
+        """Post-initialization hook for the model."""
+        self._create_id_map()
+
     def __getitem__(self, id: str) -> BaseObject:
         """Get an object by its ID."""
         if id not in self:
@@ -454,20 +551,3 @@ class Schema(BaseObject):
     def __contains__(self, id: str) -> bool:
         """Check if an object with the given ID is in the schema."""
         return id in self.id_map
-
-    @classmethod
-    def require_description(cls, rd: bool = True) -> None:
-        """Set whether a description is required for all objects.
-
-        This includes the schema, tables, columns, and constraints.
-
-        Users should call this method to set the requirement for a description
-        when validating schemas, rather than change the flag value directly.
-        """
-        logger.debug(f"Setting description requirement to '{rd}'")
-        cls.ValidationConfig._require_description = rd
-
-    @classmethod
-    def is_description_required(cls) -> bool:
-        """Return whether a description is required for all objects."""
-        return cls.ValidationConfig._require_description

felis/db/_variants.py

@@ -40,10 +40,10 @@ TABLE_OPTS = {
 }
 
 COLUMN_VARIANT_OVERRIDE = {
-    "mysql:datatype": "mysql",
-    "oracle:datatype": "oracle",
-    "postgresql:datatype": "postgresql",
-    "sqlite:datatype": "sqlite",
+    "mysql_datatype": "mysql",
+    "oracle_datatype": "oracle",
+    "postgresql_datatype": "postgresql",
+    "sqlite_datatype": "sqlite",
 }
 
 DIALECT_MODULES = {MYSQL: mysql, ORACLE: oracle, SQLITE: sqlite, POSTGRES: postgresql}
@@ -87,7 +87,7 @@ def make_variant_dict(column_obj: Column) -> dict[str, TypeEngine[Any]]:
     """
     variant_dict = {}
     for field_name, value in iter(column_obj):
-        if field_name in COLUMN_VARIANT_OVERRIDE:
+        if field_name in COLUMN_VARIANT_OVERRIDE and value is not None:
             dialect = COLUMN_VARIANT_OVERRIDE[field_name]
             variant: TypeEngine = process_variant_override(dialect, value)
             variant_dict[dialect] = variant

felis/db/sqltypes.py

@@ -21,9 +21,9 @@
 
 import builtins
 from collections.abc import Mapping
-from typing import Any
+from typing import Any, Callable
 
-from sqlalchemy import Float, SmallInteger, types
+from sqlalchemy import SmallInteger, types
 from sqlalchemy.dialects import mysql, oracle, postgresql
 from sqlalchemy.ext.compiler import compiles
 
@@ -39,24 +39,12 @@ class TINYINT(SmallInteger):
     __visit_name__ = "TINYINT"
 
 
-class DOUBLE(Float):
-    """The non-standard DOUBLE type."""
-
-    __visit_name__ = "DOUBLE"
-
-
 @compiles(TINYINT)
 def compile_tinyint(type_: Any, compiler: Any, **kw: Any) -> str:
     """Return type name for TINYINT."""
     return "TINYINT"
 
 
-@compiles(DOUBLE)
-def compile_double(type_: Any, compiler: Any, **kw: Any) -> str:
-    """Return type name for double precision type."""
-    return "DOUBLE"
-
-
 _TypeMap = Mapping[str, types.TypeEngine | type[types.TypeEngine]]
 
 boolean_map: _TypeMap = {MYSQL: mysql.BIT(1), ORACLE: oracle.NUMBER(1), POSTGRES: postgresql.BOOLEAN()}
@@ -160,7 +148,7 @@ def float(**kwargs: Any) -> types.TypeEngine:
 
 def double(**kwargs: Any) -> types.TypeEngine:
     """Return SQLAlchemy type for double precision float."""
-    return _vary(DOUBLE(), double_map, kwargs)
+    return _vary(types.DOUBLE(), double_map, kwargs)
 
 
 def char(length: builtins.int, **kwargs: Any) -> types.TypeEngine:
@@ -178,9 +166,9 @@ def unicode(length: builtins.int, **kwargs: Any) -> types.TypeEngine:
     return _vary(types.NVARCHAR(length), unicode_map, kwargs, length)
 
 
-def text(length: builtins.int, **kwargs: Any) -> types.TypeEngine:
+def text(**kwargs: Any) -> types.TypeEngine:
     """Return SQLAlchemy type for text."""
-    return _vary(types.CLOB(length), text_map, kwargs, length)
+    return _vary(types.TEXT(), text_map, kwargs)
 
 
 def binary(length: builtins.int, **kwargs: Any) -> types.TypeEngine:
@@ -193,6 +181,13 @@ def timestamp(**kwargs: Any) -> types.TypeEngine:
     return types.TIMESTAMP()
 
 
+def get_type_func(type_name: str) -> Callable:
+    """Return the function for the type with the given name."""
+    if type_name not in globals():
+        raise ValueError(f"Unknown type: {type_name}")
+    return globals()[type_name]
+
+
 def _vary(
     type_: types.TypeEngine,
     variant_map: _TypeMap,
@@ -203,7 +198,7 @@ def _vary(
     variants.update(overrides)
     for dialect, variant in variants.items():
         # If this is a class and not an instance, instantiate
-        if isinstance(variant, type):
+        if callable(variant):
             variant = variant(*args)
         type_ = type_.with_variant(variant, dialect)
     return type_

felis/types.py

@@ -125,7 +125,7 @@ class Unicode(FelisType, felis_name="unicode", votable_name="unicodeChar", is_si
     """Felis definition of unicode string type."""
 
 
-class Text(FelisType, felis_name="text", votable_name="unicodeChar", is_sized=True):
+class Text(FelisType, felis_name="text", votable_name="char"):
     """Felis definition of text type."""
 
 

felis/version.py

@@ -1,2 +1,2 @@
 __all__ = ["__version__"]
-__version__ = "26.2024.1400"
+__version__ = "26.2024.1600"

{lsst_felis-26.2024.1400.dist-info → lsst_felis-26.2024.1600.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lsst-felis
-Version: 26.2024.1400
+Version: 26.2024.1600
 Summary: A vocabulary for describing catalogs and acting on those descriptions
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: GNU General Public License v3 or later (GPLv3+)

lsst_felis-26.2024.1600.dist-info/RECORD

@@ -0,0 +1,24 @@
+felis/__init__.py,sha256=_Pw-QKMYj0WRgE8fW2N2pBXJUj-Pjv8dSKJBzykjyZU,1842
+felis/check.py,sha256=RBxXq7XwPGIucrs1PPgPtgk8MrWAJlOmoxCNySEz9-I,13892
+felis/cli.py,sha256=IG_G41LRcEa7Y_6iFQ4i44LmyARhiqiv_0gFvLeJzpA,17041
+felis/datamodel.py,sha256=Wv7SAOTahIMWWpYFT_VxBD4WDRYlcm83SGAAy30Ly0c,19293
+felis/metadata.py,sha256=5DE2YMnu6YuhwntBSe-OheCD7C2-vA4yb64BpjTC68A,18542
+felis/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+felis/simple.py,sha256=yzv_aoZrZhfakd1Xm7gLDeVKyJjCDZ7wAyYYp-l_Sxs,14414
+felis/tap.py,sha256=RBwEKyU3S0oXSNIMoI2WRAuC9WB5eai9BdQQUYN5Qdc,17704
+felis/types.py,sha256=z_ECfSxpqiFSGppjxKwCO4fPP7TLBaIN3Qo1AGF16Go,4418
+felis/utils.py,sha256=tYxr0xFdPN4gDHibeAD9d5DFgU8hKlSZVKmZoDzi4e8,4164
+felis/validation.py,sha256=f9VKvp7q-cnim2D5voTKwCdt0NRsYBpTwom1Z_3OKkc,3469
+felis/version.py,sha256=H15ZOvw6LpBJgcJl5X5QV2o57W94MWuGRM3CPP_z0Xk,55
+felis/visitor.py,sha256=EazU4nYbkKBj3mCZYvsTCBTNmh0qRaUNZIzCcM3dqOQ,6439
+felis/db/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+felis/db/_variants.py,sha256=zCuXDgU_x_pTZcWkBLgqQCiOhlA6y2tBt-PUQfafwmM,3368
+felis/db/sqltypes.py,sha256=-YPkU5yS7udcPZTg__G_85tH6oMqVm6hEcHUgYpgiT8,5734
+lsst_felis-26.2024.1600.dist-info/COPYRIGHT,sha256=bUmNy19uUxqITMpjeHFe69q3IzQpjxvvBw6oV7kR7ho,129
+lsst_felis-26.2024.1600.dist-info/LICENSE,sha256=jOtLnuWt7d5Hsx6XXB2QxzrSe2sWWh3NgMfFRetluQM,35147
+lsst_felis-26.2024.1600.dist-info/METADATA,sha256=7F-m_tITJUAB08GAsCdFA2b7kmga_AZqtk8-B0uTQ3g,1101
+lsst_felis-26.2024.1600.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+lsst_felis-26.2024.1600.dist-info/entry_points.txt,sha256=Gk2XFujA_Gp52VBk45g5kim8TDoMDJFPctsMqiq72EM,40
+lsst_felis-26.2024.1600.dist-info/top_level.txt,sha256=F4SvPip3iZRVyISi50CHhwTIAokAhSxjWiVcn4IVWRI,6
+lsst_felis-26.2024.1600.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+lsst_felis-26.2024.1600.dist-info/RECORD,,

lsst_felis-26.2024.1400.dist-info/RECORD

@@ -1,24 +0,0 @@
-felis/__init__.py,sha256=_Pw-QKMYj0WRgE8fW2N2pBXJUj-Pjv8dSKJBzykjyZU,1842
-felis/check.py,sha256=RBxXq7XwPGIucrs1PPgPtgk8MrWAJlOmoxCNySEz9-I,13892
-felis/cli.py,sha256=YeGSiA3ywPVMMdB1YxH1_Gdac1kl4oPJvJtajfCs5VU,16637
-felis/datamodel.py,sha256=ooNSg68OuNk89EVu1MtxupLUWgSyzmb50wjza1joDO4,16002
-felis/metadata.py,sha256=5DE2YMnu6YuhwntBSe-OheCD7C2-vA4yb64BpjTC68A,18542
-felis/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-felis/simple.py,sha256=yzv_aoZrZhfakd1Xm7gLDeVKyJjCDZ7wAyYYp-l_Sxs,14414
-felis/tap.py,sha256=RBwEKyU3S0oXSNIMoI2WRAuC9WB5eai9BdQQUYN5Qdc,17704
-felis/types.py,sha256=1GL6IkHcIsIydiyw1eX98REh-lWCVIRO-9qjmaZfqvw,4440
-felis/utils.py,sha256=tYxr0xFdPN4gDHibeAD9d5DFgU8hKlSZVKmZoDzi4e8,4164
-felis/validation.py,sha256=f9VKvp7q-cnim2D5voTKwCdt0NRsYBpTwom1Z_3OKkc,3469
-felis/version.py,sha256=Bkf7vQojVYVWgjqmEFJRzUA7IlOUtWz7QH7SQ12F5YA,55
-felis/visitor.py,sha256=EazU4nYbkKBj3mCZYvsTCBTNmh0qRaUNZIzCcM3dqOQ,6439
-felis/db/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-felis/db/_variants.py,sha256=aW0Q7R4KEtxLR7VMashQjDLWdzDNrMVAH521MSvMey0,3346
-felis/db/sqltypes.py,sha256=0HOEqvL0OailGP-j6Jj5tnOSu_Pt7Hi29PPof4Q5d2c,5787
-lsst_felis-26.2024.1400.dist-info/COPYRIGHT,sha256=bUmNy19uUxqITMpjeHFe69q3IzQpjxvvBw6oV7kR7ho,129
-lsst_felis-26.2024.1400.dist-info/LICENSE,sha256=jOtLnuWt7d5Hsx6XXB2QxzrSe2sWWh3NgMfFRetluQM,35147
-lsst_felis-26.2024.1400.dist-info/METADATA,sha256=y4cVRA28EHR2jiWXFDDzSRCm16TH4YnJrept5w3fkMM,1101
-lsst_felis-26.2024.1400.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-lsst_felis-26.2024.1400.dist-info/entry_points.txt,sha256=Gk2XFujA_Gp52VBk45g5kim8TDoMDJFPctsMqiq72EM,40
-lsst_felis-26.2024.1400.dist-info/top_level.txt,sha256=F4SvPip3iZRVyISi50CHhwTIAokAhSxjWiVcn4IVWRI,6
-lsst_felis-26.2024.1400.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-lsst_felis-26.2024.1400.dist-info/RECORD,,