lsst-felis 26.2024.1400__tar.gz → 26.2024.1600__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of lsst-felis might be problematic. Click here for more details.
- {lsst-felis-26.2024.1400/python/lsst_felis.egg-info → lsst_felis-26.2024.1600}/PKG-INFO +1 -1
- {lsst-felis-26.2024.1400 → lsst_felis-26.2024.1600}/python/felis/cli.py +22 -7
- {lsst-felis-26.2024.1400 → lsst_felis-26.2024.1600}/python/felis/datamodel.py +141 -61
- {lsst-felis-26.2024.1400 → lsst_felis-26.2024.1600}/python/felis/db/_variants.py +5 -5
- {lsst-felis-26.2024.1400 → lsst_felis-26.2024.1600}/python/felis/db/sqltypes.py +13 -18
- {lsst-felis-26.2024.1400 → lsst_felis-26.2024.1600}/python/felis/types.py +1 -1
- lsst_felis-26.2024.1600/python/felis/version.py +2 -0
- {lsst-felis-26.2024.1400 → lsst_felis-26.2024.1600/python/lsst_felis.egg-info}/PKG-INFO +1 -1
- {lsst-felis-26.2024.1400 → lsst_felis-26.2024.1600}/python/lsst_felis.egg-info/SOURCES.txt +1 -0
- {lsst-felis-26.2024.1400 → lsst_felis-26.2024.1600}/tests/test_cli.py +0 -4
- {lsst-felis-26.2024.1400 → lsst_felis-26.2024.1600}/tests/test_datamodel.py +61 -31
- lsst_felis-26.2024.1600/tests/test_datatypes.py +117 -0
- lsst-felis-26.2024.1400/python/felis/version.py +0 -2
- {lsst-felis-26.2024.1400 → lsst_felis-26.2024.1600}/COPYRIGHT +0 -0
- {lsst-felis-26.2024.1400 → lsst_felis-26.2024.1600}/LICENSE +0 -0
- {lsst-felis-26.2024.1400 → lsst_felis-26.2024.1600}/README.rst +0 -0
- {lsst-felis-26.2024.1400 → lsst_felis-26.2024.1600}/pyproject.toml +0 -0
- {lsst-felis-26.2024.1400 → lsst_felis-26.2024.1600}/python/felis/__init__.py +0 -0
- {lsst-felis-26.2024.1400 → lsst_felis-26.2024.1600}/python/felis/check.py +0 -0
- {lsst-felis-26.2024.1400 → lsst_felis-26.2024.1600}/python/felis/db/__init__.py +0 -0
- {lsst-felis-26.2024.1400 → lsst_felis-26.2024.1600}/python/felis/metadata.py +0 -0
- {lsst-felis-26.2024.1400 → lsst_felis-26.2024.1600}/python/felis/py.typed +0 -0
- {lsst-felis-26.2024.1400 → lsst_felis-26.2024.1600}/python/felis/simple.py +0 -0
- {lsst-felis-26.2024.1400 → lsst_felis-26.2024.1600}/python/felis/tap.py +0 -0
- {lsst-felis-26.2024.1400 → lsst_felis-26.2024.1600}/python/felis/utils.py +0 -0
- {lsst-felis-26.2024.1400 → lsst_felis-26.2024.1600}/python/felis/validation.py +0 -0
- {lsst-felis-26.2024.1400 → lsst_felis-26.2024.1600}/python/felis/visitor.py +0 -0
- {lsst-felis-26.2024.1400 → lsst_felis-26.2024.1600}/python/lsst_felis.egg-info/dependency_links.txt +0 -0
- {lsst-felis-26.2024.1400 → lsst_felis-26.2024.1600}/python/lsst_felis.egg-info/entry_points.txt +0 -0
- {lsst-felis-26.2024.1400 → lsst_felis-26.2024.1600}/python/lsst_felis.egg-info/requires.txt +0 -0
- {lsst-felis-26.2024.1400 → lsst_felis-26.2024.1600}/python/lsst_felis.egg-info/top_level.txt +0 -0
- {lsst-felis-26.2024.1400 → lsst_felis-26.2024.1600}/python/lsst_felis.egg-info/zip-safe +0 -0
- {lsst-felis-26.2024.1400 → lsst_felis-26.2024.1600}/setup.cfg +0 -0
- {lsst-felis-26.2024.1400 → lsst_felis-26.2024.1600}/tests/test_check.py +0 -0
- {lsst-felis-26.2024.1400 → lsst_felis-26.2024.1600}/tests/test_metadata.py +0 -0
- {lsst-felis-26.2024.1400 → lsst_felis-26.2024.1600}/tests/test_simple.py +0 -0
- {lsst-felis-26.2024.1400 → lsst_felis-26.2024.1600}/tests/test_tap.py +0 -0
- {lsst-felis-26.2024.1400 → lsst_felis-26.2024.1600}/tests/test_utils.py +0 -0
- {lsst-felis-26.2024.1400 → lsst_felis-26.2024.1600}/tests/test_validation.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: lsst-felis
|
|
3
|
-
Version: 26.2024.
|
|
3
|
+
Version: 26.2024.1600
|
|
4
4
|
Summary: A vocabulary for describing catalogs and acting on those descriptions
|
|
5
5
|
Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
|
|
6
6
|
License: GNU General Public License v3 or later (GPLv3+)
|
|
@@ -373,22 +373,37 @@ def merge(files: Iterable[io.TextIOBase]) -> None:
|
|
|
373
373
|
type=click.Choice(["RSP", "default"]),
|
|
374
374
|
default="default",
|
|
375
375
|
)
|
|
376
|
-
@click.option(
|
|
376
|
+
@click.option(
|
|
377
|
+
"-d", "--require-description", is_flag=True, help="Require description for all objects", default=False
|
|
378
|
+
)
|
|
379
|
+
@click.option(
|
|
380
|
+
"-t", "--check-redundant-datatypes", is_flag=True, help="Check for redundant datatypes", default=False
|
|
381
|
+
)
|
|
377
382
|
@click.argument("files", nargs=-1, type=click.File())
|
|
378
|
-
def validate(
|
|
383
|
+
def validate(
|
|
384
|
+
schema_name: str,
|
|
385
|
+
require_description: bool,
|
|
386
|
+
check_redundant_datatypes: bool,
|
|
387
|
+
files: Iterable[io.TextIOBase],
|
|
388
|
+
) -> None:
|
|
379
389
|
"""Validate one or more felis YAML files."""
|
|
380
390
|
schema_class = get_schema(schema_name)
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
if require_description:
|
|
384
|
-
Schema.require_description(True)
|
|
391
|
+
if schema_name != "default":
|
|
392
|
+
logger.info(f"Using schema '{schema_class.__name__}'")
|
|
385
393
|
|
|
386
394
|
rc = 0
|
|
387
395
|
for file in files:
|
|
388
396
|
file_name = getattr(file, "name", None)
|
|
389
397
|
logger.info(f"Validating {file_name}")
|
|
390
398
|
try:
|
|
391
|
-
|
|
399
|
+
data = yaml.load(file, Loader=yaml.SafeLoader)
|
|
400
|
+
schema_class.model_validate(
|
|
401
|
+
data,
|
|
402
|
+
context={
|
|
403
|
+
"check_redundant_datatypes": check_redundant_datatypes,
|
|
404
|
+
"require_description": require_description,
|
|
405
|
+
},
|
|
406
|
+
)
|
|
392
407
|
except ValidationError as e:
|
|
393
408
|
logger.error(e)
|
|
394
409
|
rc = 1
|
|
@@ -22,13 +22,22 @@
|
|
|
22
22
|
from __future__ import annotations
|
|
23
23
|
|
|
24
24
|
import logging
|
|
25
|
+
import re
|
|
25
26
|
from collections.abc import Mapping, Sequence
|
|
26
|
-
from enum import
|
|
27
|
+
from enum import StrEnum, auto
|
|
27
28
|
from typing import Annotated, Any, Literal, TypeAlias
|
|
28
29
|
|
|
29
30
|
from astropy import units as units # type: ignore
|
|
30
31
|
from astropy.io.votable import ucd # type: ignore
|
|
31
|
-
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
|
32
|
+
from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validator, model_validator
|
|
33
|
+
from sqlalchemy import dialects
|
|
34
|
+
from sqlalchemy import types as sqa_types
|
|
35
|
+
from sqlalchemy.engine import create_mock_engine
|
|
36
|
+
from sqlalchemy.engine.interfaces import Dialect
|
|
37
|
+
from sqlalchemy.types import TypeEngine
|
|
38
|
+
|
|
39
|
+
from .db.sqltypes import get_type_func
|
|
40
|
+
from .types import FelisType
|
|
32
41
|
|
|
33
42
|
logger = logging.getLogger(__name__)
|
|
34
43
|
|
|
@@ -49,7 +58,6 @@ __all__ = (
|
|
|
49
58
|
CONFIG = ConfigDict(
|
|
50
59
|
populate_by_name=True, # Populate attributes by name.
|
|
51
60
|
extra="forbid", # Do not allow extra fields.
|
|
52
|
-
validate_assignment=True, # Validate assignments after model is created.
|
|
53
61
|
str_strip_whitespace=True, # Strip whitespace from string fields.
|
|
54
62
|
)
|
|
55
63
|
"""Pydantic model configuration as described in:
|
|
@@ -83,40 +91,83 @@ class BaseObject(BaseModel):
|
|
|
83
91
|
"""
|
|
84
92
|
|
|
85
93
|
description: DescriptionStr | None = None
|
|
86
|
-
"""A description of the database object.
|
|
87
|
-
|
|
88
|
-
By default, the description is optional but will be required if
|
|
89
|
-
`BaseObject.Config.require_description` is set to `True` by the user.
|
|
90
|
-
"""
|
|
94
|
+
"""A description of the database object."""
|
|
91
95
|
|
|
92
|
-
@model_validator(mode="
|
|
96
|
+
@model_validator(mode="after") # type: ignore[arg-type]
|
|
93
97
|
@classmethod
|
|
94
|
-
def check_description(cls,
|
|
98
|
+
def check_description(cls, object: BaseObject, info: ValidationInfo) -> BaseObject:
|
|
95
99
|
"""Check that the description is present if required."""
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
100
|
+
context = info.context
|
|
101
|
+
if not context or not context.get("require_description", False):
|
|
102
|
+
return object
|
|
103
|
+
if object.description is None or object.description == "":
|
|
104
|
+
raise ValueError("Description is required and must be non-empty")
|
|
105
|
+
if len(object.description) < DESCR_MIN_LENGTH:
|
|
106
|
+
raise ValueError(f"Description must be at least {DESCR_MIN_LENGTH} characters long")
|
|
107
|
+
return object
|
|
102
108
|
|
|
103
109
|
|
|
104
|
-
class DataType(
|
|
110
|
+
class DataType(StrEnum):
|
|
105
111
|
"""`Enum` representing the data types supported by Felis."""
|
|
106
112
|
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
113
|
+
boolean = auto()
|
|
114
|
+
byte = auto()
|
|
115
|
+
short = auto()
|
|
116
|
+
int = auto()
|
|
117
|
+
long = auto()
|
|
118
|
+
float = auto()
|
|
119
|
+
double = auto()
|
|
120
|
+
char = auto()
|
|
121
|
+
string = auto()
|
|
122
|
+
unicode = auto()
|
|
123
|
+
text = auto()
|
|
124
|
+
binary = auto()
|
|
125
|
+
timestamp = auto()
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
_DIALECTS = {
|
|
129
|
+
"mysql": create_mock_engine("mysql://", executor=None).dialect,
|
|
130
|
+
"postgresql": create_mock_engine("postgresql://", executor=None).dialect,
|
|
131
|
+
}
|
|
132
|
+
"""Dictionary of dialect names to SQLAlchemy dialects."""
|
|
133
|
+
|
|
134
|
+
_DIALECT_MODULES = {"mysql": getattr(dialects, "mysql"), "postgresql": getattr(dialects, "postgresql")}
|
|
135
|
+
"""Dictionary of dialect names to SQLAlchemy dialect modules."""
|
|
136
|
+
|
|
137
|
+
_DATATYPE_REGEXP = re.compile(r"(\w+)(\((.*)\))?")
|
|
138
|
+
"""Regular expression to match data types in the form "type(length)"""
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def string_to_typeengine(
|
|
142
|
+
type_string: str, dialect: Dialect | None = None, length: int | None = None
|
|
143
|
+
) -> TypeEngine:
|
|
144
|
+
match = _DATATYPE_REGEXP.search(type_string)
|
|
145
|
+
if not match:
|
|
146
|
+
raise ValueError(f"Invalid type string: {type_string}")
|
|
147
|
+
|
|
148
|
+
type_name, _, params = match.groups()
|
|
149
|
+
if dialect is None:
|
|
150
|
+
type_class = getattr(sqa_types, type_name.upper(), None)
|
|
151
|
+
else:
|
|
152
|
+
try:
|
|
153
|
+
dialect_module = _DIALECT_MODULES[dialect.name]
|
|
154
|
+
except KeyError:
|
|
155
|
+
raise ValueError(f"Unsupported dialect: {dialect}")
|
|
156
|
+
type_class = getattr(dialect_module, type_name.upper(), None)
|
|
157
|
+
|
|
158
|
+
if not type_class:
|
|
159
|
+
raise ValueError(f"Unsupported type: {type_class}")
|
|
160
|
+
|
|
161
|
+
if params:
|
|
162
|
+
params = [int(param) if param.isdigit() else param for param in params.split(",")]
|
|
163
|
+
type_obj = type_class(*params)
|
|
164
|
+
else:
|
|
165
|
+
type_obj = type_class()
|
|
166
|
+
|
|
167
|
+
if hasattr(type_obj, "length") and getattr(type_obj, "length") is None and length is not None:
|
|
168
|
+
type_obj.length = length
|
|
169
|
+
|
|
170
|
+
return type_obj
|
|
120
171
|
|
|
121
172
|
|
|
122
173
|
class Column(BaseObject):
|
|
@@ -207,6 +258,53 @@ class Column(BaseObject):
|
|
|
207
258
|
|
|
208
259
|
return values
|
|
209
260
|
|
|
261
|
+
@model_validator(mode="after") # type: ignore[arg-type]
|
|
262
|
+
@classmethod
|
|
263
|
+
def validate_datatypes(cls, col: Column, info: ValidationInfo) -> Column:
|
|
264
|
+
"""Check for redundant datatypes on columns."""
|
|
265
|
+
context = info.context
|
|
266
|
+
if not context or not context.get("check_redundant_datatypes", False):
|
|
267
|
+
return col
|
|
268
|
+
if all(getattr(col, f"{dialect}:datatype", None) is not None for dialect in _DIALECTS.keys()):
|
|
269
|
+
return col
|
|
270
|
+
|
|
271
|
+
datatype = col.datatype
|
|
272
|
+
length: int | None = col.length or None
|
|
273
|
+
|
|
274
|
+
datatype_func = get_type_func(datatype)
|
|
275
|
+
felis_type = FelisType.felis_type(datatype)
|
|
276
|
+
if felis_type.is_sized:
|
|
277
|
+
if length is not None:
|
|
278
|
+
datatype_obj = datatype_func(length)
|
|
279
|
+
else:
|
|
280
|
+
raise ValueError(f"Length must be provided for sized type '{datatype}' in column '{col.id}'")
|
|
281
|
+
else:
|
|
282
|
+
datatype_obj = datatype_func()
|
|
283
|
+
|
|
284
|
+
for dialect_name, dialect in _DIALECTS.items():
|
|
285
|
+
db_annotation = f"{dialect_name}_datatype"
|
|
286
|
+
if datatype_string := col.model_dump().get(db_annotation):
|
|
287
|
+
db_datatype_obj = string_to_typeengine(datatype_string, dialect, length)
|
|
288
|
+
if datatype_obj.compile(dialect) == db_datatype_obj.compile(dialect):
|
|
289
|
+
raise ValueError(
|
|
290
|
+
"'{}: {}' is the same as 'datatype: {}' in column '{}'".format(
|
|
291
|
+
db_annotation, datatype_string, col.datatype, col.id
|
|
292
|
+
)
|
|
293
|
+
)
|
|
294
|
+
else:
|
|
295
|
+
logger.debug(
|
|
296
|
+
"Valid type override of 'datatype: {}' with '{}: {}' in column '{}'".format(
|
|
297
|
+
col.datatype, db_annotation, datatype_string, col.id
|
|
298
|
+
)
|
|
299
|
+
)
|
|
300
|
+
logger.debug(
|
|
301
|
+
"Compiled datatype '{}' with {} compiled override '{}'".format(
|
|
302
|
+
datatype_obj.compile(dialect), dialect_name, db_datatype_obj.compile(dialect)
|
|
303
|
+
)
|
|
304
|
+
)
|
|
305
|
+
|
|
306
|
+
return col
|
|
307
|
+
|
|
210
308
|
|
|
211
309
|
class Constraint(BaseObject):
|
|
212
310
|
"""A database table constraint."""
|
|
@@ -404,15 +502,6 @@ class SchemaIdVisitor:
|
|
|
404
502
|
class Schema(BaseObject):
|
|
405
503
|
"""The database schema containing the tables."""
|
|
406
504
|
|
|
407
|
-
class ValidationConfig:
|
|
408
|
-
"""Validation configuration which is specific to Felis."""
|
|
409
|
-
|
|
410
|
-
_require_description = False
|
|
411
|
-
"""Flag to require a description for all objects.
|
|
412
|
-
|
|
413
|
-
This is set by the `require_description` class method.
|
|
414
|
-
"""
|
|
415
|
-
|
|
416
505
|
version: SchemaVersion | str | None = None
|
|
417
506
|
"""The version of the schema."""
|
|
418
507
|
|
|
@@ -430,21 +519,29 @@ class Schema(BaseObject):
|
|
|
430
519
|
raise ValueError("Table names must be unique")
|
|
431
520
|
return tables
|
|
432
521
|
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
522
|
+
def _create_id_map(self: Schema) -> Schema:
|
|
523
|
+
"""Create a map of IDs to objects.
|
|
524
|
+
|
|
525
|
+
This method should not be called by users. It is called automatically
|
|
526
|
+
by the ``model_post_init()`` method. If the ID map is already
|
|
527
|
+
populated, this method will return immediately.
|
|
528
|
+
"""
|
|
436
529
|
if len(self.id_map):
|
|
437
|
-
logger.debug("ID map was already populated")
|
|
530
|
+
logger.debug("Ignoring call to create_id_map() - ID map was already populated")
|
|
438
531
|
return self
|
|
439
532
|
visitor: SchemaIdVisitor = SchemaIdVisitor()
|
|
440
533
|
visitor.visit_schema(self)
|
|
441
|
-
logger.debug(f"ID map
|
|
534
|
+
logger.debug(f"Created schema ID map with {len(self.id_map.keys())} objects")
|
|
442
535
|
if len(visitor.duplicates):
|
|
443
536
|
raise ValueError(
|
|
444
537
|
"Duplicate IDs found in schema:\n " + "\n ".join(visitor.duplicates) + "\n"
|
|
445
538
|
)
|
|
446
539
|
return self
|
|
447
540
|
|
|
541
|
+
def model_post_init(self, ctx: Any) -> None:
|
|
542
|
+
"""Post-initialization hook for the model."""
|
|
543
|
+
self._create_id_map()
|
|
544
|
+
|
|
448
545
|
def __getitem__(self, id: str) -> BaseObject:
|
|
449
546
|
"""Get an object by its ID."""
|
|
450
547
|
if id not in self:
|
|
@@ -454,20 +551,3 @@ class Schema(BaseObject):
|
|
|
454
551
|
def __contains__(self, id: str) -> bool:
|
|
455
552
|
"""Check if an object with the given ID is in the schema."""
|
|
456
553
|
return id in self.id_map
|
|
457
|
-
|
|
458
|
-
@classmethod
|
|
459
|
-
def require_description(cls, rd: bool = True) -> None:
|
|
460
|
-
"""Set whether a description is required for all objects.
|
|
461
|
-
|
|
462
|
-
This includes the schema, tables, columns, and constraints.
|
|
463
|
-
|
|
464
|
-
Users should call this method to set the requirement for a description
|
|
465
|
-
when validating schemas, rather than change the flag value directly.
|
|
466
|
-
"""
|
|
467
|
-
logger.debug(f"Setting description requirement to '{rd}'")
|
|
468
|
-
cls.ValidationConfig._require_description = rd
|
|
469
|
-
|
|
470
|
-
@classmethod
|
|
471
|
-
def is_description_required(cls) -> bool:
|
|
472
|
-
"""Return whether a description is required for all objects."""
|
|
473
|
-
return cls.ValidationConfig._require_description
|
|
@@ -40,10 +40,10 @@ TABLE_OPTS = {
|
|
|
40
40
|
}
|
|
41
41
|
|
|
42
42
|
COLUMN_VARIANT_OVERRIDE = {
|
|
43
|
-
"
|
|
44
|
-
"
|
|
45
|
-
"
|
|
46
|
-
"
|
|
43
|
+
"mysql_datatype": "mysql",
|
|
44
|
+
"oracle_datatype": "oracle",
|
|
45
|
+
"postgresql_datatype": "postgresql",
|
|
46
|
+
"sqlite_datatype": "sqlite",
|
|
47
47
|
}
|
|
48
48
|
|
|
49
49
|
DIALECT_MODULES = {MYSQL: mysql, ORACLE: oracle, SQLITE: sqlite, POSTGRES: postgresql}
|
|
@@ -87,7 +87,7 @@ def make_variant_dict(column_obj: Column) -> dict[str, TypeEngine[Any]]:
|
|
|
87
87
|
"""
|
|
88
88
|
variant_dict = {}
|
|
89
89
|
for field_name, value in iter(column_obj):
|
|
90
|
-
if field_name in COLUMN_VARIANT_OVERRIDE:
|
|
90
|
+
if field_name in COLUMN_VARIANT_OVERRIDE and value is not None:
|
|
91
91
|
dialect = COLUMN_VARIANT_OVERRIDE[field_name]
|
|
92
92
|
variant: TypeEngine = process_variant_override(dialect, value)
|
|
93
93
|
variant_dict[dialect] = variant
|
|
@@ -21,9 +21,9 @@
|
|
|
21
21
|
|
|
22
22
|
import builtins
|
|
23
23
|
from collections.abc import Mapping
|
|
24
|
-
from typing import Any
|
|
24
|
+
from typing import Any, Callable
|
|
25
25
|
|
|
26
|
-
from sqlalchemy import
|
|
26
|
+
from sqlalchemy import SmallInteger, types
|
|
27
27
|
from sqlalchemy.dialects import mysql, oracle, postgresql
|
|
28
28
|
from sqlalchemy.ext.compiler import compiles
|
|
29
29
|
|
|
@@ -39,24 +39,12 @@ class TINYINT(SmallInteger):
|
|
|
39
39
|
__visit_name__ = "TINYINT"
|
|
40
40
|
|
|
41
41
|
|
|
42
|
-
class DOUBLE(Float):
|
|
43
|
-
"""The non-standard DOUBLE type."""
|
|
44
|
-
|
|
45
|
-
__visit_name__ = "DOUBLE"
|
|
46
|
-
|
|
47
|
-
|
|
48
42
|
@compiles(TINYINT)
|
|
49
43
|
def compile_tinyint(type_: Any, compiler: Any, **kw: Any) -> str:
|
|
50
44
|
"""Return type name for TINYINT."""
|
|
51
45
|
return "TINYINT"
|
|
52
46
|
|
|
53
47
|
|
|
54
|
-
@compiles(DOUBLE)
|
|
55
|
-
def compile_double(type_: Any, compiler: Any, **kw: Any) -> str:
|
|
56
|
-
"""Return type name for double precision type."""
|
|
57
|
-
return "DOUBLE"
|
|
58
|
-
|
|
59
|
-
|
|
60
48
|
_TypeMap = Mapping[str, types.TypeEngine | type[types.TypeEngine]]
|
|
61
49
|
|
|
62
50
|
boolean_map: _TypeMap = {MYSQL: mysql.BIT(1), ORACLE: oracle.NUMBER(1), POSTGRES: postgresql.BOOLEAN()}
|
|
@@ -160,7 +148,7 @@ def float(**kwargs: Any) -> types.TypeEngine:
|
|
|
160
148
|
|
|
161
149
|
def double(**kwargs: Any) -> types.TypeEngine:
|
|
162
150
|
"""Return SQLAlchemy type for double precision float."""
|
|
163
|
-
return _vary(DOUBLE(), double_map, kwargs)
|
|
151
|
+
return _vary(types.DOUBLE(), double_map, kwargs)
|
|
164
152
|
|
|
165
153
|
|
|
166
154
|
def char(length: builtins.int, **kwargs: Any) -> types.TypeEngine:
|
|
@@ -178,9 +166,9 @@ def unicode(length: builtins.int, **kwargs: Any) -> types.TypeEngine:
|
|
|
178
166
|
return _vary(types.NVARCHAR(length), unicode_map, kwargs, length)
|
|
179
167
|
|
|
180
168
|
|
|
181
|
-
def text(
|
|
169
|
+
def text(**kwargs: Any) -> types.TypeEngine:
|
|
182
170
|
"""Return SQLAlchemy type for text."""
|
|
183
|
-
return _vary(types.
|
|
171
|
+
return _vary(types.TEXT(), text_map, kwargs)
|
|
184
172
|
|
|
185
173
|
|
|
186
174
|
def binary(length: builtins.int, **kwargs: Any) -> types.TypeEngine:
|
|
@@ -193,6 +181,13 @@ def timestamp(**kwargs: Any) -> types.TypeEngine:
|
|
|
193
181
|
return types.TIMESTAMP()
|
|
194
182
|
|
|
195
183
|
|
|
184
|
+
def get_type_func(type_name: str) -> Callable:
|
|
185
|
+
"""Return the function for the type with the given name."""
|
|
186
|
+
if type_name not in globals():
|
|
187
|
+
raise ValueError(f"Unknown type: {type_name}")
|
|
188
|
+
return globals()[type_name]
|
|
189
|
+
|
|
190
|
+
|
|
196
191
|
def _vary(
|
|
197
192
|
type_: types.TypeEngine,
|
|
198
193
|
variant_map: _TypeMap,
|
|
@@ -203,7 +198,7 @@ def _vary(
|
|
|
203
198
|
variants.update(overrides)
|
|
204
199
|
for dialect, variant in variants.items():
|
|
205
200
|
# If this is a class and not an instance, instantiate
|
|
206
|
-
if
|
|
201
|
+
if callable(variant):
|
|
207
202
|
variant = variant(*args)
|
|
208
203
|
type_ = type_.with_variant(variant, dialect)
|
|
209
204
|
return type_
|
|
@@ -125,7 +125,7 @@ class Unicode(FelisType, felis_name="unicode", votable_name="unicodeChar", is_si
|
|
|
125
125
|
"""Felis definition of unicode string type."""
|
|
126
126
|
|
|
127
127
|
|
|
128
|
-
class Text(FelisType, felis_name="text", votable_name="
|
|
128
|
+
class Text(FelisType, felis_name="text", votable_name="char"):
|
|
129
129
|
"""Felis definition of text type."""
|
|
130
130
|
|
|
131
131
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: lsst-felis
|
|
3
|
-
Version: 26.2024.
|
|
3
|
+
Version: 26.2024.1600
|
|
4
4
|
Summary: A vocabulary for describing catalogs and acting on those descriptions
|
|
5
5
|
Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
|
|
6
6
|
License: GNU General Public License v3 or later (GPLv3+)
|
|
@@ -29,7 +29,6 @@ from typing import Any
|
|
|
29
29
|
from click.testing import CliRunner
|
|
30
30
|
|
|
31
31
|
from felis.cli import cli
|
|
32
|
-
from felis.datamodel import Schema
|
|
33
32
|
|
|
34
33
|
TESTDIR = os.path.abspath(os.path.dirname(__file__))
|
|
35
34
|
TEST_YAML = os.path.join(TESTDIR, "data", "test.yml")
|
|
@@ -145,9 +144,6 @@ class CliTestCase(unittest.TestCase):
|
|
|
145
144
|
except Exception as e:
|
|
146
145
|
# Reraise exception.
|
|
147
146
|
raise e
|
|
148
|
-
finally:
|
|
149
|
-
# Turn the flag off so it does not effect subsequent tests.
|
|
150
|
-
Schema.require_description(False)
|
|
151
147
|
|
|
152
148
|
self.assertEqual(result.exit_code, 0)
|
|
153
149
|
|
|
@@ -75,14 +75,14 @@ class ColumnTestCase(unittest.TestCase):
|
|
|
75
75
|
col = Column(name="testColumn", id="#test_id", datatype="string")
|
|
76
76
|
self.assertEqual(col.name, "testColumn", "name should be 'testColumn'")
|
|
77
77
|
self.assertEqual(col.id, "#test_id", "id should be '#test_id'")
|
|
78
|
-
self.assertEqual(col.datatype, DataType.
|
|
78
|
+
self.assertEqual(col.datatype, DataType.string, "datatype should be 'DataType.string'")
|
|
79
79
|
|
|
80
80
|
# Creating from data dictionary should work and load data correctly.
|
|
81
81
|
data = {"name": "testColumn", "id": "#test_id", "datatype": "string"}
|
|
82
82
|
col = Column(**data)
|
|
83
83
|
self.assertEqual(col.name, "testColumn", "name should be 'testColumn'")
|
|
84
84
|
self.assertEqual(col.id, "#test_id", "id should be '#test_id'")
|
|
85
|
-
self.assertEqual(col.datatype, DataType.
|
|
85
|
+
self.assertEqual(col.datatype, DataType.string, "datatype should be 'DataType.string'")
|
|
86
86
|
|
|
87
87
|
# Setting a bad IVOA UCD should throw an error.
|
|
88
88
|
with self.assertRaises(ValidationError):
|
|
@@ -123,44 +123,74 @@ class ColumnTestCase(unittest.TestCase):
|
|
|
123
123
|
|
|
124
124
|
def test_require_description(self) -> None:
|
|
125
125
|
"""Test the require_description flag for the `Column` class."""
|
|
126
|
-
# Turn on description requirement for this test.
|
|
127
|
-
Schema.require_description(True)
|
|
128
126
|
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
self.assertTrue(Schema.is_description_required(), "description should be required")
|
|
127
|
+
class MockValidationInfo:
|
|
128
|
+
"""Mock context object for passing to validation method."""
|
|
132
129
|
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
with self.assertRaises(ValidationError):
|
|
136
|
-
Column(
|
|
137
|
-
**{
|
|
138
|
-
"name": "testColumn",
|
|
139
|
-
"@id": "#test_col_id",
|
|
140
|
-
"datatype": "string",
|
|
141
|
-
}
|
|
142
|
-
)
|
|
130
|
+
def __init__(self):
|
|
131
|
+
self.context = {"require_description": True}
|
|
143
132
|
|
|
144
|
-
|
|
145
|
-
with self.assertRaises(ValidationError):
|
|
146
|
-
Column(**{"name": "testColumn", "@id": "#test_col_id", "datatype": "string", "description": None})
|
|
133
|
+
info = MockValidationInfo()
|
|
147
134
|
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
with self.assertRaises(ValidationError):
|
|
151
|
-
Column(**{"name": "testColumn", "@id": "#test_col_id", "datatype": "string", "description": ""})
|
|
135
|
+
def _check_description(col: Column):
|
|
136
|
+
Schema.check_description(col, info)
|
|
152
137
|
|
|
153
|
-
# Creating a column
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
138
|
+
# Creating a column without a description should throw.
|
|
139
|
+
with self.assertRaises(ValueError):
|
|
140
|
+
_check_description(
|
|
141
|
+
Column(
|
|
142
|
+
**{
|
|
143
|
+
"name": "testColumn",
|
|
144
|
+
"@id": "#test_col_id",
|
|
145
|
+
"datatype": "string",
|
|
146
|
+
}
|
|
147
|
+
)
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
# Creating a column with a description of 'None' should throw.
|
|
151
|
+
with self.assertRaises(ValueError):
|
|
152
|
+
_check_description(
|
|
153
|
+
Column(
|
|
154
|
+
**{
|
|
155
|
+
"name": "testColumn",
|
|
156
|
+
"@id": "#test_col_id",
|
|
157
|
+
"datatype": "string",
|
|
158
|
+
"description": None,
|
|
159
|
+
}
|
|
160
|
+
)
|
|
161
|
+
)
|
|
157
162
|
|
|
158
|
-
#
|
|
159
|
-
|
|
163
|
+
# Creating a column with an empty description should throw.
|
|
164
|
+
with self.assertRaises(ValueError):
|
|
165
|
+
_check_description(
|
|
166
|
+
Column(
|
|
167
|
+
**{
|
|
168
|
+
"name": "testColumn",
|
|
169
|
+
"@id": "#test_col_id",
|
|
170
|
+
"datatype": "string",
|
|
171
|
+
"require_description": True,
|
|
172
|
+
"description": "",
|
|
173
|
+
}
|
|
174
|
+
)
|
|
175
|
+
)
|
|
176
|
+
|
|
177
|
+
# Creating a column with a description that is too short should throw.
|
|
178
|
+
with self.assertRaises(ValidationError):
|
|
179
|
+
_check_description(
|
|
180
|
+
Column(
|
|
181
|
+
**{
|
|
182
|
+
"name": "testColumn",
|
|
183
|
+
"@id": "#test_col_id",
|
|
184
|
+
"datatype": "string",
|
|
185
|
+
"require_description": True,
|
|
186
|
+
"description": "xy",
|
|
187
|
+
}
|
|
188
|
+
)
|
|
189
|
+
)
|
|
160
190
|
|
|
161
191
|
|
|
162
192
|
class ConstraintTestCase(unittest.TestCase):
|
|
163
|
-
"""Test the `UniqueConstraint`, `Index`, `
|
|
193
|
+
"""Test the `UniqueConstraint`, `Index`, `CheckConstraint`, and
|
|
164
194
|
`ForeignKeyConstraint` classes.
|
|
165
195
|
"""
|
|
166
196
|
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
# This file is part of felis.
|
|
2
|
+
#
|
|
3
|
+
# Developed for the LSST Data Management System.
|
|
4
|
+
# This product includes software developed by the LSST Project
|
|
5
|
+
# (https://www.lsst.org).
|
|
6
|
+
# See the COPYRIGHT file at the top-level directory of this distribution
|
|
7
|
+
# for details of code ownership.
|
|
8
|
+
#
|
|
9
|
+
# This program is free software: you can redistribute it and/or modify
|
|
10
|
+
# it under the terms of the GNU General Public License as published by
|
|
11
|
+
# the Free Software Foundation, either version 3 of the License, or
|
|
12
|
+
# (at your option) any later version.
|
|
13
|
+
#
|
|
14
|
+
# This program is distributed in the hope that it will be useful,
|
|
15
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
16
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
17
|
+
# GNU General Public License for more details.
|
|
18
|
+
#
|
|
19
|
+
# You should have received a copy of the GNU General Public License
|
|
20
|
+
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
21
|
+
|
|
22
|
+
import unittest
|
|
23
|
+
|
|
24
|
+
from pydantic import ValidationError
|
|
25
|
+
|
|
26
|
+
from felis.datamodel import Column
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class ColumnGenerator:
|
|
30
|
+
"""Generate column data for testing."""
|
|
31
|
+
|
|
32
|
+
def __init__(self, name, id, db_name):
|
|
33
|
+
self.name = name
|
|
34
|
+
self.id = id
|
|
35
|
+
self.db_name = db_name
|
|
36
|
+
self.context = {"check_redundant_datatypes": True}
|
|
37
|
+
|
|
38
|
+
def col(self, datatype: str, db_datatype: str, length=None):
|
|
39
|
+
return Column.model_validate(
|
|
40
|
+
{
|
|
41
|
+
"name": self.name,
|
|
42
|
+
"@id": self.id,
|
|
43
|
+
"datatype": datatype,
|
|
44
|
+
f"{self.db_name}:datatype": db_datatype,
|
|
45
|
+
"length": length,
|
|
46
|
+
},
|
|
47
|
+
context=self.context,
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class RedundantDatatypesTest(unittest.TestCase):
|
|
52
|
+
"""Test validation of redundant datatype definitions."""
|
|
53
|
+
|
|
54
|
+
def test_mysql_datatypes(self) -> None:
|
|
55
|
+
"""Test that redundant datatype definitions raise an error."""
|
|
56
|
+
coldata = ColumnGenerator("test_col", "#test_col_id", "mysql")
|
|
57
|
+
|
|
58
|
+
with self.assertRaises(ValidationError):
|
|
59
|
+
coldata.col("double", "DOUBLE")
|
|
60
|
+
|
|
61
|
+
with self.assertRaises(ValidationError):
|
|
62
|
+
coldata.col("int", "INTEGER")
|
|
63
|
+
|
|
64
|
+
with self.assertRaises(ValidationError):
|
|
65
|
+
coldata.col("boolean", "BIT(1)")
|
|
66
|
+
|
|
67
|
+
with self.assertRaises(ValidationError):
|
|
68
|
+
coldata.col("float", "FLOAT")
|
|
69
|
+
|
|
70
|
+
with self.assertRaises(ValidationError):
|
|
71
|
+
coldata.col("char", "CHAR", length=8)
|
|
72
|
+
|
|
73
|
+
with self.assertRaises(ValidationError):
|
|
74
|
+
coldata.col("string", "VARCHAR", length=32)
|
|
75
|
+
|
|
76
|
+
with self.assertRaises(ValidationError):
|
|
77
|
+
coldata.col("byte", "TINYINT")
|
|
78
|
+
|
|
79
|
+
with self.assertRaises(ValidationError):
|
|
80
|
+
coldata.col("short", "SMALLINT")
|
|
81
|
+
|
|
82
|
+
with self.assertRaises(ValidationError):
|
|
83
|
+
coldata.col("long", "BIGINT")
|
|
84
|
+
|
|
85
|
+
# These look like they should be equivalent, but the default is
|
|
86
|
+
# actually ``BIT(1)`` for MySQL.
|
|
87
|
+
coldata.col("boolean", "BOOLEAN")
|
|
88
|
+
|
|
89
|
+
with self.assertRaises(ValidationError):
|
|
90
|
+
coldata.col("unicode", "NVARCHAR", length=32)
|
|
91
|
+
|
|
92
|
+
with self.assertRaises(ValidationError):
|
|
93
|
+
coldata.col("timestamp", "TIMESTAMP")
|
|
94
|
+
|
|
95
|
+
# DM-42257: Felis does not handle unbounded text types properly.
|
|
96
|
+
# coldata.col("text", "TEXT", length=32)
|
|
97
|
+
|
|
98
|
+
with self.assertRaises(ValidationError):
|
|
99
|
+
coldata.col("binary", "LONGBLOB", length=1024)
|
|
100
|
+
|
|
101
|
+
with self.assertRaises(ValidationError):
|
|
102
|
+
# Same type and length
|
|
103
|
+
coldata.col("string", "VARCHAR(128)", length=128)
|
|
104
|
+
|
|
105
|
+
# Different types, which is okay
|
|
106
|
+
coldata.col("double", "FLOAT")
|
|
107
|
+
|
|
108
|
+
# Same base type with different lengths, which is okay
|
|
109
|
+
coldata.col("string", "VARCHAR(128)", length=32)
|
|
110
|
+
|
|
111
|
+
# Different string types, which is okay
|
|
112
|
+
coldata.col("string", "CHAR", length=32)
|
|
113
|
+
coldata.col("unicode", "CHAR", length=32)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
if __name__ == "__main__":
|
|
117
|
+
unittest.main()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{lsst-felis-26.2024.1400 → lsst_felis-26.2024.1600}/python/lsst_felis.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
{lsst-felis-26.2024.1400 → lsst_felis-26.2024.1600}/python/lsst_felis.egg-info/entry_points.txt
RENAMED
|
File without changes
|
|
File without changes
|
{lsst-felis-26.2024.1400 → lsst_felis-26.2024.1600}/python/lsst_felis.egg-info/top_level.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|