lsst-felis 27.2024.2400__tar.gz → 27.2024.2500__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of lsst-felis might be problematic. Click here for more details.
- {lsst_felis-27.2024.2400/python/lsst_felis.egg-info → lsst_felis-27.2024.2500}/PKG-INFO +1 -1
- {lsst_felis-27.2024.2400 → lsst_felis-27.2024.2500}/python/felis/cli.py +17 -16
- {lsst_felis-27.2024.2400 → lsst_felis-27.2024.2500}/python/felis/datamodel.py +45 -8
- lsst_felis-27.2024.2500/python/felis/version.py +2 -0
- {lsst_felis-27.2024.2400 → lsst_felis-27.2024.2500/python/lsst_felis.egg-info}/PKG-INFO +1 -1
- {lsst_felis-27.2024.2400 → lsst_felis-27.2024.2500}/python/lsst_felis.egg-info/SOURCES.txt +1 -3
- {lsst_felis-27.2024.2400 → lsst_felis-27.2024.2500}/tests/test_cli.py +12 -20
- {lsst_felis-27.2024.2400 → lsst_felis-27.2024.2500}/tests/test_datamodel.py +115 -54
- lsst_felis-27.2024.2400/python/felis/validation.py +0 -103
- lsst_felis-27.2024.2400/python/felis/version.py +0 -2
- lsst_felis-27.2024.2400/tests/test_validation.py +0 -233
- {lsst_felis-27.2024.2400 → lsst_felis-27.2024.2500}/COPYRIGHT +0 -0
- {lsst_felis-27.2024.2400 → lsst_felis-27.2024.2500}/LICENSE +0 -0
- {lsst_felis-27.2024.2400 → lsst_felis-27.2024.2500}/README.rst +0 -0
- {lsst_felis-27.2024.2400 → lsst_felis-27.2024.2500}/pyproject.toml +0 -0
- {lsst_felis-27.2024.2400 → lsst_felis-27.2024.2500}/python/felis/__init__.py +0 -0
- {lsst_felis-27.2024.2400 → lsst_felis-27.2024.2500}/python/felis/db/__init__.py +0 -0
- {lsst_felis-27.2024.2400 → lsst_felis-27.2024.2500}/python/felis/db/dialects.py +0 -0
- {lsst_felis-27.2024.2400 → lsst_felis-27.2024.2500}/python/felis/db/sqltypes.py +0 -0
- {lsst_felis-27.2024.2400 → lsst_felis-27.2024.2500}/python/felis/db/utils.py +0 -0
- {lsst_felis-27.2024.2400 → lsst_felis-27.2024.2500}/python/felis/db/variants.py +0 -0
- {lsst_felis-27.2024.2400 → lsst_felis-27.2024.2500}/python/felis/metadata.py +0 -0
- {lsst_felis-27.2024.2400 → lsst_felis-27.2024.2500}/python/felis/py.typed +0 -0
- {lsst_felis-27.2024.2400 → lsst_felis-27.2024.2500}/python/felis/tap.py +0 -0
- {lsst_felis-27.2024.2400 → lsst_felis-27.2024.2500}/python/felis/types.py +0 -0
- {lsst_felis-27.2024.2400 → lsst_felis-27.2024.2500}/python/lsst_felis.egg-info/dependency_links.txt +0 -0
- {lsst_felis-27.2024.2400 → lsst_felis-27.2024.2500}/python/lsst_felis.egg-info/entry_points.txt +0 -0
- {lsst_felis-27.2024.2400 → lsst_felis-27.2024.2500}/python/lsst_felis.egg-info/requires.txt +0 -0
- {lsst_felis-27.2024.2400 → lsst_felis-27.2024.2500}/python/lsst_felis.egg-info/top_level.txt +0 -0
- {lsst_felis-27.2024.2400 → lsst_felis-27.2024.2500}/python/lsst_felis.egg-info/zip-safe +0 -0
- {lsst_felis-27.2024.2400 → lsst_felis-27.2024.2500}/setup.cfg +0 -0
- {lsst_felis-27.2024.2400 → lsst_felis-27.2024.2500}/tests/test_datatypes.py +0 -0
- {lsst_felis-27.2024.2400 → lsst_felis-27.2024.2500}/tests/test_metadata.py +0 -0
- {lsst_felis-27.2024.2400 → lsst_felis-27.2024.2500}/tests/test_tap.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: lsst-felis
|
|
3
|
-
Version: 27.2024.
|
|
3
|
+
Version: 27.2024.2500
|
|
4
4
|
Summary: A vocabulary for describing catalogs and acting on those descriptions
|
|
5
5
|
Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
|
|
6
6
|
License: GNU General Public License v3 or later (GPLv3+)
|
|
@@ -37,7 +37,6 @@ from .datamodel import Schema
|
|
|
37
37
|
from .db.utils import DatabaseContext
|
|
38
38
|
from .metadata import MetaDataBuilder
|
|
39
39
|
from .tap import Tap11Base, TapLoadingVisitor, init_tables
|
|
40
|
-
from .validation import get_schema
|
|
41
40
|
|
|
42
41
|
logger = logging.getLogger("felis")
|
|
43
42
|
|
|
@@ -241,42 +240,44 @@ def load_tap(
|
|
|
241
240
|
|
|
242
241
|
|
|
243
242
|
@cli.command("validate")
|
|
243
|
+
@click.option("--check-description", is_flag=True, help="Require description for all objects", default=False)
|
|
244
244
|
@click.option(
|
|
245
|
-
"-
|
|
246
|
-
"--schema-name",
|
|
247
|
-
help="Schema name for validation",
|
|
248
|
-
type=click.Choice(["RSP", "default"]),
|
|
249
|
-
default="default",
|
|
245
|
+
"--check-redundant-datatypes", is_flag=True, help="Check for redundant datatypes", default=False
|
|
250
246
|
)
|
|
251
247
|
@click.option(
|
|
252
|
-
"
|
|
248
|
+
"--check-tap-table-indexes",
|
|
249
|
+
is_flag=True,
|
|
250
|
+
help="Check that every table has a unique TAP table index",
|
|
251
|
+
default=False,
|
|
253
252
|
)
|
|
254
253
|
@click.option(
|
|
255
|
-
"
|
|
254
|
+
"--check-tap-principal",
|
|
255
|
+
is_flag=True,
|
|
256
|
+
help="Check that at least one column per table is flagged as TAP principal",
|
|
257
|
+
default=False,
|
|
256
258
|
)
|
|
257
259
|
@click.argument("files", nargs=-1, type=click.File())
|
|
258
260
|
def validate(
|
|
259
|
-
|
|
260
|
-
require_description: bool,
|
|
261
|
+
check_description: bool,
|
|
261
262
|
check_redundant_datatypes: bool,
|
|
263
|
+
check_tap_table_indexes: bool,
|
|
264
|
+
check_tap_principal: bool,
|
|
262
265
|
files: Iterable[io.TextIOBase],
|
|
263
266
|
) -> None:
|
|
264
267
|
"""Validate one or more felis YAML files."""
|
|
265
|
-
schema_class = get_schema(schema_name)
|
|
266
|
-
if schema_name != "default":
|
|
267
|
-
logger.info(f"Using schema '{schema_class.__name__}'")
|
|
268
|
-
|
|
269
268
|
rc = 0
|
|
270
269
|
for file in files:
|
|
271
270
|
file_name = getattr(file, "name", None)
|
|
272
271
|
logger.info(f"Validating {file_name}")
|
|
273
272
|
try:
|
|
274
273
|
data = yaml.load(file, Loader=yaml.SafeLoader)
|
|
275
|
-
|
|
274
|
+
Schema.model_validate(
|
|
276
275
|
data,
|
|
277
276
|
context={
|
|
277
|
+
"check_description": check_description,
|
|
278
278
|
"check_redundant_datatypes": check_redundant_datatypes,
|
|
279
|
-
"
|
|
279
|
+
"check_tap_table_indexes": check_tap_table_indexes,
|
|
280
|
+
"check_tap_principal": check_tap_principal,
|
|
280
281
|
},
|
|
281
282
|
)
|
|
282
283
|
except ValidationError as e:
|
|
@@ -96,7 +96,7 @@ class BaseObject(BaseModel):
|
|
|
96
96
|
def check_description(self, info: ValidationInfo) -> BaseObject:
|
|
97
97
|
"""Check that the description is present if required."""
|
|
98
98
|
context = info.context
|
|
99
|
-
if not context or not context.get("
|
|
99
|
+
if not context or not context.get("check_description", False):
|
|
100
100
|
return self
|
|
101
101
|
if self.description is None or self.description == "":
|
|
102
102
|
raise ValueError("Description is required and must be non-empty")
|
|
@@ -208,12 +208,11 @@ class Column(BaseObject):
|
|
|
208
208
|
raise ValueError(f"Invalid IVOA UCD: {e}")
|
|
209
209
|
return ivoa_ucd
|
|
210
210
|
|
|
211
|
-
@model_validator(mode="
|
|
212
|
-
|
|
213
|
-
def check_units(cls, values: dict[str, Any]) -> dict[str, Any]:
|
|
211
|
+
@model_validator(mode="after")
|
|
212
|
+
def check_units(self) -> Column:
|
|
214
213
|
"""Check that units are valid."""
|
|
215
|
-
fits_unit =
|
|
216
|
-
ivoa_unit =
|
|
214
|
+
fits_unit = self.fits_tunit
|
|
215
|
+
ivoa_unit = self.ivoa_unit
|
|
217
216
|
|
|
218
217
|
if fits_unit and ivoa_unit:
|
|
219
218
|
raise ValueError("Column cannot have both FITS and IVOA units")
|
|
@@ -225,7 +224,7 @@ class Column(BaseObject):
|
|
|
225
224
|
except ValueError as e:
|
|
226
225
|
raise ValueError(f"Invalid unit: {e}")
|
|
227
226
|
|
|
228
|
-
return
|
|
227
|
+
return self
|
|
229
228
|
|
|
230
229
|
@model_validator(mode="before")
|
|
231
230
|
@classmethod
|
|
@@ -250,7 +249,7 @@ class Column(BaseObject):
|
|
|
250
249
|
return values
|
|
251
250
|
|
|
252
251
|
@model_validator(mode="after")
|
|
253
|
-
def
|
|
252
|
+
def check_redundant_datatypes(self, info: ValidationInfo) -> Column:
|
|
254
253
|
"""Check for redundant datatypes on columns."""
|
|
255
254
|
context = info.context
|
|
256
255
|
if not context or not context.get("check_redundant_datatypes", False):
|
|
@@ -419,6 +418,29 @@ class Table(BaseObject):
|
|
|
419
418
|
raise ValueError("Column names must be unique")
|
|
420
419
|
return columns
|
|
421
420
|
|
|
421
|
+
@model_validator(mode="after")
|
|
422
|
+
def check_tap_table_index(self, info: ValidationInfo) -> Table:
|
|
423
|
+
"""Check that the table has a TAP table index."""
|
|
424
|
+
context = info.context
|
|
425
|
+
if not context or not context.get("check_tap_table_indexes", False):
|
|
426
|
+
return self
|
|
427
|
+
if self.tap_table_index is None:
|
|
428
|
+
raise ValueError("Table is missing a TAP table index")
|
|
429
|
+
return self
|
|
430
|
+
|
|
431
|
+
@model_validator(mode="after")
|
|
432
|
+
def check_tap_principal(self, info: ValidationInfo) -> Table:
|
|
433
|
+
"""Check that at least one column is flagged as 'principal' for TAP
|
|
434
|
+
purposes.
|
|
435
|
+
"""
|
|
436
|
+
context = info.context
|
|
437
|
+
if not context or not context.get("check_tap_principal", False):
|
|
438
|
+
return self
|
|
439
|
+
for col in self.columns:
|
|
440
|
+
if col.tap_principal == 1:
|
|
441
|
+
return self
|
|
442
|
+
raise ValueError(f"Table '{self.name}' is missing at least one column designated as 'tap:principal'")
|
|
443
|
+
|
|
422
444
|
|
|
423
445
|
class SchemaVersion(BaseModel):
|
|
424
446
|
"""The version of the schema."""
|
|
@@ -508,6 +530,21 @@ class Schema(BaseObject):
|
|
|
508
530
|
raise ValueError("Table names must be unique")
|
|
509
531
|
return tables
|
|
510
532
|
|
|
533
|
+
@model_validator(mode="after")
|
|
534
|
+
def check_tap_table_indexes(self, info: ValidationInfo) -> Schema:
|
|
535
|
+
"""Check that the TAP table indexes are unique."""
|
|
536
|
+
context = info.context
|
|
537
|
+
if not context or not context.get("check_tap_table_indexes", False):
|
|
538
|
+
return self
|
|
539
|
+
table_indicies = set()
|
|
540
|
+
for table in self.tables:
|
|
541
|
+
table_index = table.tap_table_index
|
|
542
|
+
if table_index is not None:
|
|
543
|
+
if table_index in table_indicies:
|
|
544
|
+
raise ValueError(f"Duplicate 'tap:table_index' value {table_index} found in schema")
|
|
545
|
+
table_indicies.add(table_index)
|
|
546
|
+
return self
|
|
547
|
+
|
|
511
548
|
def _create_id_map(self: Schema) -> Schema:
|
|
512
549
|
"""Create a map of IDs to objects.
|
|
513
550
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: lsst-felis
|
|
3
|
-
Version: 27.2024.
|
|
3
|
+
Version: 27.2024.2500
|
|
4
4
|
Summary: A vocabulary for describing catalogs and acting on those descriptions
|
|
5
5
|
Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
|
|
6
6
|
License: GNU General Public License v3 or later (GPLv3+)
|
|
@@ -10,7 +10,6 @@ python/felis/metadata.py
|
|
|
10
10
|
python/felis/py.typed
|
|
11
11
|
python/felis/tap.py
|
|
12
12
|
python/felis/types.py
|
|
13
|
-
python/felis/validation.py
|
|
14
13
|
python/felis/version.py
|
|
15
14
|
python/felis/db/__init__.py
|
|
16
15
|
python/felis/db/dialects.py
|
|
@@ -28,5 +27,4 @@ tests/test_cli.py
|
|
|
28
27
|
tests/test_datamodel.py
|
|
29
28
|
tests/test_datatypes.py
|
|
30
29
|
tests/test_metadata.py
|
|
31
|
-
tests/test_tap.py
|
|
32
|
-
tests/test_validation.py
|
|
30
|
+
tests/test_tap.py
|
|
@@ -23,8 +23,6 @@ import os
|
|
|
23
23
|
import shutil
|
|
24
24
|
import tempfile
|
|
25
25
|
import unittest
|
|
26
|
-
from collections.abc import MutableMapping
|
|
27
|
-
from typing import Any
|
|
28
26
|
|
|
29
27
|
from click.testing import CliRunner
|
|
30
28
|
|
|
@@ -38,8 +36,6 @@ TEST_MERGE_YAML = os.path.join(TESTDIR, "data", "test-merge.yml")
|
|
|
38
36
|
class CliTestCase(unittest.TestCase):
|
|
39
37
|
"""Tests for CLI commands."""
|
|
40
38
|
|
|
41
|
-
schema_obj: MutableMapping[str, Any] | None = None
|
|
42
|
-
|
|
43
39
|
def setUp(self) -> None:
|
|
44
40
|
self.tmpdir = tempfile.mkdtemp(dir=TESTDIR)
|
|
45
41
|
|
|
@@ -107,24 +103,20 @@ class CliTestCase(unittest.TestCase):
|
|
|
107
103
|
result = runner.invoke(cli, ["validate", TEST_YAML], catch_exceptions=False)
|
|
108
104
|
self.assertEqual(result.exit_code, 0)
|
|
109
105
|
|
|
110
|
-
def
|
|
111
|
-
"""Test validate command with description required."""
|
|
112
|
-
runner = CliRunner()
|
|
113
|
-
try:
|
|
114
|
-
# Wrap this in a try/catch in case an exception is thrown.
|
|
115
|
-
result = runner.invoke(
|
|
116
|
-
cli, ["validate", "--require-description", TEST_YAML], catch_exceptions=False
|
|
117
|
-
)
|
|
118
|
-
except Exception as e:
|
|
119
|
-
# Reraise exception.
|
|
120
|
-
raise e
|
|
121
|
-
|
|
122
|
-
self.assertEqual(result.exit_code, 0)
|
|
123
|
-
|
|
124
|
-
def test_validate_rsp(self) -> None:
|
|
106
|
+
def test_validation_flags(self) -> None:
|
|
125
107
|
"""Test RSP schema type validation."""
|
|
126
108
|
runner = CliRunner()
|
|
127
|
-
result = runner.invoke(
|
|
109
|
+
result = runner.invoke(
|
|
110
|
+
cli,
|
|
111
|
+
[
|
|
112
|
+
"validate",
|
|
113
|
+
"--check-description",
|
|
114
|
+
"--check-tap-principal",
|
|
115
|
+
"--check-tap-table-indexes",
|
|
116
|
+
TEST_YAML,
|
|
117
|
+
],
|
|
118
|
+
catch_exceptions=False,
|
|
119
|
+
)
|
|
128
120
|
self.assertEqual(result.exit_code, 0)
|
|
129
121
|
|
|
130
122
|
|
|
@@ -122,72 +122,42 @@ class ColumnTestCase(unittest.TestCase):
|
|
|
122
122
|
with self.assertRaises(ValidationError):
|
|
123
123
|
Column(**units_data)
|
|
124
124
|
|
|
125
|
-
def
|
|
126
|
-
"""Test the
|
|
127
|
-
|
|
128
|
-
class MockValidationInfo:
|
|
129
|
-
"""Mock context object for passing to validation method."""
|
|
130
|
-
|
|
131
|
-
def __init__(self):
|
|
132
|
-
self.context = {"require_description": True}
|
|
133
|
-
|
|
134
|
-
info = MockValidationInfo()
|
|
135
|
-
|
|
136
|
-
def _check_description(col: Column):
|
|
137
|
-
Schema.check_description(col, info)
|
|
138
|
-
|
|
139
|
-
# Creating a column without a description should throw.
|
|
140
|
-
with self.assertRaises(ValueError):
|
|
141
|
-
_check_description(
|
|
142
|
-
Column(
|
|
143
|
-
**{
|
|
144
|
-
"name": "testColumn",
|
|
145
|
-
"@id": "#test_col_id",
|
|
146
|
-
"datatype": "string",
|
|
147
|
-
}
|
|
148
|
-
)
|
|
149
|
-
)
|
|
150
|
-
|
|
125
|
+
def test_description(self) -> None:
|
|
126
|
+
"""Test the validation of the description attribute."""
|
|
151
127
|
# Creating a column with a description of 'None' should throw.
|
|
152
128
|
with self.assertRaises(ValueError):
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
}
|
|
161
|
-
)
|
|
129
|
+
Column(
|
|
130
|
+
**{
|
|
131
|
+
"name": "testColumn",
|
|
132
|
+
"@id": "#test_col_id",
|
|
133
|
+
"datatype": "string",
|
|
134
|
+
"description": None,
|
|
135
|
+
}
|
|
162
136
|
)
|
|
163
137
|
|
|
164
138
|
# Creating a column with an empty description should throw.
|
|
165
139
|
with self.assertRaises(ValueError):
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
}
|
|
174
|
-
)
|
|
140
|
+
Column(
|
|
141
|
+
**{
|
|
142
|
+
"name": "testColumn",
|
|
143
|
+
"@id": "#test_col_id",
|
|
144
|
+
"datatype": "string",
|
|
145
|
+
"description": "",
|
|
146
|
+
}
|
|
175
147
|
)
|
|
176
148
|
|
|
177
149
|
# Creating a column with a description that is too short should throw.
|
|
178
150
|
with self.assertRaises(ValidationError):
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
}
|
|
187
|
-
)
|
|
151
|
+
Column(
|
|
152
|
+
**{
|
|
153
|
+
"name": "testColumn",
|
|
154
|
+
"@id": "#test_col_id",
|
|
155
|
+
"datatype": "string",
|
|
156
|
+
"description": "xy",
|
|
157
|
+
}
|
|
188
158
|
)
|
|
189
159
|
|
|
190
|
-
def test_values(self):
|
|
160
|
+
def test_values(self) -> None:
|
|
191
161
|
"""Test the `value` field of the `Column` class."""
|
|
192
162
|
|
|
193
163
|
# Define a function to return the default column data
|
|
@@ -549,5 +519,96 @@ class SchemaVersionTest(unittest.TestCase):
|
|
|
549
519
|
self.assertEqual(schema.version.read_compatible, ["1.1.0", "1.1.1"])
|
|
550
520
|
|
|
551
521
|
|
|
522
|
+
class ValidationFlagsTest(unittest.TestCase):
|
|
523
|
+
"""Test the validation flags for the `Schema` class."""
|
|
524
|
+
|
|
525
|
+
def test_check_tap_table_indexes(self) -> None:
|
|
526
|
+
"""Test the `check_tap_table_indexes` validation flag."""
|
|
527
|
+
cxt = {"check_tap_table_indexes": True}
|
|
528
|
+
schema_dict = {
|
|
529
|
+
"name": "testSchema",
|
|
530
|
+
"id": "#test_schema_id",
|
|
531
|
+
"tables": [
|
|
532
|
+
{
|
|
533
|
+
"name": "test_table",
|
|
534
|
+
"id": "#test_table_id",
|
|
535
|
+
"columns": [{"name": "test_col", "id": "#test_col", "datatype": "int"}],
|
|
536
|
+
}
|
|
537
|
+
],
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
# Creating a schema without a TAP table index should throw.
|
|
541
|
+
with self.assertRaises(ValidationError):
|
|
542
|
+
Schema.model_validate(schema_dict, context=cxt)
|
|
543
|
+
|
|
544
|
+
# Creating a schema with a TAP table index should not throw.
|
|
545
|
+
schema_dict["tables"][0]["tap_table_index"] = 1
|
|
546
|
+
Schema.model_validate(schema_dict, context=cxt)
|
|
547
|
+
schema_dict["tables"].append(
|
|
548
|
+
{
|
|
549
|
+
"name": "test_table2",
|
|
550
|
+
"id": "#test_table2",
|
|
551
|
+
"tap_table_index": 1,
|
|
552
|
+
"columns": [{"name": "test_col2", "id": "#test_col2", "datatype": "int"}],
|
|
553
|
+
}
|
|
554
|
+
)
|
|
555
|
+
|
|
556
|
+
# Creating a schema with a duplicate TAP table index should throw.
|
|
557
|
+
with self.assertRaises(ValidationError):
|
|
558
|
+
Schema.model_validate(schema_dict, context=cxt)
|
|
559
|
+
|
|
560
|
+
# Multiple, unique TAP table indexes should not throw.
|
|
561
|
+
schema_dict["tables"][1]["tap_table_index"] = 2
|
|
562
|
+
Schema.model_validate(schema_dict, context=cxt)
|
|
563
|
+
|
|
564
|
+
def test_check_tap_principal(self) -> None:
|
|
565
|
+
"""Test the validation flags for the `Schema` class."""
|
|
566
|
+
cxt = {"check_tap_principal": True}
|
|
567
|
+
schema_dict = {
|
|
568
|
+
"name": "testSchema",
|
|
569
|
+
"id": "#test_schema_id",
|
|
570
|
+
"tables": [
|
|
571
|
+
{
|
|
572
|
+
"name": "test_table",
|
|
573
|
+
"id": "#test_table_id",
|
|
574
|
+
"columns": [{"name": "test_col", "id": "#test_col", "datatype": "int"}],
|
|
575
|
+
}
|
|
576
|
+
],
|
|
577
|
+
}
|
|
578
|
+
|
|
579
|
+
# Creating a table without a TAP table principal column should throw.
|
|
580
|
+
with self.assertRaises(ValidationError):
|
|
581
|
+
Schema.model_validate(schema_dict, context=cxt)
|
|
582
|
+
|
|
583
|
+
# Creating a table with a TAP table principal column should not throw.
|
|
584
|
+
schema_dict["tables"][0]["columns"][0]["tap_principal"] = 1
|
|
585
|
+
Schema.model_validate(schema_dict, context=cxt)
|
|
586
|
+
|
|
587
|
+
def test_check_description(self) -> None:
|
|
588
|
+
"""Test the `check_description` flag for the `Column` class."""
|
|
589
|
+
cxt = {"check_description": True}
|
|
590
|
+
schema_dict = {
|
|
591
|
+
"name": "testSchema",
|
|
592
|
+
"id": "#test_schema_id",
|
|
593
|
+
"tables": [
|
|
594
|
+
{
|
|
595
|
+
"name": "test_table",
|
|
596
|
+
"id": "#test_table_id",
|
|
597
|
+
"columns": [{"name": "test_col", "id": "#test_col", "datatype": "int"}],
|
|
598
|
+
}
|
|
599
|
+
],
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
# Creating a schema without object descriptions should throw.
|
|
603
|
+
with self.assertRaises(ValidationError):
|
|
604
|
+
Schema.model_validate(schema_dict, context=cxt)
|
|
605
|
+
|
|
606
|
+
# Creating a schema with object descriptions should not throw.
|
|
607
|
+
schema_dict["description"] = "Test schema"
|
|
608
|
+
schema_dict["tables"][0]["description"] = "Test table"
|
|
609
|
+
schema_dict["tables"][0]["columns"][0]["description"] = "Test column"
|
|
610
|
+
Schema.model_validate(schema_dict, context=cxt)
|
|
611
|
+
|
|
612
|
+
|
|
552
613
|
if __name__ == "__main__":
|
|
553
614
|
unittest.main()
|
|
@@ -1,103 +0,0 @@
|
|
|
1
|
-
# This file is part of felis.
|
|
2
|
-
#
|
|
3
|
-
# Developed for the LSST Data Management System.
|
|
4
|
-
# This product includes software developed by the LSST Project
|
|
5
|
-
# (https://www.lsst.org).
|
|
6
|
-
# See the COPYRIGHT file at the top-level directory of this distribution
|
|
7
|
-
# for details of code ownership.
|
|
8
|
-
#
|
|
9
|
-
# This program is free software: you can redistribute it and/or modify
|
|
10
|
-
# it under the terms of the GNU General Public License as published by
|
|
11
|
-
# the Free Software Foundation, either version 3 of the License, or
|
|
12
|
-
# (at your option) any later version.
|
|
13
|
-
#
|
|
14
|
-
# This program is distributed in the hope that it will be useful,
|
|
15
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
16
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
17
|
-
# GNU General Public License for more details.
|
|
18
|
-
#
|
|
19
|
-
# You should have received a copy of the GNU General Public License
|
|
20
|
-
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
21
|
-
|
|
22
|
-
from __future__ import annotations
|
|
23
|
-
|
|
24
|
-
import logging
|
|
25
|
-
from collections.abc import Sequence
|
|
26
|
-
from typing import Any
|
|
27
|
-
|
|
28
|
-
from pydantic import Field, model_validator
|
|
29
|
-
|
|
30
|
-
from .datamodel import Column, DescriptionStr, Schema, Table
|
|
31
|
-
|
|
32
|
-
logger = logging.getLogger(__name__)
|
|
33
|
-
|
|
34
|
-
__all__ = ["RspColumn", "RspSchema", "RspTable", "get_schema"]
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
class RspColumn(Column):
|
|
38
|
-
"""Column for RSP data validation."""
|
|
39
|
-
|
|
40
|
-
description: DescriptionStr
|
|
41
|
-
"""Redefine description to make it required."""
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
class RspTable(Table):
|
|
45
|
-
"""Table for RSP data validation.
|
|
46
|
-
|
|
47
|
-
The list of columns is overridden to use RspColumn instead of Column.
|
|
48
|
-
|
|
49
|
-
Tables for the RSP must have a TAP table index and a valid description.
|
|
50
|
-
"""
|
|
51
|
-
|
|
52
|
-
description: DescriptionStr
|
|
53
|
-
"""Redefine description to make it required."""
|
|
54
|
-
|
|
55
|
-
tap_table_index: int = Field(..., alias="tap:table_index")
|
|
56
|
-
"""Redefine the TAP_SCHEMA table index so that it is required."""
|
|
57
|
-
|
|
58
|
-
columns: Sequence[RspColumn]
|
|
59
|
-
"""Redefine columns to include RSP validation."""
|
|
60
|
-
|
|
61
|
-
@model_validator(mode="after") # type: ignore[arg-type]
|
|
62
|
-
@classmethod
|
|
63
|
-
def check_tap_principal(cls: Any, tbl: RspTable) -> RspTable:
|
|
64
|
-
"""Check that at least one column is flagged as 'principal' for
|
|
65
|
-
TAP purposes.
|
|
66
|
-
"""
|
|
67
|
-
for col in tbl.columns:
|
|
68
|
-
if col.tap_principal == 1:
|
|
69
|
-
return tbl
|
|
70
|
-
raise ValueError(f"Table '{tbl.name}' is missing at least one column designated as 'tap:principal'")
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
class RspSchema(Schema):
|
|
74
|
-
"""Schema for RSP data validation.
|
|
75
|
-
|
|
76
|
-
TAP table indexes must be unique across all tables.
|
|
77
|
-
"""
|
|
78
|
-
|
|
79
|
-
tables: Sequence[RspTable]
|
|
80
|
-
"""Redefine tables to include RSP validation."""
|
|
81
|
-
|
|
82
|
-
@model_validator(mode="after") # type: ignore[arg-type]
|
|
83
|
-
@classmethod
|
|
84
|
-
def check_tap_table_indexes(cls: Any, sch: RspSchema) -> RspSchema:
|
|
85
|
-
"""Check that the TAP table indexes are unique."""
|
|
86
|
-
table_indicies = set()
|
|
87
|
-
for table in sch.tables:
|
|
88
|
-
table_index = table.tap_table_index
|
|
89
|
-
if table_index is not None:
|
|
90
|
-
if table_index in table_indicies:
|
|
91
|
-
raise ValueError(f"Duplicate 'tap:table_index' value {table_index} found in schema")
|
|
92
|
-
table_indicies.add(table_index)
|
|
93
|
-
return sch
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
def get_schema(schema_name: str) -> type[Schema]:
|
|
97
|
-
"""Get the schema class for the given name."""
|
|
98
|
-
if schema_name == "default":
|
|
99
|
-
return Schema
|
|
100
|
-
elif schema_name == "RSP":
|
|
101
|
-
return RspSchema
|
|
102
|
-
else:
|
|
103
|
-
raise ValueError(f"Unknown schema name '{schema_name}'")
|
|
@@ -1,233 +0,0 @@
|
|
|
1
|
-
# This file is part of felis.
|
|
2
|
-
#
|
|
3
|
-
# Developed for the LSST Data Management System.
|
|
4
|
-
# This product includes software developed by the LSST Project
|
|
5
|
-
# (https://www.lsst.org).
|
|
6
|
-
# See the COPYRIGHT file at the top-level directory of this distribution
|
|
7
|
-
# for details of code ownership.
|
|
8
|
-
#
|
|
9
|
-
# This program is free software: you can redistribute it and/or modify
|
|
10
|
-
# it under the terms of the GNU General Public License as published by
|
|
11
|
-
# the Free Software Foundation, either version 3 of the License, or
|
|
12
|
-
# (at your option) any later version.
|
|
13
|
-
#
|
|
14
|
-
# This program is distributed in the hope that it will be useful,
|
|
15
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
16
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
17
|
-
# GNU General Public License for more details.
|
|
18
|
-
#
|
|
19
|
-
# You should have received a copy of the GNU General Public License
|
|
20
|
-
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
21
|
-
|
|
22
|
-
import unittest
|
|
23
|
-
|
|
24
|
-
from pydantic import ValidationError
|
|
25
|
-
|
|
26
|
-
from felis.datamodel import Schema
|
|
27
|
-
from felis.validation import RspColumn, RspSchema, RspTable, get_schema
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
class RSPSchemaTestCase(unittest.TestCase):
|
|
31
|
-
"""Test validation of RSP schema data."""
|
|
32
|
-
|
|
33
|
-
def test_rsp_validation(self) -> None:
|
|
34
|
-
# Creating an empty RSP column should throw an exception.
|
|
35
|
-
with self.assertRaises(ValidationError):
|
|
36
|
-
RspColumn()
|
|
37
|
-
|
|
38
|
-
# Missing column description should throw an exception.
|
|
39
|
-
with self.assertRaises(ValidationError):
|
|
40
|
-
RspColumn(name="testColumn", id="#test_col_id", datatype="string")
|
|
41
|
-
|
|
42
|
-
# A column description with only whitespace should throw an exception.
|
|
43
|
-
with self.assertRaises(ValidationError):
|
|
44
|
-
RspColumn(name="testColumn", id="#test_col_id", datatype="string", description=" ")
|
|
45
|
-
|
|
46
|
-
# A column description of `None` should throw an exception.
|
|
47
|
-
with self.assertRaises(ValidationError):
|
|
48
|
-
RspColumn(name="testColumn", id="#test_col_id", datatype="string", description=None)
|
|
49
|
-
|
|
50
|
-
# A column description which is not long enough should throw.
|
|
51
|
-
with self.assertRaises(ValidationError):
|
|
52
|
-
RspColumn(name="testColumn", id="#test_col_id", datatype="string", description="xy")
|
|
53
|
-
|
|
54
|
-
# Creating a valid RSP column should not throw an exception.
|
|
55
|
-
col = RspColumn(
|
|
56
|
-
**{
|
|
57
|
-
"name": "testColumn",
|
|
58
|
-
"@id": "#test_col_id",
|
|
59
|
-
"datatype": "string",
|
|
60
|
-
"description": "test column",
|
|
61
|
-
"length": 256,
|
|
62
|
-
"tap:principal": 1,
|
|
63
|
-
}
|
|
64
|
-
)
|
|
65
|
-
|
|
66
|
-
# Creating an empty RSP table should throw an exception.
|
|
67
|
-
with self.assertRaises(ValidationError):
|
|
68
|
-
RspTable()
|
|
69
|
-
|
|
70
|
-
# Missing table description should throw an exception.
|
|
71
|
-
with self.assertRaises(ValidationError):
|
|
72
|
-
RspTable(**{"name": "testTable", "@id": "#test_table_id", "tap:table_index": 1}, columns=[col])
|
|
73
|
-
|
|
74
|
-
# A table description with only whitespace should throw an exception.
|
|
75
|
-
with self.assertRaises(ValidationError):
|
|
76
|
-
RspTable(
|
|
77
|
-
**{"name": "testTable", "@id": "#test_table_id", "tap:table_index": 1, "description": " "},
|
|
78
|
-
columns=[col],
|
|
79
|
-
)
|
|
80
|
-
|
|
81
|
-
# A table description of `None` should throw an exception.
|
|
82
|
-
with self.assertRaises(ValidationError):
|
|
83
|
-
RspTable(
|
|
84
|
-
**{"name": "testTable", "@id": "#test_table_id", "tap:table_index": 1, "description": None},
|
|
85
|
-
columns=[col],
|
|
86
|
-
)
|
|
87
|
-
|
|
88
|
-
# Missing TAP table index should throw an exception.
|
|
89
|
-
with self.assertRaises(ValidationError):
|
|
90
|
-
RspTable(name="testTable", id="#test_table_id", description="test table", columns=[col])
|
|
91
|
-
|
|
92
|
-
# Missing at least one column flagged as TAP principal should throw an
|
|
93
|
-
# exception.
|
|
94
|
-
with self.assertRaises(ValidationError):
|
|
95
|
-
RspTable(
|
|
96
|
-
**{
|
|
97
|
-
"name": "testTable",
|
|
98
|
-
"@id": "#test_table_id",
|
|
99
|
-
"description": "test table",
|
|
100
|
-
"tap:table_index": 1,
|
|
101
|
-
"columns": [
|
|
102
|
-
RspColumn(
|
|
103
|
-
**{
|
|
104
|
-
"name": "testColumn",
|
|
105
|
-
"@id": "#test_col_id",
|
|
106
|
-
"datatype": "string",
|
|
107
|
-
"description": "test column",
|
|
108
|
-
}
|
|
109
|
-
)
|
|
110
|
-
],
|
|
111
|
-
}
|
|
112
|
-
)
|
|
113
|
-
|
|
114
|
-
# Creating a valid RSP table should not throw an exception.
|
|
115
|
-
tbl = RspTable(
|
|
116
|
-
**{
|
|
117
|
-
"name": "testTable",
|
|
118
|
-
"@id": "#test_table_id",
|
|
119
|
-
"description": "test table",
|
|
120
|
-
"tap:table_index": 1,
|
|
121
|
-
"columns": [col],
|
|
122
|
-
}
|
|
123
|
-
)
|
|
124
|
-
|
|
125
|
-
# Creating an empty RSP table schema throw an exception.
|
|
126
|
-
with self.assertRaises(ValidationError):
|
|
127
|
-
RspSchema(tables=[tbl])
|
|
128
|
-
|
|
129
|
-
# Creating a schema with duplicate TAP table indices should throw an
|
|
130
|
-
# exception.
|
|
131
|
-
with self.assertRaises(ValidationError):
|
|
132
|
-
RspSchema(
|
|
133
|
-
**{"name": "testSchema", "@id": "#test_schema_id", "description": "test schema"},
|
|
134
|
-
tables=[
|
|
135
|
-
RspTable(
|
|
136
|
-
**{
|
|
137
|
-
"name": "testTable1",
|
|
138
|
-
"@id": "#test_table1_id",
|
|
139
|
-
"description": "test table",
|
|
140
|
-
"tap:table_index": 1,
|
|
141
|
-
"columns": [
|
|
142
|
-
RspColumn(
|
|
143
|
-
**{
|
|
144
|
-
"name": "testColumn",
|
|
145
|
-
"@id": "#test_col1_id",
|
|
146
|
-
"datatype": "string",
|
|
147
|
-
"description": "test column",
|
|
148
|
-
"tap:principal": 1,
|
|
149
|
-
}
|
|
150
|
-
)
|
|
151
|
-
],
|
|
152
|
-
}
|
|
153
|
-
),
|
|
154
|
-
RspTable(
|
|
155
|
-
**{
|
|
156
|
-
"name": "testTable2",
|
|
157
|
-
"@id": "#test_table2_id",
|
|
158
|
-
"description": "test table",
|
|
159
|
-
"tap:table_index": 1,
|
|
160
|
-
"columns": [
|
|
161
|
-
RspColumn(
|
|
162
|
-
**{
|
|
163
|
-
"name": "testColumn",
|
|
164
|
-
"@id": "#test_col2_id",
|
|
165
|
-
"datatype": "string",
|
|
166
|
-
"description": "test column",
|
|
167
|
-
"tap:principal": 1,
|
|
168
|
-
}
|
|
169
|
-
)
|
|
170
|
-
],
|
|
171
|
-
}
|
|
172
|
-
),
|
|
173
|
-
],
|
|
174
|
-
)
|
|
175
|
-
|
|
176
|
-
# Creating a valid schema with multiple tables having unique TAP table
|
|
177
|
-
# indices should not throw a exception.
|
|
178
|
-
RspSchema(
|
|
179
|
-
**{"name": "testSchema", "@id": "#test_schema_id", "description": "test schema"},
|
|
180
|
-
tables=[
|
|
181
|
-
RspTable(
|
|
182
|
-
**{
|
|
183
|
-
"name": "testTable",
|
|
184
|
-
"@id": "#test_table_id",
|
|
185
|
-
"description": "test table",
|
|
186
|
-
"tap:table_index": 1,
|
|
187
|
-
"columns": [
|
|
188
|
-
RspColumn(
|
|
189
|
-
**{
|
|
190
|
-
"name": "testColumn",
|
|
191
|
-
"@id": "#test_col1_id",
|
|
192
|
-
"datatype": "string",
|
|
193
|
-
"description": "test column",
|
|
194
|
-
"tap:principal": 1,
|
|
195
|
-
"length": 256,
|
|
196
|
-
}
|
|
197
|
-
)
|
|
198
|
-
],
|
|
199
|
-
}
|
|
200
|
-
),
|
|
201
|
-
RspTable(
|
|
202
|
-
**{
|
|
203
|
-
"name": "testTable2",
|
|
204
|
-
"@id": "#test_table2_id",
|
|
205
|
-
"description": "test table",
|
|
206
|
-
"tap:table_index": 2,
|
|
207
|
-
"columns": [
|
|
208
|
-
RspColumn(
|
|
209
|
-
**{
|
|
210
|
-
"name": "testColumn",
|
|
211
|
-
"@id": "#test_col2_id",
|
|
212
|
-
"datatype": "string",
|
|
213
|
-
"description": "test column",
|
|
214
|
-
"tap:principal": 1,
|
|
215
|
-
"length": 256,
|
|
216
|
-
}
|
|
217
|
-
)
|
|
218
|
-
],
|
|
219
|
-
}
|
|
220
|
-
),
|
|
221
|
-
],
|
|
222
|
-
)
|
|
223
|
-
|
|
224
|
-
def test_get_schema(self) -> None:
|
|
225
|
-
"""Test that get_schema() returns the correct schema types."""
|
|
226
|
-
rsp_schema: RspSchema = get_schema("RSP")
|
|
227
|
-
self.assertEqual(rsp_schema.__name__, "RspSchema")
|
|
228
|
-
|
|
229
|
-
default_schema: Schema = get_schema("default")
|
|
230
|
-
self.assertEqual(default_schema.__name__, "Schema")
|
|
231
|
-
|
|
232
|
-
with self.assertRaises(ValueError):
|
|
233
|
-
get_schema("invalid_schema")
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{lsst_felis-27.2024.2400 → lsst_felis-27.2024.2500}/python/lsst_felis.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
{lsst_felis-27.2024.2400 → lsst_felis-27.2024.2500}/python/lsst_felis.egg-info/entry_points.txt
RENAMED
|
File without changes
|
|
File without changes
|
{lsst_felis-27.2024.2400 → lsst_felis-27.2024.2500}/python/lsst_felis.egg-info/top_level.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|