lsst-felis 27.0.0rc3__tar.gz → 27.2024.1900__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of lsst-felis might be problematic. Click here for more details.
- {lsst_felis-27.0.0rc3/python/lsst_felis.egg-info → lsst_felis-27.2024.1900}/PKG-INFO +5 -3
- {lsst_felis-27.0.0rc3 → lsst_felis-27.2024.1900}/pyproject.toml +22 -19
- {lsst_felis-27.0.0rc3 → lsst_felis-27.2024.1900}/python/felis/__init__.py +0 -25
- {lsst_felis-27.0.0rc3 → lsst_felis-27.2024.1900}/python/felis/cli.py +23 -195
- {lsst_felis-27.0.0rc3 → lsst_felis-27.2024.1900}/python/felis/datamodel.py +81 -46
- {lsst_felis-27.0.0rc3 → lsst_felis-27.2024.1900}/python/felis/db/sqltypes.py +3 -3
- {lsst_felis-27.0.0rc3 → lsst_felis-27.2024.1900}/python/felis/metadata.py +21 -12
- {lsst_felis-27.0.0rc3 → lsst_felis-27.2024.1900}/python/felis/tap.py +67 -80
- {lsst_felis-27.0.0rc3 → lsst_felis-27.2024.1900}/python/felis/validation.py +1 -1
- lsst_felis-27.2024.1900/python/felis/version.py +2 -0
- {lsst_felis-27.0.0rc3 → lsst_felis-27.2024.1900/python/lsst_felis.egg-info}/PKG-INFO +5 -3
- {lsst_felis-27.0.0rc3 → lsst_felis-27.2024.1900}/python/lsst_felis.egg-info/SOURCES.txt +0 -7
- {lsst_felis-27.0.0rc3 → lsst_felis-27.2024.1900}/python/lsst_felis.egg-info/requires.txt +3 -1
- {lsst_felis-27.0.0rc3 → lsst_felis-27.2024.1900}/tests/test_cli.py +0 -26
- {lsst_felis-27.0.0rc3 → lsst_felis-27.2024.1900}/tests/test_datamodel.py +83 -7
- {lsst_felis-27.0.0rc3 → lsst_felis-27.2024.1900}/tests/test_datatypes.py +5 -6
- {lsst_felis-27.0.0rc3 → lsst_felis-27.2024.1900}/tests/test_tap.py +5 -10
- {lsst_felis-27.0.0rc3 → lsst_felis-27.2024.1900}/tests/test_validation.py +3 -0
- lsst_felis-27.0.0rc3/python/felis/check.py +0 -381
- lsst_felis-27.0.0rc3/python/felis/simple.py +0 -424
- lsst_felis-27.0.0rc3/python/felis/utils.py +0 -100
- lsst_felis-27.0.0rc3/python/felis/version.py +0 -2
- lsst_felis-27.0.0rc3/python/felis/visitor.py +0 -180
- lsst_felis-27.0.0rc3/tests/test_check.py +0 -235
- lsst_felis-27.0.0rc3/tests/test_simple.py +0 -178
- lsst_felis-27.0.0rc3/tests/test_utils.py +0 -79
- {lsst_felis-27.0.0rc3 → lsst_felis-27.2024.1900}/COPYRIGHT +0 -0
- {lsst_felis-27.0.0rc3 → lsst_felis-27.2024.1900}/LICENSE +0 -0
- {lsst_felis-27.0.0rc3 → lsst_felis-27.2024.1900}/README.rst +0 -0
- {lsst_felis-27.0.0rc3 → lsst_felis-27.2024.1900}/python/felis/db/__init__.py +0 -0
- {lsst_felis-27.0.0rc3 → lsst_felis-27.2024.1900}/python/felis/db/_variants.py +0 -0
- {lsst_felis-27.0.0rc3 → lsst_felis-27.2024.1900}/python/felis/py.typed +0 -0
- {lsst_felis-27.0.0rc3 → lsst_felis-27.2024.1900}/python/felis/types.py +0 -0
- {lsst_felis-27.0.0rc3 → lsst_felis-27.2024.1900}/python/lsst_felis.egg-info/dependency_links.txt +0 -0
- {lsst_felis-27.0.0rc3 → lsst_felis-27.2024.1900}/python/lsst_felis.egg-info/entry_points.txt +0 -0
- {lsst_felis-27.0.0rc3 → lsst_felis-27.2024.1900}/python/lsst_felis.egg-info/top_level.txt +0 -0
- {lsst_felis-27.0.0rc3 → lsst_felis-27.2024.1900}/python/lsst_felis.egg-info/zip-safe +0 -0
- {lsst_felis-27.0.0rc3 → lsst_felis-27.2024.1900}/setup.cfg +0 -0
- {lsst_felis-27.0.0rc3 → lsst_felis-27.2024.1900}/tests/test_metadata.py +0 -0
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: lsst-felis
|
|
3
|
-
Version: 27.
|
|
3
|
+
Version: 27.2024.1900
|
|
4
4
|
Summary: A vocabulary for describing catalogs and acting on those descriptions
|
|
5
5
|
Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
|
|
6
6
|
License: GNU General Public License v3 or later (GPLv3+)
|
|
7
|
-
Project-URL: Homepage, https://
|
|
7
|
+
Project-URL: Homepage, https://felis.lsst.io
|
|
8
|
+
Project-URL: Source, https://github.com/lsst/felis
|
|
8
9
|
Keywords: lsst
|
|
9
10
|
Classifier: Intended Audience :: Science/Research
|
|
10
11
|
Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
|
|
@@ -21,8 +22,9 @@ Requires-Dist: astropy>=4
|
|
|
21
22
|
Requires-Dist: sqlalchemy>=1.4
|
|
22
23
|
Requires-Dist: click>=7
|
|
23
24
|
Requires-Dist: pyyaml>=6
|
|
24
|
-
Requires-Dist: pyld>=2
|
|
25
25
|
Requires-Dist: pydantic<3,>=2
|
|
26
26
|
Requires-Dist: lsst-utils
|
|
27
27
|
Provides-Extra: test
|
|
28
28
|
Requires-Dist: pytest>=3.2; extra == "test"
|
|
29
|
+
Provides-Extra: dev
|
|
30
|
+
Requires-Dist: documenteer[guide]; extra == "dev"
|
|
@@ -25,7 +25,6 @@ dependencies = [
|
|
|
25
25
|
"sqlalchemy >= 1.4",
|
|
26
26
|
"click >= 7",
|
|
27
27
|
"pyyaml >= 6",
|
|
28
|
-
"pyld >= 2",
|
|
29
28
|
"pydantic >= 2, < 3",
|
|
30
29
|
"lsst-utils"
|
|
31
30
|
]
|
|
@@ -33,12 +32,16 @@ requires-python = ">=3.11.0"
|
|
|
33
32
|
dynamic = ["version"]
|
|
34
33
|
|
|
35
34
|
[project.urls]
|
|
36
|
-
|
|
35
|
+
Homepage = "https://felis.lsst.io"
|
|
36
|
+
Source = "https://github.com/lsst/felis"
|
|
37
37
|
|
|
38
38
|
[project.optional-dependencies]
|
|
39
39
|
test = [
|
|
40
40
|
"pytest >= 3.2"
|
|
41
41
|
]
|
|
42
|
+
dev = [
|
|
43
|
+
"documenteer[guide]"
|
|
44
|
+
]
|
|
42
45
|
|
|
43
46
|
[tool.pytest.ini_options]
|
|
44
47
|
|
|
@@ -110,12 +113,27 @@ line_length = 110
|
|
|
110
113
|
write_to = "python/felis/version.py"
|
|
111
114
|
|
|
112
115
|
[tool.ruff]
|
|
116
|
+
line-length = 110
|
|
117
|
+
target-version = "py311"
|
|
113
118
|
exclude = [
|
|
114
119
|
"__init__.py",
|
|
115
120
|
"lex.py",
|
|
116
121
|
"yacc.py",
|
|
117
122
|
]
|
|
123
|
+
|
|
124
|
+
[tool.ruff.lint]
|
|
118
125
|
ignore = [
|
|
126
|
+
"D100",
|
|
127
|
+
"D102",
|
|
128
|
+
"D104",
|
|
129
|
+
"D105",
|
|
130
|
+
"D107",
|
|
131
|
+
"D200",
|
|
132
|
+
"D203",
|
|
133
|
+
"D205",
|
|
134
|
+
"D213",
|
|
135
|
+
"D400",
|
|
136
|
+
"D413",
|
|
119
137
|
"N802",
|
|
120
138
|
"N803",
|
|
121
139
|
"N806",
|
|
@@ -123,30 +141,15 @@ ignore = [
|
|
|
123
141
|
"N815",
|
|
124
142
|
"N816",
|
|
125
143
|
"N999",
|
|
126
|
-
"
|
|
127
|
-
"D105",
|
|
128
|
-
"D102",
|
|
129
|
-
"D104",
|
|
130
|
-
"D100",
|
|
131
|
-
"D200",
|
|
132
|
-
"D205",
|
|
133
|
-
"D203",
|
|
134
|
-
"D213",
|
|
135
|
-
"D400",
|
|
144
|
+
"UP007", # Allow UNION in type annotation
|
|
136
145
|
]
|
|
137
|
-
line-length = 110
|
|
138
146
|
select = [
|
|
139
147
|
"E", # pycodestyle
|
|
140
148
|
"F", # pycodestyle
|
|
141
149
|
"N", # pep8-naming
|
|
142
150
|
"W", # pycodestyle
|
|
143
151
|
"D", # pydocstyle
|
|
144
|
-
|
|
145
|
-
target-version = "py311"
|
|
146
|
-
# Commented out to suppress "unused noqa" in jenkins which has older ruff not
|
|
147
|
-
# generating E721.
|
|
148
|
-
extend-select = [
|
|
149
|
-
"RUF100", # Warn about unused noqa
|
|
152
|
+
"UP", # pyupgrade
|
|
150
153
|
]
|
|
151
154
|
|
|
152
155
|
[tool.pydeps]
|
|
@@ -19,29 +19,4 @@
|
|
|
19
19
|
# You should have received a copy of the GNU General Public License
|
|
20
20
|
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
21
21
|
|
|
22
|
-
from . import types
|
|
23
|
-
from .check import *
|
|
24
22
|
from .version import *
|
|
25
|
-
from .visitor import *
|
|
26
|
-
|
|
27
|
-
DEFAULT_CONTEXT = {
|
|
28
|
-
"@vocab": "http://lsst.org/felis/",
|
|
29
|
-
"mysql": "http://mysql.com/",
|
|
30
|
-
"postgres": "http://posgresql.org/",
|
|
31
|
-
"oracle": "http://oracle.com/database/",
|
|
32
|
-
"sqlite": "http://sqlite.org/",
|
|
33
|
-
"fits": "http://fits.gsfc.nasa.gov/FITS/4.0/",
|
|
34
|
-
"ivoa": "http://ivoa.net/rdf/",
|
|
35
|
-
"votable": "http://ivoa.net/rdf/VOTable/",
|
|
36
|
-
"tap": "http://ivoa.net/documents/TAP/",
|
|
37
|
-
"tables": {"@container": "@list", "@type": "@id", "@id": "felis:Table"},
|
|
38
|
-
"columns": {"@container": "@list", "@type": "@id", "@id": "felis:Column"},
|
|
39
|
-
"constraints": {"@container": "@list", "@type": "@id"},
|
|
40
|
-
"indexes": {"@container": "@list", "@type": "@id", "@id": "felis:Index"},
|
|
41
|
-
"referencedColumns": {"@container": "@list", "@type": "@id"},
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
DEFAULT_FRAME = {
|
|
45
|
-
"@context": DEFAULT_CONTEXT,
|
|
46
|
-
"@type": "felis:Schema",
|
|
47
|
-
}
|
|
@@ -22,25 +22,20 @@
|
|
|
22
22
|
from __future__ import annotations
|
|
23
23
|
|
|
24
24
|
import io
|
|
25
|
-
import json
|
|
26
25
|
import logging
|
|
27
|
-
import
|
|
28
|
-
from
|
|
29
|
-
from typing import IO, Any
|
|
26
|
+
from collections.abc import Iterable
|
|
27
|
+
from typing import IO
|
|
30
28
|
|
|
31
29
|
import click
|
|
32
30
|
import yaml
|
|
33
31
|
from pydantic import ValidationError
|
|
34
|
-
from pyld import jsonld
|
|
35
32
|
from sqlalchemy.engine import Engine, create_engine, create_mock_engine, make_url
|
|
36
33
|
from sqlalchemy.engine.mock import MockConnection
|
|
37
34
|
|
|
38
|
-
from . import
|
|
39
|
-
from .check import CheckingVisitor
|
|
35
|
+
from . import __version__
|
|
40
36
|
from .datamodel import Schema
|
|
41
37
|
from .metadata import DatabaseContext, InsertDump, MetaDataBuilder
|
|
42
38
|
from .tap import Tap11Base, TapLoadingVisitor, init_tables
|
|
43
|
-
from .utils import ReorderingVisitor
|
|
44
39
|
from .validation import get_schema
|
|
45
40
|
|
|
46
41
|
logger = logging.getLogger("felis")
|
|
@@ -183,6 +178,7 @@ def init_tap(
|
|
|
183
178
|
@click.option("--tap-columns-table", help="Alt Table Name for TAP_SCHEMA.columns")
|
|
184
179
|
@click.option("--tap-keys-table", help="Alt Table Name for TAP_SCHEMA.keys")
|
|
185
180
|
@click.option("--tap-key-columns-table", help="Alt Table Name for TAP_SCHEMA.key_columns")
|
|
181
|
+
@click.option("--tap-schema-index", type=int, help="TAP_SCHEMA index of the schema")
|
|
186
182
|
@click.argument("file", type=click.File())
|
|
187
183
|
def load_tap(
|
|
188
184
|
engine_url: str,
|
|
@@ -196,6 +192,7 @@ def load_tap(
|
|
|
196
192
|
tap_columns_table: str,
|
|
197
193
|
tap_keys_table: str,
|
|
198
194
|
tap_key_columns_table: str,
|
|
195
|
+
tap_schema_index: int,
|
|
199
196
|
file: io.TextIOBase,
|
|
200
197
|
) -> None:
|
|
201
198
|
"""Load TAP metadata from a Felis FILE.
|
|
@@ -203,28 +200,8 @@ def load_tap(
|
|
|
203
200
|
This command loads the associated TAP metadata from a Felis FILE
|
|
204
201
|
to the TAP_SCHEMA tables.
|
|
205
202
|
"""
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
if isinstance(top_level_object, dict):
|
|
209
|
-
schema_obj = top_level_object
|
|
210
|
-
if "@graph" not in schema_obj:
|
|
211
|
-
schema_obj["@type"] = "felis:Schema"
|
|
212
|
-
schema_obj["@context"] = DEFAULT_CONTEXT
|
|
213
|
-
elif isinstance(top_level_object, list):
|
|
214
|
-
schema_obj = {"@context": DEFAULT_CONTEXT, "@graph": top_level_object}
|
|
215
|
-
else:
|
|
216
|
-
logger.error("Schema object not of recognizable type")
|
|
217
|
-
raise click.exceptions.Exit(1)
|
|
218
|
-
|
|
219
|
-
normalized = _normalize(schema_obj, embed="@always")
|
|
220
|
-
if len(normalized["@graph"]) > 1 and (schema_name or catalog_name):
|
|
221
|
-
logger.error("--schema-name and --catalog-name incompatible with multiple schemas")
|
|
222
|
-
raise click.exceptions.Exit(1)
|
|
223
|
-
|
|
224
|
-
# Force normalized["@graph"] to a list, which is what happens when there's
|
|
225
|
-
# multiple schemas
|
|
226
|
-
if isinstance(normalized["@graph"], dict):
|
|
227
|
-
normalized["@graph"] = [normalized["@graph"]]
|
|
203
|
+
yaml_data = yaml.load(file, Loader=yaml.SafeLoader)
|
|
204
|
+
schema = Schema.model_validate(yaml_data)
|
|
228
205
|
|
|
229
206
|
tap_tables = init_tables(
|
|
230
207
|
tap_schema_name,
|
|
@@ -243,126 +220,28 @@ def load_tap(
|
|
|
243
220
|
# In Memory SQLite - Mostly used to test
|
|
244
221
|
Tap11Base.metadata.create_all(engine)
|
|
245
222
|
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
223
|
+
tap_visitor = TapLoadingVisitor(
|
|
224
|
+
engine,
|
|
225
|
+
catalog_name=catalog_name,
|
|
226
|
+
schema_name=schema_name,
|
|
227
|
+
tap_tables=tap_tables,
|
|
228
|
+
tap_schema_index=tap_schema_index,
|
|
229
|
+
)
|
|
230
|
+
tap_visitor.visit_schema(schema)
|
|
254
231
|
else:
|
|
255
232
|
_insert_dump = InsertDump()
|
|
256
233
|
conn = create_mock_engine(make_url(engine_url), executor=_insert_dump.dump, paramstyle="pyformat")
|
|
257
234
|
# After the engine is created, update the executor with the dialect
|
|
258
235
|
_insert_dump.dialect = conn.dialect
|
|
259
236
|
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
@cli.command("modify-tap")
|
|
271
|
-
@click.option("--start-schema-at", type=int, help="Rewrite index for tap:schema_index", default=0)
|
|
272
|
-
@click.argument("files", nargs=-1, type=click.File())
|
|
273
|
-
def modify_tap(start_schema_at: int, files: Iterable[io.TextIOBase]) -> None:
|
|
274
|
-
"""Modify TAP information in Felis schema FILES.
|
|
275
|
-
|
|
276
|
-
This command has some utilities to aid in rewriting felis FILES
|
|
277
|
-
in specific ways. It will write out a merged version of these files.
|
|
278
|
-
"""
|
|
279
|
-
count = 0
|
|
280
|
-
graph = []
|
|
281
|
-
for file in files:
|
|
282
|
-
schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
|
|
283
|
-
if "@graph" not in schema_obj:
|
|
284
|
-
schema_obj["@type"] = "felis:Schema"
|
|
285
|
-
schema_obj["@context"] = DEFAULT_CONTEXT
|
|
286
|
-
schema_index = schema_obj.get("tap:schema_index")
|
|
287
|
-
if not schema_index or (schema_index and schema_index > start_schema_at):
|
|
288
|
-
schema_index = start_schema_at + count
|
|
289
|
-
count += 1
|
|
290
|
-
schema_obj["tap:schema_index"] = schema_index
|
|
291
|
-
graph.extend(jsonld.flatten(schema_obj))
|
|
292
|
-
merged = {"@context": DEFAULT_CONTEXT, "@graph": graph}
|
|
293
|
-
normalized = _normalize(merged, embed="@always")
|
|
294
|
-
_dump(normalized)
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
@cli.command("basic-check")
|
|
298
|
-
@click.argument("file", type=click.File())
|
|
299
|
-
def basic_check(file: io.TextIOBase) -> None:
|
|
300
|
-
"""Perform a basic check on a felis FILE.
|
|
301
|
-
|
|
302
|
-
This performs a very check to ensure required fields are
|
|
303
|
-
populated and basic semantics are okay. It does not ensure semantics
|
|
304
|
-
are valid for other commands like create-all or load-tap.
|
|
305
|
-
"""
|
|
306
|
-
schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
|
|
307
|
-
schema_obj["@type"] = "felis:Schema"
|
|
308
|
-
# Force Context and Schema Type
|
|
309
|
-
schema_obj["@context"] = DEFAULT_CONTEXT
|
|
310
|
-
check_visitor = CheckingVisitor()
|
|
311
|
-
check_visitor.visit_schema(schema_obj)
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
@cli.command("normalize")
|
|
315
|
-
@click.argument("file", type=click.File())
|
|
316
|
-
def normalize(file: io.TextIOBase) -> None:
|
|
317
|
-
"""Normalize a Felis FILE.
|
|
318
|
-
|
|
319
|
-
Takes a felis schema FILE, expands it (resolving the full URLs),
|
|
320
|
-
then compacts it, and finally produces output in the canonical
|
|
321
|
-
format.
|
|
322
|
-
|
|
323
|
-
(This is most useful in some debugging scenarios)
|
|
324
|
-
|
|
325
|
-
See Also :
|
|
326
|
-
|
|
327
|
-
https://json-ld.org/spec/latest/json-ld/#expanded-document-form
|
|
328
|
-
https://json-ld.org/spec/latest/json-ld/#compacted-document-form
|
|
329
|
-
"""
|
|
330
|
-
schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
|
|
331
|
-
schema_obj["@type"] = "felis:Schema"
|
|
332
|
-
# Force Context and Schema Type
|
|
333
|
-
schema_obj["@context"] = DEFAULT_CONTEXT
|
|
334
|
-
expanded = jsonld.expand(schema_obj)
|
|
335
|
-
normalized = _normalize(expanded, embed="@always")
|
|
336
|
-
_dump(normalized)
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
@cli.command("merge")
|
|
340
|
-
@click.argument("files", nargs=-1, type=click.File())
|
|
341
|
-
def merge(files: Iterable[io.TextIOBase]) -> None:
|
|
342
|
-
"""Merge a set of Felis FILES.
|
|
343
|
-
|
|
344
|
-
This will expand out the felis FILES so that it is easy to
|
|
345
|
-
override values (using @Id), then normalize to a single
|
|
346
|
-
output.
|
|
347
|
-
"""
|
|
348
|
-
graph = []
|
|
349
|
-
for file in files:
|
|
350
|
-
schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
|
|
351
|
-
if "@graph" not in schema_obj:
|
|
352
|
-
schema_obj["@type"] = "felis:Schema"
|
|
353
|
-
schema_obj["@context"] = DEFAULT_CONTEXT
|
|
354
|
-
graph.extend(jsonld.flatten(schema_obj))
|
|
355
|
-
updated_map: MutableMapping[str, Any] = {}
|
|
356
|
-
for item in graph:
|
|
357
|
-
_id = item["@id"]
|
|
358
|
-
item_to_update = updated_map.get(_id, item)
|
|
359
|
-
if item_to_update and item_to_update != item:
|
|
360
|
-
logger.debug(f"Overwriting {_id}")
|
|
361
|
-
item_to_update.update(item)
|
|
362
|
-
updated_map[_id] = item_to_update
|
|
363
|
-
merged = {"@context": DEFAULT_CONTEXT, "@graph": list(updated_map.values())}
|
|
364
|
-
normalized = _normalize(merged, embed="@always")
|
|
365
|
-
_dump(normalized)
|
|
237
|
+
tap_visitor = TapLoadingVisitor.from_mock_connection(
|
|
238
|
+
conn,
|
|
239
|
+
catalog_name=catalog_name,
|
|
240
|
+
schema_name=schema_name,
|
|
241
|
+
tap_tables=tap_tables,
|
|
242
|
+
tap_schema_index=tap_schema_index,
|
|
243
|
+
)
|
|
244
|
+
tap_visitor.visit_schema(schema)
|
|
366
245
|
|
|
367
246
|
|
|
368
247
|
@cli.command("validate")
|
|
@@ -411,56 +290,5 @@ def validate(
|
|
|
411
290
|
raise click.exceptions.Exit(rc)
|
|
412
291
|
|
|
413
292
|
|
|
414
|
-
@cli.command("dump-json")
|
|
415
|
-
@click.option("-x", "--expanded", is_flag=True, help="Extended schema before dumping.")
|
|
416
|
-
@click.option("-f", "--framed", is_flag=True, help="Frame schema before dumping.")
|
|
417
|
-
@click.option("-c", "--compacted", is_flag=True, help="Compact schema before dumping.")
|
|
418
|
-
@click.option("-g", "--graph", is_flag=True, help="Pass graph option to compact.")
|
|
419
|
-
@click.argument("file", type=click.File())
|
|
420
|
-
def dump_json(
|
|
421
|
-
file: io.TextIOBase,
|
|
422
|
-
expanded: bool = False,
|
|
423
|
-
compacted: bool = False,
|
|
424
|
-
framed: bool = False,
|
|
425
|
-
graph: bool = False,
|
|
426
|
-
) -> None:
|
|
427
|
-
"""Dump JSON representation using various JSON-LD options."""
|
|
428
|
-
schema_obj = yaml.load(file, Loader=yaml.SafeLoader)
|
|
429
|
-
schema_obj["@type"] = "felis:Schema"
|
|
430
|
-
# Force Context and Schema Type
|
|
431
|
-
schema_obj["@context"] = DEFAULT_CONTEXT
|
|
432
|
-
|
|
433
|
-
if expanded:
|
|
434
|
-
schema_obj = jsonld.expand(schema_obj)
|
|
435
|
-
if framed:
|
|
436
|
-
schema_obj = jsonld.frame(schema_obj, DEFAULT_FRAME)
|
|
437
|
-
if compacted:
|
|
438
|
-
options = {}
|
|
439
|
-
if graph:
|
|
440
|
-
options["graph"] = True
|
|
441
|
-
schema_obj = jsonld.compact(schema_obj, DEFAULT_CONTEXT, options=options)
|
|
442
|
-
json.dump(schema_obj, sys.stdout, indent=4)
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
def _dump(obj: Mapping[str, Any]) -> None:
|
|
446
|
-
class OrderedDumper(yaml.Dumper):
|
|
447
|
-
pass
|
|
448
|
-
|
|
449
|
-
def _dict_representer(dumper: yaml.Dumper, data: Any) -> Any:
|
|
450
|
-
return dumper.represent_mapping(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, data.items())
|
|
451
|
-
|
|
452
|
-
OrderedDumper.add_representer(dict, _dict_representer)
|
|
453
|
-
print(yaml.dump(obj, Dumper=OrderedDumper, default_flow_style=False))
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
def _normalize(schema_obj: Mapping[str, Any], embed: str = "@last") -> MutableMapping[str, Any]:
|
|
457
|
-
framed = jsonld.frame(schema_obj, DEFAULT_FRAME, options=dict(embed=embed))
|
|
458
|
-
compacted = jsonld.compact(framed, DEFAULT_CONTEXT, options=dict(graph=True))
|
|
459
|
-
graph = compacted["@graph"]
|
|
460
|
-
graph = [ReorderingVisitor(add_type=True).visit_schema(schema_obj) for schema_obj in graph]
|
|
461
|
-
compacted["@graph"] = graph if len(graph) > 1 else graph[0]
|
|
462
|
-
return compacted
|
|
463
|
-
|
|
464
|
-
|
|
465
293
|
if __name__ == "__main__":
|
|
466
294
|
cli()
|
|
@@ -37,7 +37,7 @@ from sqlalchemy.engine.interfaces import Dialect
|
|
|
37
37
|
from sqlalchemy.types import TypeEngine
|
|
38
38
|
|
|
39
39
|
from .db.sqltypes import get_type_func
|
|
40
|
-
from .types import FelisType
|
|
40
|
+
from .types import Boolean, Byte, Char, Double, FelisType, Float, Int, Long, Short, String, Text, Unicode
|
|
41
41
|
|
|
42
42
|
logger = logging.getLogger(__name__)
|
|
43
43
|
|
|
@@ -93,18 +93,20 @@ class BaseObject(BaseModel):
|
|
|
93
93
|
description: DescriptionStr | None = None
|
|
94
94
|
"""A description of the database object."""
|
|
95
95
|
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
96
|
+
votable_utype: str | None = Field(None, alias="votable:utype")
|
|
97
|
+
"""The VOTable utype (usage-specific or unique type) of the object."""
|
|
98
|
+
|
|
99
|
+
@model_validator(mode="after")
|
|
100
|
+
def check_description(self, info: ValidationInfo) -> BaseObject:
|
|
99
101
|
"""Check that the description is present if required."""
|
|
100
102
|
context = info.context
|
|
101
103
|
if not context or not context.get("require_description", False):
|
|
102
|
-
return
|
|
103
|
-
if
|
|
104
|
+
return self
|
|
105
|
+
if self.description is None or self.description == "":
|
|
104
106
|
raise ValueError("Description is required and must be non-empty")
|
|
105
|
-
if len(
|
|
107
|
+
if len(self.description) < DESCR_MIN_LENGTH:
|
|
106
108
|
raise ValueError(f"Description must be at least {DESCR_MIN_LENGTH} characters long")
|
|
107
|
-
return
|
|
109
|
+
return self
|
|
108
110
|
|
|
109
111
|
|
|
110
112
|
class DataType(StrEnum):
|
|
@@ -176,18 +178,13 @@ class Column(BaseObject):
|
|
|
176
178
|
datatype: DataType
|
|
177
179
|
"""The datatype of the column."""
|
|
178
180
|
|
|
179
|
-
length: int | None = None
|
|
181
|
+
length: int | None = Field(None, gt=0)
|
|
180
182
|
"""The length of the column."""
|
|
181
183
|
|
|
182
|
-
nullable: bool
|
|
183
|
-
"""Whether the column can be ``NULL``.
|
|
184
|
-
|
|
185
|
-
If `None`, this value was not set explicitly in the YAML data. In this
|
|
186
|
-
case, it will be set to `False` for columns with numeric types and `True`
|
|
187
|
-
otherwise.
|
|
188
|
-
"""
|
|
184
|
+
nullable: bool = True
|
|
185
|
+
"""Whether the column can be ``NULL``."""
|
|
189
186
|
|
|
190
|
-
value:
|
|
187
|
+
value: str | int | float | bool | None = None
|
|
191
188
|
"""The default value of the column."""
|
|
192
189
|
|
|
193
190
|
autoincrement: bool | None = None
|
|
@@ -222,12 +219,33 @@ class Column(BaseObject):
|
|
|
222
219
|
"""TAP_SCHEMA indication that this column is defined by an IVOA standard.
|
|
223
220
|
"""
|
|
224
221
|
|
|
225
|
-
votable_utype: str | None = Field(None, alias="votable:utype")
|
|
226
|
-
"""The VOTable utype (usage-specific or unique type) of the column."""
|
|
227
|
-
|
|
228
222
|
votable_xtype: str | None = Field(None, alias="votable:xtype")
|
|
229
223
|
"""The VOTable xtype (extended type) of the column."""
|
|
230
224
|
|
|
225
|
+
votable_datatype: str | None = Field(None, alias="votable:datatype")
|
|
226
|
+
"""The VOTable datatype of the column."""
|
|
227
|
+
|
|
228
|
+
@model_validator(mode="after")
|
|
229
|
+
def check_value(self) -> Column:
|
|
230
|
+
"""Check that the default value is valid."""
|
|
231
|
+
if (value := self.value) is not None:
|
|
232
|
+
if value is not None and self.autoincrement is True:
|
|
233
|
+
raise ValueError("Column cannot have both a default value and be autoincremented")
|
|
234
|
+
felis_type = FelisType.felis_type(self.datatype)
|
|
235
|
+
if felis_type.is_numeric:
|
|
236
|
+
if felis_type in (Byte, Short, Int, Long) and not isinstance(value, int):
|
|
237
|
+
raise ValueError("Default value must be an int for integer type columns")
|
|
238
|
+
elif felis_type in (Float, Double) and not isinstance(value, float):
|
|
239
|
+
raise ValueError("Default value must be a decimal number for float and double columns")
|
|
240
|
+
elif felis_type in (String, Char, Unicode, Text):
|
|
241
|
+
if not isinstance(value, str):
|
|
242
|
+
raise ValueError("Default value must be a string for string columns")
|
|
243
|
+
if not len(value):
|
|
244
|
+
raise ValueError("Default value must be a non-empty string for string columns")
|
|
245
|
+
elif felis_type is Boolean and not isinstance(value, bool):
|
|
246
|
+
raise ValueError("Default value must be a boolean for boolean columns")
|
|
247
|
+
return self
|
|
248
|
+
|
|
231
249
|
@field_validator("ivoa_ucd")
|
|
232
250
|
@classmethod
|
|
233
251
|
def check_ivoa_ucd(cls, ivoa_ucd: str) -> str:
|
|
@@ -258,52 +276,69 @@ class Column(BaseObject):
|
|
|
258
276
|
|
|
259
277
|
return values
|
|
260
278
|
|
|
261
|
-
@model_validator(mode="
|
|
279
|
+
@model_validator(mode="before")
|
|
262
280
|
@classmethod
|
|
263
|
-
def
|
|
281
|
+
def check_length(cls, values: dict[str, Any]) -> dict[str, Any]:
|
|
282
|
+
"""Check that a valid length is provided for sized types."""
|
|
283
|
+
datatype = values.get("datatype")
|
|
284
|
+
if datatype is None:
|
|
285
|
+
# Skip this validation if datatype is not provided
|
|
286
|
+
return values
|
|
287
|
+
length = values.get("length")
|
|
288
|
+
felis_type = FelisType.felis_type(datatype)
|
|
289
|
+
if felis_type.is_sized and length is None:
|
|
290
|
+
raise ValueError(
|
|
291
|
+
f"Length must be provided for type '{datatype}'"
|
|
292
|
+
+ (f" in column '{values['@id']}'" if "@id" in values else "")
|
|
293
|
+
)
|
|
294
|
+
elif not felis_type.is_sized and length is not None:
|
|
295
|
+
logger.warning(
|
|
296
|
+
f"The datatype '{datatype}' does not support a specified length"
|
|
297
|
+
+ (f" in column '{values['@id']}'" if "@id" in values else "")
|
|
298
|
+
)
|
|
299
|
+
return values
|
|
300
|
+
|
|
301
|
+
@model_validator(mode="after")
|
|
302
|
+
def check_datatypes(self, info: ValidationInfo) -> Column:
|
|
264
303
|
"""Check for redundant datatypes on columns."""
|
|
265
304
|
context = info.context
|
|
266
305
|
if not context or not context.get("check_redundant_datatypes", False):
|
|
267
|
-
return
|
|
268
|
-
if all(getattr(
|
|
269
|
-
return
|
|
306
|
+
return self
|
|
307
|
+
if all(getattr(self, f"{dialect}:datatype", None) is not None for dialect in _DIALECTS.keys()):
|
|
308
|
+
return self
|
|
270
309
|
|
|
271
|
-
datatype =
|
|
272
|
-
length: int | None =
|
|
310
|
+
datatype = self.datatype
|
|
311
|
+
length: int | None = self.length or None
|
|
273
312
|
|
|
274
313
|
datatype_func = get_type_func(datatype)
|
|
275
314
|
felis_type = FelisType.felis_type(datatype)
|
|
276
315
|
if felis_type.is_sized:
|
|
277
|
-
|
|
278
|
-
datatype_obj = datatype_func(length)
|
|
279
|
-
else:
|
|
280
|
-
raise ValueError(f"Length must be provided for sized type '{datatype}' in column '{col.id}'")
|
|
316
|
+
datatype_obj = datatype_func(length)
|
|
281
317
|
else:
|
|
282
318
|
datatype_obj = datatype_func()
|
|
283
319
|
|
|
284
320
|
for dialect_name, dialect in _DIALECTS.items():
|
|
285
321
|
db_annotation = f"{dialect_name}_datatype"
|
|
286
|
-
if datatype_string :=
|
|
322
|
+
if datatype_string := self.model_dump().get(db_annotation):
|
|
287
323
|
db_datatype_obj = string_to_typeengine(datatype_string, dialect, length)
|
|
288
324
|
if datatype_obj.compile(dialect) == db_datatype_obj.compile(dialect):
|
|
289
325
|
raise ValueError(
|
|
290
|
-
"'{}: {}' is
|
|
291
|
-
db_annotation,
|
|
326
|
+
"'{}: {}' is a redundant override of 'datatype: {}' in column '{}'{}".format(
|
|
327
|
+
db_annotation,
|
|
328
|
+
datatype_string,
|
|
329
|
+
self.datatype,
|
|
330
|
+
self.id,
|
|
331
|
+
"" if length is None else f" with length {length}",
|
|
292
332
|
)
|
|
293
333
|
)
|
|
294
334
|
else:
|
|
295
335
|
logger.debug(
|
|
296
|
-
"
|
|
297
|
-
|
|
298
|
-
)
|
|
299
|
-
|
|
300
|
-
logger.debug(
|
|
301
|
-
"Compiled datatype '{}' with {} compiled override '{}'".format(
|
|
302
|
-
datatype_obj.compile(dialect), dialect_name, db_datatype_obj.compile(dialect)
|
|
303
|
-
)
|
|
336
|
+
f"Type override of 'datatype: {self.datatype}' "
|
|
337
|
+
f"with '{db_annotation}: {datatype_string}' in column '{self.id}' "
|
|
338
|
+
f"compiled to '{datatype_obj.compile(dialect)}' and "
|
|
339
|
+
f"'{db_datatype_obj.compile(dialect)}'"
|
|
304
340
|
)
|
|
305
|
-
|
|
306
|
-
return col
|
|
341
|
+
return self
|
|
307
342
|
|
|
308
343
|
|
|
309
344
|
class Constraint(BaseObject):
|
|
@@ -458,7 +493,7 @@ class SchemaIdVisitor:
|
|
|
458
493
|
|
|
459
494
|
def __init__(self) -> None:
|
|
460
495
|
"""Create a new SchemaVisitor."""
|
|
461
|
-
self.schema:
|
|
496
|
+
self.schema: Schema | None = None
|
|
462
497
|
self.duplicates: set[str] = set()
|
|
463
498
|
|
|
464
499
|
def add(self, obj: BaseObject) -> None:
|
|
@@ -471,7 +506,7 @@ class SchemaIdVisitor:
|
|
|
471
506
|
else:
|
|
472
507
|
self.schema.id_map[obj_id] = obj
|
|
473
508
|
|
|
474
|
-
def visit_schema(self, schema:
|
|
509
|
+
def visit_schema(self, schema: Schema) -> None:
|
|
475
510
|
"""Visit the schema object that was added during initialization.
|
|
476
511
|
|
|
477
512
|
This will set an internal variable pointing to the schema object.
|
|
@@ -20,8 +20,8 @@
|
|
|
20
20
|
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
21
21
|
|
|
22
22
|
import builtins
|
|
23
|
-
from collections.abc import Mapping
|
|
24
|
-
from typing import Any
|
|
23
|
+
from collections.abc import Callable, Mapping
|
|
24
|
+
from typing import Any
|
|
25
25
|
|
|
26
26
|
from sqlalchemy import SmallInteger, types
|
|
27
27
|
from sqlalchemy.dialects import mysql, oracle, postgresql
|
|
@@ -47,7 +47,7 @@ def compile_tinyint(type_: Any, compiler: Any, **kw: Any) -> str:
|
|
|
47
47
|
|
|
48
48
|
_TypeMap = Mapping[str, types.TypeEngine | type[types.TypeEngine]]
|
|
49
49
|
|
|
50
|
-
boolean_map: _TypeMap = {MYSQL: mysql.
|
|
50
|
+
boolean_map: _TypeMap = {MYSQL: mysql.BOOLEAN, ORACLE: oracle.NUMBER(1), POSTGRES: postgresql.BOOLEAN()}
|
|
51
51
|
|
|
52
52
|
byte_map: _TypeMap = {
|
|
53
53
|
MYSQL: mysql.TINYINT(),
|