linkml 1.8.3__py3-none-any.whl → 1.8.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- linkml/generators/__init__.py +2 -0
- linkml/generators/common/ifabsent_processor.py +98 -21
- linkml/generators/common/naming.py +106 -0
- linkml/generators/docgen/index.md.jinja2 +6 -6
- linkml/generators/docgen.py +80 -21
- linkml/generators/golanggen.py +3 -1
- linkml/generators/graphqlgen.py +34 -2
- linkml/generators/jsonschemagen.py +4 -2
- linkml/generators/owlgen.py +36 -17
- linkml/generators/projectgen.py +13 -11
- linkml/generators/pydanticgen/array.py +340 -56
- linkml/generators/pydanticgen/build.py +4 -2
- linkml/generators/pydanticgen/pydanticgen.py +35 -16
- linkml/generators/pydanticgen/template.py +119 -3
- linkml/generators/pydanticgen/templates/imports.py.jinja +11 -3
- linkml/generators/pydanticgen/templates/module.py.jinja +1 -3
- linkml/generators/pydanticgen/templates/validator.py.jinja +2 -2
- linkml/generators/python/python_ifabsent_processor.py +1 -1
- linkml/generators/pythongen.py +135 -31
- linkml/generators/shaclgen.py +34 -10
- linkml/generators/sparqlgen.py +3 -1
- linkml/generators/sqlalchemygen.py +5 -3
- linkml/generators/sqltablegen.py +4 -2
- linkml/generators/typescriptgen.py +13 -6
- linkml/linter/linter.py +2 -1
- linkml/transformers/logical_model_transformer.py +3 -3
- linkml/transformers/relmodel_transformer.py +18 -4
- linkml/utils/converter.py +3 -1
- linkml/utils/exceptions.py +11 -0
- linkml/utils/execute_tutorial.py +22 -20
- linkml/utils/generator.py +6 -4
- linkml/utils/mergeutils.py +4 -2
- linkml/utils/schema_fixer.py +5 -5
- linkml/utils/schemaloader.py +5 -3
- linkml/utils/sqlutils.py +3 -1
- linkml/validator/plugins/pydantic_validation_plugin.py +1 -1
- linkml/validators/jsonschemavalidator.py +3 -1
- linkml/validators/sparqlvalidator.py +5 -3
- linkml/workspaces/example_runner.py +3 -1
- {linkml-1.8.3.dist-info → linkml-1.8.5.dist-info}/METADATA +3 -1
- {linkml-1.8.3.dist-info → linkml-1.8.5.dist-info}/RECORD +44 -42
- {linkml-1.8.3.dist-info → linkml-1.8.5.dist-info}/LICENSE +0 -0
- {linkml-1.8.3.dist-info → linkml-1.8.5.dist-info}/WHEEL +0 -0
- {linkml-1.8.3.dist-info → linkml-1.8.5.dist-info}/entry_points.txt +0 -0
linkml/generators/shaclgen.py
CHANGED
@@ -5,19 +5,21 @@ from typing import Callable, List
|
|
5
5
|
|
6
6
|
import click
|
7
7
|
from jsonasobj2 import JsonObj, as_dict
|
8
|
-
from linkml_runtime.linkml_model.meta import ElementName
|
8
|
+
from linkml_runtime.linkml_model.meta import ClassDefinition, ElementName
|
9
9
|
from linkml_runtime.utils.formatutils import underscore
|
10
10
|
from linkml_runtime.utils.schemaview import SchemaView
|
11
11
|
from linkml_runtime.utils.yamlutils import TypedNode, extended_float, extended_int, extended_str
|
12
12
|
from rdflib import BNode, Graph, Literal, URIRef
|
13
13
|
from rdflib.collection import Collection
|
14
|
-
from rdflib.namespace import RDF,
|
14
|
+
from rdflib.namespace import RDF, SH, XSD
|
15
15
|
|
16
16
|
from linkml._version import __version__
|
17
17
|
from linkml.generators.shacl.shacl_data_type import ShaclDataType
|
18
18
|
from linkml.generators.shacl.shacl_ifabsent_processor import ShaclIfAbsentProcessor
|
19
19
|
from linkml.utils.generator import Generator, shared_arguments
|
20
20
|
|
21
|
+
logger = logging.getLogger(__name__)
|
22
|
+
|
21
23
|
|
22
24
|
@dataclass
|
23
25
|
class ShaclGenerator(Generator):
|
@@ -74,9 +76,6 @@ class ShaclGenerator(Generator):
|
|
74
76
|
shape_pv(RDF.type, SH.NodeShape)
|
75
77
|
shape_pv(SH.targetClass, class_uri) # TODO
|
76
78
|
|
77
|
-
if c.is_a:
|
78
|
-
shape_pv(RDFS.subClassOf, URIRef(sv.get_uri(c.is_a, expand=True)))
|
79
|
-
|
80
79
|
if self.closed:
|
81
80
|
if c.mixin or c.abstract:
|
82
81
|
shape_pv(SH.closed, Literal(False))
|
@@ -88,9 +87,9 @@ class ShaclGenerator(Generator):
|
|
88
87
|
shape_pv(SH.name, Literal(c.title))
|
89
88
|
if c.description is not None:
|
90
89
|
shape_pv(SH.description, Literal(c.description))
|
91
|
-
|
92
|
-
|
93
|
-
|
90
|
+
|
91
|
+
shape_pv(SH.ignoredProperties, self._build_ignored_properties(g, c))
|
92
|
+
|
94
93
|
if c.annotations and self.include_annotations:
|
95
94
|
self._add_annotations(shape_pv, c)
|
96
95
|
order = 0
|
@@ -130,7 +129,7 @@ class ShaclGenerator(Generator):
|
|
130
129
|
# slot definition, as both are mapped to sh:in in SHACL
|
131
130
|
if s.equals_string or s.equals_string_in:
|
132
131
|
error = "'equals_string'/'equals_string_in' and 'any_of' are mutually exclusive"
|
133
|
-
raise ValueError(f'{TypedNode.yaml_loc(s, suffix="")} {error}')
|
132
|
+
raise ValueError(f'{TypedNode.yaml_loc(str(s), suffix="")} {error}')
|
134
133
|
|
135
134
|
or_node = BNode()
|
136
135
|
prop_pv(SH["or"], or_node)
|
@@ -244,7 +243,7 @@ class ShaclGenerator(Generator):
|
|
244
243
|
if rt.annotations and self.include_annotations:
|
245
244
|
self._add_annotations(func, rt)
|
246
245
|
else:
|
247
|
-
|
246
|
+
logger.error(f"No URI for type {rt.name}")
|
248
247
|
|
249
248
|
def _and_equals_string(self, g: Graph, func: Callable, values: List) -> None:
|
250
249
|
pv_node = BNode()
|
@@ -301,6 +300,31 @@ class ShaclGenerator(Generator):
|
|
301
300
|
)
|
302
301
|
func(SH["in"], pv_node)
|
303
302
|
|
303
|
+
def _build_ignored_properties(self, g: Graph, c: ClassDefinition) -> BNode:
|
304
|
+
def collect_child_properties(class_name: str, output: set) -> None:
|
305
|
+
for childName in self.schemaview.class_children(class_name, imports=True, mixins=False, is_a=True):
|
306
|
+
output.update(
|
307
|
+
{
|
308
|
+
URIRef(self.schemaview.get_uri(prop, expand=True))
|
309
|
+
for prop in self.schemaview.class_slots(childName)
|
310
|
+
}
|
311
|
+
)
|
312
|
+
collect_child_properties(childName, output)
|
313
|
+
|
314
|
+
child_properties = set()
|
315
|
+
collect_child_properties(c.name, child_properties)
|
316
|
+
|
317
|
+
class_slot_uris = {
|
318
|
+
URIRef(self.schemaview.get_uri(prop, expand=True)) for prop in self.schemaview.class_slots(c.name)
|
319
|
+
}
|
320
|
+
ignored_properties = child_properties.difference(class_slot_uris)
|
321
|
+
|
322
|
+
list_node = BNode()
|
323
|
+
ignored_properties.add(RDF.type)
|
324
|
+
Collection(g, list_node, list(ignored_properties))
|
325
|
+
|
326
|
+
return list_node
|
327
|
+
|
304
328
|
|
305
329
|
def add_simple_data_type(func: Callable, r: ElementName) -> None:
|
306
330
|
for datatype in list(ShaclDataType):
|
linkml/generators/sparqlgen.py
CHANGED
@@ -14,6 +14,8 @@ from linkml_runtime.utils.schemaview import SchemaView
|
|
14
14
|
from linkml._version import __version__
|
15
15
|
from linkml.utils.generator import Generator, shared_arguments
|
16
16
|
|
17
|
+
logger = logging.getLogger(__name__)
|
18
|
+
|
17
19
|
template = """
|
18
20
|
{% for pfxn, pfx in schema.prefixes.items() -%}
|
19
21
|
PREFIX {{pfxn}}: <{{pfx.prefix_reference}}>
|
@@ -150,7 +152,7 @@ class SparqlGenerator(Generator):
|
|
150
152
|
extra = ""
|
151
153
|
if named_graphs is not None:
|
152
154
|
extra += f'FILTER( ?graph in ( {",".join(named_graphs)} ))'
|
153
|
-
|
155
|
+
logger.info(f"Named Graphs = {named_graphs} // extra={extra}")
|
154
156
|
if limit is not None and isinstance(limit, int):
|
155
157
|
limit = f"LIMIT {limit}"
|
156
158
|
else:
|
@@ -21,6 +21,8 @@ from linkml.generators.sqltablegen import SQLTableGenerator
|
|
21
21
|
from linkml.transformers.relmodel_transformer import ForeignKeyPolicy, RelationalModelTransformer
|
22
22
|
from linkml.utils.generator import Generator, shared_arguments
|
23
23
|
|
24
|
+
logger = logging.getLogger(__name__)
|
25
|
+
|
24
26
|
|
25
27
|
class TemplateEnum(Enum):
|
26
28
|
DECLARATIVE = "declarative"
|
@@ -84,7 +86,7 @@ class SQLAlchemyGenerator(Generator):
|
|
84
86
|
template_obj = Template(template_str)
|
85
87
|
if model_path is None:
|
86
88
|
model_path = self.schema.name
|
87
|
-
|
89
|
+
logger.info(f"Package for dataclasses == {model_path}")
|
88
90
|
backrefs = defaultdict(list)
|
89
91
|
for m in tr_result.mappings:
|
90
92
|
backrefs[m.source_class].append(m)
|
@@ -113,7 +115,7 @@ class SQLAlchemyGenerator(Generator):
|
|
113
115
|
is_join_table=lambda c: any(tag for tag in c.annotations.keys() if tag == "linkml:derived_from"),
|
114
116
|
classes=rel_schema_classes_ordered,
|
115
117
|
)
|
116
|
-
|
118
|
+
logger.debug(f"# Generated code:\n{code}")
|
117
119
|
return code
|
118
120
|
|
119
121
|
def serialize(self, **kwargs) -> str:
|
@@ -173,7 +175,7 @@ class SQLAlchemyGenerator(Generator):
|
|
173
175
|
def skip(cls: ClassDefinition) -> bool:
|
174
176
|
is_skip = len(cls.attributes) == 0
|
175
177
|
if is_skip:
|
176
|
-
|
178
|
+
logger.error(f"SKIPPING: {cls.name}")
|
177
179
|
return is_skip
|
178
180
|
|
179
181
|
# TODO: move this
|
linkml/generators/sqltablegen.py
CHANGED
@@ -16,6 +16,8 @@ from linkml.transformers.relmodel_transformer import ForeignKeyPolicy, Relationa
|
|
16
16
|
from linkml.utils.generator import Generator, shared_arguments
|
17
17
|
from linkml.utils.schemaloader import SchemaLoader
|
18
18
|
|
19
|
+
logger = logging.getLogger(__name__)
|
20
|
+
|
19
21
|
|
20
22
|
class SqlNamingPolicy(Enum):
|
21
23
|
preserve = "preserve"
|
@@ -262,12 +264,12 @@ class SQLTableGenerator(Generator):
|
|
262
264
|
elif range is None:
|
263
265
|
return Text()
|
264
266
|
else:
|
265
|
-
|
267
|
+
logger.error(f"Unknown range: {range} for {slot.name} = {slot.range}")
|
266
268
|
return Text()
|
267
269
|
if range_base in RANGEMAP:
|
268
270
|
return RANGEMAP[range_base]
|
269
271
|
else:
|
270
|
-
|
272
|
+
logger.error(f"UNKNOWN range base: {range_base} for {slot.name} = {slot.range}")
|
271
273
|
return Text()
|
272
274
|
|
273
275
|
@staticmethod
|
@@ -19,6 +19,9 @@ from linkml._version import __version__
|
|
19
19
|
from linkml.generators.oocodegen import OOCodeGenerator
|
20
20
|
from linkml.utils.generator import shared_arguments
|
21
21
|
|
22
|
+
logger = logging.getLogger(__name__)
|
23
|
+
|
24
|
+
|
22
25
|
type_map = {
|
23
26
|
"str": "string",
|
24
27
|
"int": "number",
|
@@ -120,7 +123,7 @@ class TypescriptGenerator(OOCodeGenerator):
|
|
120
123
|
gen_type_utils: bool = False
|
121
124
|
include_induced_slots: bool = False
|
122
125
|
|
123
|
-
def serialize(self) -> str:
|
126
|
+
def serialize(self, output=None) -> str:
|
124
127
|
"""Serialize a schema to typescript string"""
|
125
128
|
|
126
129
|
sv: SchemaView = self.schemaview
|
@@ -132,6 +135,9 @@ class TypescriptGenerator(OOCodeGenerator):
|
|
132
135
|
view=self.schemaview,
|
133
136
|
enums=enums,
|
134
137
|
)
|
138
|
+
if output is not None:
|
139
|
+
with open(output, "w") as out:
|
140
|
+
out.write(out_str)
|
135
141
|
return out_str
|
136
142
|
|
137
143
|
@staticmethod
|
@@ -207,7 +213,7 @@ class TypescriptGenerator(OOCodeGenerator):
|
|
207
213
|
elif t.typeof and t.typeof in type_map:
|
208
214
|
tsrange = type_map[t.typeof]
|
209
215
|
else:
|
210
|
-
|
216
|
+
logger.warning(f"Unknown type.base: {t.name}")
|
211
217
|
if slot.multivalued:
|
212
218
|
tsrange = f"{tsrange}[]"
|
213
219
|
return tsrange
|
@@ -241,7 +247,7 @@ class TypescriptGenerator(OOCodeGenerator):
|
|
241
247
|
elif t.typeof and t.typeof in type_map:
|
242
248
|
return type_init_map[t.typeof]
|
243
249
|
else:
|
244
|
-
|
250
|
+
logger.warning(f"Unknown type.base: {t.name}")
|
245
251
|
return "null"
|
246
252
|
return "null"
|
247
253
|
|
@@ -264,8 +270,9 @@ class TypescriptGenerator(OOCodeGenerator):
|
|
264
270
|
@click.version_option(__version__, "-V", "--version")
|
265
271
|
@click.option("--gen-type-utils/", "-u", help="Generate Type checking utils", is_flag=True)
|
266
272
|
@click.option("--include-induced-slots/", help="Generate slots induced through inheritance", is_flag=True)
|
267
|
-
@click.
|
268
|
-
|
273
|
+
@click.option("--output", type=click.Path(dir_okay=False))
|
274
|
+
@click.command()
|
275
|
+
def cli(yamlfile, gen_type_utils=False, include_induced_slots=False, output=None, **args):
|
269
276
|
"""Generate typescript interfaces and types
|
270
277
|
|
271
278
|
See https://github.com/linkml/linkml-runtime.js
|
@@ -273,7 +280,7 @@ def cli(yamlfile, gen_type_utils=False, include_induced_slots=False, **args):
|
|
273
280
|
gen = TypescriptGenerator(
|
274
281
|
yamlfile, gen_type_utils=gen_type_utils, include_induced_slots=include_induced_slots, **args
|
275
282
|
)
|
276
|
-
|
283
|
+
gen.serialize(output=output)
|
277
284
|
|
278
285
|
|
279
286
|
if __name__ == "__main__":
|
linkml/linter/linter.py
CHANGED
@@ -8,6 +8,7 @@ from typing import Any, Dict, Iterable, Union
|
|
8
8
|
import jsonschema
|
9
9
|
import yaml
|
10
10
|
from jsonschema.exceptions import best_match
|
11
|
+
from jsonschema.protocols import Validator
|
11
12
|
from linkml_runtime import SchemaView
|
12
13
|
from linkml_runtime.dumpers import yaml_dumper
|
13
14
|
from linkml_runtime.linkml_model import SchemaDefinition
|
@@ -35,7 +36,7 @@ def get_named_config(name: str) -> Dict[str, Any]:
|
|
35
36
|
|
36
37
|
|
37
38
|
@lru_cache
|
38
|
-
def get_metamodel_validator() ->
|
39
|
+
def get_metamodel_validator() -> Validator:
|
39
40
|
meta_json_gen = JsonSchemaGenerator(LOCAL_METAMODEL_YAML_FILE, not_closed=False)
|
40
41
|
meta_json_schema = meta_json_gen.generate()
|
41
42
|
validator_cls = jsonschema.validators.validator_for(meta_json_schema, default=jsonschema.Draft7Validator)
|
@@ -360,13 +360,13 @@ class LogicalModelTransformer(ModelTransformer):
|
|
360
360
|
target_schema = target_schemaview.schema
|
361
361
|
for tn, typ in target_schema.types.items():
|
362
362
|
ancs = sv.type_ancestors(tn, reflexive=False)
|
363
|
-
|
363
|
+
logger.debug(f"Unrolling type {tn}, merging {len(ancs)}")
|
364
364
|
if ancs:
|
365
365
|
for type_anc in ancs:
|
366
366
|
self._merge_type_ancestors(target=typ, source=sv.get_type(type_anc))
|
367
367
|
for sn, slot in target_schema.slots.items():
|
368
368
|
ancs = sv.slot_ancestors(sn, reflexive=False)
|
369
|
-
|
369
|
+
logger.debug(f"Unrolling slot {sn}, merging {len(ancs)}")
|
370
370
|
if ancs:
|
371
371
|
for slot_anc in ancs:
|
372
372
|
self._merge_slot_ancestors(target=slot, source=target_schema.slots[slot_anc])
|
@@ -379,7 +379,7 @@ class LogicalModelTransformer(ModelTransformer):
|
|
379
379
|
depth_first=False,
|
380
380
|
)
|
381
381
|
ancs = list(reversed(ancs))
|
382
|
-
|
382
|
+
logger.debug(f"Unrolling class {cn}, merging {len(ancs)}")
|
383
383
|
self._roll_down(target_schema, cn, ancs)
|
384
384
|
self.apply_defaults(target_schema)
|
385
385
|
if simplify:
|
@@ -12,9 +12,12 @@ from linkml_runtime.linkml_model import (
|
|
12
12
|
SchemaDefinition,
|
13
13
|
SlotDefinition,
|
14
14
|
)
|
15
|
+
from linkml_runtime.linkml_model.meta import UniqueKey
|
15
16
|
from linkml_runtime.utils.schemaview import SchemaView, SlotDefinitionName
|
16
17
|
from sqlalchemy import Enum
|
17
18
|
|
19
|
+
logger = logging.getLogger(__name__)
|
20
|
+
|
18
21
|
|
19
22
|
class RelationalAnnotations(Enum):
|
20
23
|
PRIMARY_KEY = "primary_key"
|
@@ -215,11 +218,11 @@ class RelationalModelTransformer:
|
|
215
218
|
for cn in target_sv.all_classes():
|
216
219
|
pk = self.get_direct_identifier_attribute(target_sv, cn)
|
217
220
|
if self.foreign_key_policy == ForeignKeyPolicy.NO_FOREIGN_KEYS:
|
218
|
-
|
221
|
+
logger.info(f"Will not inject any PKs, and policy == {self.foreign_key_policy}")
|
219
222
|
else:
|
220
223
|
if pk is None:
|
221
224
|
pk = self.add_primary_key(cn, target_sv)
|
222
|
-
|
225
|
+
logger.info(f"Added primary key {cn}.{pk.name}")
|
223
226
|
for link in links:
|
224
227
|
if link.target_class == cn:
|
225
228
|
link.target_slot = pk.name
|
@@ -233,7 +236,7 @@ class RelationalModelTransformer:
|
|
233
236
|
continue
|
234
237
|
pk_slot = self.get_direct_identifier_attribute(target_sv, cn)
|
235
238
|
# if self.is_skip(c) and len(incoming_links) == 0:
|
236
|
-
#
|
239
|
+
# logger.info(f'Skipping class: {c.name}')
|
237
240
|
# del target.classes[cn]
|
238
241
|
# continue
|
239
242
|
for src_slot in list(c.attributes.values()):
|
@@ -278,6 +281,17 @@ class RelationalModelTransformer:
|
|
278
281
|
target_slot=backref_slot.name,
|
279
282
|
)
|
280
283
|
)
|
284
|
+
backref_key_slots = [s for s in backref_class.attributes.values() if s.key]
|
285
|
+
if backref_key_slots:
|
286
|
+
if len(backref_key_slots) > 1:
|
287
|
+
raise ValueError(f"Multiple keys for {c.name}: {backref_key_slots}")
|
288
|
+
backref_key_slot = backref_key_slots[0]
|
289
|
+
unique_key_name = f"{c.name}_{backref_key_slot.name}"
|
290
|
+
backref_class.unique_keys[unique_key_name] = UniqueKey(
|
291
|
+
unique_key_name=unique_key_name,
|
292
|
+
unique_key_slots=[backref_slot.name, backref_key_slot.name],
|
293
|
+
)
|
294
|
+
|
281
295
|
else:
|
282
296
|
# MANY-TO-MANY
|
283
297
|
# create new linking table
|
@@ -375,7 +389,7 @@ class RelationalModelTransformer:
|
|
375
389
|
removed_ucs = []
|
376
390
|
for uc_name, uc in c.unique_keys.items():
|
377
391
|
if any(sn in multivalued_slots_original for sn in uc.unique_key_slots):
|
378
|
-
|
392
|
+
logger.warning(
|
379
393
|
f"Cannot represent uniqueness constraint {uc_name}. "
|
380
394
|
f"one of the slots {uc.unique_key_slots} is multivalued"
|
381
395
|
)
|
linkml/utils/converter.py
CHANGED
@@ -23,6 +23,8 @@ from linkml.utils.datautils import (
|
|
23
23
|
infer_root_class,
|
24
24
|
)
|
25
25
|
|
26
|
+
logger = logging.getLogger(__name__)
|
27
|
+
|
26
28
|
|
27
29
|
@click.command(name="convert")
|
28
30
|
@click.option("--module", "-m", help="Path to python datamodel module")
|
@@ -119,7 +121,7 @@ def cli(
|
|
119
121
|
sv.set_modified()
|
120
122
|
if target_class is None and target_class_from_path:
|
121
123
|
target_class = os.path.basename(input).split("-")[0]
|
122
|
-
|
124
|
+
logger.info(f"inferred target class = {target_class} from {input}")
|
123
125
|
if target_class is None:
|
124
126
|
target_class = infer_root_class(sv)
|
125
127
|
if target_class is None:
|
linkml/utils/execute_tutorial.py
CHANGED
@@ -10,6 +10,8 @@ import click
|
|
10
10
|
|
11
11
|
from linkml._version import __version__
|
12
12
|
|
13
|
+
logger = logging.getLogger(__name__)
|
14
|
+
|
13
15
|
re_decl = re.compile(r"^(\S+):$")
|
14
16
|
re_start_yaml = re.compile(r"^```(\w+)$")
|
15
17
|
re_end_yaml = re.compile(r"^```$")
|
@@ -46,17 +48,17 @@ def execute_blocks(directory: str, blocks: List[Block]) -> List[str]:
|
|
46
48
|
:return: errors
|
47
49
|
"""
|
48
50
|
Path(directory).mkdir(parents=True, exist_ok=True)
|
49
|
-
|
51
|
+
logger.info(f"Executing in: {directory}")
|
50
52
|
last_block = None
|
51
53
|
errs = []
|
52
54
|
|
53
55
|
def err(e):
|
54
56
|
errs.append(e)
|
55
|
-
|
57
|
+
logger.error(e)
|
56
58
|
|
57
59
|
for block in blocks:
|
58
60
|
write_lines(block.prior_lines)
|
59
|
-
|
61
|
+
logger.info(f"# Block: {block.category} {block.title}")
|
60
62
|
if block.is_file_block():
|
61
63
|
path = PurePath(directory, block.title)
|
62
64
|
with open(path, "w", encoding="UTF-8") as stream:
|
@@ -73,44 +75,44 @@ def execute_blocks(directory: str, blocks: List[Block]) -> List[str]:
|
|
73
75
|
if len(outpath) > 1:
|
74
76
|
raise Exception(f"Maximum 1 token after > in {block.content}. Got: {outpath}")
|
75
77
|
outpath = str(Path(directory, *outpath))
|
76
|
-
|
78
|
+
logger.info(f"OUTPATH = {outpath}")
|
77
79
|
else:
|
78
80
|
outpath = None
|
79
|
-
|
81
|
+
logger.info(f"Executing: {cmd}")
|
80
82
|
r = subprocess.run(cmd, cwd=directory, capture_output=True)
|
81
83
|
block.output = r.stdout.decode("utf-8")
|
82
84
|
if outpath:
|
83
85
|
with open(outpath, "w", encoding="UTF-8") as stream:
|
84
|
-
|
86
|
+
logger.info(f"WRITING {len(block.output)} TO = {outpath}")
|
85
87
|
stream.write(block.output)
|
86
88
|
block.error = r.stderr.decode("utf-8")
|
87
|
-
|
89
|
+
logger.info(f"OUT [sample] = {block.output[0:30]}")
|
88
90
|
if block.expected_fail:
|
89
91
|
if r.returncode == 0:
|
90
92
|
err(f"Command unexpectedly succeeded: {cmd}")
|
91
93
|
else:
|
92
|
-
|
94
|
+
logger.info("Failed as expected")
|
93
95
|
if block.error:
|
94
|
-
|
96
|
+
logger.info(f"ERR [sample] = ...{block.error[-200:]}")
|
95
97
|
else:
|
96
98
|
if block.error:
|
97
|
-
|
99
|
+
logger.info(f"ERR = {block.error}")
|
98
100
|
if r.returncode != 0:
|
99
101
|
err(f"Command failed: {cmd}")
|
100
102
|
else:
|
101
|
-
|
103
|
+
logger.info("Success!")
|
102
104
|
elif block.is_stdout():
|
103
105
|
if "compare_rdf" in block.annotations:
|
104
|
-
|
106
|
+
logger.warning("SKIPPING RDF COMPARISON. TODO: https://github.com/linkml/linkml/issues/427")
|
105
107
|
elif last_block.output:
|
106
108
|
if last_block.output.strip() != block.content.strip():
|
107
109
|
err(f"Mismatch: {str(last_block.output)} != {block.content}")
|
108
110
|
else:
|
109
|
-
|
111
|
+
logger.info("Hurray! Contents match!")
|
110
112
|
else:
|
111
|
-
|
113
|
+
logger.info("No comparison performed")
|
112
114
|
else:
|
113
|
-
|
115
|
+
logger.warning(f"Ignoring block: {block}")
|
114
116
|
last_block = block
|
115
117
|
return errs
|
116
118
|
|
@@ -201,20 +203,20 @@ def cli(inputs, directory):
|
|
201
203
|
logging.basicConfig(level=logging.INFO)
|
202
204
|
errs = []
|
203
205
|
for input in inputs:
|
204
|
-
|
206
|
+
logger.info(f"INPUT={input}")
|
205
207
|
blocks = parse_file_to_blocks(input)
|
206
208
|
print(f"## {len(blocks)} Blocks")
|
207
209
|
localdir = Path(input).stem
|
208
210
|
subdir = PurePath(directory, localdir)
|
209
211
|
input_errs = execute_blocks(str(subdir), blocks)
|
210
212
|
if len(input_errs) > 0:
|
211
|
-
|
213
|
+
logger.error(f"TUTORIAL {input} FAILURES: {len(input_errs)}")
|
212
214
|
errs += input_errs
|
213
|
-
|
215
|
+
logger.info(f"Errors = {len(errs)}")
|
214
216
|
if len(errs) > 0:
|
215
|
-
|
217
|
+
logger.error(f"Encountered {len(errs)} Errors")
|
216
218
|
for err in errs:
|
217
|
-
|
219
|
+
logger.error(f"Error: {err}")
|
218
220
|
sys.exit(1)
|
219
221
|
|
220
222
|
|
linkml/utils/generator.py
CHANGED
@@ -54,6 +54,8 @@ from linkml.utils.mergeutils import alias_root
|
|
54
54
|
from linkml.utils.schemaloader import SchemaLoader
|
55
55
|
from linkml.utils.typereferences import References
|
56
56
|
|
57
|
+
logger = logging.getLogger(__name__)
|
58
|
+
|
57
59
|
|
58
60
|
@lru_cache
|
59
61
|
def _resolved_metamodel(mergeimports):
|
@@ -61,7 +63,7 @@ def _resolved_metamodel(mergeimports):
|
|
61
63
|
raise AssertionError(f"{LOCAL_METAMODEL_YAML_FILE} not found")
|
62
64
|
|
63
65
|
base_dir = str(Path(str(LOCAL_METAMODEL_YAML_FILE)).parent)
|
64
|
-
|
66
|
+
logger.debug(f"BASE={base_dir}")
|
65
67
|
metamodel = SchemaLoader(
|
66
68
|
LOCAL_METAMODEL_YAML_FILE,
|
67
69
|
importmap={"linkml": base_dir},
|
@@ -179,7 +181,7 @@ class Generator(metaclass=abc.ABCMeta):
|
|
179
181
|
|
180
182
|
def __post_init__(self) -> None:
|
181
183
|
if not self.logger:
|
182
|
-
self.logger =
|
184
|
+
self.logger = logger
|
183
185
|
if self.log_level is not None:
|
184
186
|
self.logger.setLevel(self.log_level)
|
185
187
|
if self.format is None:
|
@@ -203,7 +205,7 @@ class Generator(metaclass=abc.ABCMeta):
|
|
203
205
|
if self.uses_schemaloader:
|
204
206
|
self._initialize_using_schemaloader(schema)
|
205
207
|
else:
|
206
|
-
|
208
|
+
self.logger.info(f"Using SchemaView with im={self.importmap} // base_dir={self.base_dir}")
|
207
209
|
self.schemaview = SchemaView(schema, importmap=self.importmap, base_dir=self.base_dir)
|
208
210
|
if self.include:
|
209
211
|
if isinstance(self.include, (str, Path)):
|
@@ -840,7 +842,7 @@ class Generator(metaclass=abc.ABCMeta):
|
|
840
842
|
if ":" not in mapping or len(mapping.split(":")) != 2:
|
841
843
|
raise ValueError(f"Definition {defn.name} - unrecognized mapping: {mapping}")
|
842
844
|
ns = mapping.split(":")[0]
|
843
|
-
|
845
|
+
self.logger.debug(f"Adding {ns} from {mapping} // {defn}")
|
844
846
|
if ns:
|
845
847
|
self.add_prefix(ns)
|
846
848
|
|
linkml/utils/mergeutils.py
CHANGED
@@ -18,6 +18,8 @@ from linkml_runtime.utils.namespaces import Namespaces
|
|
18
18
|
from linkml_runtime.utils.yamlutils import extended_str
|
19
19
|
from rdflib import URIRef
|
20
20
|
|
21
|
+
logger = logging.getLogger(__name__)
|
22
|
+
|
21
23
|
|
22
24
|
def merge_schemas(
|
23
25
|
target: SchemaDefinition,
|
@@ -72,7 +74,7 @@ def merge_namespaces(target: SchemaDefinition, mergee: SchemaDefinition, namespa
|
|
72
74
|
# We cannot resolve this to an absolute path, so we have to assume that
|
73
75
|
# this prefix is already defined correctly in the target
|
74
76
|
if prefix.prefix_prefix not in namespaces:
|
75
|
-
|
77
|
+
logger.info(
|
76
78
|
"Adding an unadjusted relative prefix for %s from %s, "
|
77
79
|
+ "as the prefix is not yet defined, even as we cannot adjust it relative to the final file. "
|
78
80
|
+ "If it cannot be resolved, add the prefix definition to the input schema!",
|
@@ -85,7 +87,7 @@ def merge_namespaces(target: SchemaDefinition, mergee: SchemaDefinition, namespa
|
|
85
87
|
prefix.prefix_prefix in target.prefixes
|
86
88
|
and target.prefixes[prefix.prefix_prefix].prefix_reference != prefix.prefix_reference
|
87
89
|
):
|
88
|
-
|
90
|
+
logger.info(
|
89
91
|
"Ignoring different relative prefix for %s from %s, "
|
90
92
|
+ "as we cannot adjust it relative to the final file. "
|
91
93
|
+ "Assuming the first found location is correct: %s!",
|
linkml/utils/schema_fixer.py
CHANGED
@@ -94,7 +94,7 @@ class SchemaFixer:
|
|
94
94
|
tree_roots = [c for c in sv.all_classes().values() if c.tree_root]
|
95
95
|
if len(tree_roots) > 0:
|
96
96
|
if force:
|
97
|
-
|
97
|
+
logger.info("Forcing addition of containers")
|
98
98
|
else:
|
99
99
|
raise ValueError(f"Schema already has containers: {tree_roots}")
|
100
100
|
container = ClassDefinition(class_name, tree_root=True)
|
@@ -228,7 +228,7 @@ class SchemaFixer:
|
|
228
228
|
# slots within that are redundant
|
229
229
|
slot_usage_keys = list(cls.slot_usage.keys())
|
230
230
|
for slot_usage_key in slot_usage_keys:
|
231
|
-
|
231
|
+
logger.debug(f"TESTING: {class_name}.{slot_usage_key}")
|
232
232
|
slot_usage_value = cls.slot_usage[slot_usage_key]
|
233
233
|
# perform a deletion test: what can be retrieved by inference
|
234
234
|
del cls.slot_usage[slot_usage_key]
|
@@ -236,7 +236,7 @@ class SchemaFixer:
|
|
236
236
|
try:
|
237
237
|
induced_slot = sv.induced_slot(slot_usage_key, class_name)
|
238
238
|
except ValueError:
|
239
|
-
|
239
|
+
logger.warning(f"slot_usage with no slot: {slot_usage_key}")
|
240
240
|
continue
|
241
241
|
# restore value
|
242
242
|
cls.slot_usage[slot_usage_key] = slot_usage_value
|
@@ -258,7 +258,7 @@ class SchemaFixer:
|
|
258
258
|
continue
|
259
259
|
induced_v = getattr(induced_slot, metaslot_name, None)
|
260
260
|
if v is not None and v != [] and v != {} and v == induced_v:
|
261
|
-
|
261
|
+
logger.info(f"REDUNDANT: {class_name}.{slot_usage_key}[{metaslot_name}] = {v}")
|
262
262
|
to_delete.append(metaslot_name)
|
263
263
|
for metaslot_name in to_delete:
|
264
264
|
del slot_usage_value[metaslot_name]
|
@@ -302,7 +302,7 @@ class SchemaFixer:
|
|
302
302
|
if len(vals_strs) == 1:
|
303
303
|
harmonized_slot[k] = vals.pop()
|
304
304
|
elif len(vals_strs) > 1:
|
305
|
-
|
305
|
+
logger.info(f"Variable values in {slot_name}.{k}: {vals_strs}")
|
306
306
|
new_slots[str(slot_name)] = harmonized_slot
|
307
307
|
return new_slots
|
308
308
|
|
linkml/utils/schemaloader.py
CHANGED
@@ -29,6 +29,8 @@ from linkml.utils.mergeutils import merge_classes, merge_schemas, merge_slots, s
|
|
29
29
|
from linkml.utils.rawloader import load_raw_schema
|
30
30
|
from linkml.utils.schemasynopsis import SchemaSynopsis
|
31
31
|
|
32
|
+
lgr = logging.getLogger(__name__)
|
33
|
+
|
32
34
|
|
33
35
|
class SchemaLoader:
|
34
36
|
def __init__(
|
@@ -57,7 +59,7 @@ class SchemaLoader:
|
|
57
59
|
:param source_file_date: modification of source file
|
58
60
|
:param source_file_size: size of source file
|
59
61
|
"""
|
60
|
-
self.logger = logger if logger is not None else
|
62
|
+
self.logger = logger if logger is not None else lgr
|
61
63
|
if isinstance(data, SchemaDefinition):
|
62
64
|
self.schema = data
|
63
65
|
else:
|
@@ -176,7 +178,7 @@ class SchemaLoader:
|
|
176
178
|
# mangled names are overwritten if a schema with attributes is passed in
|
177
179
|
# TODO: handle this in a more graceful way
|
178
180
|
# see https://github.com/linkml/linkml/issues/872
|
179
|
-
|
181
|
+
self.logger.warning(
|
180
182
|
f'Class: "{cls.name}" attribute "{attribute.name}" - '
|
181
183
|
f"mangled name: {mangled_slot_name} already exists",
|
182
184
|
)
|
@@ -770,7 +772,7 @@ class SchemaLoader:
|
|
770
772
|
if slotname in self.schema.slots:
|
771
773
|
base_slot = self.schema.slots[slotname]
|
772
774
|
else:
|
773
|
-
|
775
|
+
self.logger.error(f"slot_usage for undefined slot: {slotname}")
|
774
776
|
base_slot = None
|
775
777
|
parent_slot = self.schema.slots.get(slot_usage.is_a)
|
776
778
|
# Follow the ancestry of the class to get the most proximal parent
|
linkml/utils/sqlutils.py
CHANGED
@@ -40,6 +40,8 @@ from linkml.utils.datautils import (
|
|
40
40
|
infer_root_class,
|
41
41
|
)
|
42
42
|
|
43
|
+
logger = logging.getLogger(__name__)
|
44
|
+
|
43
45
|
|
44
46
|
@dataclass
|
45
47
|
class SQLStore:
|
@@ -376,7 +378,7 @@ def dump(
|
|
376
378
|
|
377
379
|
inputs = [item for input in inputs for item in glob.glob(input)]
|
378
380
|
for input in inputs:
|
379
|
-
|
381
|
+
logger.info(f"Loading: {input}")
|
380
382
|
input_format = _get_format(input, input_format)
|
381
383
|
loader = get_loader(input_format)
|
382
384
|
|