linkml-map 0.1.0__py3-none-any.whl

linkml_map/__init__.py

@@ -0,0 +1,6 @@
+from linkml_map.transformer.object_transformer import ObjectTransformer
+
+__all__ = [
+    "Session",
+    "ObjectTransformer",
+]

linkml_map/cli/cli.py

@@ -0,0 +1,205 @@
+"""Command line interface for linkml-transformer."""
+
+import logging
+import sys
+
+import click
+
+__all__ = [
+    "main",
+]
+
+import yaml
+from linkml_runtime import SchemaView
+from linkml_runtime.dumpers import yaml_dumper
+
+from linkml_map.compiler.markdown_compiler import MarkdownCompiler
+from linkml_map.compiler.python_compiler import PythonCompiler
+from linkml_map.inference.inverter import TransformationSpecificationInverter
+from linkml_map.inference.schema_mapper import SchemaMapper
+from linkml_map.transformer.object_transformer import ObjectTransformer
+
+schema_option = click.option("-s", "--schema", help="Path to source schema.")
+transformer_specification_option = click.option(
+    "-T", "--transformer-specification", help="Path to transformer specification."
+)
+output_option_wb = click.option(
+    "-o",
+    "--output",
+    type=click.File(mode="wb"),
+    default=sys.stdout,
+    help="Output file.",
+)
+output_option = click.option("-o", "--output", help="Output file.")
+output_format_options = click.option(
+    "-O",
+    "--output-format",
+    type=click.Choice(["json", "yaml", "pickle", "md"]),
+    default="yaml",
+    help="Output format.",
+)
+
+
+@click.group()
+@click.option("-v", "--verbose", count=True)
+@click.option("-q", "--quiet")
+# @click.version_option(__version__)
+def main(verbose: int, quiet: bool):
+    """CLI for linkml-transformer."""
+    logger = logging.getLogger()
+    if verbose >= 2:
+        logger.setLevel(level=logging.DEBUG)
+    elif verbose == 1:
+        logger.setLevel(level=logging.INFO)
+    else:
+        logger.setLevel(level=logging.WARNING)
+    if quiet:
+        logger.setLevel(level=logging.ERROR)
+    logger.info(f"Logger {logger.name} set to level {logger.level}")
+
+
+@main.command()
+@output_option
+@transformer_specification_option
+@schema_option
+@output_format_options
+@click.option("--source-type")
+@click.option(
+    "--unrestricted-eval/--no-unrestricted-eval",
+    default=False,
+    show_default=True,
+    help="Allow unrestricted eval of python expressions.",
+)
+@click.argument("input")
+def map_data(
+    input,
+    schema,
+    source_type,
+    transformer_specification,
+    output,
+    output_format,
+    **kwargs,
+):
+    """
+    Map data from a source schema to a target schema using a transformation specification.
+
+    Example:
+
+        linkml-tr map-data -T X-to-Y-tr.yaml -s X.yaml  X-data.yaml
+    """
+    logging.info(f"Transforming {input} conforming to {schema} using {transformer_specification}")
+    tr = ObjectTransformer(**kwargs)
+    tr.source_schemaview = SchemaView(schema)
+    tr.load_transformer_specification(transformer_specification)
+    with open(input) as file:
+        input_obj = yaml.safe_load(file)
+    tr.index(input_obj, source_type)
+    tr_obj = tr.map_object(input_obj, source_type)
+    if output:
+        outfile = open(output, "w", encoding="utf-8")
+    else:
+        outfile = sys.stdout
+    outfile.write(yaml_dumper.dumps(tr_obj))
+
+
+@main.command()
+@output_option
+@transformer_specification_option
+@schema_option
+@click.option("--source-type")
+@click.option("--target", default="python", show_default=True, help="Target representation.")
+def compile(
+    schema,
+    source_type,
+    transformer_specification,
+    target,
+    output,
+    **kwargs,
+):
+    """
+    Compiles a schema to another representation.
+
+    Example:
+
+        linkml-tr compile -T X-to-Y-tr.yaml -s X.yaml
+    """
+    logging.info(f"Compiling {transformer_specification} with {schema}")
+    compiler_args = {}
+    if schema:
+        compiler_args["source_schemaview"] = SchemaView(schema)
+    if target == "python":
+        compiler = PythonCompiler(**compiler_args)
+    elif target == "markdown":
+        compiler = MarkdownCompiler(**compiler_args)
+    else:
+        raise NotImplementedError(f"Compiler {target} not implemented")
+    tr = ObjectTransformer()
+    tr.source_schemaview = SchemaView(schema)
+    tr.load_transformer_specification(transformer_specification)
+    result = compiler.compile(tr.specification)
+    print(result.serialization)
+
+
+@main.command()
+@output_option
+@transformer_specification_option
+@output_format_options
+@click.argument("schema")
+def derive_schema(schema, transformer_specification, output, output_format, **kwargs):
+    """Derive a schema from a source schema and a transformation specification.
+
+    This can be thought of as "copying" the source to a target, using the transformation
+    specification as a "patch"
+
+    Notes:
+
+        the implementation is currently incomplete; the derived schema may not be valid
+        linkml, e.g. there may be "dangling" references.
+
+    Example:
+
+        linkml-tr derive-schema -T transform/personinfo-to-agent.transform.yaml source/personinfo.yaml
+    """
+    logging.info(f"Transforming {schema} using {transformer_specification}")
+    tr = ObjectTransformer()
+    tr.load_transformer_specification(transformer_specification)
+    mapper = SchemaMapper(transformer=tr)
+    mapper.source_schemaview = SchemaView(schema)
+    target_schema = mapper.derive_schema()
+    if output:
+        outfile = open(output, "w", encoding="utf-8")
+    else:
+        outfile = sys.stdout
+    outfile.write(yaml_dumper.dumps(target_schema))
+
+
+@main.command()
+@output_option
+@transformer_specification_option
+@output_format_options
+@click.option("--strict/--no-strict", default=True, show_default=True, help="Strict mode.")
+@click.argument("schema")
+def invert(schema, transformer_specification, output, output_format, **kwargs):
+    """Invert a transformation specification.
+
+    Example:
+
+        linkml-tr invert -T transform/personinfo-to-agent.transform.yaml source/personinfo.yaml
+    """
+    logging.info(f"Inverting {transformer_specification} using {schema} as source")
+    tr = ObjectTransformer()
+    tr.load_transformer_specification(transformer_specification)
+    inverter = TransformationSpecificationInverter(
+        source_schemaview=SchemaView(schema),
+        **kwargs,
+    )
+    inverted_spec = inverter.invert(tr.specification)
+    if output:
+        outfile = open(output, "w", encoding="utf-8")
+    else:
+        outfile = sys.stdout
+    outfile.write(yaml_dumper.dumps(inverted_spec))
+
+
+if __name__ == "__main__":
+    main()

linkml_map/compiler/__init__.py

@@ -0,0 +1 @@
+"""Compiles schemas into other transformation frameworks."""

linkml_map/compiler/awk_compiler.py

@@ -0,0 +1,13 @@
+from linkml_map.compiler.compiler import CompiledSpecification, Compiler
+from linkml_map.datamodel.transformer_model import TransformationSpecification
+
+
+class AWKCompiler(Compiler):
+    """
+    Compiles a Transformation Specification to an Awk script.
+
+    Note: this is only expected to work for flat schemas.
+    """
+
+    def compile(self, specification: TransformationSpecification) -> CompiledSpecification:
+        raise NotImplementedError

linkml_map/compiler/compiler.py

@@ -0,0 +1,88 @@
+"""
+Compilers are responsible for compiling a transformation specification into an alternative representation.
+
+This is the opposite of an importer.
+
+For example:
+
+- LinkML-Transformer Specifications to R2RML
+- LinkML-Transformer Specifications to awk scripts
+- LinkML-Transformer Specifications to SQL
+- LinkML-Transformer Specifications to Python (OO)
+- LinkML-Transformer Specifications to Pandas
+- LinkML-Transformer Specifications to Hamilton
+"""
+
+from abc import ABC
+from dataclasses import dataclass, field
+from types import ModuleType
+from typing import Iterator, Optional
+
+from linkml_runtime import SchemaView
+from linkml_runtime.dumpers import yaml_dumper
+from linkml_runtime.utils.compile_python import compile_python
+
+from linkml_map.datamodel.transformer_model import TransformationSpecification
+from linkml_map.inference.schema_mapper import SchemaMapper
+
+
+@dataclass
+class CompiledSpecification:
+    serialization: str = field(default="")
+
+    _module: Optional[ModuleType] = None
+
+    @property
+    def module(self) -> ModuleType:
+        if not self._module:
+            self._module = compile_python(self.serialization)
+        return self._module
+
+
+@dataclass
+class Compiler(ABC):
+    """
+    Base class for all compilers.
+
+    A compiler will compile a transformation specification into
+    an alternative representation.
+
+    An example compiler would be a R2RML compiler.
+
+    Note: Compilers and Importers will in general be implemented by providing
+    mapping specifications
+    """
+
+    source_schemaview: SchemaView = None
+    """A view over the schema describing the source."""
+
+    source_python_module: str = None
+    """The python module containing the source classes."""
+
+    target_python_module: str = None
+    """The python module containing the target classes."""
+
+    def compile(self, specification: TransformationSpecification) -> CompiledSpecification:
+        """
+        Transform source object into an instance of the target class.
+
+        :param specification:
+        :return:
+        """
+        s = self._compile_header(specification)
+        for chunk in self._compile_iterator(specification):
+            s += chunk
+        return CompiledSpecification(serialization=s)
+
+    def _compile_header(self, specification: TransformationSpecification) -> str:
+        return ""
+
+    def _compile_iterator(self, specification: TransformationSpecification) -> Iterator[str]:
+        raise NotImplementedError
+
+    def derived_target_schemaview(self, specification: TransformationSpecification):
+        """
+        Returns a view over the target schema, including any derived classes.
+        """
+        mapper = SchemaMapper(source_schemaview=self.source_schemaview)
+        return SchemaView(yaml_dumper.dumps(mapper.derive_schema(specification)))

linkml_map/compiler/graphviz_compiler.py

@@ -0,0 +1,109 @@
+import re
+from dataclasses import dataclass
+from typing import List, Optional, Tuple
+
+from graphviz import Digraph
+from linkml_runtime import SchemaView
+from pydantic import BaseModel
+
+from linkml_map.compiler.compiler import CompiledSpecification, Compiler
+from linkml_map.datamodel.transformer_model import TransformationSpecification
+
+
+class Record(BaseModel):
+    """
+    A simplified representation of a class, UML-style.
+    """
+
+    name: str
+    source: str
+    fields: List[Tuple[str, str]] = []
+
+    @property
+    def id(self):
+        return f"{self.source}{self.name}"
+
+    def __str__(self):
+        return (
+            f"""<
+        <TABLE BORDER="0" CELLBORDER="1" CELLSPACING="0">
+            <TR><TD><B>{self.name}</B></TD></TR>
+        """
+            + "".join(
+                [
+                    f'<TR><TD SIDES="LRB" PORT="{f[0]}">{f[0]} : {f[1]}</TD></TR>'
+                    for f in self.fields
+                ]
+            )
+            + """
+        </TABLE>>"""
+        )
+
+
+@dataclass
+class GraphvizObject(CompiledSpecification):
+    digraph: Digraph = None
+
+    def render(self, file_path: str, format="png", view=False) -> None:
+        """
+        Renders a graphviz graph to a file.
+        :param file_path:
+        :return:
+        """
+        self.digraph.render(file_path, format=format, view=view)
+
+
+class GraphvizCompiler(Compiler):
+    """
+    Compiles a Transformation Specification to GraphViz.
+    """
+
+    def compile(
+        self, specification: TransformationSpecification, elements: Optional[List[str]] = None
+    ) -> GraphvizObject:
+        dg = Digraph(comment="UML Class Diagram", format="png")
+        dg.attr(rankdir="LR")  # Set graph direction from left to right
+        target_schemaview = self.derived_target_schemaview(specification)
+        source_schemaview = self.source_schemaview
+
+        records = []
+        records += self.add_records(source_schemaview, "source")
+        records += self.add_records(target_schemaview, "target")
+
+        for record in records:
+            dg.node(record.id, str(record), shape="plaintext")
+
+        # Define the class nodes with fields in UML format using HTML-like labels
+        # for precise control over the stacking of the fields
+        for target_cn, cd in specification.class_derivations.items():
+            source_cn = cd.populated_from
+            if source_cn is None:
+                source_cn = cd.name
+            if elements is not None and target_cn not in elements:
+                continue
+            source_record = Record(name=source_cn, source="source")
+            target_record = Record(name=target_cn, source="target")
+            for sd in cd.slot_derivations.values():
+                target_slot = sd.name
+                target_id = f"{target_record.id}:{target_slot}"
+                source_slot = sd.populated_from
+                if source_slot:
+                    source_id = f"{source_record.id}:{source_slot}"
+                    dg.edge(source_id, target_id)
+                elif sd.expr:
+                    # TODO: do this in a less hacky way
+                    tokens = re.findall(r"\w+", sd.expr)
+                    for token in tokens:
+                        if token not in source_schemaview.all_slots():
+                            continue
+                        dg.edge(f"{source_record.id}:{token}", target_id, style="dashed")
+        return GraphvizObject(digraph=dg, serialization=dg.source)
+
+    def add_records(self, schemaview: SchemaView, source: str) -> List[Record]:
+        records = []
+        for cn in schemaview.all_classes():
+            record = Record(name=cn, source=source)
+            for induced_slot in schemaview.class_induced_slots(cn):
+                record.fields.append((induced_slot.name, induced_slot.range))
+            records.append(record)
+        return records

linkml_map/compiler/j2_based_compiler.py

@@ -0,0 +1,36 @@
+from dataclasses import dataclass
+
+from jinja2 import Environment, FileSystemLoader
+
+from linkml_map.compiler.compiler import CompiledSpecification, Compiler
+from linkml_map.compiler.templates import TEMPLATE_DIR
+from linkml_map.datamodel.transformer_model import TransformationSpecification
+
+
+@dataclass
+class J2BasedCompiler(Compiler):
+    """
+    Compiles a Transformation Specification using a Jinja2 template.
+    """
+
+    template_dir: str = None
+    """The directory containing the Jinja2 template."""
+
+    template_name: str = None
+    """The name of the Jinja2 template."""
+
+    def compile(self, specification: TransformationSpecification) -> CompiledSpecification:
+        template_dir = self.template_dir
+        if not template_dir:
+            template_dir = TEMPLATE_DIR
+        if not template_dir:
+            raise ValueError("template_dir must be set")
+        loader = FileSystemLoader(template_dir)
+        env = Environment(loader=loader, autoescape=True)
+        if not self.template_name:
+            raise ValueError("template_name must be set")
+        template = env.get_template(self.template_name)
+        rendered = template.render(
+            spec=specification,
+        )
+        return CompiledSpecification(serialization=rendered)

linkml_map/compiler/markdown_compiler.py

@@ -0,0 +1,12 @@
+from dataclasses import dataclass
+
+from linkml_map.compiler.j2_based_compiler import J2BasedCompiler
+
+
+@dataclass
+class MarkdownCompiler(J2BasedCompiler):
+    """
+    Compiles a Transformation Specification to Markdown.
+    """
+
+    template_name: str = "markdown.j2"

linkml_map/compiler/python_compiler.py

@@ -0,0 +1,113 @@
+from copy import deepcopy
+from dataclasses import dataclass
+from typing import Iterator
+
+from jinja2 import Template
+
+from linkml_map.compiler.compiler import Compiler
+from linkml_map.datamodel.transformer_model import (
+    ClassDerivation,
+    TransformationSpecification,
+)
+from linkml_map.inference.inference import induce_missing_values
+
+CD_TEMPLATE = """
+{% macro gen_slot_derivation_value(sd, var) -%}
+{%- if sd.range -%}
+derive_{{ sd.range }}({{ var }})
+{%- else -%}
+{%- if var is not none -%}
+{{ var }}
+{%- else -%}
+None
+{%- endif -%}
+{%- endif -%}
+{%- endmacro %}
+{% macro gen_slot_derivation(sd, force_singlevalued=False) -%}
+{%- if not force_singlevalued and sd.populated_from and induced_slots[sd.populated_from].multivalued -%}
+ [ {{ gen_slot_derivation_value(sd, "x") }} for x in {{ gen_slot_derivation(sd, force_singlevalued=True) }} ]
+{%- else -%}
+ {%- if sd.populated_from -%}
+  source_object.{{ sd.populated_from }}
+ {%- elif sd.expr -%}
+  {%- if '\n' in sd.expr -%}
+ gen_{{ sd.name }}(source_object)
+  {%- elif '{' in sd.expr and '}' in sd.expr -%}
+  {{ sd.expr|replace('{', '')|replace('}', '') }}
+  {%- else -%}
+  {{ sd.expr }}
+  {%- endif -%}
+ {%- else -%}
+  None
+ {%- endif -%}
+{%- endif -%}
+{%- endmacro %}
+{% macro gen_slot_derivation_defs(sd) -%}
+{% if sd.expr and '\n' in sd.expr %}
+
+    def gen_{{ sd.name }}(src):
+        target = None
+   {%- for line in sd.expr.split('\n') %}
+        {{ line }}
+   {%- endfor -%}
+        return target
+{% endif %}
+{%- endmacro %}
+def derive_{{ cd.name }}(
+        source_object: {{ source_module }}.{{ cd.populated_from }}
+    ) -> {{ target_module }}.{{ cd.name }}:
+    # assign slots
+{%- for slot in source_slots %}
+    {{ slot.name }} = source_object.{{ slot.name }}
+{%- endfor %}
+{%-  for sd in cd.slot_derivations.values() -%}
+    {{  gen_slot_derivation_defs(sd) }}
+{%-  endfor %}
+
+    return tgt.{{ cd.name }}(
+        {%- for sd in cd.slot_derivations.values() %}
+        {{ sd.name }}={{ gen_slot_derivation(sd) }},
+        {%- endfor %}
+    )
+"""
+
+
+@dataclass
+class PythonCompiler(Compiler):
+    """
+    Compiles a Transformation Specification to Python code.
+    """
+
+    def _compile_header(self, specification: TransformationSpecification) -> str:
+        s = ""
+        if self.source_python_module:
+            s += f"import {self.source_python_module} as src\n"
+        if self.target_python_module:
+            s += f"import {self.target_python_module} as tgt\n"
+        s += "\nNULL = None\n\n"
+        return s
+
+    def _compile_iterator(self, specification: TransformationSpecification) -> Iterator[str]:
+        specification = deepcopy(specification)
+        induce_missing_values(specification, self.source_schemaview)
+        for cd in specification.class_derivations.values():
+            yield from self._compiled_class_derivations_iter(cd)
+
+    def _compiled_class_derivations_iter(self, cd: ClassDerivation) -> Iterator[str]:
+        sv = self.source_schemaview
+        if cd.populated_from:
+            populated_from = cd.populated_from
+        else:
+            populated_from = cd.name
+        if populated_from not in sv.all_classes():
+            return
+        induced_slots = {s.name: s for s in sv.class_induced_slots(populated_from)}
+        t = Template(CD_TEMPLATE)
+        yield t.render(
+            cd=cd,
+            source_module="src",
+            target_module="tgt",
+            induced_slots=induced_slots,
+            schemaview=sv,
+            source_slots=sv.class_induced_slots(populated_from),
+        )

linkml_map/compiler/r2rml_compiler.py

@@ -0,0 +1,12 @@
+from linkml_map.compiler.compiler import CompiledSpecification, Compiler
+from linkml_map.datamodel.transformer_model import TransformationSpecification
+
+
+class R2RMLCompiler(Compiler):
+    """
+    Compiles a Transformation Specification to R2RML.
+
+    """
+
+    def compile(self, specification: TransformationSpecification) -> CompiledSpecification:
+        raise NotImplementedError

linkml_map/compiler/sparql_compiler.py

@@ -0,0 +1,12 @@
+from linkml_map.compiler.compiler import CompiledSpecification, Compiler
+from linkml_map.datamodel.transformer_model import TransformationSpecification
+
+
+class SPARQLCompiler(Compiler):
+    """
+    Compiles a Transformation Specification to SPARQL Construct.
+
+    """
+
+    def compile(self, specification: TransformationSpecification) -> CompiledSpecification:
+        raise NotImplementedError