linkml-map 0.1.0__tar.gz

linkml_map-0.1.0/PKG-INFO

@@ -0,0 +1,56 @@
+Metadata-Version: 2.1
+Name: linkml-map
+Version: 0.1.0
+Summary: 
+Author: cmungall
+Author-email: cjm@berkeleybop.org
+Requires-Python: >=3.9,<4.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: asteval (>=0.9.29,<0.10.0)
+Requires-Dist: deepdiff (>=6.7.1,<7.0.0)
+Requires-Dist: linkml-runtime (>=1.7.2)
+Requires-Dist: pydantic (>=2.5.3,<3.0.0)
+Requires-Dist: ucumvert (>=0.1.1,<0.2.0)
+Description-Content-Type: text/markdown
+
+# linkml-map
+
+[![Pyversions](https://img.shields.io/pypi/pyversions/linkml-map.svg)](https://pypi.python.org/pypi/linkml-map)
+![](https://github.com/linkml/linkml-map/workflows/Build/badge.svg)
+[![PyPi](https://img.shields.io/pypi/v/linkml-map.svg)](https://pypi.python.org/pypi/linkml-map)
+[![codecov](https://codecov.io/gh/linkml/linkml-map/branch/main/graph/badge.svg?token=WNQNG986UN)](https://codecov.io/gh/linkml/linkml-map)
+
+LinkML Map is a framework for specifying and executing mappings between data models.
+
+Features:
+
+- YAML-based lightweight syntax
+- Python library for executing mappings on data files
+- Ability to compile to other frameworks (forthcoming)
+- Derivation of target (implicit) schemas, allowing easy customization of data models (*profiling*)
+- Simple YAML dictionaries for simple mappings
+- Automatic unit conversion
+- Use of subset of Python to specify complex mappings
+- Visualizations of mappings
+- Mappings are reversible (provided all expressions used are reversible)
+
+For full documentation see:
+
+- [linkml.io/linkml-map/](https://linkml.io/linkml-map/)
+
+Status:
+
+The transformation data model is not yet fully stable, and may be subject to change.
+Not all parts of the model are implemented in the reference Python framework.
+
+## Quickstart
+
+* [Tutorial Notebook](src/docs/examples/Tutorial.ipynb)
+* [Generated Docs](https://linkml.github.io/linkml-map/)
+* [Compliance Suite](https://linkml.github.io/linkml-map/specification/compliance)
+
+

linkml_map-0.1.0/README.md

@@ -0,0 +1,36 @@
+# linkml-map
+
+[![Pyversions](https://img.shields.io/pypi/pyversions/linkml-map.svg)](https://pypi.python.org/pypi/linkml-map)
+![](https://github.com/linkml/linkml-map/workflows/Build/badge.svg)
+[![PyPi](https://img.shields.io/pypi/v/linkml-map.svg)](https://pypi.python.org/pypi/linkml-map)
+[![codecov](https://codecov.io/gh/linkml/linkml-map/branch/main/graph/badge.svg?token=WNQNG986UN)](https://codecov.io/gh/linkml/linkml-map)
+
+LinkML Map is a framework for specifying and executing mappings between data models.
+
+Features:
+
+- YAML-based lightweight syntax
+- Python library for executing mappings on data files
+- Ability to compile to other frameworks (forthcoming)
+- Derivation of target (implicit) schemas, allowing easy customization of data models (*profiling*)
+- Simple YAML dictionaries for simple mappings
+- Automatic unit conversion
+- Use of subset of Python to specify complex mappings
+- Visualizations of mappings
+- Mappings are reversible (provided all expressions used are reversible)
+
+For full documentation see:
+
+- [linkml.io/linkml-map/](https://linkml.io/linkml-map/)
+
+Status:
+
+The transformation data model is not yet fully stable, and may be subject to change.
+Not all parts of the model are implemented in the reference Python framework.
+
+## Quickstart
+
+* [Tutorial Notebook](src/docs/examples/Tutorial.ipynb)
+* [Generated Docs](https://linkml.github.io/linkml-map/)
+* [Compliance Suite](https://linkml.github.io/linkml-map/specification/compliance)
+

linkml_map-0.1.0/pyproject.toml

@@ -0,0 +1,63 @@
+[tool.poetry]
+name = "linkml-map"
+version = "0.1.0"
+description = ""
+authors = ["cmungall <cjm@berkeleybop.org>"]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.9"
+linkml-runtime = ">=1.7.2"
+asteval = "^0.9.29"
+deepdiff = "^6.7.1"
+pydantic = "^2.5.3"
+ucumvert = "^0.1.1"
+
+[tool.poetry.dev-dependencies]
+pytest = "^7.3.1"
+pytest-cov = "^4.0.0"
+linkml = ">=1.7.0"
+mkdocs-mermaid2-plugin = "^0.6.0"
+mknotebooks = "^0.8.0"
+mkdocs-windmill = "*"
+tox = "*"
+
+[tool.poetry.scripts]
+linkml-map = "linkml_map.cli.cli:main"
+
+[tool.poetry.group.dev.dependencies]
+jupyter = "^1.0.0"
+linkml = "^1.6.7"
+
+
+[tool.poetry.group.units.dependencies]
+pint = "^0.23"
+
+
+[tool.poetry.group.graphviz.dependencies]
+graphviz = "^0.20.1"
+
+
+[tool.poetry.group.duckdb.dependencies]
+duckdb = "^0.10.1"
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.black]
+line-length = 100
+target-version = ["py39", "py310", "py311"]
+
+[tool.isort]
+profile = "black"
+multi_line_output = 3
+include_trailing_comma = true
+reverse_relative = true
+
+# Ref: https://github.com/codespell-project/codespell#using-a-config-file
+[tool.codespell]
+skip = '.git,*.pdf,*.svg,go.sum,*.lock'
+check-hidden = true
+ignore-regex = '(^\s*"image/\S+": ".*|\b(KEGG.BRITE|mor.nlm.nih.gov)\b)'
+ignore-words-list = 'infarction,amination'

linkml_map-0.1.0/src/linkml_map/__init__.py

@@ -0,0 +1,6 @@
+from linkml_map.transformer.object_transformer import ObjectTransformer
+
+__all__ = [
+    "Session",
+    "ObjectTransformer",
+]

linkml_map-0.1.0/src/linkml_map/cli/cli.py

@@ -0,0 +1,205 @@
+"""Command line interface for linkml-transformer."""
+
+import logging
+import sys
+
+import click
+
+__all__ = [
+    "main",
+]
+
+import yaml
+from linkml_runtime import SchemaView
+from linkml_runtime.dumpers import yaml_dumper
+
+from linkml_map.compiler.markdown_compiler import MarkdownCompiler
+from linkml_map.compiler.python_compiler import PythonCompiler
+from linkml_map.inference.inverter import TransformationSpecificationInverter
+from linkml_map.inference.schema_mapper import SchemaMapper
+from linkml_map.transformer.object_transformer import ObjectTransformer
+
+schema_option = click.option("-s", "--schema", help="Path to source schema.")
+transformer_specification_option = click.option(
+    "-T", "--transformer-specification", help="Path to transformer specification."
+)
+output_option_wb = click.option(
+    "-o",
+    "--output",
+    type=click.File(mode="wb"),
+    default=sys.stdout,
+    help="Output file.",
+)
+output_option = click.option("-o", "--output", help="Output file.")
+output_format_options = click.option(
+    "-O",
+    "--output-format",
+    type=click.Choice(["json", "yaml", "pickle", "md"]),
+    default="yaml",
+    help="Output format.",
+)
+
+
+@click.group()
+@click.option("-v", "--verbose", count=True)
+@click.option("-q", "--quiet")
+# @click.version_option(__version__)
+def main(verbose: int, quiet: bool):
+    """CLI for linkml-transformer."""
+    logger = logging.getLogger()
+    if verbose >= 2:
+        logger.setLevel(level=logging.DEBUG)
+    elif verbose == 1:
+        logger.setLevel(level=logging.INFO)
+    else:
+        logger.setLevel(level=logging.WARNING)
+    if quiet:
+        logger.setLevel(level=logging.ERROR)
+    logger.info(f"Logger {logger.name} set to level {logger.level}")
+
+
+@main.command()
+@output_option
+@transformer_specification_option
+@schema_option
+@output_format_options
+@click.option("--source-type")
+@click.option(
+    "--unrestricted-eval/--no-unrestricted-eval",
+    default=False,
+    show_default=True,
+    help="Allow unrestricted eval of python expressions.",
+)
+@click.argument("input")
+def map_data(
+    input,
+    schema,
+    source_type,
+    transformer_specification,
+    output,
+    output_format,
+    **kwargs,
+):
+    """
+    Map data from a source schema to a target schema using a transformation specification.
+
+    Example:
+
+        linkml-tr map-data -T X-to-Y-tr.yaml -s X.yaml  X-data.yaml
+    """
+    logging.info(f"Transforming {input} conforming to {schema} using {transformer_specification}")
+    tr = ObjectTransformer(**kwargs)
+    tr.source_schemaview = SchemaView(schema)
+    tr.load_transformer_specification(transformer_specification)
+    with open(input) as file:
+        input_obj = yaml.safe_load(file)
+    tr.index(input_obj, source_type)
+    tr_obj = tr.map_object(input_obj, source_type)
+    if output:
+        outfile = open(output, "w", encoding="utf-8")
+    else:
+        outfile = sys.stdout
+    outfile.write(yaml_dumper.dumps(tr_obj))
+
+
+@main.command()
+@output_option
+@transformer_specification_option
+@schema_option
+@click.option("--source-type")
+@click.option("--target", default="python", show_default=True, help="Target representation.")
+def compile(
+    schema,
+    source_type,
+    transformer_specification,
+    target,
+    output,
+    **kwargs,
+):
+    """
+    Compiles a schema to another representation.
+
+    Example:
+
+        linkml-tr compile -T X-to-Y-tr.yaml -s X.yaml
+    """
+    logging.info(f"Compiling {transformer_specification} with {schema}")
+    compiler_args = {}
+    if schema:
+        compiler_args["source_schemaview"] = SchemaView(schema)
+    if target == "python":
+        compiler = PythonCompiler(**compiler_args)
+    elif target == "markdown":
+        compiler = MarkdownCompiler(**compiler_args)
+    else:
+        raise NotImplementedError(f"Compiler {target} not implemented")
+    tr = ObjectTransformer()
+    tr.source_schemaview = SchemaView(schema)
+    tr.load_transformer_specification(transformer_specification)
+    result = compiler.compile(tr.specification)
+    print(result.serialization)
+
+
+@main.command()
+@output_option
+@transformer_specification_option
+@output_format_options
+@click.argument("schema")
+def derive_schema(schema, transformer_specification, output, output_format, **kwargs):
+    """Derive a schema from a source schema and a transformation specification.
+
+    This can be thought of as "copying" the source to a target, using the transformation
+    specification as a "patch"
+
+    Notes:
+
+        the implementation is currently incomplete; the derived schema may not be valid
+        linkml, e.g. there may be "dangling" references.
+
+    Example:
+
+        linkml-tr derive-schema -T transform/personinfo-to-agent.transform.yaml source/personinfo.yaml
+    """
+    logging.info(f"Transforming {schema} using {transformer_specification}")
+    tr = ObjectTransformer()
+    tr.load_transformer_specification(transformer_specification)
+    mapper = SchemaMapper(transformer=tr)
+    mapper.source_schemaview = SchemaView(schema)
+    target_schema = mapper.derive_schema()
+    if output:
+        outfile = open(output, "w", encoding="utf-8")
+    else:
+        outfile = sys.stdout
+    outfile.write(yaml_dumper.dumps(target_schema))
+
+
+@main.command()
+@output_option
+@transformer_specification_option
+@output_format_options
+@click.option("--strict/--no-strict", default=True, show_default=True, help="Strict mode.")
+@click.argument("schema")
+def invert(schema, transformer_specification, output, output_format, **kwargs):
+    """Invert a transformation specification.
+
+    Example:
+
+        linkml-tr invert -T transform/personinfo-to-agent.transform.yaml source/personinfo.yaml
+    """
+    logging.info(f"Inverting {transformer_specification} using {schema} as source")
+    tr = ObjectTransformer()
+    tr.load_transformer_specification(transformer_specification)
+    inverter = TransformationSpecificationInverter(
+        source_schemaview=SchemaView(schema),
+        **kwargs,
+    )
+    inverted_spec = inverter.invert(tr.specification)
+    if output:
+        outfile = open(output, "w", encoding="utf-8")
+    else:
+        outfile = sys.stdout
+    outfile.write(yaml_dumper.dumps(inverted_spec))
+
+
+if __name__ == "__main__":
+    main()

linkml_map-0.1.0/src/linkml_map/compiler/__init__.py

@@ -0,0 +1 @@
+"""Compiles schemas into other transformation frameworks."""

linkml_map-0.1.0/src/linkml_map/compiler/awk_compiler.py

@@ -0,0 +1,13 @@
+from linkml_map.compiler.compiler import CompiledSpecification, Compiler
+from linkml_map.datamodel.transformer_model import TransformationSpecification
+
+
+class AWKCompiler(Compiler):
+    """
+    Compiles a Transformation Specification to an Awk script.
+
+    Note: this is only expected to work for flat schemas.
+    """
+
+    def compile(self, specification: TransformationSpecification) -> CompiledSpecification:
+        raise NotImplementedError

linkml_map-0.1.0/src/linkml_map/compiler/compiler.py

@@ -0,0 +1,88 @@
+"""
+Compilers are responsible for compiling a transformation specification into an alternative representation.
+
+This is the opposite of an importer.
+
+For example:
+
+- LinkML-Transformer Specifications to R2RML
+- LinkML-Transformer Specifications to awk scripts
+- LinkML-Transformer Specifications to SQL
+- LinkML-Transformer Specifications to Python (OO)
+- LinkML-Transformer Specifications to Pandas
+- LinkML-Transformer Specifications to Hamilton
+"""
+
+from abc import ABC
+from dataclasses import dataclass, field
+from types import ModuleType
+from typing import Iterator, Optional
+
+from linkml_runtime import SchemaView
+from linkml_runtime.dumpers import yaml_dumper
+from linkml_runtime.utils.compile_python import compile_python
+
+from linkml_map.datamodel.transformer_model import TransformationSpecification
+from linkml_map.inference.schema_mapper import SchemaMapper
+
+
+@dataclass
+class CompiledSpecification:
+    serialization: str = field(default="")
+
+    _module: Optional[ModuleType] = None
+
+    @property
+    def module(self) -> ModuleType:
+        if not self._module:
+            self._module = compile_python(self.serialization)
+        return self._module
+
+
+@dataclass
+class Compiler(ABC):
+    """
+    Base class for all compilers.
+
+    A compiler will compile a transformation specification into
+    an alternative representation.
+
+    An example compiler would be a R2RML compiler.
+
+    Note: Compilers and Importers will in general be implemented by providing
+    mapping specifications
+    """
+
+    source_schemaview: SchemaView = None
+    """A view over the schema describing the source."""
+
+    source_python_module: str = None
+    """The python module containing the source classes."""
+
+    target_python_module: str = None
+    """The python module containing the target classes."""
+
+    def compile(self, specification: TransformationSpecification) -> CompiledSpecification:
+        """
+        Transform source object into an instance of the target class.
+
+        :param specification:
+        :return:
+        """
+        s = self._compile_header(specification)
+        for chunk in self._compile_iterator(specification):
+            s += chunk
+        return CompiledSpecification(serialization=s)
+
+    def _compile_header(self, specification: TransformationSpecification) -> str:
+        return ""
+
+    def _compile_iterator(self, specification: TransformationSpecification) -> Iterator[str]:
+        raise NotImplementedError
+
+    def derived_target_schemaview(self, specification: TransformationSpecification):
+        """
+        Returns a view over the target schema, including any derived classes.
+        """
+        mapper = SchemaMapper(source_schemaview=self.source_schemaview)
+        return SchemaView(yaml_dumper.dumps(mapper.derive_schema(specification)))

linkml_map-0.1.0/src/linkml_map/compiler/graphviz_compiler.py

@@ -0,0 +1,109 @@
+import re
+from dataclasses import dataclass
+from typing import List, Optional, Tuple
+
+from graphviz import Digraph
+from linkml_runtime import SchemaView
+from pydantic import BaseModel
+
+from linkml_map.compiler.compiler import CompiledSpecification, Compiler
+from linkml_map.datamodel.transformer_model import TransformationSpecification
+
+
+class Record(BaseModel):
+    """
+    A simplified representation of a class, UML-style.
+    """
+
+    name: str
+    source: str
+    fields: List[Tuple[str, str]] = []
+
+    @property
+    def id(self):
+        return f"{self.source}{self.name}"
+
+    def __str__(self):
+        return (
+            f"""<
+        <TABLE BORDER="0" CELLBORDER="1" CELLSPACING="0">
+            <TR><TD><B>{self.name}</B></TD></TR>
+        """
+            + "".join(
+                [
+                    f'<TR><TD SIDES="LRB" PORT="{f[0]}">{f[0]} : {f[1]}</TD></TR>'
+                    for f in self.fields
+                ]
+            )
+            + """
+        </TABLE>>"""
+        )
+
+
+@dataclass
+class GraphvizObject(CompiledSpecification):
+    digraph: Digraph = None
+
+    def render(self, file_path: str, format="png", view=False) -> None:
+        """
+        Renders a graphviz graph to a file.
+        :param file_path:
+        :return:
+        """
+        self.digraph.render(file_path, format=format, view=view)
+
+
+class GraphvizCompiler(Compiler):
+    """
+    Compiles a Transformation Specification to GraphViz.
+    """
+
+    def compile(
+        self, specification: TransformationSpecification, elements: Optional[List[str]] = None
+    ) -> GraphvizObject:
+        dg = Digraph(comment="UML Class Diagram", format="png")
+        dg.attr(rankdir="LR")  # Set graph direction from left to right
+        target_schemaview = self.derived_target_schemaview(specification)
+        source_schemaview = self.source_schemaview
+
+        records = []
+        records += self.add_records(source_schemaview, "source")
+        records += self.add_records(target_schemaview, "target")
+
+        for record in records:
+            dg.node(record.id, str(record), shape="plaintext")
+
+        # Define the class nodes with fields in UML format using HTML-like labels
+        # for precise control over the stacking of the fields
+        for target_cn, cd in specification.class_derivations.items():
+            source_cn = cd.populated_from
+            if source_cn is None:
+                source_cn = cd.name
+            if elements is not None and target_cn not in elements:
+                continue
+            source_record = Record(name=source_cn, source="source")
+            target_record = Record(name=target_cn, source="target")
+            for sd in cd.slot_derivations.values():
+                target_slot = sd.name
+                target_id = f"{target_record.id}:{target_slot}"
+                source_slot = sd.populated_from
+                if source_slot:
+                    source_id = f"{source_record.id}:{source_slot}"
+                    dg.edge(source_id, target_id)
+                elif sd.expr:
+                    # TODO: do this in a less hacky way
+                    tokens = re.findall(r"\w+", sd.expr)
+                    for token in tokens:
+                        if token not in source_schemaview.all_slots():
+                            continue
+                        dg.edge(f"{source_record.id}:{token}", target_id, style="dashed")
+        return GraphvizObject(digraph=dg, serialization=dg.source)
+
+    def add_records(self, schemaview: SchemaView, source: str) -> List[Record]:
+        records = []
+        for cn in schemaview.all_classes():
+            record = Record(name=cn, source=source)
+            for induced_slot in schemaview.class_induced_slots(cn):
+                record.fields.append((induced_slot.name, induced_slot.range))
+            records.append(record)
+        return records

linkml_map-0.1.0/src/linkml_map/compiler/j2_based_compiler.py

@@ -0,0 +1,36 @@
+from dataclasses import dataclass
+
+from jinja2 import Environment, FileSystemLoader
+
+from linkml_map.compiler.compiler import CompiledSpecification, Compiler
+from linkml_map.compiler.templates import TEMPLATE_DIR
+from linkml_map.datamodel.transformer_model import TransformationSpecification
+
+
+@dataclass
+class J2BasedCompiler(Compiler):
+    """
+    Compiles a Transformation Specification using a Jinja2 template.
+    """
+
+    template_dir: str = None
+    """The directory containing the Jinja2 template."""
+
+    template_name: str = None
+    """The name of the Jinja2 template."""
+
+    def compile(self, specification: TransformationSpecification) -> CompiledSpecification:
+        template_dir = self.template_dir
+        if not template_dir:
+            template_dir = TEMPLATE_DIR
+        if not template_dir:
+            raise ValueError("template_dir must be set")
+        loader = FileSystemLoader(template_dir)
+        env = Environment(loader=loader, autoescape=True)
+        if not self.template_name:
+            raise ValueError("template_name must be set")
+        template = env.get_template(self.template_name)
+        rendered = template.render(
+            spec=specification,
+        )
+        return CompiledSpecification(serialization=rendered)

linkml_map-0.1.0/src/linkml_map/compiler/markdown_compiler.py

@@ -0,0 +1,12 @@
+from dataclasses import dataclass
+
+from linkml_map.compiler.j2_based_compiler import J2BasedCompiler
+
+
+@dataclass
+class MarkdownCompiler(J2BasedCompiler):
+    """
+    Compiles a Transformation Specification to Markdown.
+    """
+
+    template_name: str = "markdown.j2"