maplib 0.19.3__cp310-abi3-win_amd64.whl

maplib/adding_triples.py

@@ -0,0 +1,29 @@
+from maplib.maplib import Model, Template, IRI, Triple, Variable
+
+
+def add_triples(
+    source: Model, target: Model, source_graph: str = None, target_graph: str = None
+):
+    """(Zero) copy the triples from one Model into another.
+
+    :param source: The source model
+    :param target: The target model
+    :param source_graph: The named graph in the source model to copy from. None means default graph.
+    :param target_graph: The named graph in the target model to copy into. None means default graph.
+    """
+    for p in source.get_predicate_iris(source_graph):
+        subject = Variable("subject")
+        object = Variable("object")
+        template = Template(
+            iri=IRI("urn:maplib:tmp"),
+            parameters=[subject, object],
+            instances=[Triple(subject, p, object)],
+        )
+        sms = source.get_predicate(p, source_graph)
+        for sm in sms:
+            target.map(
+                template,
+                sm.mappings,
+                types=sm.rdf_types,
+                graph=target_graph,
+            )

maplib/template_generator/__init__.py

@@ -0,0 +1 @@
+from .generate import generate_templates

maplib/template_generator/generate.py

@@ -0,0 +1,234 @@
+from typing import Dict, Optional
+from .ordering import topological_sort
+import polars as pl
+
+from maplib import Model, Variable, RDFType, Parameter, Triple, IRI, Template
+
+
+def generate_templates(m: Model, graph: Optional[str]) -> Dict[str, Template]:
+    """Generate templates for instantiating the classes in an ontology
+
+    :param m: The model where the ontology is stored. We mainly rely on rdfs:subClassOf, rdfs:range and rdfs:domain.
+    :param graph: The named graph where the ontology is stored.
+
+    :return A dictionary of templates for instantiating the classes in the ontology, where the keys are the class URIs.
+
+    Usage example - note that it is important to add the templates to the Model you want to populate.
+    >>> from maplib import Model, create_templates
+    >>>
+    >>> m_ont = Model()
+    >>> m_ont.read("my_ontology.ttl")
+    >>> templates = generate_templates(m_ont)
+    >>> m = Model()
+    >>> for t in templates.values():
+    >>>     m.add_template(t)
+    >>> m.map("https://example.net/MyClass", df)
+    """
+
+    properties = get_properties(m, graph=graph)
+    properties_by_domain = {}
+    properties_by_range = {}
+    for r in properties.iter_rows(named=True):
+        dom = r["domain"]
+        if dom in properties_by_domain:
+            properties_by_domain[dom].append(r)
+        else:
+            properties_by_domain[dom] = [r]
+
+        ran = r["range"]
+        if ran in properties_by_range:
+            properties_by_range[ran].append(r)
+        else:
+            properties_by_range[ran] = [r]
+
+    subclasses = get_subclasses(m, graph=graph)
+
+    subclass_of = {}
+    for r in (
+        subclasses.group_by("child")
+        .agg(pl.col("parent").alias("parents"))
+        .iter_rows(named=True)
+    ):
+        subclass_of[r["child"]] = r["parents"]
+
+    class_ordering = topological_sort(subclasses)
+
+    templates_without_typing = generate_templates_without_typing(
+        properties_by_domain, properties_by_range, class_ordering, subclass_of
+    )
+    templates_with_typing = generate_templates_with_typing(templates_without_typing)
+    templates = {}
+    for t, template in templates_without_typing.items():
+
+        templates[t + "_notype"] = template
+    for t, template in templates_with_typing.items():
+        templates[t] = template
+
+    return templates
+
+
+def get_properties(m: Model, graph: Optional[str]) -> pl.DataFrame:
+    properties = m.query(
+        """
+        PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#> 
+        PREFIX owl:  <http://www.w3.org/2002/07/owl#> 
+
+        SELECT ?property ?property_type ?domain ?range WHERE { 
+            ?property a ?property_type .
+            ?property rdfs:domain ?domain .
+            ?property rdfs:range ?range .
+            FILTER(ISIRI(?domain) && ISIRI(?range))
+        }
+        """,
+        native_dataframe=True,
+        graph=graph,
+    )
+    pl.Config.set_fmt_str_lengths(100)
+    properties_by_subclass_restriction = m.query(
+        """
+        PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#> 
+        PREFIX owl:  <http://www.w3.org/2002/07/owl#> 
+
+        SELECT ?property ?property_type ?domain WHERE { 
+            ?domain rdfs:subClassOf ?restr .
+            ?restr a owl:Restriction .
+            ?restr owl:onProperty ?property .
+            #?property a ?property_type .
+        }
+        """,
+        native_dataframe=True,
+        graph=graph,
+    )
+    properties_by_subclass_restriction = (
+        properties_by_subclass_restriction.with_columns(
+            pl.lit("http://www.w3.org/2000/01/rdf-schema#ObjectType").alias(
+                "property_type"
+            ),
+            pl.lit(None).cast(pl.String).alias("range"),
+        )
+    )
+    if properties.height == 0:
+        return properties_by_subclass_restriction
+    elif properties_by_subclass_restriction.height == 0:
+        return properties
+    else:
+        return properties.vstack(properties_by_subclass_restriction)
+
+def get_subclasses(m: Model, graph: Optional[str]) -> pl.DataFrame:
+    subclasses = m.query(
+        """
+        PREFIX rdfs:<http://www.w3.org/2000/01/rdf-schema#> 
+        SELECT ?child ?parent WHERE { 
+            ?child rdfs:subClassOf ?parent .
+            FILTER(ISIRI(?child) && ISIRI(?parent))
+        }
+        """,
+        native_dataframe=True,
+        graph=graph,
+    )
+    return subclasses
+
+
+def uri_to_variable(uri: str) -> Variable:
+    split = uri.split("/")
+    name = split[-1].split("#")[-1]
+    cleaned_name = name.replace("-", "_").replace(".", "_")
+    return Variable(cleaned_name)
+
+
+def generate_templates_without_typing(
+    properties_by_domain,
+    properties_by_range,
+    class_ordering,
+    subclass_of: Dict[str, str],
+) -> Dict[str, Template]:
+    templates = {}
+    i = 0
+    subj = get_subj_var()
+
+    for c in class_ordering:
+        i += 1
+
+        parameters = []
+        instances = []
+        existing_preds = set()
+        existing_varnames = set()
+
+        # Check dupe!!
+        subj_parameter = Parameter(
+            variable=subj, optional=False, rdf_type=RDFType.IRI
+        )
+        parameters.append(subj_parameter)
+        if c in properties_by_domain:
+            for p in properties_by_domain[c]:
+                if p["property"] in existing_preds:
+                    # print("dupe: ", str(p))
+                    continue
+                existing_preds.add(p["property"])
+                v = uri_to_variable(p["property"])
+                existing_varnames.add(v.name)
+                if p["property_type"] == "http://www.w3.org/2002/07/owl#ObjectProperty":
+                    t = RDFType.IRI
+                elif p["range"]:
+                    t = RDFType.Literal(p["range"])
+                else:
+                    t = None
+                param = Parameter(variable=v, optional=True, rdf_type=t)
+                parameters.append(param)
+                predicate = IRI(p["property"])
+                triple = Triple(subj, predicate, v)
+                instances.append(triple)
+        if c in properties_by_range:
+            for p in properties_by_range[c]:
+                if p["property"] in existing_preds:
+                    # print("dupe: ", str(p))
+                    continue
+                existing_preds.add(p["property"])
+                v = uri_to_variable(p["property"])
+                existing_varnames.add(v.name)
+                t = RDFType.IRI
+                param = Parameter(variable=v, optional=True, rdf_type=t)
+                parameters.append(param)
+                predicate = IRI(p["property"])
+                triple = Triple(v, predicate, subj)
+                instances.append(triple)
+        c_tpl_iri = c + "_notype"
+        if c in subclass_of:
+            for sc in subclass_of[c]:
+                if sc in templates:
+                    sct = templates[sc]
+                    variables = []
+                    for p in sct.parameters:
+                        if p.variable.name not in existing_varnames:
+                            variables.append(p.variable)
+                            if p.variable.name != "id":
+                                parameters.append(p)
+                        else:
+                            variables.append(None)
+                            # print(f"Duplicate variable: {str(p.variable.name)}")
+                    instances.append(sct.instance(variables))
+
+        tpl = Template(IRI(c_tpl_iri), parameters=parameters, instances=instances)
+        templates[c] = tpl
+    return templates
+
+
+def generate_templates_with_typing(
+    templates: Dict[str, Template],
+) -> Dict[str, Template]:
+    subj = get_subj_var()
+    with_type_templates = {}
+    for t, template in templates.items():
+        instances = template.instances
+        instances.append(
+            Triple(subj, IRI("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), IRI(t))
+        )
+        new_template = Template(
+            iri=IRI(t), parameters=template.parameters, instances=instances
+        )
+        with_type_templates[t] = new_template
+    return with_type_templates
+
+
+def get_subj_var() -> Variable:
+    return Variable("id")

maplib/template_generator/ordering.py

@@ -0,0 +1,40 @@
+import polars as pl
+from typing import List
+
+
+def topological_sort(subclasses: pl.DataFrame) -> List[str]:
+    in_degree_df = subclasses.group_by("child").agg([pl.col("parent").count()])
+    in_degree_dict = {}
+    for d in in_degree_df.iter_rows(named=True):
+        in_degree_dict[d["child"]] = d["parent"]
+
+    out_edges_df = subclasses.group_by("parent").agg([pl.col("child")])
+    out_edges_dict = {}
+    for d in out_edges_df.iter_rows(named=True):
+        out_edges_dict[d["parent"]] = d["child"]
+
+    sorting = []
+
+    # First, we can add those parents who have no parents.
+    for p in subclasses.select("parent").unique().iter_rows(named=True):
+        if p["parent"] not in in_degree_dict:
+            sorting.append(p["parent"])
+            for out_edge in out_edges_dict[p["parent"]]:
+                in_degree_dict[out_edge] = in_degree_dict[out_edge] - 1
+
+    # Next, a topological sort defines ordering.
+    while len(in_degree_dict) > 0:
+        found_p = None
+        for p, i in in_degree_dict.items():
+            if i == 0:
+                found_p = p
+                break
+        if found_p is None:
+            assert False, "Error in the algorithm.. "
+
+        sorting.append(found_p)
+        in_degree_dict.pop(found_p)
+        if found_p in out_edges_dict:
+            for out_edge in out_edges_dict[found_p]:
+                in_degree_dict[out_edge] = in_degree_dict[out_edge] - 1
+    return sorting

maplib-0.19.3.dist-info/METADATA

@@ -0,0 +1,207 @@
+Metadata-Version: 2.4
+Name: maplib
+Version: 0.19.3
+Classifier: Development Status :: 4 - Beta
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Rust
+Classifier: Topic :: Database :: Database Engines/Servers
+Classifier: Topic :: Scientific/Engineering
+Requires-Dist: polars>=0.20.13
+Requires-Dist: pyarrow>=7.0.0
+Requires-Dist: fastapi[standard]>=0.115
+License-File: LICENSE
+Summary: Dataframe-based interactive knowledge graph construction
+Keywords: rdf,graph,dataframe,sparql,ottr
+Author-email: Magnus Bakken <magnus@data-treehouse.com>
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+Project-URL: Homepage, https://github.com/DataTreehouse/maplib
+Project-URL: Documentation, https://datatreehouse.github.io/maplib/maplib/maplib.html
+Project-URL: Repository, https://github.com/DataTreehouse/maplib
+Project-URL: Changelog, https://github.com/DataTreehouse/maplib/releases
+
+## maplib: High-performance RDF knowledge graph construction, SHACL validation and SPARQL-based enrichment in Python
+maplib is a knowledge graph construction library for building RDF knowledge graphs using template expansion ([OTTR](https://ottr.xyz/) Templates). Maplib features SPARQL- and SHACL-engines that are available as the graph is being constructed, allowing enrichment and validation. It can construct and validate knowledge graphs with millions of nodes in seconds.
+
+maplib allows you to leverage your existing skills with Pandas or Polars to extract and wrangle data from existing databases and spreadsheets, before applying simple templates to them to build a knowledge graph. 
+
+Template expansion is typically zero-copy and nearly instantaneous, and the built-in SPARQL and SHACL engines means you can query, inspect, enrich and validate the knowledge graph immediately.      
+
+maplib is written in Rust, it is built on [Apache Arrow](https://arrow.apache.org/) using [Pola.rs](https://www.pola.rs/) and uses libraries from [Oxigraph](https://github.com/oxigraph/oxigraph) for handling linked data as well as parsing SPARQL queries.
+
+## Installing
+The package is published on [PyPi](https://pypi.org/project/maplib/) and the API documented [here](https://datatreehouse.github.io/maplib/maplib.html):
+```shell
+pip install maplib
+```
+Please send us a message, e.g. on LinkedIn (search for Data Treehouse) or on our [webpage](https://www.data-treehouse.com/contact-8) if you want to try out SHACL.  
+
+## Model
+We can easily map DataFrames to RDF-graphs using the Python library. Below is a reproduction of the example in the paper [1]. Assume that we have a DataFrame given by: 
+
+```python
+import polars as pl
+pl.Config.set_fmt_str_lengths(150)
+
+pi = "https://github.com/DataTreehouse/maplib/pizza#"
+df = pl.DataFrame({
+    "p":[pi + "Hawaiian", pi + "Grandiosa"],
+    "c":[pi + "CAN", pi + "NOR"],
+    "ings": [[pi + "Pineapple", pi + "Ham"],
+             [pi + "Pepper", pi + "Meat"]]
+})
+print(df)
+```
+That is, our DataFrame is:
+
+| p                             | c                              | ings                                     |
+|-------------------------------|--------------------------------|------------------------------------------|
+| str                           | str                            | list[str]                                |
+| "https://.../pizza#Hawaiian"  | "https://.../maplib/pizza#CAN" | [".../pizza#Pineapple", ".../pizza#Ham"] |
+| "https://.../pizza#Grandiosa" | "https://.../maplib/pizza#NOR" | [".../pizza#Pepper", ".../pizza#Meat"]   |
+
+Then we can define a OTTR template, and create our knowledge graph by expanding this template with our DataFrame as input:
+```python
+from maplib import Model, Prefix, Template, Argument, Parameter, Variable, RDFType, Triple, a
+pi = Prefix(pi)
+
+p_var = Variable("p")
+c_var = Variable("c")
+ings_var = Variable("ings")
+
+template = Template(
+    iri= pi.suf("PizzaTemplate"),
+    parameters= [
+        Parameter(variable=p_var, rdf_type=RDFType.IRI()),
+        Parameter(variable=c_var, rdf_type=RDFType.IRI()),
+        Parameter(variable=ings_var, rdf_type=RDFType.Nested(RDFType.IRI()))
+    ],
+    instances= [
+        Triple(p_var, a, pi.suf("Pizza")),
+        Triple(p_var, pi.suf("fromCountry"), c_var),
+        Triple(
+            p_var, 
+            pi.suf("hasIngredient"), 
+            Argument(term=ings_var, list_expand=True), 
+            list_expander="cross")
+    ]
+)
+
+m = Model()
+m.map(template, df)
+hpizzas = """
+    PREFIX pi:<https://github.com/DataTreehouse/maplib/pizza#>
+    CONSTRUCT { ?p a pi:HeterodoxPizza } 
+    WHERE {
+        ?p a pi:Pizza .
+        ?p pi:hasIngredient pi:Pineapple .
+    }"""
+m.insert(hpizzas)
+return m
+```
+
+We can immediately query the mapped knowledge graph:
+
+```python
+m.query("""
+PREFIX pi:<https://github.com/DataTreehouse/maplib/pizza#>
+SELECT ?p ?i WHERE {
+?p a pi:Pizza .
+?p pi:hasIngredient ?i .
+}
+""")
+```
+
+The query gives the following result (a DataFrame):
+
+| p                               | i                                     |
+|---------------------------------|---------------------------------------|
+| str                             | str                                   |
+| "<https://.../pizza#Grandiosa>" | "<https://.../pizza#Meat>"      |
+| "<https://.../pizza#Grandiosa>" | "<https://.../pizza#Pepper>"    |
+| "<https://.../pizza#Hawaiian>"  | "<https://.../pizza#Pineapple>" |
+| "<https://.../pizza#Hawaiian>"  | "<https://.../pizza#Ham>"       |
+
+Next, we are able to perform a construct query, which creates new triples but does not insert them. 
+
+```python
+hpizzas = """
+PREFIX pi:<https://github.com/DataTreehouse/maplib/pizza#>
+CONSTRUCT { ?p a pi:UnorthodoxPizza } 
+WHERE {
+    ?p a pi:Pizza .
+    ?p pi:hasIngredient pi:Pineapple .
+}"""
+res = m.query(hpizzas)
+res[0]
+```
+
+The resulting triples are given below:
+
+| subject                        | verb                                 | object                                |
+|--------------------------------|--------------------------------------|---------------------------------------|
+| str                            | str                                  | str                                   |
+| "<https://.../pizza#Hawaiian>" | "<http://.../22-rdf-syntax-ns#type>" | "<https://.../pizza#UnorthodoxPizza>" |
+
+If we are happy with the output of this construct-query, we can insert it in the model state. Afterwards we check that the triple is added with a query.
+
+```python
+m.insert(hpizzas)
+m.query("""
+PREFIX pi:<https://github.com/DataTreehouse/maplib/pizza#>
+
+SELECT ?p WHERE {
+?p a pi:UnorthodoxPizza
+}
+""")
+```
+
+Indeed, we have added the triple: 
+
+| p                                                          |
+|------------------------------------------------------------|
+| str                                                        |
+| "<https://github.com/DataTreehouse/maplib/pizza#Hawaiian>" |
+
+## API
+The [API](https://datatreehouse.github.io/maplib/maplib.html) is simple, and contains only one class and a few methods for:
+- expanding templates
+- querying with SPARQL
+- validating with SHACL
+- importing triples (Turtle, RDF/XML, NTriples)
+- writing triples (Turtle, RDF/XML, NTriples)
+- creating a new Model object (sprout) based on queries over the current Model object.
+
+The API is documented [HERE](https://datatreehouse.github.io/maplib/maplib.html)
+
+## Roadmap of features and optimizations
+Spring 2025
+- Datalog reasoning support ✅
+- Reduced memory footprint ✅
+- Further SPARQL optimizations
+- JSON-LD support
+
+Fall 2025
+- SHACL rules support
+- Improved TTL serialization (prettier and faster)
++++
+
+Roadmap is subject to changes,particularly user and customer requests. 
+
+## References
+There is an associated paper [1] with associated benchmarks showing superior performance and scalability that can be found [here](https://ieeexplore.ieee.org/document/10106242). OTTR is described in [2].
+
+[1] M. Bakken, "maplib: Interactive, literal RDF model model for industry," in IEEE Access, doi: 10.1109/ACCESS.2023.3269093.
+
+[2] M. G. Skjæveland, D. P. Lupp, L. H. Karlsen, and J. W. Klüwer, “Ottr: Formal templates for pattern-based ontology engineering.” in WOP (Book),
+2021, pp. 349–377.
+
+## Licensing
+All code produced since August 1st. 2023 is copyrighted to [Data Treehouse AS](https://www.data-treehouse.com/) with an Apache 2.0 license unless otherwise noted. 
+
+All code which was produced before August 1st. 2023 copyrighted to [Prediktor AS](https://www.prediktor.com/) with an Apache 2.0 license unless otherwise noted, and has been financed by [The Research Council of Norway](https://www.forskningsradet.no/en/) (grant no. 316656) and [Prediktor AS](https://www.prediktor.com/) as part of a PhD Degree. The code at this state is archived in the repository at [https://github.com/magbak/maplib](https://github.com/magbak/maplib).
+

maplib-0.19.3.dist-info/RECORD

@@ -0,0 +1,13 @@
+maplib-0.19.3.dist-info/METADATA,sha256=mwSXi6090Hcuo4h8zNTWVUyIWKtceG7cJNCIIOFeMNU,9258
+maplib-0.19.3.dist-info/WHEEL,sha256=MF1HCxdpioEZC6lqltB_WBzy4tzMPetoB2hZW2ZWEzg,95
+maplib-0.19.3.dist-info/licenses/LICENSE,sha256=8f_rikNX2RHmVhT1CFq1M2itL6kTpawNjNTHUFCB870,11661
+maplib/.gitignore,sha256=88KgwL2QsVFk7EKzNn65u6Z-5ibwf9RPU6J68KuZotY,6
+maplib/__init__.py,sha256=XolwCUgnC2mTW9Z1YvmHdicsV-Z4r3RHr03KwWflMUE,1789
+maplib/__init__.pyi,sha256=DOgWoWRUtgGt0VZA3Z0NIQ_rmpP8Pj3NEwMvU3yxlN8,34745
+maplib/adding_triples.py,sha256=BSQQQs4-XgpGLax9UXC-GKZ2fu6BidvtgsBlRJJqxek,1084
+maplib/maplib.pyd,sha256=nDXa2taPxl1VCk1zXNl9dCHZdIvS0okGmlrwQY8OS3M,97077760
+maplib/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+maplib/template_generator/__init__.py,sha256=9w78SbwEDZY2moDzpiy0axVxIy5qWouoPqIFfLvKXKQ,42
+maplib/template_generator/generate.py,sha256=TjurFj-RyGziryFi0tcXWIqK6LGaaJRk09oSyvITlVg,8314
+maplib/template_generator/ordering.py,sha256=0yvKst8bp0df8cpiCEVljb0WOcvxD9y_XdOpg8SogtU,1468
+maplib-0.19.3.dist-info/RECORD,,

maplib-0.19.3.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: maturin (1.9.4)
+Root-Is-Purelib: false
+Tag: cp310-abi3-win_amd64