setlr 1.0.2__py3-none-any.whl

setlr/trig_store.py

@@ -0,0 +1,158 @@
+from os.path import abspath
+from rdflib.store import Store, VALID_STORE, NO_STORE
+from rdflib.term import URIRef
+from urllib.request import pathname2url
+from rdflib import ConjunctiveGraph
+from rdflib.term import BNode
+
+def bb(u):
+    return u.encode("utf-8")
+
+
+class TrigStore(Store):
+    """\
+    """
+
+    context_aware = True
+    formula_aware = True
+    transaction_aware = False
+    graph_aware = True
+    db_env = None
+
+    def __init__(self, configuration=None, identifier=None):
+        self.__open = False
+        self.__identifier = identifier
+        super(TrigStore, self).__init__(configuration)
+        self.__namespace = {}
+        self.__prefix = {}
+
+    def __get_identifier(self):
+        return self.__identifier
+
+    identifier = property(__get_identifier)
+
+    def _init_db_environment(self, file, create=True):
+        file_obj = open(file, "a+")
+        return file_obj
+
+    def is_open(self):
+        return self.__open
+
+    def open(self, path, create=True):
+        homeDir = path
+
+        if self.__identifier is None:
+            self.__identifier = URIRef(pathname2url(abspath(homeDir)))
+
+        db_env = self._init_db_environment(homeDir, create)
+        if db_env == NO_STORE:
+            return NO_STORE
+        self.db_env = db_env
+        self.__open = True
+
+        return VALID_STORE
+
+    def sync(self):
+        pass
+
+    def close(self, commit_pending_transaction=False):
+        self.__open = False
+        self.db_env.close()
+
+    def add(self, triple, context, quoted=False, txn=None):
+        """\
+        Add a triple to the store of triples.
+        """
+        (subject, predicate, object) = triple
+        assert self.__open, "The Store must be open."
+        assert context != self, "Can not add triple directly to store"
+        value = "%s { %s %s %s .}\n" % (context.identifier.n3(), subject.n3(), predicate.n3(), object.n3())
+        self.db_env.seek(0,2) # seek to end
+        self.db_env.write(value)
+
+    def __remove(self, spo, c, quoted=False, txn=None):
+        pass
+
+    def remove(self, spo, context, txn=None):
+        pass
+
+    def triples(self, spo, context=None, txn=None):
+        """A generator over all the triples matching"""
+        assert self.__open, "The Store must be open."
+
+        if context is None or isinstance(context.identifier, BNode):
+            c = None
+        else:
+            c = context.identifier
+        subject, predicate, object = spo
+        self.db_env.seek(0)
+        for line in self.db_env:
+            g = ConjunctiveGraph()
+            g.parse(data=line, format="trig")
+
+            for s, p, o, ctx in g.quads((None, None, None, None)):
+                #print(s, p, o, ctx.identifier)
+                if subject is not None and subject != s:
+                    continue
+                if predicate is not None and predicate != p:
+                    continue
+                if object is not None and object != o:
+                    continue
+                if c is not None :
+                    if c != ctx.identifier:
+                        continue
+                yield (s, p, o), ctx.identifier
+
+    def __len__(self, context=None):
+        def blocks(files, size=65536):
+            while True:
+                b = files.read(size)
+                if not b:
+                    break
+                yield b
+
+        self.db_env.seek(0)
+        return (sum(bl.count("\n") for bl in blocks(self.db_env)))
+
+    def contexts(self, triple=None):
+        for context in []:
+            yield context
+
+
+    def add_graph(self, graph):
+        pass
+
+    def remove_graph(self, graph):
+        pass
+
+    def __ctx_to_str(self, ctx):
+        if ctx is None:
+            return None
+        try:
+            # ctx could be a graph. In that case, use its identifier
+            ctx_str = "{}:{}".format(ctx.identifier.__class__.__name__, ctx.identifier)
+            self.__context_obj_map[ctx_str] = ctx
+            return ctx_str
+        except AttributeError:
+            # otherwise, ctx should be a URIRef or BNode or str
+            if isinstance(ctx, str):
+                ctx_str = "{}:{}".format(ctx.__class__.__name__, ctx)
+                if ctx_str in self.__context_obj_map:
+                    return ctx_str
+                self.__context_obj_map[ctx_str] = ctx
+                return ctx_str
+            raise RuntimeError("Cannot use that type of object as a Graph context")
+
+    def bind(self, prefix, namespace):
+        self.__prefix[namespace] = prefix
+        self.__namespace[prefix] = namespace
+
+    def namespace(self, prefix):
+        return self.__namespace.get(prefix, None)
+
+    def prefix(self, namespace):
+        return self.__prefix.get(namespace, None)
+
+    def namespaces(self):
+        for prefix, namespace in self.__namespace.items():
+            yield prefix, namespace

setlr-1.0.2.dist-info/METADATA

@@ -0,0 +1,209 @@
+Metadata-Version: 2.4
+Name: setlr
+Version: 1.0.2
+Summary: setlr is a tool for Semantic Extraction, Transformation, and Loading.
+Author-email: Jamie McCusker <mccusj@cs.rpi.edu>
+Project-URL: Homepage, http://packages.python.org/setlr
+Keywords: rdf,semantic,etl
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Topic :: Utilities
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: future
+Requires-Dist: cython
+Requires-Dist: numpy
+Requires-Dist: rdflib>=6.0.0
+Requires-Dist: pandas>=0.23.0
+Requires-Dist: requests
+Requires-Dist: toposort
+Requires-Dist: beautifulsoup4
+Requires-Dist: jinja2
+Requires-Dist: lxml
+Requires-Dist: six
+Requires-Dist: xlrd
+Requires-Dist: ijson
+Requires-Dist: click
+Requires-Dist: tqdm
+Requires-Dist: requests-testadapter
+Requires-Dist: python-slugify
+Requires-Dist: pyshacl[js]
+Dynamic: license-file
+
+# setlr: Semantic Extract, Transform and Load
+
+[![Unit Tests](https://github.com/tetherless-world/setlr/actions/workflows/test.yml/badge.svg)](https://github.com/tetherless-world/setlr/actions/workflows/test.yml)
+[![Lint](https://github.com/tetherless-world/setlr/actions/workflows/lint.yml/badge.svg)](https://github.com/tetherless-world/setlr/actions/workflows/lint.yml)
+[![codecov](https://codecov.io/gh/tetherless-world/setlr/branch/main/graph/badge.svg)](https://codecov.io/gh/tetherless-world/setlr)
+
+**SETLr** is a powerful Python tool for generating RDF graphs from tabular data using declarative SETL (Semantic Extract, Transform, Load) scripts.
+
+## Features
+
+✨ **Multiple Data Sources**: CSV, Excel, JSON, XML, RDF, SAS files  
+🔄 **Flexible Transformations**: JSON-LD templates with Jinja2, Python functions, SPARQL  
+⚡ **High Performance**: Streaming XML parsing, pandas DataFrames, progress tracking  
+🐍 **Python Integration**: Use as library or CLI tool  
+✅ **Validation**: Built-in SHACL validation  
+📝 **Well Documented**: Comprehensive guides and API reference  
+
+## Quick Start
+
+### Installation
+
+```bash
+pip install setlr
+```
+
+### Simple Example
+
+Create `data.csv`:
+```csv
+ID,Name,Email
+1,Alice,alice@example.com
+2,Bob,bob@example.com
+```
+
+Create `transform.setl.ttl`:
+```turtle
+@prefix setl: <http://purl.org/twc/vocab/setl/> .
+@prefix prov: <http://www.w3.org/ns/prov#> .
+@prefix csvw: <http://www.w3.org/ns/csvw#> .
+@prefix void: <http://rdfs.org/ns/void#> .
+@prefix : <http://example.com/> .
+
+:table a csvw:Table, setl:Table ;
+    prov:wasGeneratedBy [ a setl:Extract ; prov:used <data.csv> ] .
+
+:output a void:Dataset ;
+    prov:wasGeneratedBy [
+        a setl:Transform, setl:JSLDT ;
+        prov:used :table ;
+        prov:value '''[{
+            "@id": "http://example.com/person/{{row.ID}}",
+            "@type": "http://xmlns.com/foaf/0.1/Person",
+            "http://xmlns.com/foaf/0.1/name": "{{row.Name}}",
+            "http://xmlns.com/foaf/0.1/mbox": "mailto:{{row.Email}}"
+        }]'''
+    ] .
+```
+
+Run SETLr:
+```bash
+setlr transform.setl.ttl
+```
+
+### Using from Python
+
+```python
+from rdflib import Graph, URIRef
+import setlr
+
+# Load SETL script
+setl_graph = Graph()
+setl_graph.parse("transform.setl.ttl", format="turtle")
+
+# Execute ETL pipeline
+resources = setlr.run_setl(setl_graph)
+
+# Access generated RDF
+output = resources[URIRef('http://example.com/output')]
+print(f"Generated {len(output)} RDF triples")
+```
+
+## Documentation
+
+📚 **[Complete Documentation](docs/README.md)** - Full guides and references
+
+**Quick Links:**
+- [Tutorial](docs/tutorial.md) - Step-by-step guide to SETLr
+- [JSLDT Template Language](docs/jsldt.md) - Transform syntax reference
+- [Python API](docs/python-api.md) - Using SETLr from Python
+- [Quick Start](docs/quickstart.md) - Get started in 5 minutes
+- [Examples](docs/examples.md) - Real-world examples
+
+**Advanced Topics:**
+- [Streaming XML with XPath](docs/streaming-xml.md) - Efficient large file processing
+- [Python Functions](docs/python-functions.md) - Custom Python transforms
+- [SPARQL Support](docs/sparql.md) - Query and update endpoints
+- [SHACL Validation](docs/shacl.md) - Validate your RDF output
+
+## Key Concepts
+
+SETLr uses RDF (with PROV-O vocabulary) to describe ETL workflows:
+
+1. **Extract**: Load data from sources (CSV, Excel, JSON, XML, RDF, SAS)
+2. **Transform**: Apply templates or Python scripts to generate RDF
+3. **Load**: Save to files or SPARQL endpoints
+
+## Supported Formats
+
+**Input:**
+- Tabular: CSV, TSV, Excel (XLS/XLSX), SAS (XPORT/SAS7BDAT)
+- Structured: JSON (with ijson selectors), XML (with XPath streaming)
+- Semantic: RDF (Turtle, JSON-LD, RDF/XML, etc.), OWL Ontologies
+
+**Output:**
+- RDF: Turtle, TriG, N-Triples, N3, RDF/XML, JSON-LD
+- Destinations: Files, SPARQL Update endpoints
+
+## Examples
+
+See the [examples/](example/) directory for complete working examples:
+
+- `social.setl.ttl` - Basic CSV to RDF with conditionals and loops
+- `ontology.setl.ttl` - OWL ontology transformation with SHACL shapes
+
+## Development
+
+```bash
+# Clone repository
+git clone https://github.com/tetherless-world/setlr.git
+cd setlr
+
+# Bootstrap (creates venv and installs dependencies)
+./script/bootstrap
+
+# Activate virtual environment  
+source venv/bin/activate
+
+# Run tests
+./script/build
+
+# Run linter
+flake8 setlr/
+```
+
+## Contributing
+
+Contributions are welcome! Please see our [Contributing Guide](CONTRIBUTING.md) for details on:
+- Development setup and workflow
+- Code standards and style guidelines
+- Testing requirements
+- Pull request process
+
+Please note that this project follows a [Code of Conduct](CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code.
+
+## License
+
+Apache License 2.0 - see [LICENSE](LICENSE) file for details.
+
+## Citation
+
+If you use SETLr in your research, please cite:
+
+```bibtex
+@software{setlr,
+  title = {SETLr: Semantic Extract, Transform and Load},
+  author = {McCusker, Jamie},
+  year = {2024},
+  url = {https://github.com/tetherless-world/setlr}
+}
+```
+
+## Support
+
+- 📖 [Documentation](docs/README.md)
+- 🐛 [Issue Tracker](https://github.com/tetherless-world/setlr/issues)
+- 💬 [Discussions](https://github.com/tetherless-world/setlr/discussions)
+- 🔒 [Security Policy](SECURITY.md) - Report security vulnerabilities

setlr-1.0.2.dist-info/RECORD

@@ -0,0 +1,10 @@
+setlr/__init__.py,sha256=Q1qXaRi8kFYylbK484DSB4OBhtVHthnzOpyNz8B2g-U,1653
+setlr/core.py,sha256=9p_EgNhGrpi2iyl7Y__jdCpNHabG2X7_6EDcrhvtSH8,39889
+setlr/iterparse_filter.py,sha256=I7lx4DCF3jKIFdOSyls0EwrUhWcyxQPiec4oQuvBjoA,24371
+setlr/trig_store.py,sha256=3xAodK-tIddD_0X3WAWpeiUbKXSrqGoWTLUkg4QobNk,4810
+setlr-1.0.2.dist-info/licenses/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
+setlr-1.0.2.dist-info/METADATA,sha256=X3bGWxxNHJXXVT2vrXCrvp2Eva4obyIvl06rpyRTN4Q,6202
+setlr-1.0.2.dist-info/WHEEL,sha256=qELbo2s1Yzl39ZmrAibXA2jjPLUYfnVhUNTlyF1rq0Y,92
+setlr-1.0.2.dist-info/entry_points.txt,sha256=qBRdeA-Gu1SMKniJptXGe2oo3IPOPABTQPMFwfux8fw,37
+setlr-1.0.2.dist-info/top_level.txt,sha256=x23SyJAUjYkpfhcZIwivm0nYNx6TW3Zj7aiQMLQ3hWc,6
+setlr-1.0.2.dist-info/RECORD,,

setlr-1.0.2.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.10.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

setlr-1.0.2.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+setlr = setlr:main

setlr-1.0.2.dist-info/licenses/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright {yyyy} {name of copyright owner}
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

setlr-1.0.2.dist-info/top_level.txt

@@ -0,0 +1 @@
+setlr