setlr 0.2.17__tar.gz → 0.2.19__tar.gz

setlr-0.2.19/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright {yyyy} {name of copyright owner}
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

{setlr-0.2.17 → setlr-0.2.19}/PKG-INFO

@@ -1,14 +1,17 @@
-Metadata-Version: 1.1
+Metadata-Version: 2.1
 Name: setlr
-Version: 0.2.17
+Version: 0.2.19
 Summary: setlr is a tool for Semantic Extraction, Transformation, and Loading.
 Home-page: http://packages.python.org/setlr
 Author: Jamie McCusker
 Author-email: mccusj@cs.rpi.edu
 License: Apache License 2.0
-Description: SETLr is a tool for generating RDF graphs, including named graphs, from almost any kind of tabular data.
 Keywords: rdf semantic etl
 Platform: UNKNOWN
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Topic :: Utilities
 Classifier: License :: OSI Approved :: Apache Software License
+License-File: LICENSE
+
+SETLr is a tool for generating RDF graphs, including named graphs, from almost any kind of tabular data.
+

{setlr-0.2.17 → setlr-0.2.19}/setlr/__init__.py

@@ -36,6 +36,8 @@ import logging
 import hashlib
 from slugify import slugify
 
+from .trig_store import TrigStore
+
 def hash(value):
     m = hashlib.sha256()
     m.update(value.encode('utf-8'))
@@ -414,9 +416,11 @@ def clone(value):
 
 functions = {}
 def get_function(expr, local_keys):
-    key = tuple([expr]+sorted(local_keys))
+    used_local_keys = [k for k in local_keys if k in expr]
+    key = tuple([expr]+sorted(used_local_keys))
     if key not in functions:
-        script = '''lambda %s: %s'''% (', '.join(sorted(local_keys)), expr)
+        script = '''lambda %s,**kwargs: %s'''% (', '.join(sorted(used_local_keys)), expr)
+        #print(script)
         fn = eval(script)
         fn.__name__ = expr.encode("ascii", "ignore").decode('utf8')
         functions[key] = fn
@@ -429,6 +433,23 @@ def get_template(templ):
         templates[templ] = t
     return templates[templ]
 
+def flatten_lists(o):
+    if isinstance(o, list):
+        result = []
+        for x in o:
+            flattened = flatten_lists(x)
+            if isinstance(flattened, list):
+                result.extend(flattened)
+            else:
+                result.append(flattened)
+        return result
+    elif isinstance(o, dict):
+        for key in o.keys():
+            o[key] = flatten_lists(o[key])
+        return o
+    else:
+        return o
+
 def process_row(row, template, rowname, table, resources, transform, variables):
     result = []
     e = {'row':row,
@@ -532,7 +553,9 @@ def process_row(row, template, rowname, table, resources, transform, variables):
                     fn = get_function(expression, list(env.keys()))
                     v = fn(**env)
                     if v is not None:
-                        if len(variable_list) == 1:
+                        if len(variable_list) == 1 and not (
+                                isinstance(v, collections.Iterable)
+                                and not isinstance(v, str)):
                             v = [v]
                         new_env = dict(env)
                         for i, variable in enumerate(variable_list):
@@ -579,7 +602,8 @@ def process_row(row, template, rowname, table, resources, transform, variables):
             parent[key] = this
         else:
             parent.append(this)
-    return result
+
+    return flatten_lists(result)
 
 def json_transform(transform, resources):
     logger.info("Transforming %s", transform.identifier)
@@ -600,9 +624,10 @@ def json_transform(transform, resources):
     else:
         result = ConjunctiveGraph()
         if generated[RDF.type : setl.Persisted]:
-            result = ConjunctiveGraph(store="Sleepycat")
+            store = TrigStore()
+            result = ConjunctiveGraph(store=store)
         if generated[RDF.type : setl.Persisted]:
-            tempdir = tempfile.mkdtemp()
+            tempdir = tempfile.mktemp()
             logger.info("Persisting %s to %s", generated.identifier, tempdir)
             result.store.open(tempdir, True)
     s = transform.value(prov.value).value
@@ -643,15 +668,16 @@ def json_transform(transform, resources):
                 }
                 if context is not None:
                     root['@context'] = context
-                before = len(result)
+                #logger.debug(json.dumps(root, indent=4))
+                #before = len(result)
                 #graph = ConjunctiveGraph(identifier=generated.identifier)
                 #graph.parse(data=json.dumps(root),format="json-ld")
                 data = json.dumps(root)
                 #del root
                 result.parse(data=data, format="json-ld")
                 #del data
-                after = len(result)
-                logger.debug("Row "+str(rowname)+" added "+str(after-before)+" triples.")
+                #after = len(result)
+                logger.debug("Row "+str(rowname))#+" added "+str(after-before)+" triples.")
                 sys.stdout.flush()
             except Exception as e:
                 trace = sys.exc_info()[2]
@@ -758,8 +784,8 @@ def load(load_resource, resources):
                 fmt = formats[fmt]
                 #print fmt
             with _load_open(generated) as o:
-                o.write(file_graph.serialize(format=fmt))
-                o.close()
+                file_graph.serialize(o, format=fmt)
+
         elif generated[RDF.type:sd.Service]:
             from rdflib.plugins.stores.sparqlstore import SPARQLUpdateStore
             endpoint = generated.value(sd.endpoint, default=generated).identifier

setlr-0.2.19/setlr/trig_store.py

@@ -0,0 +1,160 @@
+import logging
+from threading import Thread
+from os.path import exists, abspath
+from os import mkdir
+from rdflib.store import Store, VALID_STORE, NO_STORE
+from rdflib.term import URIRef
+from urllib.request import pathname2url
+from rdflib import ConjunctiveGraph
+from rdflib.term import BNode
+
+def bb(u):
+    return u.encode("utf-8")
+
+
+class TrigStore(Store):
+    """\
+    """
+
+    context_aware = True
+    formula_aware = True
+    transaction_aware = False
+    graph_aware = True
+    db_env = None
+
+    def __init__(self, configuration=None, identifier=None):
+        self.__open = False
+        self.__identifier = identifier
+        super(TrigStore, self).__init__(configuration)
+        self.__namespace = {}
+        self.__prefix = {}
+
+    def __get_identifier(self):
+        return self.__identifier
+
+    identifier = property(__get_identifier)
+
+    def _init_db_environment(self, file, create=True):
+        file_obj = open(file, "a+")
+        return file_obj
+
+    def is_open(self):
+        return self.__open
+
+    def open(self, path, create=True):
+        homeDir = path
+
+        if self.__identifier is None:
+            self.__identifier = URIRef(pathname2url(abspath(homeDir)))
+
+        db_env = self._init_db_environment(homeDir, create)
+        if db_env == NO_STORE:
+            return NO_STORE
+        self.db_env = db_env
+        self.__open = True
+
+        return VALID_STORE
+
+    def sync(self):
+        pass
+
+    def close(self, commit_pending_transaction=False):
+        self.__open = False
+        self.db_env.close()
+
+    def add(self, triple, context, quoted=False, txn=None):
+        """\
+        Add a triple to the store of triples.
+        """
+        (subject, predicate, object) = triple
+        assert self.__open, "The Store must be open."
+        assert context != self, "Can not add triple directly to store"
+        value = "%s { %s %s %s .}\n" % (context.identifier.n3(), subject.n3(), predicate.n3(), object.n3())
+        self.db_env.seek(0,2) # seek to end
+        self.db_env.write(value)
+
+    def __remove(self, spo, c, quoted=False, txn=None):
+        pass
+
+    def remove(self, spo, context, txn=None):
+        pass
+
+    def triples(self, spo, context=None, txn=None):
+        """A generator over all the triples matching"""
+        assert self.__open, "The Store must be open."
+
+        if context is None or isinstance(context.identifier, BNode):
+            c = None
+        else:
+            c = context.identifier
+        subject, predicate, object = spo
+        self.db_env.seek(0)
+        for line in self.db_env:
+            g = ConjunctiveGraph()
+            g.parse(data=line, format="trig")
+
+            for s, p, o, ctx in g.quads((None, None, None, None)):
+                #print(s, p, o, ctx.identifier)
+                if subject is not None and subject != s:
+                    continue
+                if predicate is not None and predicate != p:
+                    continue
+                if object is not None and object != o:
+                    continue
+                if c is not None :
+                    if c != ctx.identifier:
+                        continue
+                yield (s, p, o), ctx.identifier
+
+    def __len__(self, context=None):
+        def blocks(files, size=65536):
+            while True:
+                b = files.read(size)
+                if not b: break
+                yield b
+
+        self.db_env.seek(0)
+        return (sum(bl.count("\n") for bl in blocks(self.db_env)))
+
+    def contexts(self, triple=None):
+        for context in []:
+            yield context
+
+
+    def add_graph(self, graph):
+        pass
+
+    def remove_graph(self, graph):
+        pass
+
+    def __ctx_to_str(self, ctx):
+        if ctx is None:
+            return None
+        try:
+            # ctx could be a graph. In that case, use its identifier
+            ctx_str = "{}:{}".format(ctx.identifier.__class__.__name__, ctx.identifier)
+            self.__context_obj_map[ctx_str] = ctx
+            return ctx_str
+        except AttributeError:
+            # otherwise, ctx should be a URIRef or BNode or str
+            if isinstance(ctx, str):
+                ctx_str = "{}:{}".format(ctx.__class__.__name__, ctx)
+                if ctx_str in self.__context_obj_map:
+                    return ctx_str
+                self.__context_obj_map[ctx_str] = ctx
+                return ctx_str
+            raise RuntimeError("Cannot use that type of object as a Graph context")
+
+    def bind(self, prefix, namespace):
+        self.__prefix[namespace] = prefix
+        self.__namespace[prefix] = namespace
+
+    def namespace(self, prefix):
+        return self.__namespace.get(prefix, None)
+
+    def prefix(self, namespace):
+        return self.__prefix.get(namespace, None)
+
+    def namespaces(self):
+        for prefix, namespace in self.__namespace.items():
+            yield prefix, namespace

{setlr-0.2.17 → setlr-0.2.19}/setlr.egg-info/PKG-INFO

@@ -1,14 +1,17 @@
-Metadata-Version: 1.1
+Metadata-Version: 2.1
 Name: setlr
-Version: 0.2.17
+Version: 0.2.19
 Summary: setlr is a tool for Semantic Extraction, Transformation, and Loading.
 Home-page: http://packages.python.org/setlr
 Author: Jamie McCusker
 Author-email: mccusj@cs.rpi.edu
 License: Apache License 2.0
-Description: SETLr is a tool for generating RDF graphs, including named graphs, from almost any kind of tabular data.
 Keywords: rdf semantic etl
 Platform: UNKNOWN
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Topic :: Utilities
 Classifier: License :: OSI Approved :: Apache Software License
+License-File: LICENSE
+
+SETLr is a tool for generating RDF graphs, including named graphs, from almost any kind of tabular data.
+

{setlr-0.2.17 → setlr-0.2.19}/setlr.egg-info/SOURCES.txt

@@ -1,9 +1,11 @@
+LICENSE
 README.md
 setup.cfg
 setup.py
 setlr/__init__.py
 setlr/iterparse_filter.py
 setlr/sqlite-store.py
+setlr/trig_store.py
 setlr.egg-info/PKG-INFO
 setlr.egg-info/SOURCES.txt
 setlr.egg-info/dependency_links.txt

{setlr-0.2.17 → setlr-0.2.19}/setlr.egg-info/requires.txt

@@ -2,8 +2,7 @@ future
 pip>=9.0.0
 cython
 numpy
-rdflib
-rdflib-jsonld
+rdflib>=6.0.0
 pandas>=0.23.0
 requests
 toposort

{setlr-0.2.17 → setlr-0.2.19}/setup.py

@@ -10,7 +10,7 @@ def read(fname):
 
 setup(
     name = "setlr",
-    version = "0.2.17",
+    version = "0.2.19",
     author = "Jamie McCusker",
     author_email = "mccusj@cs.rpi.edu",
     description = ("setlr is a tool for Semantic Extraction, Transformation, and Loading."),
@@ -25,8 +25,7 @@ setup(
         'pip>=9.0.0',
         'cython',
         'numpy',
-        'rdflib',
-        'rdflib-jsonld',
+        'rdflib>=6.0.0',
         'pandas>=0.23.0',
         'requests',
         'toposort',