csvpath 0.0.586__tar.gz → 0.0.588__tar.gz

{csvpath-0.0.586 → csvpath-0.0.588}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: csvpath
-Version: 0.0.586
+Version: 0.0.588
 Summary: A data preboarding framework for ingesting, managing, and validating CSV, Excel, and other tabular data files using a Collect, Store, Validate, Publish architecture to create a trusted publisher for downstream data consumers.
 Author: David Kershaw
 Author-email: info@csvpath.org

{csvpath-0.0.586 → csvpath-0.0.588}/csvpath/matching/functions/function_factory.py

@@ -139,6 +139,7 @@ class FunctionFactory:
     @classmethod
     def add_function(cls, name: str, function: Function) -> None:
         """use to add a new, external function at runtime"""
+
         if name is None:
             name = function.name
         if name is None:
@@ -150,18 +151,31 @@ class FunctionFactory:
             raise InvalidNameException("Name must not be an empty string")
         if not cls.valid_function_name(name):  # name.isalpha():
             raise InvalidNameException(f"Name {name} is not valid")
+        cls._debug(function.matcher, "Looking for an existing function named %s", name)
         if cls.get_function(None, name=name, find_external_functions=False) is not None:
             function.matcher.csvpath.logger.warning(
                 "Internal function is overriden by external function: %s", name
             )
-            # raise InvalidNameException("Built-in functions cannot be overriden")
         if not isinstance(function, Function):
             # pass as an instance, not a class, for specificity. good to do?
             raise InvalidChildException(
                 "Function being registered must be passed as an instance"
             )
+        cls._debug(function.matcher, "Adding %s as key to %s", name, function.__class__)
         cls.NOT_MY_FUNCTION[name] = function.__class__
 
+    @classmethod
+    def _debug(cls, matcher, txt: str, obj=None, obj2=None) -> None:
+        if matcher is None:
+            return
+        if matcher.csvpath is None:
+            return
+        if obj2 is None:
+            matcher.csvpath.logger.debug(txt, str(obj))
+        else:
+            matcher.csvpath.logger.debug(txt, str(obj), str(obj2))
+
+
     #
     # valid function names start with a letter and contain only letters,
     # numbers, periods, and/or underscores. the grammar allows '.' and '_' at the
@@ -287,9 +301,15 @@ class FunctionFactory:
             # if proj not set we check for name
             #
             qname = cls.qname(matcher=matcher, name=name)
+            cls._debug(matcher, "Qualified name of %s is %s", name, qname)
             if qname in FunctionFactory.NOT_MY_FUNCTION:
                 f = cls.NOT_MY_FUNCTION[qname]
                 f = f(matcher, name, child)
+            cls._debug(matcher, "Found %s as %s", qname, f)
+            if f is None:
+                cls._debug(
+                    matcher, "%s not in: %s", qname, FunctionFactory.NOT_MY_FUNCTION
+                )
         return f
 
     @classmethod
@@ -308,10 +328,10 @@ class FunctionFactory:
         return qname
 
     @classmethod
-    def improve_name(cls, name:str) -> str:
+    def improve_name(cls, name: str) -> str:
         ffrom = " `~!@#$%^&*()_+-=[]\\{}|;':\",./<>?"
-        to    = "1234567890abcdefghijklmnopqrstuvw"
-        table = str.maketrans(ffrom,to)
+        to = "1234567890abcdefghijklmnopqrstuvw"
+        table = str.maketrans(ffrom, to)
         name = name.translate(table)
         return name
 

{csvpath-0.0.586 → csvpath-0.0.588}/csvpath/matching/functions/function_finder.py

@@ -24,6 +24,7 @@ class FunctionFinder:  # pylint: disable=R0903
             if path is None or path.strip() == "":
                 matcher.csvpath.logger.error("No [functions][imports] in config.ini")
                 return
+            cls._debug(matcher, "Functions import file is at %s", path)
             if not os.path.exists(path):
                 matcher.csvpath.logger.error(
                     f"[functions][imports] path {path} in {config.configpath} does not exist"
@@ -35,15 +36,32 @@ class FunctionFinder:  # pylint: disable=R0903
                 i = 0
                 for line in file:
                     i += 1
-                    if str(line).strip() == "":
+                    line = str(line).strip()
+                    if line == "" or line.startswith("#"):
                         continue
-                    cls._add_function(matcher, func_fact, line)
+                    cls._debug(matcher, "Adding function %s", line)
+                    try:
+                        cls._add_function(matcher, func_fact, line)
+                    except Exception as ex:
+                        matcher.csvpath.logger.exception(f"Could not add function for '{line}'")
                 matcher.csvpath.logger.info("Added %s external functions", i)
         # add a sentinel to keep us from attempting reload.
         # this instance will never be found, but the dict will
         # never be empty
         e = cls.externals_sentinel(matcher)
         func_fact.add_function(e, Yes(None, e))
+        cls._debug(matcher, "Added sentinel function %s", e)
+
+    @classmethod
+    def _debug(cls, matcher, txt: str, obj=None, obj2=None) -> None:
+        if matcher is None:
+            return
+        if matcher.csvpath is None:
+            return
+        if obj2 is None:
+            matcher.csvpath.logger.debug(txt, str(obj))
+        else:
+            matcher.csvpath.logger.debug(txt, str(obj), str(obj2))
 
     @classmethod
     def externals_sentinel(cls, matcher) -> str:
@@ -79,10 +97,20 @@ class FunctionFinder:  # pylint: disable=R0903
         # lines in config are like:
         #   from module import class as function-name
         #
+        cls._debug(matcher, "Import line is %s", cs)
         if len(cs) == 6 and cs[0] == "from" and cs[2] == "import" and cs[4] == "as":
             config = matcher.csvpath.config
-            instance = ClassLoader.load_private_function(config, s, matcher, cs[5])
-            qname = func_fact.qname(matcher=matcher, name=cs[5])
-            func_fact.add_function(qname, instance)
+            try:
+                instance = ClassLoader.load_private_function(config, s, matcher, cs[5])
+                qname = func_fact.qname(matcher=matcher, name=cs[5])
+                cls._debug(matcher, "Adding custom function %s: %s", qname, instance)
+                func_fact.add_function(qname, instance)
+            except Exception as ex:
+                if matcher is not None and matcher.csvpath is not None:
+                    matcher.csvpath.logger.exception(f"Cannot load class {cs}")
+                #
+                # matcher and csvpath will be present, but we check to be sure. there's no
+                # fallback because it is an unlikely scenario that would have its own logic.
+                #
         else:
             raise ConfigurationException("Unclear external function imports: {s}")

csvpath-0.0.588/csvpath/util/class_loader.py

@@ -0,0 +1,215 @@
+import importlib
+import os
+from typing import Any
+
+
+class ClassLoadingError(RuntimeError):
+    ...
+
+
+class ClassLoader:
+    #
+    # load is fine for classes we expect. e.g. we load specific types of data readers and writers
+    # using this method without any concerns about different projects needing to not collide. today
+    # we treat integration listeners the same way, but longer term we need those to be able to be
+    # custom loaded too.
+    #
+    @classmethod
+    def load(cls, s: str, args: list = None, kwargs: dict = None) -> Any:
+        s = s.strip()
+        if s != "":
+            instance = None
+            cs = s.split(" ")
+            #
+            # lines in config are like:
+            #   from module import class
+            #
+            if len(cs) == 4 and cs[0] == "from" and cs[2] == "import":
+                module = importlib.import_module(cs[1])
+                class_ = getattr(module, cs[3])
+                args = args if args is not None else []
+                kwargs = kwargs if kwargs is not None else {}
+                instance = class_(*args, **kwargs)
+                return instance
+            else:
+                raise ClassLoadingError(f"Unclear class loading import statement: {s}")
+        return None
+
+    #
+    # this loads a custom function using a name that is distinct across projects so there
+    # is no risk of name collisions.
+    #
+    @classmethod
+    def load_private_function(cls, config, stmt: str, *args, **kwargs):
+        imports = config.get(section="functions", name="imports")
+        if not imports or str(imports).strip() == "":
+            raise ValueError("Imports cannot be None or ''")
+        imports = os.path.dirname(imports)
+        return cls.load_private_class(imports, stmt, *args, **kwargs)
+
+
+    """
+    @classmethod
+    def load_private_class(cls, base_path: str, stmt: str, *args, **kwargs):
+        if not stmt or stmt.strip() == "":
+            raise ValueError("Load statement cannot be None or ''")
+        module_name = None
+        class_name = None
+        #
+        # remove the imports file name. functions must be in or below the dir
+        # where the functions file is.
+        #
+        # /x/y/z/function.imports
+        # /x/y/z
+        # from a.b.c import C as cone
+        #
+        # we look in /x/y/z/a/b/c.py for class C
+        #             base  module             clss
+        #
+        cs = stmt.split(" ")
+        if len(cs) >= 4 and cs[0] == "from" and cs[2] == "import":
+            module_name = cs[1]
+            class_name = cs[3]
+        else:
+            raise ClassLoadingError(f"Unclear class loading import statement: {stmt}")
+        #
+        # Resolve the file path for the module
+        #
+        module_path = os.path.join(base_path, module_name)
+        module_path = module_path.replace(".", os.sep)
+        module_path = f"{module_path}.py"
+        if not os.path.exists(module_path):
+            raise ImportError(f"Module {module_name} not found in {module_path}")
+        #
+        # create a unique module spec. we hash the full path to the bytes in order
+        # to make sure we have a distinct class, even if one project names and locates
+        # a class in the exact same relative location.
+        #
+        # e.g. project A has a function.imports file at config/function.imports that defines a class B as:
+        #           from b import B as bee
+        #      project B has a function.imports file at config/function.imports that defines a class B as:
+        #           from b import B as bee
+        #      These two class Bs would be identified exactly the same way even though they are different. We
+        #      don't ever want a request for project B's class B to be returned when project A requests a
+        #      class B. The hash keeps the names distinct.
+        #
+        try:
+            import py_compile
+            py_compile.compile(str(module_path), doraise=True)
+            print(f"Compiled fine")
+        except Exception as ex:
+            print(f"Cannot compile: {ex}")
+
+        try:
+            print("ABS:", os.path.abspath(module_path))
+            print("REAL:", os.path.realpath(module_path))
+            print("EXISTS:", os.path.exists(module_path))
+            print("ISFILE:", os.path.isfile(module_path))
+            print("ACCESS_R:", os.access(module_path, os.R_OK))
+            print("ACCESS_X:", os.access(module_path, os.X_OK))
+        except Exception as ex:
+            print(f"Cannot print module_path attributes: {ex}")
+
+        spec = importlib.util.spec_from_file_location(
+            f"{module_name}_{hash(module_path)}",  # unique name per loader instance
+            str(module_path),
+        )
+        module = importlib.util.module_from_spec(spec)
+        loader = spec.loader
+        if loader is None:
+            raise ImportError(f"Could not load spec for {module_name} at {module_path}")
+        loader.exec_module(module)
+        cls = getattr(module, class_name)
+        instance = cls(*args, **kwargs)
+        return instance
+    """
+
+    @classmethod
+    def load_private_class(cls, base_path: str, stmt: str, *args, **kwargs):
+        if not stmt or stmt.strip() == "":
+            raise ValueError("Load statement cannot be None or ''")
+        module_name = None
+        class_name = None
+        #
+        # remove the imports file name. functions must be in or below the dir
+        # where the functions file is.
+        #
+        # /x/y/z/function.imports
+        # /x/y/z
+        # from a.b.c import C as cone
+        #
+        # we look in /x/y/z/a/b/c.py for class C
+        #             base  module             clss
+        #
+        cs = stmt.split(" ")
+        if len(cs) >= 4 and cs[0] == "from" and cs[2] == "import":
+            module_name = cs[1]
+            class_name = cs[3]
+        else:
+            raise ClassLoadingError(f"Unclear class loading import statement: {stmt}")
+        #
+        # Resolve the file path for the module
+        #
+        module_path = os.path.join(base_path, module_name)
+        module_path = module_path.replace(".", os.sep)
+        module_path = f"{module_path}.py"
+        if not os.path.exists(module_path):
+            raise ImportError(f"Module {module_name} not found in {module_path}")
+        #
+        # create a unique module spec. we hash the full path to the bytes in order
+        # to make sure we have a distinct class, even if one project names and locates
+        # a class in the exact same relative location.
+        #
+        # e.g. project A has a function.imports file at config/function.imports that defines a class B as:
+        #           from b import B as bee
+        #      project B has a function.imports file at config/function.imports that defines a class B as:
+        #           from b import B as bee
+        #      These two class Bs would be identified exactly the same way even though they are different. We
+        #      don't ever want a request for project B's class B to be returned when project A requests a
+        #      class B. The hash keeps the names distinct.
+        #
+        compiled_path = None
+        try:
+            import py_compile
+            #py_compile.compile(str(module_path), doraise=True)
+
+            compiled_path = py_compile.compile(
+                str(module_path),
+                cfile=str(module_path) + "c",   # or any path you choose
+                doraise=True
+            )
+
+            print(f"Compiled fine to: {compiled_path}")
+
+        except Exception as ex:
+            print(f"Cannot compile: {ex}")
+        try:
+            print("ABS:", os.path.abspath(module_path))
+            print("REAL:", os.path.realpath(module_path))
+            print("EXISTS:", os.path.exists(module_path))
+            print("ISFILE:", os.path.isfile(module_path))
+            print("ACCESS_R:", os.access(module_path, os.R_OK))
+            print("ACCESS_X:", os.access(module_path, os.X_OK))
+        except Exception as ex:
+            print(f"Cannot print module_path attributes: {ex}")
+        """
+        spec = importlib.util.spec_from_file_location(
+            f"{module_name}_{hash(module_path)}",  # unique name per loader instance
+            str(module_path),
+        )
+        """
+        spec = importlib.util.spec_from_file_location(
+            f"{module_name}_{hash(compiled_path)}",
+            compiled_path
+        )
+        #module = importlib.util.module_from_spec(spec)
+        #spec.loader.exec_module(module)
+
+        module = importlib.util.module_from_spec(spec)
+        loader = spec.loader
+        if loader is None:
+            raise ImportError(f"Could not load spec for {module_name} at {module_path}")
+        loader.exec_module(module)
+        cls = getattr(module, class_name)
+        instance = cls(*args, **kwargs)
+        return instance

{csvpath-0.0.586 → csvpath-0.0.588}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "csvpath"
-version = "0.0.586"
+version = "0.0.588"
 description = "A data preboarding framework for ingesting, managing, and validating CSV, Excel, and other tabular data files using a Collect, Store, Validate, Publish architecture to create a trusted publisher for downstream data consumers."
 authors = ["David Kershaw <info@csvpath.org>"]
 readme = "README.md"

csvpath-0.0.586/csvpath/util/class_loader.py

@@ -1,105 +0,0 @@
-import importlib
-import os
-from typing import Any
-
-
-class ClassLoadingError(RuntimeError):
-    ...
-
-
-class ClassLoader:
-    #
-    # load is fine for classes we expect. e.g. we load specific types of data readers and writers
-    # using this method without any concerns about different projects needing to not collide. today
-    # we treat integration listeners the same way, but longer term we need those to be able to be
-    # custom loaded too.
-    #
-    @classmethod
-    def load(cls, s: str, args: list = None, kwargs: dict = None) -> Any:
-        s = s.strip()
-        if s != "":
-            instance = None
-            cs = s.split(" ")
-            #
-            # lines in config are like:
-            #   from module import class
-            #
-            if len(cs) == 4 and cs[0] == "from" and cs[2] == "import":
-                module = importlib.import_module(cs[1])
-                class_ = getattr(module, cs[3])
-                args = args if args is not None else []
-                kwargs = kwargs if kwargs is not None else {}
-                instance = class_(*args, **kwargs)
-                return instance
-            else:
-                raise ClassLoadingError(f"Unclear class loading import statement: {s}")
-        return None
-
-    #
-    # this loads a custom function using a name that is distinct across projects so there
-    # is no risk of name collisions.
-    #
-    @classmethod
-    def load_private_function(cls, config, stmt: str, *args, **kwargs):
-        imports = config.get(section="functions", name="imports")
-        if not imports or str(imports).strip() == "":
-            raise ValueError("Imports cannot be None or ''")
-        imports = os.path.dirname(imports)
-        return cls.load_private_class(imports, stmt, *args, **kwargs)
-
-    @classmethod
-    def load_private_class(cls, base_path: str, stmt: str, *args, **kwargs):
-        if not stmt or stmt.strip() == "":
-            raise ValueError("Load statement cannot be None or ''")
-        module_name = None
-        class_name = None
-        #
-        # remove the imports file name. functions must be in or below the dir
-        # where the functions file is.
-        #
-        # /x/y/z/function.imports
-        # /x/y/z
-        # from a.b.c import C as cone
-        #
-        # we look in /x/y/z/a/b/c.py for class C
-        #             base  module             clss
-        #
-        cs = stmt.split(" ")
-        if len(cs) >= 4 and cs[0] == "from" and cs[2] == "import":
-            module_name = cs[1]
-            class_name = cs[3]
-        else:
-            raise ClassLoadingError(f"Unclear class loading import statement: {stmt}")
-        #
-        # Resolve the file path for the module
-        #
-        module_path = os.path.join(base_path, module_name)
-        module_path = module_path.replace(".", os.sep)
-        module_path = f"{module_path}.py"
-        if not os.path.exists(module_path):
-            raise ImportError(f"Module {module_name} not found in {module_path}")
-        #
-        # create a unique module spec. we hash the full path to the bytes in order
-        # to make sure we have a distinct class, even if one project names and locates
-        # a class in the exact same relative location.
-        #
-        # e.g. project A has a function.imports file at config/function.imports that defines a class B as:
-        #           from b import B as bee
-        #      project B has a function.imports file at config/function.imports that defines a class B as:
-        #           from b import B as bee
-        #      These two class Bs would be identified exactly the same way even though they are different. We
-        #      don't ever want a request for project B's class B to be returned when project A requests a
-        #      class B. The hash keeps the names distinct.
-        #
-        spec = importlib.util.spec_from_file_location(
-            f"{module_name}_{hash(module_path)}",  # unique name per loader instance
-            str(module_path),
-        )
-        module = importlib.util.module_from_spec(spec)
-        loader = spec.loader
-        if loader is None:
-            raise ImportError(f"Could not load spec for {module_name} at {module_path}")
-        loader.exec_module(module)
-        cls = getattr(module, class_name)
-        instance = cls(*args, **kwargs)
-        return instance