csvpath 0.0.473__tar.gz → 0.0.474__tar.gz

{csvpath-0.0.473 → csvpath-0.0.474}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: csvpath
-Version: 0.0.473
+Version: 0.0.474
 Summary: A declarative language for data extraction and validation of CSV files
 Author: David Kershaw
 Author-email: dk107dk@hotmail.com
@@ -22,7 +22,6 @@ Requires-Dist: inflect (>=7.3.1,<8.0.0)
 Requires-Dist: jinja2 (>=3.1.4,<4.0.0)
 Requires-Dist: lark (>=1.2.2,<2.0.0)
 Requires-Dist: metaphone (>=0.6,<0.7)
-Requires-Dist: pandas (>=2.2.2,<3.0.0)
 Requires-Dist: ply (>=3.11,<4.0)
 Requires-Dist: python-dateutil (>=2.9.0.post0,<3.0.0)
 Requires-Dist: tabulate (>=0.9.0,<0.10.0)

{csvpath-0.0.473 → csvpath-0.0.474}/config/config.ini

@@ -20,3 +20,7 @@ path = config/config.ini
 
 [cache]
 path = cache
+
+[functions]
+imports =
+

csvpath-0.0.474/config/imports.txt

@@ -0,0 +1 @@
+from csvpath.matching.functions.boolean.yes import Yes as sure

{csvpath-0.0.473 → csvpath-0.0.474}/csvpath/matching/functions/function_factory.py

@@ -1,6 +1,7 @@
 # pylint: disable=C0114
 from csvpath.matching.productions.expression import Matchable
 from .function import Function
+from .function_finder import FunctionFinder
 from .dates.now import Now
 from .dates.datef import Date
 from .strings.lower import Lower
@@ -56,7 +57,8 @@ from .stats.percent import Percent
 from .stats.minf import Min, Max, Average
 from .stats.percent_unique import PercentUnique
 from .stats.stdev import Stdev
-from .stats.correlate import Correlate
+
+# from .stats.correlate import Correlate
 from .print.printf import Print
 from .print.table import HeaderTable, RowTable, VarTable, RunTable
 from .print.print_line import PrintLine
@@ -260,8 +262,8 @@ class FunctionFactory:
             f = Strip(matcher, name, child)
         elif name == "jinja":
             f = Jinjaf(matcher, name, child)
-        elif name == "correlate":
-            f = Correlate(matcher, name, child)
+        #        elif name == "correlate":
+        #            f = Correlate(matcher, name, child)
         elif name in ["count_headers", "count_headers_in_line"]:
             f = CountHeaders(matcher, name, child)
         elif name == "percent_unique":
@@ -292,9 +294,6 @@ class FunctionFactory:
             f = Stack(matcher, name, child)
         elif name in ["stdev", "pstdev"]:
             f = Stdev(matcher, name, child)
-        #
-        # dup_lines can also decide matches
-        #
         elif name == "has_dups":
             f = HasDups(matcher, name, child)
         elif name == "count_dups":
@@ -373,15 +372,13 @@ class FunctionFactory:
             f = Num(matcher, name, child)
         elif name == "counter":
             f = Counter(matcher, name, child)
-
         else:
-            if (
-                f is None
-                and find_external_functions
-                and name in FunctionFactory.NOT_MY_FUNCTION
-            ):
-                f = cls.NOT_MY_FUNCTION[name]
-                f = f(matcher, name, child)
+            if f is None and find_external_functions:
+                if FunctionFinder.EXTERNALS not in FunctionFactory.NOT_MY_FUNCTION:
+                    FunctionFinder().load(matcher, cls)
+                if name in FunctionFactory.NOT_MY_FUNCTION:
+                    f = cls.NOT_MY_FUNCTION[name]
+                    f = f(matcher, name, child)
             if not find_external_functions:
                 return None
             if f is None:

csvpath-0.0.474/csvpath/matching/functions/function_finder.py

@@ -0,0 +1,55 @@
+# pylint: disable=C0114
+import os
+import importlib
+from csvpath.util.config_exception import ConfigurationException
+from .boolean.yes import Yes
+
+
+class FunctionFinder:
+    EXTERNALS = "externalfunctionsloaded"
+
+    @classmethod
+    def load(cls, matcher, func_fact) -> None:
+        # any problems loading will bubble up to the nearest
+        # expression and be handled there.
+        config = matcher.csvpath.config
+        # find the list
+        path = config.function_imports
+        # read the list
+        if path is None:
+            matcher.csvpath.logger.error("No [functions][imports] in config.ini")
+            return
+        if not os.path.exists(path):
+            matcher.csvpath.logger.error(
+                "No [functions][imports] path from config.ini does not exist"
+            )
+            return
+        with open(path, "r") as file:
+            i = 0
+            for line in file:
+                i += 1
+                cls._add_function(matcher, func_fact, line)
+            matcher.csvpath.logger.info("Added %s external functions", i)
+        # add a sentinel to keep us from attempting reload.
+        # this instance will never be found, but the dict will
+        # never be empty
+        func_fact.add_function(cls.EXTERNALS, Yes(None, cls.EXTERNALS))
+
+    @classmethod
+    def _add_function(cls, matcher, func_fact, s: str) -> None:
+        s = s.strip()
+        if s != "":
+            instance = None
+            # instantiate the classes
+            # function_name module classname
+            cs = s.split(" ")
+            if len(cs) == 6 and cs[0] == "from" and cs[2] == "import" and cs[4] == "as":
+                module = importlib.import_module(cs[1])
+                class_ = getattr(module, cs[3])
+                instance = class_(matcher, cs[5])
+                # load to FunctionFactory.add
+                func_fact.add_function(cs[5], instance)
+            else:
+                raise ConfigurationException(
+                    "Unclear external function import setup: {s}"
+                )

{csvpath-0.0.473 → csvpath-0.0.474}/csvpath/util/config.py

@@ -42,6 +42,7 @@ class Sections(Enum):
     CSV_FILES = "csv_files"
     ERRORS = "errors"
     LOGGING = "logging"
+    FUNCTIONS = "functions"
 
 
 class Config:
@@ -172,6 +173,17 @@ path =
         self.log_file_size = self._get(
             Sections.LOGGING.value, LogFile.LOG_FILE_SIZE.value
         )
+        #
+        # path to external functions list. external functions are very optional.
+        # not blowing up when absent seems reasonable.
+        #
+        try:
+            self.function_imports = self._get(Sections.FUNCTIONS.value, "imports")
+        except Exception:
+            pass
+        #
+        # reload if another config path is set
+        #
         path = self._get("config", "path")
         if path:
             path = path.strip().lower()
@@ -262,17 +274,32 @@ path =
 
     # ======================================
 
+    @property
+    def configpath(self) -> str:
+        return self._configpath
+
+    @configpath.setter
+    def configpath(self, path: str) -> None:
+        self._configpath = path
+
     @property
     def cache_dir_path(self) -> str:
         try:
             path = self._get("cache", "path")
         except Exception:
-            print("No cache path in config.ini at [cache][path]. Using 'cache'.")
             path = "config"
             self._config.add_section("cache")
             self._config.set("cache", "path", path)
         return path
 
+    @property
+    def function_imports(self) -> str:
+        return self._function_imports
+
+    @function_imports.setter
+    def function_imports(self, path: str) -> None:
+        self._function_imports = path
+
     @property
     def csvpath_file_extensions(self) -> list[str]:
         return self._csvpath_file_extensions

{csvpath-0.0.473 → csvpath-0.0.474}/docs/functions/implementing_functions.md

@@ -80,17 +80,26 @@ When your function needs its arguments you can call:
 - self._value_two(skip=skip)
 - self._siblings()
 
-When you need to get a value or a match from a child object you use `to_value()` or `matches()`. Remember to pass the skip list as a named argument "skip".
+When you need to get a value or a match from a child object you use `to_value()` or `matches()`. Remember to pass the skip list as the named argument `skip`.
 
 ## Registering
 
-To register your function for use, add it to the FunctionFactory like this:
+To register your function for use, add it to the `FunctionFactory` like this:
 
 ```python
     from csvpath.matching.functions.function_factory import FunctionFactory
     FunctionFactory.add_function(name='iamafunction', function=my_function_instance)
 ```
 
+Alternatively, add a file path to your `config.ini` file under the key `[functions][imports]` that points to a list of functions to register. The key is like:
+
+    [functions]
+    imports = my_imports/functions.txt
+
+In the file list every function on its own line using a format like that of Python imports:
+
+    from a.b.c.my_function import MyFunction as iamafunction
+
 To use your function do something like:
 
 ```bash

{csvpath-0.0.473 → csvpath-0.0.474}/docs/functions.md

@@ -41,7 +41,20 @@ Some functions will optionally make use of an arbitrary name qualifier to better
 
 ## Custom Functions
 
-Creating your own function is easy. Once you create a function, you register it with the `FunctionFactory` class. You must register your functions each time you run CsvPath. Use your function in csvpaths by simply referring to it by name like any other function.
+Creating your own function is easy. Once you create a function, you register it with the `FunctionFactory` class. You can register your functions either programmatically or by creating an import file listing your functions. Your import file must be referenced in your `config.ini` file at `[functions][imports]` like this:
+
+    [functions]
+    imports = my_project_assets/my_functions.imports
+
+Each custom function has its own line in your imports file. The format is the same as you use for importing classes into Python files -- under the hood the mechanism is similar. E.g.
+
+```python
+    from my_function.for_stuff.me import Me as function_me
+```
+
+In this example your class is `Me` and the name of the function you use in your csvpath is `function_me()`
+
+Use your functions in csvpaths by simply referring to them by name like you would any built-in function.
 
 <a href='https://github.com/dk107dk/csvpath/blob/main/docs/functions/implementing_functions.md'>Read more about implementing your own functions here.</a>
 

{csvpath-0.0.473 → csvpath-0.0.474}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "csvpath"
-version = "0.0.473"
+version = "0.0.474"
 description = "A declarative language for data extraction and validation of CSV files"
 authors = ["David Kershaw <dk107dk@hotmail.com>"]
 readme = "README.md"
@@ -27,7 +27,6 @@ include = ["config", "logs", "docs"]
 python = "^3.12"
 ply = "^3.11"
 python-dateutil = "^2.9.0.post0"
-pandas = "^2.2.2"
 jinja2 = "^3.1.4"
 inflect = "^7.3.1"
 lark = "^1.2.2"

csvpath-0.0.473/csvpath/matching/functions/stats/correlate.py

@@ -1,41 +0,0 @@
-# pylint: disable=C0114
-import pandas as pd
-from ..function_focus import ValueProducer
-
-
-class Correlate(ValueProducer):
-    """does a statistical correlation test on the values of two headers"""
-
-    def check_valid(self) -> None:
-        self.validate_two_args()
-        super().check_valid()
-
-    def _produce_value(self, skip=None) -> None:
-        child = self.children[0]
-        left = child.left
-        right = child.right
-        leftlist = left.to_value(skip=skip)
-        rightlist = right.to_value(skip=skip)
-        leftlist, rightlist = self._trim(leftlist, rightlist)
-        ll = pd.Series(leftlist)
-        rl = pd.Series(rightlist)
-        corr = ll.corr(rl)
-        f = float(corr)
-        f = round(f, 2)
-        self.value = f
-
-    def _decide_match(self, skip=None) -> None:
-        self.to_value(skip=skip)
-        self.match = self._noop_match()  # pragma: no cover
-
-    def _trim(self, leftlist, rightlist):
-        n = len(leftlist) if len(leftlist) < len(rightlist) else len(rightlist)
-        ll = []
-        rl = []
-        for i in range(0, n):
-            try:
-                ll.append(float(leftlist[i]))
-                rl.append(float(rightlist[i]))
-            except (TypeError, ValueError):
-                pass
-        return ll, rl