csvpath 0.0.473__tar.gz → 0.0.475__tar.gz

{csvpath-0.0.473 → csvpath-0.0.475}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: csvpath
-Version: 0.0.473
+Version: 0.0.475
 Summary: A declarative language for data extraction and validation of CSV files
 Author: David Kershaw
 Author-email: dk107dk@hotmail.com
@@ -22,7 +22,6 @@ Requires-Dist: inflect (>=7.3.1,<8.0.0)
 Requires-Dist: jinja2 (>=3.1.4,<4.0.0)
 Requires-Dist: lark (>=1.2.2,<2.0.0)
 Requires-Dist: metaphone (>=0.6,<0.7)
-Requires-Dist: pandas (>=2.2.2,<3.0.0)
 Requires-Dist: ply (>=3.11,<4.0)
 Requires-Dist: python-dateutil (>=2.9.0.post0,<3.0.0)
 Requires-Dist: tabulate (>=0.9.0,<0.10.0)

{csvpath-0.0.473 → csvpath-0.0.475}/config/config.ini

@@ -20,3 +20,7 @@ path = config/config.ini
 
 [cache]
 path = cache
+
+[functions]
+imports = config/functions.imports
+

{csvpath-0.0.473 → csvpath-0.0.475}/csvpath/csvpath.py

@@ -318,7 +318,7 @@ class CsvPath(CsvPathPublic, ErrorCollector):  # pylint: disable=R0902, R0904
     @property
     def config(self) -> Config:  # pylint: disable=C0116
         if not self._config:
-            self._config = Config(self)
+            self._config = Config()
         return self._config
 
     def has_errors(self) -> bool:  # pylint: disable=C0116

{csvpath-0.0.473 → csvpath-0.0.475}/csvpath/csvpaths.py

@@ -132,7 +132,7 @@ class CsvPaths(CsvPathsPublic, CsvPathsCoordinator):
         self.quotechar = quotechar
         self.skip_blank_lines = skip_blank_lines
         self.current_matcher: CsvPath = None
-        self._config = Config(self)
+        self._config = Config()
         self.logger = LogUtility.logger(self)
         self.logger.info("initialized CsvPaths")
         self._errors = []

{csvpath-0.0.473 → csvpath-0.0.475}/csvpath/matching/functions/function_factory.py

@@ -1,6 +1,7 @@
 # pylint: disable=C0114
 from csvpath.matching.productions.expression import Matchable
 from .function import Function
+from .function_finder import FunctionFinder
 from .dates.now import Now
 from .dates.datef import Date
 from .strings.lower import Lower
@@ -56,7 +57,8 @@ from .stats.percent import Percent
 from .stats.minf import Min, Max, Average
 from .stats.percent_unique import PercentUnique
 from .stats.stdev import Stdev
-from .stats.correlate import Correlate
+
+# from .stats.correlate import Correlate
 from .print.printf import Print
 from .print.table import HeaderTable, RowTable, VarTable, RunTable
 from .print.print_line import PrintLine
@@ -260,8 +262,8 @@ class FunctionFactory:
             f = Strip(matcher, name, child)
         elif name == "jinja":
             f = Jinjaf(matcher, name, child)
-        elif name == "correlate":
-            f = Correlate(matcher, name, child)
+        #        elif name == "correlate":
+        #            f = Correlate(matcher, name, child)
         elif name in ["count_headers", "count_headers_in_line"]:
             f = CountHeaders(matcher, name, child)
         elif name == "percent_unique":
@@ -292,9 +294,6 @@ class FunctionFactory:
             f = Stack(matcher, name, child)
         elif name in ["stdev", "pstdev"]:
             f = Stdev(matcher, name, child)
-        #
-        # dup_lines can also decide matches
-        #
         elif name == "has_dups":
             f = HasDups(matcher, name, child)
         elif name == "count_dups":
@@ -373,15 +372,13 @@ class FunctionFactory:
             f = Num(matcher, name, child)
         elif name == "counter":
             f = Counter(matcher, name, child)
-
         else:
-            if (
-                f is None
-                and find_external_functions
-                and name in FunctionFactory.NOT_MY_FUNCTION
-            ):
-                f = cls.NOT_MY_FUNCTION[name]
-                f = f(matcher, name, child)
+            if f is None and find_external_functions:
+                if FunctionFinder.EXTERNALS not in FunctionFactory.NOT_MY_FUNCTION:
+                    FunctionFinder().load(matcher, cls)
+                if name in FunctionFactory.NOT_MY_FUNCTION:
+                    f = cls.NOT_MY_FUNCTION[name]
+                    f = f(matcher, name, child)
             if not find_external_functions:
                 return None
             if f is None:

csvpath-0.0.475/csvpath/matching/functions/function_finder.py

@@ -0,0 +1,55 @@
+# pylint: disable=C0114
+import os
+import importlib
+from csvpath.util.config_exception import ConfigurationException
+from .boolean.yes import Yes
+
+
+class FunctionFinder:
+    EXTERNALS = "externalfunctionsloaded"
+
+    @classmethod
+    def load(cls, matcher, func_fact) -> None:
+        # any problems loading will bubble up to the nearest
+        # expression and be handled there.
+        config = matcher.csvpath.config
+        # find the list
+        path = config.function_imports
+        # read the list
+        if path is None:
+            matcher.csvpath.logger.error("No [functions][imports] in config.ini")
+            return
+        if not os.path.exists(path):
+            matcher.csvpath.logger.error(
+                f"[functions][imports] path in {config.configpath} does not exist"
+            )
+            return
+        with open(path, "r") as file:
+            i = 0
+            for line in file:
+                i += 1
+                cls._add_function(matcher, func_fact, line)
+            matcher.csvpath.logger.info("Added %s external functions", i)
+        # add a sentinel to keep us from attempting reload.
+        # this instance will never be found, but the dict will
+        # never be empty
+        func_fact.add_function(cls.EXTERNALS, Yes(None, cls.EXTERNALS))
+
+    @classmethod
+    def _add_function(cls, matcher, func_fact, s: str) -> None:
+        s = s.strip()
+        if s != "":
+            instance = None
+            # instantiate the classes
+            # function_name module classname
+            cs = s.split(" ")
+            if len(cs) == 6 and cs[0] == "from" and cs[2] == "import" and cs[4] == "as":
+                module = importlib.import_module(cs[1])
+                class_ = getattr(module, cs[3])
+                instance = class_(matcher, cs[5])
+                # load to FunctionFactory.add
+                func_fact.add_function(cs[5], instance)
+            else:
+                raise ConfigurationException(
+                    "Unclear external function import setup: {s}"
+                )

{csvpath-0.0.473 → csvpath-0.0.475}/csvpath/util/config.py

@@ -42,6 +42,8 @@ class Sections(Enum):
     CSV_FILES = "csv_files"
     ERRORS = "errors"
     LOGGING = "logging"
+    FUNCTIONS = "functions"
+    CACHE = "cache"
 
 
 class Config:
@@ -50,22 +52,45 @@ class Config:
      - set a CSVPATH_CONFIG_FILE env var
      - create a Config instance set its CONFIG member and call reload
      - or set Config.CONFIG and reload to reset all instances w/o own specific settings
+    Also, you can pass Config(load=False) to give you the opportunity to set some/all
+    properties programmatically.
     """
 
     CONFIG: str = "config/config.ini"
     CSVPATH_CONFIG_FILE_ENV: str = "CSVPATH_CONFIG_PATH"
 
-    def __init__(self, holder):
-        self._holder = holder
+    def __init__(self, *, load=True):
+        self.load = load
+        self._cache_dir_path = None
+        self._function_imports = None
+        self._csvpath_file_extensions = None
+        self._csv_file_extensions = None
+        self._csvpath_errors_policy = None
+        self._csvpaths_errors_policy = None
+        self._csvpath_log_level = None
+        self._csvpaths_log_level = None
+        self._log_file = None
+        self._log_files_to_keep = None
+        self._log_file_size = None
         self._config = RawConfigParser()
         self.log_file_handler = None
         self._configpath = environ.get(Config.CSVPATH_CONFIG_FILE_ENV)
         if self._configpath is None:
             self._configpath = Config.CONFIG
-        self._load_config()
+        if self.load:
+            self._load_config()
+
+    @property
+    def load(self) -> bool:
+        return self._load
+
+    @load.setter
+    def load(self, l:bool) -> None:
+        self._load = l
 
     def reload(self):
         self._config = RawConfigParser()
+        self._load = True
         self._load_config()
 
     def set_config_path_and_reload(self, path: str) -> None:
@@ -94,10 +119,19 @@ class Config:
             )
 
     def _create_default_config(self) -> None:
-        if not path.exists("config"):
-            os.makedirs("config")
-            with open(Config.CONFIG, "w") as file:
-                c = """
+        directory = ""
+        file = ""
+        if self._configpath is None or self._configpath.strip() == "":
+            raise ConfiguationException("Config path cannot be None")
+        if self._configpath.find(os.sep) > 0:
+            s = self._configpath.rfind(os.sep)
+            directory = self._configpath[0:s]
+            name = self._configpath[s+1:]
+        if directory != "":
+            if not path.exists(directory):
+                os.makedirs(directory)
+        with open(self._configpath, "w") as file:
+            c = """
 [csvpath_files]
 extensions = txt, csvpath, csvpaths
 [csv_files]
@@ -113,20 +147,25 @@ log_files_to_keep = 100
 log_file_size = 52428800
 [config]
 path =
-                """
-                file.write(c)
-            print(f"Creating a default config file at {Config.CONFIG}.")
+[functions]
+imports =
+[cache]
+path =
+            """
+            file.write(c)
+            print(f"Created a default config file at {directory} with name {name}.")
             print("If you want your config to be somewhere else remember to")
             print("update the path in the default config.ini")
 
     def _assure_logs_path(self) -> None:
-        filepath = self.log_file
-        if not filepath:
-            filepath = "logs/csvpath.log"
-            self.log_file = filepath
-        dirpath = self._get_dir_path(filepath)
-        if dirpath and not path.exists(dirpath):
-            os.makedirs(dirpath)
+        if self.load:
+            filepath = self.log_file
+            if not filepath or filepath.strip() == "":
+                filepath = "logs/csvpath.log"
+                self.log_file = filepath
+            dirpath = self._get_dir_path(filepath)
+            if dirpath and not path.exists(dirpath):
+                os.makedirs(dirpath)
 
     def _get_dir_path(self, filepath):
         if filepath.find(os.sep) > -1:
@@ -135,20 +174,23 @@ path =
         return None
 
     def _assure_cache_path(self) -> None:
-        dirpath = self.cache_dir_path
-        if dirpath and not path.exists(dirpath):
-            os.makedirs(dirpath)
-        elif not dirpath:
-            raise ConfigurationException(
-                "No cache path available. Check config.ini [cache][path]."
-            )
+        if self.load:
+            if self.cache_dir_path is None or self.cache_dir_path.strip() == "":
+                self.cache_dir_path = "cache"
+            if not path.exists(self.cache_dir_path):
+                os.makedirs(self.cache_dir_path)
 
     def _assure_config_file_path(self) -> None:
-        if not self._configpath or not os.path.isfile(self._configpath):
-            self._configpath = Config.CONFIG
-            self._create_default_config()
+        if self.load:
+            if not self._configpath or self._configpath.strip() == "":
+                self._configpath = Config.CONFIG
+            if not os.path.isfile(self._configpath):
+                self._create_default_config()
 
     def _load_config(self, norecurse=False):
+        if self._load is False:
+            print("WARNING: _load_config called on a config instance that is set to not load")
+            return
         self._assure_config_file_path()
         #
         #
@@ -172,6 +214,23 @@ path =
         self.log_file_size = self._get(
             Sections.LOGGING.value, LogFile.LOG_FILE_SIZE.value
         )
+        # path to external functions list. external functions are very optional.
+        # not blowing up when absent seems reasonable.
+        try:
+            self.function_imports = self._get(Sections.FUNCTIONS.value, "imports")
+            print(f"config fuc im: {self.function_imports}")
+        except Exception:
+            print("WARNING: config cannot load [functions][imports] from {self.configpath}")
+            pass
+        # likewise caching.
+        try:
+            self.cache_dir_path = self._get(Sections.CACHE.value, "path")
+        except Exception:
+            print("WARNING: config cannot load [cache][path] from {self.configpath}")
+            pass
+        #
+        # reload if another config path is set
+        #
         path = self._get("config", "path")
         if path:
             path = path.strip().lower()
@@ -262,16 +321,29 @@ path =
 
     # ======================================
 
+    @property
+    def configpath(self) -> str:
+        return self._configpath
+
+    @configpath.setter
+    def configpath(self, path: str) -> None:
+        self._configpath = path
+
     @property
     def cache_dir_path(self) -> str:
-        try:
-            path = self._get("cache", "path")
-        except Exception:
-            print("No cache path in config.ini at [cache][path]. Using 'cache'.")
-            path = "config"
-            self._config.add_section("cache")
-            self._config.set("cache", "path", path)
-        return path
+        return self._cache_dir_path
+
+    @cache_dir_path.setter
+    def cache_dir_path(self, p) -> None:
+        self._cache_dir_path = p
+
+    @property
+    def function_imports(self) -> str:
+        return self._function_imports
+
+    @function_imports.setter
+    def function_imports(self, path: str) -> None:
+        self._function_imports = path
 
     @property
     def csvpath_file_extensions(self) -> list[str]:

{csvpath-0.0.473 → csvpath-0.0.475}/docs/functions/implementing_functions.md

@@ -80,17 +80,26 @@ When your function needs its arguments you can call:
 - self._value_two(skip=skip)
 - self._siblings()
 
-When you need to get a value or a match from a child object you use `to_value()` or `matches()`. Remember to pass the skip list as a named argument "skip".
+When you need to get a value or a match from a child object you use `to_value()` or `matches()`. Remember to pass the skip list as the named argument `skip`.
 
 ## Registering
 
-To register your function for use, add it to the FunctionFactory like this:
+To register your function for use, add it to the `FunctionFactory` like this:
 
 ```python
     from csvpath.matching.functions.function_factory import FunctionFactory
     FunctionFactory.add_function(name='iamafunction', function=my_function_instance)
 ```
 
+Alternatively, add a file path to your `config.ini` file under the key `[functions][imports]` that points to a list of functions to register. The key is like:
+
+    [functions]
+    imports = my_imports/functions.txt
+
+In the file list every function on its own line using a format like that of Python imports:
+
+    from a.b.c.my_function import MyFunction as iamafunction
+
 To use your function do something like:
 
 ```bash

{csvpath-0.0.473 → csvpath-0.0.475}/docs/functions.md

@@ -41,7 +41,20 @@ Some functions will optionally make use of an arbitrary name qualifier to better
 
 ## Custom Functions
 
-Creating your own function is easy. Once you create a function, you register it with the `FunctionFactory` class. You must register your functions each time you run CsvPath. Use your function in csvpaths by simply referring to it by name like any other function.
+Creating your own function is easy. Once you create a function, you register it with the `FunctionFactory` class. You can register your functions either programmatically or by creating an import file listing your functions. Your import file must be referenced in your `config.ini` file at `[functions][imports]` like this:
+
+    [functions]
+    imports = my_project_assets/my_functions.imports
+
+Each custom function has its own line in your imports file. The format is the same as you use for importing classes into Python files -- under the hood the mechanism is similar. E.g.
+
+```python
+    from my_function.for_stuff.me import Me as function_me
+```
+
+In this example your class is `Me` and the name of the function you use in your csvpath is `function_me()`
+
+Use your functions in csvpaths by simply referring to them by name like you would any built-in function.
 
 <a href='https://github.com/dk107dk/csvpath/blob/main/docs/functions/implementing_functions.md'>Read more about implementing your own functions here.</a>
 

{csvpath-0.0.473 → csvpath-0.0.475}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "csvpath"
-version = "0.0.473"
+version = "0.0.475"
 description = "A declarative language for data extraction and validation of CSV files"
 authors = ["David Kershaw <dk107dk@hotmail.com>"]
 readme = "README.md"
@@ -27,7 +27,6 @@ include = ["config", "logs", "docs"]
 python = "^3.12"
 ply = "^3.11"
 python-dateutil = "^2.9.0.post0"
-pandas = "^2.2.2"
 jinja2 = "^3.1.4"
 inflect = "^7.3.1"
 lark = "^1.2.2"

csvpath-0.0.473/csvpath/matching/functions/stats/correlate.py

@@ -1,41 +0,0 @@
-# pylint: disable=C0114
-import pandas as pd
-from ..function_focus import ValueProducer
-
-
-class Correlate(ValueProducer):
-    """does a statistical correlation test on the values of two headers"""
-
-    def check_valid(self) -> None:
-        self.validate_two_args()
-        super().check_valid()
-
-    def _produce_value(self, skip=None) -> None:
-        child = self.children[0]
-        left = child.left
-        right = child.right
-        leftlist = left.to_value(skip=skip)
-        rightlist = right.to_value(skip=skip)
-        leftlist, rightlist = self._trim(leftlist, rightlist)
-        ll = pd.Series(leftlist)
-        rl = pd.Series(rightlist)
-        corr = ll.corr(rl)
-        f = float(corr)
-        f = round(f, 2)
-        self.value = f
-
-    def _decide_match(self, skip=None) -> None:
-        self.to_value(skip=skip)
-        self.match = self._noop_match()  # pragma: no cover
-
-    def _trim(self, leftlist, rightlist):
-        n = len(leftlist) if len(leftlist) < len(rightlist) else len(rightlist)
-        ll = []
-        rl = []
-        for i in range(0, n):
-            try:
-                ll.append(float(leftlist[i]))
-                rl.append(float(rightlist[i]))
-            except (TypeError, ValueError):
-                pass
-        return ll, rl