csvpath 0.0.537__tar.gz → 0.0.538__tar.gz

{csvpath-0.0.537 → csvpath-0.0.538}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: csvpath
-Version: 0.0.537
+Version: 0.0.538
 Summary: A data preboarding framework for managing and validating CSV, Excel, and other tabular data files using a Collect, Store, Validate, Publish pattern to create a trusted publisher for downstream data consumers.
 Author: David Kershaw
 Author-email: dk107dk@hotmail.com

{csvpath-0.0.537 → csvpath-0.0.538}/assets/config/jenkins-local-filesystem-mysql.ini

@@ -36,18 +36,19 @@ csvpaths = inputs/named_paths
 on_unmatched_file_fingerprints = halt
 
 [listeners]
-groups = default
+groups = sql
+
 sql.file = from csvpath.managers.integrations.sql.sql_file_listener import SqlFileListener
 sql.paths = from csvpath.managers.integrations.sql.sql_paths_listener import SqlPathsListener
 sql.result = from csvpath.managers.integrations.sql.sql_result_listener import SqlResultListener
 sql.results = from csvpath.managers.integrations.sql.sql_results_listener import SqlResultsListener
-sqlite.result = from csvpath.managers.integrations.sqlite.sqlite_result_listener import SqliteResultListener
-sqlite.results = from csvpath.managers.integrations.sqlite.sqlite_results_listener import SqliteResultsListener
-default.file = from csvpath.managers.files.files_listener import FilesListener
-default.paths = from csvpath.managers.paths.paths_listener import PathsListener
 otlp.result = from csvpath.managers.integrations.otlp.otlp_result_listener import OpenTelemetryResultListener
 otlp.results = from csvpath.managers.integrations.otlp.otlp_results_listener import OpenTelemetryResultsListener
 otlp.errors = from csvpath.managers.integrations.otlp.otlp_error_listener import OpenTelemetryErrorListener
+default.file = from csvpath.managers.files.files_listener import FilesListener
+default.paths = from csvpath.managers.paths.paths_listener import PathsListener
+sqlite.result = from csvpath.managers.integrations.sqlite.sqlite_result_listener import SqliteResultListener
+sqlite.results = from csvpath.managers.integrations.sqlite.sqlite_results_listener import SqliteResultsListener
 sftpplus.paths = from csvpath.managers.integrations.sftpplus.sftpplus_listener import SftpPlusListener
 sftp.results = from csvpath.managers.integrations.sftp.sftp_listener import SftpListener
 ckan.results = from csvpath.managers.integrations.ckan.ckan_listener import CkanListener
@@ -68,7 +69,7 @@ password = tinpenny
 
 [sql]
 dialect = mysql
-connection_string = mysql://csvpath:hangzhou@192.168.66.1/csvpath
+connection_string = mysql://csvpath:password1@192.168.66.1/csvpath
 
 [sqlite]
 db = archive/csvpath.db
@@ -81,7 +82,7 @@ port = SFTPPLUS_PORT
 admin_username = SFTPPLUS_ADMIN_USERNAME
 admin_password = SFTPPLUS_ADMIN_PASSWORD
 api_url = https://localhost:10020/json
-scripts_dir = 
+scripts_dir =
 execute_timeout = 300
 
 [ckan]
@@ -96,5 +97,5 @@ timeout = 5
 verify = False
 
 [slack]
-webhook_url = 
+webhook_url =
 

{csvpath-0.0.537 → csvpath-0.0.538}/csvpath/csvpath.py

@@ -824,7 +824,15 @@ class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
         if self.csvpaths is None:
             return data
         name = self._get_name(data)
-        path = self.csvpaths.file_manager.get_named_file(name)
+        #
+        # this will blow up frequently when name is an actual path. ie name == path
+        # below. since we want file manager to be discriminating we have to catch
+        # the error and reset the name
+        #
+        try:
+            path = self.csvpaths.file_manager.get_named_file(name)
+        except ValueError:
+            path = name
         if path is None:
             return data
         if path == name:

{csvpath-0.0.537 → csvpath-0.0.538}/csvpath/managers/files/file_manager.py

@@ -295,6 +295,14 @@ class FileManager:
     # this feels like the better sig.
     #
     def has_named_file(self, name: NamedFileName) -> bool:
+        #
+        # cannot be a reference or part of a reference. has to just
+        # be the simple name of the named file.
+        #
+        self.legal_name(name)
+        #
+        # the home should exist if the named file exists.
+        #
         p = self.named_file_home(name)
         nos = self.nos
         nos.path = p
@@ -310,11 +318,19 @@ class FileManager:
 
     def remove_named_file(self, name: NamedFileName) -> None:
         """@private"""
+        #
+        # cannot delete any specific files. this is for the named_file
+        # as a whole. the named_file is immutable, so this is all-or-nothing
+        # and we expect it to happen rarely in a production env.
+        #
+        self.legal_name(name)
         p = os.path.join(self.named_files_dir, name)
         nos = self.nos
         nos.path = p
         if nos.exists():
             nos.remove()
+            return True
+        return False
 
     def remove_all_named_files(self) -> None:
         """@private"""
@@ -351,6 +367,20 @@ class FileManager:
             if self.csvpaths.ecoms.do_i_raise():
                 raise
 
+    def legal_name(self, name:str) -> None:
+        if name is None:
+            raise ValueError("Name cannot be None")
+        if name.strip() == "":
+            raise ValueError("Name cannot be empty")
+        if name.find("/") > -1 or name.find("\\") > -1:
+            raise ValueError(f"Not a legal name: {name}. Path seperators are not allowed.")
+        if name.find(".") > -1:
+            raise ValueError(f"Not a legal name: {name}. Periods are not allowed.")
+        if name.find("$") > -1:
+            raise ValueError(f"Not a legal name: {name}. Dollarsigns are not allowed.")
+        if name.find("#") > -1:
+            raise ValueError(f"Not a legal name: {name}. Hashmarks are not allowed.")
+
     #
     # if name is provided all files selected will be registered under the same name in
     # the order they are found; which is likely not deterministic. template, if any, is
@@ -364,8 +394,12 @@ class FileManager:
     def add_named_files_from_dir(
         self, dirname: str, *, name=None, template: str = None, recurse=True
     ):
-        if dirname is None or dirname.strip() == "":
-            raise ValueError("Dirname cannot be None or empty")
+        #
+        # legal_name handled at add_named_file
+        #
+        #self.legal_name(name)
+        #if dirname is None or dirname.strip() == "":
+        #    raise ValueError("Dirname cannot be None or empty")
         #
         # need to support adding all files from directory under the same name. preferably
         # in order of file created time, if possible.
@@ -405,8 +439,9 @@ class FileManager:
     def add_named_file(
         self, *, name: NamedFileName, path: str, template: str = None
     ) -> None:
-        if name is None or name.strip() == "":
-            raise ValueError("Name cannot be None or empty")
+        self.legal_name(name)
+        #if name is None or name.strip() == "":
+        #    raise ValueError("Name cannot be None or empty")
         if path is None or path.strip() == "":
             raise ValueError("Path cannot be None or empty")
         if template is not None:

{csvpath-0.0.537 → csvpath-0.0.538}/csvpath/util/config.py

@@ -71,8 +71,6 @@ class Config:
 
     def __init__(self, *, load=True):
         self.load = load
-        self._cache_dir_path = None
-        self._function_imports = None
         self._additional_listeners = None
         self._csvpath_file_extensions = None
         self._csv_file_extensions = None
@@ -83,12 +81,11 @@ class Config:
         self._log_file = None
         self._log_files_to_keep = None
         self._log_file_size = None
-        self._archive_path = None
-        self._transfer_root = None
-        self._inputs_files_path = None
-        self._inputs_csvpaths_path = None
         self._config = RawConfigParser()
         self.log_file_handler = None
+        #
+        # if env is set it wins out over anything else.
+        #
         self._configpath = environ.get(Config.CSVPATH_CONFIG_FILE_ENV)
         if self._configpath is None:
             self._configpath = Config.CONFIG
@@ -116,19 +113,34 @@ class Config:
     def config_path(self) -> str:
         return self._configpath
 
+    @property
+    def sections(self) -> list[str]:
+        return self._config.sections()
+
     def get(self, *, section: str, name: str, default=None):
         return self._get(section, name, default)
 
     def _get(self, section: str, name: str, default=None):
+        #
+        # TODO: we should swap all uppercase values for env var values if we find a
+        # matching env var. same as we do for metadata values
+        #
         if self._config is None:
             raise ConfigurationException("No config object available")
         try:
             s = self._config[section][name]
             ret = None
-            if s.find(",") > -1:
+            if s and isinstance(s, str) and s.find(",") > -1:
                 ret = [s.strip() for s in s.split(",")]
-            else:
+            elif isinstance(s, str):
                 ret = s.strip()
+            else:
+                ret = s
+
+            if ret and isinstance(ret, str) and ret.isupper():
+                v2 = os.getenv(ret)
+                if v2 is not None:
+                    ret = v2.strip()
             return ret
         except KeyError:
             if self.csvpath_log_level == LogLevels.DEBUG:
@@ -137,6 +149,11 @@ class Config:
                 )
             return default
 
+    def _set(self, section, key, value) -> None:
+        if isinstance( value, list ):
+            value = ",".join(value)
+        self._config.set(section, key, value)
+
     #
     # adds the value to the internal configparser object. doesn't save.
     # does a refresh to make sure any lists or other interpreted values
@@ -172,15 +189,11 @@ class Config:
                 os.makedirs(directory)
         with open(self._configpath, "w", encoding="utf-8") as file:
             c = f"""
-[csvpath_files]
-extensions = txt, csvpath, csvpaths
-
-[csv_files]
-extensions = txt, csv, tsv, dat, tab, psv, ssv
+[extensions]
+csvpath_files = csvpath, csvpaths
+csv_files = txt, csv, tsv, dat, tab, psv, ssv
 
 [errors]
-csvpath = collect, fail, print
-csvpaths = raise, collect
 use_format = full
 pattern = {{time}}:{{file}}:{{line}}:{{paths}}:{{instance}}:{{chain}}:  {{message}}
 
@@ -189,7 +202,7 @@ csvpath = info
 csvpaths = info
 log_file = logs{os.sep}csvpath.log
 log_files_to_keep = 100
-log_file_size = 52428800
+log_file_size = 50000000
 # file or rotating
 handler = file
 
@@ -314,7 +327,19 @@ shell = /bin/bash
 
 
             """
+
             file.write(c)
+            #
+            # writing a new config means we want to immediately load it?
+            # possibly not.
+            #
+            self._assure_logs_path()
+            self._assure_archive_path()
+            self._assure_transfer_root()
+            self._assure_inputs_files_path()
+            self._assure_cache_path()
+            self._assure_inputs_csvpaths_path()
+
             print(f"Created a default config file at {directory} with name {name}.")
             print("If you want your config to be somewhere else remember to")
             print("update the path in the default config.ini")
@@ -391,15 +416,18 @@ shell = /bin/bash
         if uc and uc.strip().lower() == "no":
             return
         if self.load:
-            if self.cache_dir_path is None or self.cache_dir_path.strip() == "":
-                self.cache_dir_path = "cache"
-            if self.cache_dir_path.find("://") > -1:
+            p = self._get("cache", "path", "cache")
+            if p:
+                p = p.strip()
+            if p == "":
+                self._set("cache", "use_cache", "no")
+                return
+            if p.find("://") > -1:
                 raise ConfigurationException(
-                    f"Cache dir must be on the local drive, not {self.cache_dir_path}"
+                    f"Cache dir must be on the local drive, not {p}"
                 )
-            exists = path.exists(self.cache_dir_path)
-            if not exists:
-                os.makedirs(self.cache_dir_path)
+            if not os.path.exists(p):
+                os.makedirs(p)
 
     def _assure_config_file_path(self) -> None:
         if self.load:
@@ -424,42 +452,33 @@ shell = /bin/bash
     # file, it just gathers and translates settings for external use.
     #
     def refresh(self) -> None:
-        self.csvpath_file_extensions = self._get(
-            Sections.CSVPATH_FILES.value, "extensions"
-        )
-        self.csv_file_extensions = self._get(Sections.CSV_FILES.value, "extensions")
-
-        self.csvpath_errors_policy = self._get(Sections.ERRORS.value, "csvpath")
-        self.csvpaths_errors_policy = self._get(Sections.ERRORS.value, "csvpaths")
+        #
+        # the sections: csv_files and csvpath_files are deprecated in favor of
+        # a single extensions section with csv_files and csvpath_files as keys.
+        # is is a simpler way to go and will UI better. the old way can hang about
+        # for a release or two, but should be retired soon.
+        #
+        #
+        # file extensions will reload themselves as long as we clear them
+        #
+        self.csvpath_file_extensions = None
+        self.csv_file_extensions = None
+        #
+        #
+        #
+        self.csvpath_errors_policy = self._get(Sections.ERRORS.value, "csvpath", ["raise", "print", "stop", "fail", "collect"])
+        self.csvpaths_errors_policy = self._get(Sections.ERRORS.value, "csvpaths", ["raise", "print", "stop", "fail", "collect"])
 
         self.csvpath_log_level = self._get(Sections.LOGGING.value, "csvpath")
         self.csvpaths_log_level = self._get(Sections.LOGGING.value, "csvpaths")
 
         self.log_file = self._get(Sections.LOGGING.value, LogFile.LOG_FILE.value)
         self.log_files_to_keep = self._get(
-            Sections.LOGGING.value, LogFile.LOG_FILES_TO_KEEP.value
+            Sections.LOGGING.value, LogFile.LOG_FILES_TO_KEEP.value, 10
         )
         self.log_file_size = self._get(
-            Sections.LOGGING.value, LogFile.LOG_FILE_SIZE.value
+            Sections.LOGGING.value, LogFile.LOG_FILE_SIZE.value, 12800000
         )
-        # path to external functions list. external functions are very optional.
-        # not blowing up when absent seems reasonable.
-        try:
-            self.function_imports = self._get(Sections.FUNCTIONS.value, "imports")
-        except Exception:
-            print(
-                "WARNING: config cannot load [functions][imports] from {self.configpath}"
-            )
-            pass
-        # likewise caching.
-        try:
-            self.cache_dir_path = self._get(Sections.CACHE.value, "path")
-        except Exception:
-            print("WARNING: config cannot load [cache][path] from {self.configpath}")
-            pass
-        #
-        # reload if another config path is set
-        #
         path = self._get("config", "path")
         if path:
             path = path.strip().lower()
@@ -471,23 +490,6 @@ shell = /bin/bash
 
     def validate_config(self) -> None:
         #
-        # files
-        #
-        if (
-            self.csvpath_file_extensions is None
-            or not isinstance(self.csvpath_file_extensions, list)
-            or not len(self.csvpath_file_extensions) > 0
-        ):
-            raise ConfigurationException(
-                f"CsvPath file extensions are wrong: {self.csvpath_file_extensions}"
-            )
-        if (
-            self.csv_file_extensions is None
-            or not isinstance(self.csv_file_extensions, list)
-            or not len(self.csv_file_extensions) > 0
-        ):
-            raise ConfigurationException("CSV file extensions are wrong")
-        #
         # error policies
         #
         if (
@@ -582,11 +584,11 @@ shell = /bin/bash
 
     @property
     def cache_dir_path(self) -> str:
-        return self._cache_dir_path
+        return self._get("cache", "path")
 
     @cache_dir_path.setter
     def cache_dir_path(self, p) -> None:
-        self._cache_dir_path = p
+        self._set("cache", "path", p )
 
     def halt_on_unmatched_file_fingerprints(self) -> bool:
         houf = self._get("inputs", "on_unmatched_file_fingerprints")
@@ -598,29 +600,19 @@ shell = /bin/bash
 
     @property
     def transfer_root(self) -> str:
-        if self._transfer_root is None:
-            self._transfer_root = self._get("results", "transfers")
-            if self._transfer_root is None:
-                self._transfer_root = "transfers"
-                self.add_to_config("results", "transfers", "transfers")
-        return self._transfer_root
+        return self._get("results", "transfers", "transfers")
 
     @transfer_root.setter
     def transfer_root(self, p) -> None:
-        self._transfer_root = p
+        self._set("results", "transfers", p)
 
     @property
     def archive_path(self) -> str:
-        if self._archive_path is None:
-            self._archive_path = self._get("results", "archive")
-            if self._archive_path is None:
-                self._archive_path = "archive"
-                self.add_to_config("results", "archive", "archive")
-        return self._archive_path
+        return self._get("results", "archive", "archive")
 
     @archive_path.setter
     def archive_path(self, p) -> None:
-        self._archive_path = p
+        self._set("results", "archive", p)
 
     @property
     def archive_name(self) -> str:
@@ -631,40 +623,40 @@ shell = /bin/bash
 
     @property
     def inputs_files_path(self) -> str:
-        if self._inputs_files_path is None:
-            self._inputs_files_path = self._get("inputs", "files")
-            if self._inputs_files_path is None:
-                self._inputs_files_path = "inputs"
-                self.add_to_config("inputs", "files", f"inputs{os.sep}named_files")
-        return self._inputs_files_path
+        return self._get("inputs", "files", f"inputs{os.sep}named_files")
 
     @inputs_files_path.setter
     def inputs_files_path(self, p) -> None:
-        self._inputs_files_path = p
+        self._set("inputs", "files", p)
 
     @property
     def inputs_csvpaths_path(self) -> str:
-        if self._inputs_csvpaths_path is None:
-            self._inputs_csvpaths_path = self._get("inputs", "csvpaths")
-            if self._inputs_csvpaths_path is None:
-                self._inputs_csvpaths_path = "inputs"
-                self.add_to_config("inputs", "csvpaths", f"inputs{os.sep}named_paths")
-        return self._inputs_csvpaths_path
+        return self._get("inputs", "csvpaths", f"inputs{os.sep}named_paths")
 
     @inputs_csvpaths_path.setter
     def inputs_csvpaths_path(self, p) -> None:
-        self._inputs_csvpaths_path = p
+        self._set("inputs", "csvpaths", p)
 
     @property
     def function_imports(self) -> str:
-        return self._function_imports
+        return self._get(Sections.FUNCTIONS.value, "imports", "imports")
 
     @function_imports.setter
     def function_imports(self, path: str) -> None:
-        self._function_imports = path
+        self._set( Sections.FUNCTIONS.value, "imports", path)
 
     @property
     def csvpath_file_extensions(self) -> list[str]:
+        if self._csvpath_file_extensions is None:
+            cs = self._get("extensions", "csvpath_files")
+            if not cs or len(cs) == 0:
+                #
+                # the old way
+                #
+                cs = self._get(Sections.CSVPATH_FILES.value, "extensions")
+                if not cs or len(cs) == 0:
+                    cs = ["csvpath","csvpaths"]
+            self._csvpath_file_extensions = cs
         return self._csvpath_file_extensions
 
     @csvpath_file_extensions.setter
@@ -675,6 +667,16 @@ shell = /bin/bash
 
     @property
     def csv_file_extensions(self) -> list[str]:
+        if self._csv_file_extensions is None:
+            cs = self._get("extensions", "csv_files")
+            if not cs or len(cs) == 0:
+                #
+                # the old way
+                #
+                cs = self._get(Sections.CSV_FILES.value, "extensions")
+                if not cs or len(cs) == 0:
+                    cs = ["csv","xlsx"]
+            self._csv_file_extensions = cs
         return self._csv_file_extensions
 
     @csv_file_extensions.setter
@@ -729,22 +731,37 @@ shell = /bin/bash
 
     @property
     def log_files_to_keep(self) -> int:
-        return self._log_files_to_keep
+        fs = self._get(Sections.LOGGING.value, LogFile.LOG_FILES_TO_KEEP.value, 10)
+        try:
+            fs = int(fs)
+        except ValueError:
+            return 10
+        return fs
 
     @log_files_to_keep.setter
     def log_files_to_keep(self, i: int) -> None:
         try:
-            self._log_files_to_keep = int(i)
-        except (TypeError, ValueError):
-            raise ConfigurationException("Error in log_files_to_keep config")
+            i = int(i)
+        except (ValueError, TypeError):
+            i = 10
+        self._set(Sections.LOGGING.value, LogFile.LOG_FILES_TO_KEEP.value, i)
 
     @property
     def log_file_size(self) -> int:
-        return self._log_file_size
+        fs = self._get(Sections.LOGGING.value, LogFile.LOG_FILE_SIZE.value, 12800000)
+        try:
+            fs = int(fs)
+        except ValueError:
+            fs = 12800000
+        return fs
 
     @log_file_size.setter
     def log_file_size(self, i: int) -> None:
         try:
-            self._log_file_size = int(i)
+            i = int(i)
         except (TypeError, ValueError):
-            raise ConfigurationException("Error in log_files_size config")
+            i = 12800000
+        self._set(Sections.LOGGING.value, LogFile.LOG_FILE_SIZE.value, i)
+
+
+

{csvpath-0.0.537 → csvpath-0.0.538}/csvpath/util/nos.py

@@ -88,12 +88,8 @@ class Nos:
         return self.do.dir_exists()
 
     def physical_dirs(self) -> bool:
-        """True if dirs may exist.
-        False if there is no concept of dirs
-        that are independent from objects.
-        Cloud objects stores like S3 are the
-        latter.
-        """
+        """True if dirs can exist independently. False if there is no concept of dirs
+        that are independent from objects. Cloud objects stores like S3 are the latter."""
         return self.do.physical_dirs()
 
     def rename(self, new_path: str) -> None:
@@ -108,9 +104,6 @@ class Nos:
     def makedir(self) -> None:
         self.do.makedir()
 
-    #
-    # listdir should be assumed to be unordered.
-    #
     def listdir(
         self,
         *,

{csvpath-0.0.537 → csvpath-0.0.538}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "csvpath"
-version = "0.0.537"
+version = "0.0.538"
 description = "A data preboarding framework for managing and validating CSV, Excel, and other tabular data files using a Collect, Store, Validate, Publish pattern to create a trusted publisher for downstream data consumers."
 authors = ["David Kershaw <dk107dk@hotmail.com>"]
 readme = "README.md"