csvpath 0.0.599__tar.gz → 0.0.606__tar.gz

{csvpath-0.0.599 → csvpath-0.0.606}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: csvpath
-Version: 0.0.599
+Version: 0.0.606
 Summary: CsvPath Framework is a data preboarding automation library for receiving, validating, and tracking CSV, Excel, JSONL and other tabular data files before they can corrupt downstream data consumers.
 License-File: LICENSE
 Author: David Kershaw
@@ -54,6 +54,7 @@ Requires-Dist: pytest (==8.4.2)
 Requires-Dist: python-dateutil (==2.9.0.post0)
 Requires-Dist: pytz (==2024.2)
 Requires-Dist: requests (==2.32.5)
+Requires-Dist: ruff (>=0.15.8,<0.16.0)
 Requires-Dist: smart-open[all] (==7.5.1)
 Requires-Dist: sqlalchemy (==2.0.48)
 Requires-Dist: tabulate (==0.9.0)

{csvpath-0.0.599 → csvpath-0.0.606}/assets/gocd/windows/azure.bat

@@ -30,3 +30,11 @@ cmd.exe /C c:\Users\python\.local\bin\poetry.exe install
 cmd.exe /C c:\Users\python\.local\bin\poetry.exe run pytest
 
 
+net stop w32time
+w32tm /unregister
+w32tm /register
+net start w32time
+timeout /t 1
+w32tm /resync
+
+

{csvpath-0.0.599 → csvpath-0.0.606}/assets/gocd/windows/gcs.bat

@@ -29,3 +29,11 @@ cmd.exe /C c:\Users\python\.local\bin\poetry.exe install
 cmd.exe /C c:\Users\python\.local\bin\poetry.exe run pytest
 
 
+net stop w32time
+w32tm /unregister
+w32tm /register
+net start w32time
+timeout /t 1
+w32tm /resync
+
+

{csvpath-0.0.599 → csvpath-0.0.606}/assets/gocd/windows/local.bat

@@ -53,3 +53,11 @@ cmd.exe /C c:\Users\python\.local\bin\poetry.exe install
 cmd.exe /C c:\Users\python\.local\bin\poetry.exe run pytest
 
 
+net stop w32time
+w32tm /unregister
+w32tm /register
+net start w32time
+timeout /t 1
+w32tm /resync
+
+

{csvpath-0.0.599 → csvpath-0.0.606}/assets/gocd/windows/s3.bat

@@ -31,3 +31,11 @@ cmd.exe /C c:\Users\python\.local\bin\poetry.exe install
 cmd.exe /C c:\Users\python\.local\bin\poetry.exe run pytest
 
 
+net stop w32time
+w32tm /unregister
+w32tm /register
+net start w32time
+timeout /t 1
+w32tm /resync
+
+

{csvpath-0.0.599 → csvpath-0.0.606}/assets/gocd/windows/sftp.bat

@@ -29,3 +29,11 @@ cmd.exe /C c:\Users\python\.local\bin\poetry.exe install
 cmd.exe /C c:\Users\python\.local\bin\poetry.exe run pytest
 
 
+net stop w32time
+w32tm /unregister
+w32tm /register
+net start w32time
+timeout /t 1
+w32tm /resync
+
+

{csvpath-0.0.599 → csvpath-0.0.606}/config/config.ini

@@ -35,7 +35,7 @@ path = cache
 use_cache = yes
 
 [functions]
-imports =
+imports = 
 
 [results]
 archive = tmp/local/archive
@@ -48,7 +48,7 @@ allow_http_files = True
 allow_local_files = True
 
 [listeners]
-groups = default,scripts
+groups = default
 webhook.results = from csvpath.managers.integrations.webhook.webhook_results_listener import WebhookResultsListener
 scripts.results = from csvpath.managers.integrations.scripts.scripts_results_listener import ScriptsResultsListener
 sql.file = from csvpath.managers.integrations.sql.sql_file_listener import SqlFileListener
@@ -105,7 +105,7 @@ port = SFTPPLUS_PORT
 admin_username = SFTPPLUS_ADMIN_USERNAME
 admin_password = SFTPPLUS_ADMIN_PASSWORD
 api_url = https://. . . :10020/json
-scripts_dir =
+scripts_dir = 
 execute_timeout = 300
 
 [ckan]
@@ -120,7 +120,7 @@ timeout = 5
 verify = False
 
 [slack]
-webhook_url =
+webhook_url = 
 
 [scripts]
 run_scripts = yes

{csvpath-0.0.599 → csvpath-0.0.606}/csvpath/csvpaths.py

@@ -1,15 +1,15 @@
-""" CsvPaths' intent is to help you manage and automate your use
-    of the CsvPath library. it makes it easier to scale your CSV quality control. """
+"""CsvPaths' intent is to help you manage and automate your use
+of the CsvPath library. it makes it easier to scale your CSV quality control."""
 
 import os
 import traceback
 import atexit
 import threading
 
-from uuid import uuid4, UUID
+from uuid import uuid4
 from abc import ABC, abstractmethod
 from typing import List, Any, NewType
-from datetime import datetime, timezone
+from datetime import datetime
 from .managers.errors.error import Error
 from .managers.errors.error_comms import ErrorCommunications
 from .managers.errors.error_manager import ErrorManager
@@ -26,7 +26,6 @@ from .managers.results.results_manager import ResultsManager
 from .managers.results.result import Result
 from .util.box import Box
 from . import CsvPath
-from .managers.integrations.otlp.metrics import Metrics
 
 
 # types for clarity
@@ -108,7 +107,7 @@ class CsvPaths(CsvPathsCoordinator, ErrorCollector):
         skip_blank_lines=True,
         print_default=True,
         project_context=None,
-        project=None
+        project=None,
         # config: Config = None,
     ):
         if CsvPaths.METRICS_WRAP_REG is False:
@@ -173,8 +172,8 @@ class CsvPaths(CsvPathsCoordinator, ErrorCollector):
         #
         #
         #
-        self.info_dump()
-        """ @private """
+        # self.info_dump()
+
         self._errors = []
         # coordinator attributes
         self._stop_all = False
@@ -219,7 +218,14 @@ class CsvPaths(CsvPathsCoordinator, ErrorCollector):
         # that need to know about the run during the run; e.g. parquet()
         #
         self._csvpath_instances = []
-
+        #
+        # any listeners added here will be added to the internal_listeners of
+        # the csvpath instances that run.
+        #
+        self.dynamic_csvpath_error_listeners = []
+        #
+        #
+        #
         self.logger.info(
             f"Initialized CsvPaths: {self} in thread: {threading.current_thread()}"
         )
@@ -263,8 +269,9 @@ class CsvPaths(CsvPathsCoordinator, ErrorCollector):
         if subs not in ["env", "none"]:
             subs = f"{subs} with keys:"
             e = self.config.config_env.env
-            for k, v in e.items():
-                subs = f"{subs}\n - {k}"
+            if e is not None:
+                for k, v in e.items():
+                    subs = f"{subs}\n - {k}"
         cache = self.config.get(section="cache", name="path")
         cache = cache if self.config.get(section="cache", name="use_cache") else "none"
         logstr = self.logger if self.logger else "pending"
@@ -278,40 +285,42 @@ class CsvPaths(CsvPathsCoordinator, ErrorCollector):
         return f"""
 Context: {self.project_context}.{self.project}
 Logger: {logstr}
-Configured log path: {self.config.get(section='logging', name='log_file')}
+Configured log path: {self.config.get(section="logging", name="log_file")}
 Current log paths: {logpath}
 Config file: {self.config.configpath}
-Var sub source: {self.config.get(section='config', name='var_sub_source')}
+Var sub source: {self.config.get(section="config", name="var_sub_source")}
 Var subs: {subs}
 Errors:
- - csvpath: {self.config.get(section='errors', name='csvpath')}
- - csvpaths: {self.config.get(section='errors', name='csvpaths')}
-Named-files: {self.config.get(section='inputs', name='files')}
-Named-paths: {self.config.get(section='inputs', name='files')}
-Archive: {self.config.get(section='results', name='archive')}
+ - csvpath: {self.config.get(section="errors", name="csvpath")}
+ - csvpaths: {self.config.get(section="errors", name="csvpaths")}
+Named-files: {self.config.get(section="inputs", name="files")}
+Named-paths: {self.config.get(section="inputs", name="files")}
+Archive: {self.config.get(section="results", name="archive")}
 Cache: {cache}
 {lstrs}
         """
 
-    """
-    def info_dump(self) -> None:
-        self.logger.info(
-            "Initated logging on log path: %s",
-            self.config.get(section="logging", name="log_file"),
-        )
-        self.logger.info("Config file is at: %s", self.config.configpath)
-        intgs = self.config.get(section="listeners", name="groups")
-        self.logger.debug("Active integrations:")
-        for _ in intgs:
-            self.logger.debug("  - %s", _)
-    """
-
     def _set_managers(self) -> None:
         self.paths_manager = PathsManager(csvpaths=self)
         self.file_manager = FileManager(csvpaths=self)
-        self.results_manager = ResultsManager(csvpaths=self)
         self.ecoms = ErrorCommunications(csvpaths=self)
         self.error_manager = ErrorManager(csvpaths=self)
+        #
+        # we take a bit more care with resman because it may have listeners
+        # that are set programmatically. other managers deal with their
+        # registrars differently and don't have the problem of maintaining
+        # lists of listeners at the csvpaths level
+        #
+        resman = ResultsManager(csvpaths=self)
+        if self.results_manager:
+            resman.dynamic_result_listeners = (
+                self.results_manager.dynamic_result_listeners
+            )
+            resman.dynamic_results_listeners = (
+                self.results_manager.dynamic_results_listeners
+            )
+            resman.dynamic_run_listeners = self.results_manager.dynamic_run_listeners
+        self.results_manager = resman
 
     @property
     def project(self) -> str:
@@ -432,6 +441,8 @@ Cache: {cache}
         if path.config.configpath != self.config.configpath:
             path.config.set_config_path_and_reload(self.config.configpath)
             path.logger = None
+        for _ in self.dynamic_csvpath_error_listeners:
+            path.error_manager.add_internal_listener(_)
         return path
 
     def stop_all(self) -> None:  # pragma: no cover
@@ -1448,10 +1459,8 @@ Cache: {cache}
                     #
                     # re: W0212: treating _consider_line something like package private
                     #
-                    matched = (
-                        self.current_matcher._consider_line(  # pylint:disable=W0212
-                            line
-                        )
+                    matched = self.current_matcher._consider_line(  # pylint:disable=W0212
+                        line
                     )
                     if self.current_matcher.stopped:
                         stopped_count.append(1)

{csvpath-0.0.599 → csvpath-0.0.606}/csvpath/managers/files/file_manager.py

@@ -1,6 +1,4 @@
-import os
 import json
-import csv
 import traceback
 from typing import NewType
 from json import JSONDecodeError
@@ -13,7 +11,7 @@ from csvpath.util.references.files_reference_finder_2 import (
 from csvpath.util.references.results_reference_finder_2 import (
     ResultsReferenceFinder2 as ResultsReferenceFinder,
 )
-from csvpath.util.exceptions import InputException, FileException
+from csvpath.util.exceptions import FileException
 from csvpath.util.nos import Nos
 from csvpath.util.box import Box
 from csvpath.util.path_util import PathUtility as pathu
@@ -641,10 +639,12 @@ class FileManager:
             #
             self.csvpaths.logger.debug("Path after removing mark, if any: %s", path)
             self._copy_in(path, home, template)
+            self.csvpaths.logger.debug("Done copying in")
             #
             #
             #
             self.assure_docs_and_discriptor(name)
+            self.csvpaths.logger.debug("Assured docs and discriptor")
             #
             # create the reference to the bytes/version added to the named-file.
             # this is the most specific reference possible. we return it when we're
@@ -658,6 +658,7 @@ class FileManager:
             # create the metadata event for this registration
             #
             name_home = self.named_file_home(name)
+            self.csvpaths.logger.debug("Name home is %s", name_home)
             mdata = FileMetadata(self.csvpaths.config)
             mdata.named_file_name = name
             mdata.named_file_ref = ret
@@ -678,6 +679,7 @@ class FileManager:
             mdata.name_home = name_home
             mdata.mark = mark
             mdata.template = template
+            self.csvpaths.logger.debug("Created metadata")
             #
             # TODO: add file_size. move FileInfo into Nos. for now it is 0.
             #

{csvpath-0.0.599 → csvpath-0.0.606}/csvpath/managers/integrations/webhook/webhook_results_listener.py

@@ -1,4 +1,3 @@
-import os
 import json
 from csvpath.managers.metadata import Metadata
 from csvpath.managers.results.results_metadata import ResultsMetadata
@@ -70,13 +69,13 @@ class WebhookResultsListener(WebhookListener):
         if cfg is None:
             return None
         if atype.find("all") > -1:
-            return cfg.all_url
+            return cfg.on_complete_all.url
         elif atype.find("invalid") > -1:
-            return cfg.invalid_url
+            return cfg.on_complete_invalid.url
         elif atype.find("valid") > -1:
-            return cfg.valid_url
+            return cfg.on_complete_valid.url
         elif atype.find("error") > -1:
-            return cfg.error_url
+            return cfg.on_complete_error.url
         else:
             raise ValueError(f"Unknown type: {atype}")
 
@@ -106,13 +105,13 @@ class WebhookResultsListener(WebhookListener):
             return {}
         on = None
         if atype.find("all") > -1:
-            on = cfg.on_complete_all
+            on = cfg.on_complete_all.payload
         elif atype.find("invalid") > -1:
-            on = cfg.on_complete_invalid
+            on = cfg.on_complete_invalid.payload
         elif atype.find("valid") > -1:
-            on = cfg.on_complete_valid
+            on = cfg.on_complete_valid.payload
         elif atype.find("error") > -1:
-            on = cfg.on_complete_error
+            on = cfg.on_complete_error.payload
         else:
             raise ValueError(f"Unknown type: {atype}")
         if on and len(on) > 0:
@@ -122,7 +121,6 @@ class WebhookResultsListener(WebhookListener):
         return None
 
     def _payload(self, *, mdata: Metadata, atype: str, cfg, hook: str) -> dict:
-
         metadata = self.get_data(mdata, "meta.json")
         variables = self.get_data(mdata, "vars.json")
         pairs = VarUtility.get_value_pairs_from_value(

{csvpath-0.0.599 → csvpath-0.0.606}/csvpath/managers/paths/paths_describer.py

@@ -20,15 +20,16 @@ class Scripts(BaseModel):
     on_complete_error: Optional[str] = None
 
 
+class Webhook(BaseModel):
+    url: Optional[str] = None
+    payload: Optional[str] = None
+
+
 class Webhooks(BaseModel):
-    on_complete_all: Optional[str] = None
-    on_complete_invalid: Optional[str] = None
-    on_complete_valid: Optional[str] = None
-    on_complete_error: Optional[str] = None
-    all_url: Optional[str] = None
-    valid_url: Optional[str] = None
-    invalid_url: Optional[str] = None
-    error_url: Optional[str] = None
+    on_complete_all: Optional[Webhook] = None
+    on_complete_invalid: Optional[Webhook] = None
+    on_complete_valid: Optional[Webhook] = None
+    on_complete_error: Optional[Webhook] = None
 
 
 class GroupConfig(BaseModel):
@@ -42,7 +43,6 @@ class Config(BaseModel):
 
 
 class NamedPathsDescriber:
-
     README = "README.md"
     DEFAULT_README = "# Named-Paths Documentation\n\n"
     JSON_FILE = "definition.json"
@@ -78,6 +78,23 @@ class NamedPathsDescriber:
         #
         if self.CONFIG in j:
             GroupConfig(**j[self.CONFIG])
+        #
+        # clear out the Nones
+        #
+        if self.CONFIG in j:
+            configs = j[self.CONFIG]
+            if "groups" in configs:
+                configs = configs["groups"]
+                for _k, _v in j[self.CONFIG].items():
+                    rms = []
+                    for k, v in _v.items():
+                        if v is None:
+                            rms.append(k)
+                    for rm in rms:
+                        del _v[rm]
+        #
+        #
+        #
         p = Nos(home).join(self.JSON_FILE)
         with DataFileWriter(path=p) as writer:
             json.dump(j, writer.sink, indent=2)
@@ -125,7 +142,7 @@ class NamedPathsDescriber:
             config.groups[name] = gc
         else:
             config.groups[name] = j
-        _[self.CONFIG] = config.model_dump()
+        _[self.CONFIG] = config.model_dump(exclude_none=True)
         self.store_json(name, _)
 
     def get_config(self, name: NamedPathsName) -> GroupConfig:

{csvpath-0.0.599 → csvpath-0.0.606}/csvpath/managers/paths/paths_manager.py

@@ -1,5 +1,4 @@
 # pylint: disable=C0114
-import os
 import json
 import traceback
 from typing import NewType
@@ -10,7 +9,6 @@ from csvpath.util.metadata_parser import MetadataParser
 from csvpath.util.references.reference_parser import ReferenceParser
 from csvpath.util.file_readers import DataFileReader
 from csvpath.util.file_writers import DataFileWriter
-from csvpath.util.box import Box
 from csvpath.util.nos import Nos
 from .paths_registrar import PathsRegistrar
 from .paths_metadata import PathsMetadata
@@ -392,7 +390,7 @@ class PathsManager:
         assure_definition: bool = True,
     ) -> str:
         self.last_add_metadata = None
-        if template is not None:
+        if template is not None and str(template).strip() != "":
             #
             # this will raise an error. if that's a problem use temu.validate
             #
@@ -470,7 +468,7 @@ class PathsManager:
             if assure_definition is True:
                 t = self.get_json_paths_file(name)
                 if t is None:
-                    raise ValueError(f"JSON definitionn for {name} cannot be None")
+                    raise ValueError(f"JSON definition for {name} cannot be None")
                 if name not in t:
                     t[name] = []
                     self.describer.store_json(name, t)

{csvpath-0.0.599 → csvpath-0.0.606}/csvpath/managers/results/results_manager.py

@@ -1,11 +1,7 @@
 # pylint: disable=C0114
 import os
-import json
 import re
-import traceback
 from uuid import UUID
-from pathlib import Path
-import datetime
 import dateutil.parser
 from typing import Dict, List, Any
 from csvpath.util.line_spooler import LineSpooler
@@ -40,6 +36,19 @@ class ResultsManager:  # pylint: disable=C0115
         # use property
         self.csvpaths = csvpaths
         """@private"""
+        #
+        # the dynamic listeners are added to each Registrar before
+        # it distributes updates. we don't need this for the other
+        # managers (paths, files, errors) because they don't create
+        # new registrars each time. we can just do:
+        # paths.error_manager.registrar.add_internal_listeners(lst)
+        # but with these we cannot do that. we could change how this
+        # class handles listeners but it feels like a small additive
+        # is better than an unknown impact
+        #
+        self.dynamic_result_listeners = []
+        self.dynamic_results_listeners = []
+        self.dynamic_run_listeners = []
 
     @property
     def csvpaths(self):
@@ -59,6 +68,11 @@ class ResultsManager:  # pylint: disable=C0115
             pathsname=pathsname,
             results=results,
         )
+        for _ in self.dynamic_results_listeners:
+            rr.add_internal_listener(_)
+        #
+        # add any dynamic listeners to the registrar here
+        #
         m = rr.manifest
         mdata = ResultsMetadata(self.csvpaths.config)
         if "time" not in m or m["time"] is None:
@@ -104,6 +118,12 @@ class ResultsManager:  # pylint: disable=C0115
             run_dir=run_dir,
             pathsname=pathsname,
         )
+        #
+        # add any dynamic listeners to the registrar here
+        #
+        for _ in self.dynamic_results_listeners:
+            rr.add_internal_listener(_)
+
         #
         # collect the named-paths and named-file uuids. these may
         # need to come from a different source at some point but
@@ -337,6 +357,14 @@ class ResultsManager:  # pylint: disable=C0115
         mdata.named_file_name = result.file_name
         mdata.method = result.method
         rr = RunRegistrar(self.csvpaths)
+        #
+        # add any dynamic listeners to the registrar here
+        #
+        print(f"regsman: me: {id(self)}")
+        print(f"regsman: my csvpaths: {id(self.csvpaths)}")
+        print(f"regsman: dylsi: {self.dynamic_run_listeners}")
+        for _ in self.dynamic_run_listeners:
+            rr.add_internal_listener(_)
         rr.register_start(mdata)
         #
         # we prep the results event
@@ -376,6 +404,11 @@ class ResultsManager:  # pylint: disable=C0115
         rr = ResultRegistrar(
             csvpaths=self.csvpaths, result=result, result_serializer=rs
         )
+        #
+        # add any dynamic listeners to the registrar here
+        #
+        for _ in self.dynamic_result_listeners:
+            rr.add_internal_listener(_)
         rr.register_start(mdata)
 
     def set_named_results(self, results: Dict[str, List[Result]]) -> None:
@@ -464,8 +497,12 @@ class ResultsManager:  # pylint: disable=C0115
             # exp! it would be good to offer a var sub here
             #
             if pathto:
-                result.csvpath.config.set(section="_dummy-section", name="_dummy-name", value=pathto)
-                pathto = result.csvpath.config.get(section="_dummy-section", name="_dummy-name")
+                result.csvpath.config.set(
+                    section="_dummy-section", name="_dummy-name", value=pathto
+                )
+                pathto = result.csvpath.config.get(
+                    section="_dummy-section", name="_dummy-name"
+                )
             #
             #
             #
@@ -559,9 +596,15 @@ class ResultsManager:  # pylint: disable=C0115
         self.do_transfers_if(result)
         rs = ResultSerializer(self._csvpaths.config.archive_path)
         rs.save_result(result)
-        ResultRegistrar(
+        rr = ResultRegistrar(
             csvpaths=self.csvpaths, result=result, result_serializer=rs
-        ).register_complete()
+        )
+        #
+        # add any dynamic listeners to the registrar here
+        #
+        for _ in self.dynamic_result_listeners:
+            rr.add_internal_listener(_)
+        rr.register_complete()
 
     def remove_named_results(self, name: str) -> None:
         """@private"""

{csvpath-0.0.599 → csvpath-0.0.606}/csvpath/managers/test_listener.py

@@ -1,6 +1,4 @@
-from abc import ABC, abstractmethod
 from .metadata import Metadata
-from ..util.config import Config
 from .listener import Listener
 
 
@@ -13,6 +11,8 @@ class TestListener(Listener):
     def __init__(self, config=None) -> None:
         self._config = None
         super().__init__(config)
+        self.mine = None
 
     def metadata_update(self, mdata: Metadata) -> None:
         TestListener.METADATA = mdata
+        self.mine = mdata

{csvpath-0.0.599 → csvpath-0.0.606}/csvpath/util/azure/azure_data_writer.py

@@ -1,9 +1,9 @@
 # pylint: disable=C0114
 
-import os
 from smart_open import open
 from ..file_writers import DataFileWriter
 from csvpath.util.azure.azure_utils import AzureUtility
+from csvpath.util.stopwatch import Stopwatch
 
 
 class AzureDataWriter(DataFileWriter):
@@ -13,17 +13,16 @@ class AzureDataWriter(DataFileWriter):
         if self.sink is None:
             client = AzureUtility.make_client()
             self.sink = open(
-                self.path,
-                self.mode,
-                transport_params={"client": client},
-                newline=''
+                self.path, self.mode, transport_params={"client": client}, newline=""
             )
             AzureDataWriter._write_file_count += 1
 
     def write(self, data) -> None:
         if self.is_binary and not isinstance(data, bytes):
             data = data.encode(self.encoding)
+        s = Stopwatch()
         self.sink.write(data)
+        s.end()
 
     def file_info(self) -> dict[str, str | int | float]:
         # TODO: what can/should we provide here?

{csvpath-0.0.599 → csvpath-0.0.606}/csvpath/util/azure/azure_nos.py

@@ -1,6 +1,4 @@
 # pylint: disable=C0114
-import os
-from azure.storage.blob import BlobServiceClient, ContainerClient
 from azure.storage.blob._list_blobs_helper import BlobPrefix
 from .azure_utils import AzureUtility
 from ..path_util import PathUtility as pathu
@@ -100,8 +98,6 @@ class AzureDo:
         if files_only is True and dirs_only is True:
             raise ValueError("Cannot list with neither files nor dirs")
         container, blob = AzureUtility.path_to_parts(path)
-
-        # listed = os.path.basename(self.path)
         if not blob.endswith("/"):
             blob = f"{blob}/"
         if blob == "/":