csvpath 0.0.537__tar.gz → 0.0.541__tar.gz

{csvpath-0.0.537 → csvpath-0.0.541}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: csvpath
-Version: 0.0.537
+Version: 0.0.541
 Summary: A data preboarding framework for managing and validating CSV, Excel, and other tabular data files using a Collect, Store, Validate, Publish pattern to create a trusted publisher for downstream data consumers.
 Author: David Kershaw
 Author-email: dk107dk@hotmail.com

{csvpath-0.0.537 → csvpath-0.0.541/assets}/config/config.ini

@@ -57,7 +57,7 @@ groups =
 #slack, marquez, ckan, sftp, sftpplus, otlp, default, sqlite
 
 # general purpose webhook caller
-webhook.results = from csvpath.managers.integrations.webhook.webhook_results_listener import WebhookResultListener
+webhook.results = from csvpath.managers.integrations.webhook.webhook_results_listener import WebhookResultsListener
 
 # add a listener to exec scripts at the end of named-paths group runs
 scripts.results = from csvpath.managers.integrations.scripts.scripts_results_listener import ScriptsResultsListener
@@ -91,7 +91,7 @@ otlp.errors = from csvpath.managers.integrations.otlp.otlp_error_listener import
 sftpplus.paths = from csvpath.managers.integrations.sftpplus.sftpplus_listener import SftpPlusListener
 
 # add sftp to the list of groups above to push results to an sftp account
-sftp.results = from csvpath.managers.integrations.sftp.sftp_listener import SftpListener
+sftp.results = from csvpath.managers.integrations.sftp.sftp_sender import SftpSender
 
 # add ckan to the list of groups above for alerts to slack webhooks
 ckan.results = from csvpath.managers.integrations.ckan.ckan_listener import CkanListener

{csvpath-0.0.537 → csvpath-0.0.541}/assets/config/jenkins-local-filesystem-mysql.ini

@@ -36,18 +36,19 @@ csvpaths = inputs/named_paths
 on_unmatched_file_fingerprints = halt
 
 [listeners]
-groups = default
+groups = sql
+
 sql.file = from csvpath.managers.integrations.sql.sql_file_listener import SqlFileListener
 sql.paths = from csvpath.managers.integrations.sql.sql_paths_listener import SqlPathsListener
 sql.result = from csvpath.managers.integrations.sql.sql_result_listener import SqlResultListener
 sql.results = from csvpath.managers.integrations.sql.sql_results_listener import SqlResultsListener
-sqlite.result = from csvpath.managers.integrations.sqlite.sqlite_result_listener import SqliteResultListener
-sqlite.results = from csvpath.managers.integrations.sqlite.sqlite_results_listener import SqliteResultsListener
-default.file = from csvpath.managers.files.files_listener import FilesListener
-default.paths = from csvpath.managers.paths.paths_listener import PathsListener
 otlp.result = from csvpath.managers.integrations.otlp.otlp_result_listener import OpenTelemetryResultListener
 otlp.results = from csvpath.managers.integrations.otlp.otlp_results_listener import OpenTelemetryResultsListener
 otlp.errors = from csvpath.managers.integrations.otlp.otlp_error_listener import OpenTelemetryErrorListener
+default.file = from csvpath.managers.files.files_listener import FilesListener
+default.paths = from csvpath.managers.paths.paths_listener import PathsListener
+sqlite.result = from csvpath.managers.integrations.sqlite.sqlite_result_listener import SqliteResultListener
+sqlite.results = from csvpath.managers.integrations.sqlite.sqlite_results_listener import SqliteResultsListener
 sftpplus.paths = from csvpath.managers.integrations.sftpplus.sftpplus_listener import SftpPlusListener
 sftp.results = from csvpath.managers.integrations.sftp.sftp_listener import SftpListener
 ckan.results = from csvpath.managers.integrations.ckan.ckan_listener import CkanListener
@@ -68,7 +69,7 @@ password = tinpenny
 
 [sql]
 dialect = mysql
-connection_string = mysql://csvpath:hangzhou@192.168.66.1/csvpath
+connection_string = mysql://csvpath:password1@192.168.66.1/csvpath
 
 [sqlite]
 db = archive/csvpath.db
@@ -81,7 +82,7 @@ port = SFTPPLUS_PORT
 admin_username = SFTPPLUS_ADMIN_USERNAME
 admin_password = SFTPPLUS_ADMIN_PASSWORD
 api_url = https://localhost:10020/json
-scripts_dir = 
+scripts_dir =
 execute_timeout = 300
 
 [ckan]
@@ -96,5 +97,5 @@ timeout = 5
 verify = False
 
 [slack]
-webhook_url = 
+webhook_url =
 

{csvpath-0.0.537/assets → csvpath-0.0.541}/config/config.ini

@@ -57,7 +57,7 @@ groups =
 #slack, marquez, ckan, sftp, sftpplus, otlp, default, sqlite
 
 # general purpose webhook caller
-webhook.results = from csvpath.managers.integrations.webhook.webhook_results_listener import WebhookResultListener
+webhook.results = from csvpath.managers.integrations.webhook.webhook_results_listener import WebhookResultsListener
 
 # add a listener to exec scripts at the end of named-paths group runs
 scripts.results = from csvpath.managers.integrations.scripts.scripts_results_listener import ScriptsResultsListener
@@ -91,7 +91,7 @@ otlp.errors = from csvpath.managers.integrations.otlp.otlp_error_listener import
 sftpplus.paths = from csvpath.managers.integrations.sftpplus.sftpplus_listener import SftpPlusListener
 
 # add sftp to the list of groups above to push results to an sftp account
-sftp.results = from csvpath.managers.integrations.sftp.sftp_listener import SftpListener
+sftp.results = from csvpath.managers.integrations.sftp.sftp_sender import SftpSender
 
 # add ckan to the list of groups above for alerts to slack webhooks
 ckan.results = from csvpath.managers.integrations.ckan.ckan_listener import CkanListener

{csvpath-0.0.537 → csvpath-0.0.541}/csvpath/cli/function_describer.py

@@ -31,6 +31,11 @@ class FunctionDescriber:
     def sigs(cls, function):
         sigs = []
         args = function.args
+        if not args:
+            #
+            # this is possibly due to the very small number of unrefactored functions. (3?)
+            #
+            return sigs
         argsets = args.argsets
         for ai, a in enumerate(argsets):
             pa = ""

{csvpath-0.0.537 → csvpath-0.0.541}/csvpath/csvpath.py

@@ -19,7 +19,6 @@ from .matching.matcher import Matcher
 from .scanning.scanner import Scanner
 from .util.metadata_parser import MetadataParser
 from .managers.errors.error import Error
-from .managers.errors.error_collector import ErrorCollector
 from .managers.errors.error_comms import ErrorCommunications
 from .managers.errors.error_manager import ErrorManager
 from .managers.metadata import Metadata
@@ -33,6 +32,7 @@ from .util.exceptions import (
     CsvPathsException,
 )
 from .matching.util.exceptions import MatchException
+from .managers.errors.error_collector import ErrorCollector
 
 
 class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
@@ -584,7 +584,7 @@ class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
     @property
     def explain(self) -> bool:
         """@private
-        when this property is True CsvPath dumps a match explaination
+        when this property is True CsvPath dumps a match explanation
         to INFO. this can be expensive. a 25% performance hit wouldn't
         be unexpected.
         """
@@ -672,7 +672,7 @@ class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
         #   - logic-mode: AND | OR
         #   - return-mode: matches | no-matches
         #   - print-mode: default | no-default
-        #   - validation-mode: (no-)print | log | (no-)raise | quiet | (no-)match
+        #   - validation-mode: (no-)print | log | (no-)raise | quiet | (no-)match | (no-)stop
         #   - run-mode: no-run | run
         #   - unmatched-mode: no-keep | keep
         #   - source-mode: preceding | origin
@@ -824,7 +824,15 @@ class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
         if self.csvpaths is None:
             return data
         name = self._get_name(data)
-        path = self.csvpaths.file_manager.get_named_file(name)
+        #
+        # this will blow up frequently when name is an actual path. ie name == path
+        # below. since we want file manager to be discriminating we have to catch
+        # the error and reset the name
+        #
+        try:
+            path = self.csvpaths.file_manager.get_named_file(name)
+        except ValueError:
+            path = name
         if path is None:
             return data
         if path == name:
@@ -967,7 +975,8 @@ class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
     def limit_collection_to(self, indexes: List[int]) -> None:
         """@private"""
         self._limit_collection_to = indexes
-        self.logger.warning("Setting a limit on headers collected: %s", indexes)
+        if self._limit_collection_to and self._limit_collection_to != indexes:
+            self.logger.info("Limiting headers collected: %s", indexes)
 
     def stop(self) -> None:
         """@private"""
@@ -1129,7 +1138,7 @@ class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
         #
         # DataFileReader is abstract. instantiating it results in a concrete subclass.
         # pylint doesn't like that just because it doesn't see what we're doing.
-        # otoh, is this a bad way to do it? not sure but it works fine.
+        # otoh, is this a bad way to do it? it works fine.
         #
         reader = DataFileReader(  # pylint: disable=E0110
             self.scanner.filename, delimiter=self.delimiter, quotechar=self.quotechar
@@ -1256,9 +1265,13 @@ class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
         ls = []
         for k in self.limit_collection_to:
             if k is None or k >= len(line):
-                raise InputException(
-                    f"[{self.identity}] Line {self.line_monitor.physical_line_number}: unknown header name: {k} of {self.limit_collection_to} in headers {self.headers}"
-                )
+                #
+                # FP change. didn't do do_i_raise and didn't handle w/error mgr
+                #
+                msg = f"[{self.identity}] Line {self.line_monitor.physical_line_number}: unknown header name: {k} of {self.limit_collection_to} in headers {self.headers}"
+                self.error_manager.handle_error(source=self, msg=msg)
+                if self.ecoms.do_i_raise():
+                    raise InputException(msg)
             ls.append(line[k])
         return ls
 
@@ -1313,6 +1326,12 @@ class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
         if use_cache:
             uc = self.csvpaths.config.get(section="cache", name="use_cache")
             use_cache = uc is None or uc.strip().lower() != "no"
+        #
+        # do we really want to only use cache if we're running a named-paths group?
+        # users may want to run one-offs repeatedly, particularly in FP. otoh, we
+        # don't have a file_manager and its a big change for probably a small number of
+        # bad cases and it can be worked around by simply using a CsvPaths.
+        #
         if self.csvpaths and use_cache is True:
             self.line_monitor = self.csvpaths.file_manager.lines_and_headers_cacher.get_new_line_monitor(
                 self.scanner.filename

{csvpath-0.0.537 → csvpath-0.0.541}/csvpath/managers/errors/error_comms.py

@@ -17,7 +17,7 @@ class ErrorCommunications:
     the error policy in config/config.ini (or whereever your config is)
     is the baseline. however, every csvpath can override the config
     for some of the error handling using a comment with the metadata
-    field args-validation-mode. (soon to be validation-mode). config
+    field validation-mode. config
     has two setting values not tracked in metadata: quiet and log.
     """
 
@@ -62,7 +62,15 @@ class ErrorCommunications:
     def do_i_stop(self) -> bool:
         mode = None
         if self._csvpath and self._csvpath.stop_on_validation_errors is not None:
+            #
+            # looks wrong. function in FlightPath is not correct. doesn't match other do_i_...
+            #
             mode = self._csvpath.stop_on_validation_errors
+            #
+            # this was a change made for flightpath. not sure it works. it may well work.
+            # however atm visibility isn't great. come back later.
+            #
+            # return self._csvpath.stop_on_validation_errors
         policy = self.in_policy(OnError.STOP.value)
         return mode is True or policy is True
 

{csvpath-0.0.537 → csvpath-0.0.541}/csvpath/managers/files/file_manager.py

@@ -295,6 +295,17 @@ class FileManager:
     # this feels like the better sig.
     #
     def has_named_file(self, name: NamedFileName) -> bool:
+        #
+        # cannot be a reference or part of a reference. has to just
+        # be the simple name of the named file.
+        #
+        try:
+            self.legal_name(name)
+        except Exception:
+            return False
+        #
+        # the home should exist if the named file exists.
+        #
         p = self.named_file_home(name)
         nos = self.nos
         nos.path = p
@@ -310,11 +321,19 @@ class FileManager:
 
     def remove_named_file(self, name: NamedFileName) -> None:
         """@private"""
+        #
+        # cannot delete any specific files. this is for the named_file
+        # as a whole. the named_file is immutable, so this is all-or-nothing
+        # and we expect it to happen rarely in a production env.
+        #
+        self.legal_name(name)
         p = os.path.join(self.named_files_dir, name)
         nos = self.nos
         nos.path = p
         if nos.exists():
             nos.remove()
+            return True
+        return False
 
     def remove_all_named_files(self) -> None:
         """@private"""
@@ -351,6 +370,22 @@ class FileManager:
             if self.csvpaths.ecoms.do_i_raise():
                 raise
 
+    def legal_name(self, name: str) -> None:
+        if name is None:
+            raise ValueError("Name cannot be None")
+        if name.strip() == "":
+            raise ValueError("Name cannot be empty")
+        if name.find("/") > -1 or name.find("\\") > -1:
+            raise ValueError(
+                f"Not a legal name: {name}. Path seperators are not allowed."
+            )
+        if name.find(".") > -1:
+            raise ValueError(f"Not a legal name: {name}. Periods are not allowed.")
+        if name.find("$") > -1:
+            raise ValueError(f"Not a legal name: {name}. Dollarsigns are not allowed.")
+        if name.find("#") > -1:
+            raise ValueError(f"Not a legal name: {name}. Hashmarks are not allowed.")
+
     #
     # if name is provided all files selected will be registered under the same name in
     # the order they are found; which is likely not deterministic. template, if any, is
@@ -364,8 +399,12 @@ class FileManager:
     def add_named_files_from_dir(
         self, dirname: str, *, name=None, template: str = None, recurse=True
     ):
-        if dirname is None or dirname.strip() == "":
-            raise ValueError("Dirname cannot be None or empty")
+        #
+        # legal_name handled at add_named_file
+        #
+        # self.legal_name(name)
+        # if dirname is None or dirname.strip() == "":
+        #    raise ValueError("Dirname cannot be None or empty")
         #
         # need to support adding all files from directory under the same name. preferably
         # in order of file created time, if possible.
@@ -405,8 +444,9 @@ class FileManager:
     def add_named_file(
         self, *, name: NamedFileName, path: str, template: str = None
     ) -> None:
-        if name is None or name.strip() == "":
-            raise ValueError("Name cannot be None or empty")
+        self.legal_name(name)
+        # if name is None or name.strip() == "":
+        #    raise ValueError("Name cannot be None or empty")
         if path is None or path.strip() == "":
             raise ValueError("Path cannot be None or empty")
         if template is not None:

{csvpath-0.0.537 → csvpath-0.0.541}/csvpath/managers/integrations/ol/file_listener_ol.py

@@ -1,4 +1,4 @@
-from .ol_listener import OpenLineageListener
+from csvpath.managers.integrations.ol.ol_listener import OpenLineageListener
 
 
 class OpenLineageFileListener(OpenLineageListener):

{csvpath-0.0.537 → csvpath-0.0.541}/csvpath/managers/integrations/ol/paths_listener_ol.py

@@ -1,7 +1,7 @@
 from openlineage.client import OpenLineageClient
 
-from ..metadata import Metadata
-from .ol_listener import OpenLineageListener
+from csvpath.managers.metadata import Metadata
+from csvpath.managers.integrations.ol.ol_listener import OpenLineageListener
 
 
 class OpenLineagePathsListener(OpenLineageListener):

{csvpath-0.0.537 → csvpath-0.0.541}/csvpath/managers/integrations/ol/result_listener_ol.py

@@ -1,4 +1,4 @@
-from ..ol_listener import OpenLineageListener
+from csvpath.managers.integrations.ol.ol_listener import OpenLineageListener
 
 
 class OpenLineageResultListener(OpenLineageListener):

{csvpath-0.0.537 → csvpath-0.0.541}/csvpath/managers/integrations/ol/results_listener_ol.py

@@ -1,4 +1,4 @@
-from .ol_listener import OpenLineageListener
+from csvpath.managers.integrations.ol.ol_listener import OpenLineageListener
 
 
 class OpenLineageResultsListener(OpenLineageListener):

{csvpath-0.0.537 → csvpath-0.0.541}/csvpath/managers/integrations/ol/run_listener_ol.py

@@ -1,4 +1,4 @@
-from .ol_listener import OpenLineageListener
+from csvpath.managers.integrations.ol.ol_listener import OpenLineageListener
 
 
 class OpenLineageResultsListener(OpenLineageListener):

{csvpath-0.0.537 → csvpath-0.0.541}/csvpath/managers/paths/paths_manager.py

@@ -197,12 +197,20 @@ class PathsManager:
                 raise InputException(msg)
 
     def add_named_paths_from_file(
-        self, *, name: NamedPathsName, file_path: str, template=None
+        self,
+        *,
+        name: NamedPathsName,
+        file_path: str,
+        template=None,
+        append: bool = False,
     ) -> None:
+        #
+        # change for FP: added append as a pass-through
+        #
         self.csvpaths.logger.debug("Reading csvpaths file at %s", file_path)
         _ = self._get_csvpaths_from_file(file_path)
         self.add_named_paths(
-            name=name, paths=_, source_path=file_path, template=template
+            name=name, paths=_, source_path=file_path, template=template, append=append
         )
 
     def add_named_paths_from_json(self, file_path: str) -> None:
@@ -248,12 +256,16 @@ class PathsManager:
         from_json: str = None,
         source_path: str = None,
         template: str = None,
+        append: bool = False,
     ) -> None:
         if template is not None:
             temu.valid(template)
         if from_file is not None:
+            #
+            # change for FP. added append as a pass-through
+            #
             return self.add_named_paths_from_file(
-                name=name, file_path=from_file, template=template
+                name=name, file_path=from_file, template=template, append=append
             )
         elif from_dir is not None:
             return self.add_named_paths_from_dir(
@@ -273,7 +285,7 @@ class PathsManager:
         for _ in paths:
             self.csvpaths.logger.debug("Adding %s to %s", _, name)
         s = self._str_from_list(paths)
-        t = self._copy_in(name, s)
+        t = self._copy_in(name, s, append=append)
         grp_paths = self.get_identified_paths_in(name, paths=paths)
         ids = [t[0] for t in grp_paths]
         for i, t in enumerate(ids):
@@ -630,9 +642,15 @@ class PathsManager:
         cs = s.split("---- CSVPATH ----")
         cs = [s for s in cs if s.strip() != ""]
         #
-        # this update may not happen. it depends on if the group.csvpaths file has changed.
-        # if someone put a new group.csvpaths file by hand we want to capture its fingerprint
-        # for future reference. this shouldn't happen, but it probably will happen.
+        # this update may not result in broadcasting an update event to listeners.
+        # that all depends on if group.csvpaths was changed outside the manager.
+        # if someone put a new group.csvpaths file by hand we want to capture its
+        # fingerprint for future reference. this shouldn't happen, but it probably
+        # will happen.
+        #
+        # seen 1x with FlightPath. not yet clear if FP is doing something new and
+        # desireable or if it has a bug. in principle, still don't see a reason this
+        # update should obtain, unless a user edits a file they shouldn't.
         #
         self.registrar.update_manifest_if(name=name, group_file_path=grp, paths=cs)
         return cs
@@ -644,7 +662,21 @@ class PathsManager:
             f = f"{f}\n\n---- CSVPATH ----\n\n{_}"
         return f
 
-    def _copy_in(self, name: NamedPathsName, csvpathstr: Csvpath) -> None:
+    def _copy_in(self, name: NamedPathsName, csvpathstr: Csvpath, append=False) -> None:
+        #
+        # if we have a set of paths we append these new paths
+        #
+        if self.has_named_paths(name) and append is True:
+            existing = self.get_named_paths(name)
+            estr = self._str_from_list(existing)
+            if not estr.strip().endswith(
+                "---- CSVPATH ----"
+            ) and not csvpathstr.strip().startswith("---- CSVPATH ----"):
+                csvpathstr = f"{estr}\n\n---- CSVPATH ----\n\n"
+            csvpathstr = f"{estr}{csvpathstr}"
+        #
+        # continue with the write
+        #
         temp = self._group_file_path(name)
         with DataFileWriter(path=temp, mode="w") as writer:
             writer.append(csvpathstr)
@@ -691,6 +723,32 @@ class PathsManager:
             ps.append(path[0])
         return ps
 
+    #
+    # this version correctly picks up csvpaths that the Framework identifies by index
+    # because they don't have user assigned identities. this was seen in FlightPath
+    # but applies in general.
+    #
+    def _get_from(self, npn: NamedPathsName, identity: Identity) -> list[Csvpath]:
+        index = expu.to_int(identity)
+        ps = []
+        paths = self.get_identified_paths_in(npn)
+        for i, path in enumerate(paths):
+            #
+            # if we have the identity or the identity is the index or we are collecting
+            # after our first collection because :from collects all csvpaths coming after
+            # a starting csvpath
+            #
+            if (path[0] == identity or (path[0] is None and index == i)) or len(ps) > 0:
+                ps.append(path[1])
+            """
+            if path[0] != identity and len(ps) == 0:
+                continue
+            ps.append(path[1])
+            """
+        return ps
+
+    """
+    original version
     def _get_from(self, npn: NamedPathsName, identity: Identity) -> list[Csvpath]:
         ps = []
         paths = self.get_identified_paths_in(npn)
@@ -699,6 +757,7 @@ class PathsManager:
                 continue
             ps.append(path[1])
         return ps
+    """
 
     def get_preceeding_instance_identity(self, name, index: int) -> str:
         if index <= 0:
@@ -735,6 +794,7 @@ class PathsManager:
             idps.append((c.identity, path))
         return idps
 
+    """
     def _find_one(self, npn: NamedPathsName, identity: Identity) -> Csvpath:
         if npn is not None:
             paths = self.get_identified_paths_in(npn)
@@ -744,6 +804,29 @@ class PathsManager:
         raise InputException(
             f"Path identified as '{identity}' must be in the group identitied as '{npn}'"
         )
+    """
+
+    def _find_one(self, npn: NamedPathsName, identity: Identity) -> Csvpath:
+        #
+        # this version of the method is a change for FlightPath. the change is correct for
+        # all purposes, but showed up clearly in FP. the problem is that we don't test index
+        # for unidentitied paths. luckily we do return all paths with None as the identity
+        # for those w/o user specified id. for those we just need to test the index.
+        #
+        # this version assumes that if a csvpath is identitied we do not allow using the
+        # index to point to it. this may not be the right assumption, but it is arguably a
+        # good way to go. if the writer bothered to identify, the id is more specific than
+        # the index and so less error-prone.
+        #
+        index = expu.to_int(identity)
+        if npn is not None:
+            paths = self.get_identified_paths_in(npn)
+            for i, path in enumerate(paths):
+                if path[0] == identity or (path[0] is None and index == i):
+                    return path[1]
+        raise InputException(
+            f"Path identified as '{identity}' must be in the group identitied as '{npn}'"
+        )
 
     def _name_from_name_part(self, name):
         i = name.rfind(".")

{csvpath-0.0.537 → csvpath-0.0.541}/csvpath/managers/paths/paths_registrar.py

@@ -35,7 +35,7 @@ class PathsRegistrar(Registrar, Listener):
         mdata.fingerprint = self._fingerprint(name=mdata.named_paths_name)
         self.distribute_update(mdata)
 
-    def update_manifest_if(self, *, group_file_path, name, paths=None):
+    def update_manifest_if(self, *, group_file_path, name, paths):
         #
         # if we find that the current group file does not have the same
         # fingerprint as the most recent on file, we register a new version.
@@ -47,20 +47,19 @@ class PathsRegistrar(Registrar, Listener):
         mpath = self.manifest_path(name)
         cf = self._most_recent_fingerprint(mpath)
         if f != cf:
-            mdata = PathsMetadata()
+            mdata = PathsMetadata(self.csvpaths.config)
             mdata.archive_name = self.csvpaths.config.archive_name
             mdata.named_paths_name = name
-            #
-            # why two of these? :/
-            #
-            # mdata.named_paths_file = group_file_path
             mdata.group_file_path = group_file_path
             mdata.named_paths = paths
             mdata.named_paths_identities = [
-                t[0] for t in self.manager.get_identified_paths_in(name)
+                #
+                # if we don't pass paths to get_identified_paths_in we will infinite loop
+                #
+                t[0]
+                for t in self.manager.get_identified_paths_in(name, paths)
             ]
-            if paths:
-                mdata.named_paths_count = len(paths)
+            mdata.named_paths_count = len(paths)
             mdata.manifest_path = mpath
             mdata.fingerprint = f
             self.distribute_update(mdata)

{csvpath-0.0.537 → csvpath-0.0.541}/csvpath/managers/results/readers/readers.py

@@ -63,7 +63,7 @@ class PrintoutsReader(ResultReader):
 
 class ResultReadersFacade(ErrorsReader, UnmatchedReader, LinesReader, PrintoutsReader):
     #
-    # as soonn as we get a run_dir and an instance we'll create the
+    # as soon as we get a run_dir and an instance we'll create the
     # instance_dir and load the readers
     #
     def __init__(self, result):

{csvpath-0.0.537 → csvpath-0.0.541}/csvpath/managers/results/results_manager.py

@@ -309,7 +309,12 @@ class ResultsManager:  # pylint: disable=C0115
         path = self._csvpaths.config.archive_path
         if Nos(path).dir_exists():
             names = Nos(path).listdir()
-            names = [n for n in names if not n.startswith(".")]
+            #
+            # listing dir shouldn't return manifest.json or any file. can do better here.
+            #
+            names = [
+                n for n in names if not n.startswith(".") and not n.endswith(".json")
+            ]
             names.sort()
         else:
             self._csvpaths.logger.warning(
@@ -513,7 +518,14 @@ class ResultsManager:  # pylint: disable=C0115
         # CsvPaths instance, we would create a new one.
         #
         if name in self.named_results:
-            return self.named_results[name]
+            #
+            # exp. removed 4 May.
+            # was/is this ever a good idea?
+            #
+            # seems to be important for a handful of unit tests.
+            #
+            rs = self.named_results[name]
+            return rs
         #
         # find and load the result, if exists. we find results home with the name. run_home is the
         # last run dir. the results we're looking for are the instance dirs in the run dir.
@@ -573,7 +585,9 @@ class ResultsManager:  # pylint: disable=C0115
             nos = Nos(path)
             exists = nos.dir_exists()
             nonphy = nos.physical_dirs()
+            #
             # is not nonphy needed?
+            #
             if exists or not nonphy:
                 runs = nos.listdir()
                 if len(runs) > 0:

{csvpath-0.0.537 → csvpath-0.0.541}/csvpath/matching/functions/args_helper.py

@@ -26,7 +26,7 @@ class ArgumentValidationHelper:
         # Validate each argument against the argset.
         # this part gets ugly. we should instead just say:
         #   f"invalid {argset.matchable.name}. {argset.explain}"
-        # and then write the explaination into the classes. (OMG! but it would be best, imho).
+        # and then write the explanation into the classes. (OMG! but it would be best, imho).
         #
         results = []
         for va in valid_argsets:

{csvpath-0.0.537 → csvpath-0.0.541}/csvpath/matching/functions/boolean/all.py

@@ -16,11 +16,26 @@ class All(MatchDecider):
     """checks that a number of match components return True"""
 
     def check_valid(self) -> None:  # pragma: no cover
+        self.description = [
+            self._cap_name(),
+            self.wrap(
+                """\
+            Tests if all contained or referenced match components evaluate to True.
+            If all() has no arguments the check is if all headers have values.
+        """
+            ),
+        ]
         self.args = Args(matchable=self)
-        self.args.argset(0)
-        self.args.argset(1).arg(types=[None, Variables, Headers], actuals=[])
+        self.args.argset(0)  # what would the function of all() w/o args be?
+        self.args.argset(1).arg(
+            name="a function indicating all headers or all variables",
+            types=[None, Variables, Headers],
+            actuals=[],
+        )
         self.args.argset().arg(
-            types=[None, Function, Variable, Header], actuals=[None, Any]
+            name="one of a set of match components",
+            types=[None, Function, Variable, Header],
+            actuals=[None, Any],
         )
         self.args.validate(self.siblings())
         super().check_valid()