csvpath 0.0.620__tar.gz → 0.0.621__tar.gz

{csvpath-0.0.620 → csvpath-0.0.621}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: csvpath
-Version: 0.0.620
+Version: 0.0.621
 Summary: CsvPath Framework is a data preboarding automation library for receiving, validating, and tracking CSV, Excel, JSONL and other tabular data files before they can corrupt downstream data consumers.
 License-File: LICENSE
 Author: David Kershaw
@@ -33,6 +33,7 @@ Requires-Dist: jinja2 (==3.1.6)
 Requires-Dist: jsonlines (==4.0.0)
 Requires-Dist: jsonpath-ng (==1.8.0)
 Requires-Dist: jsonpickle (==4.1.1)
+Requires-Dist: jsonschema (>=4.26.0,<5.0.0)
 Requires-Dist: lark (==1.3.1)
 Requires-Dist: lxml (==6.0.2)
 Requires-Dist: marquez-python (==0.50.0)
@@ -59,6 +60,7 @@ Requires-Dist: smart-open[all] (==7.5.1)
 Requires-Dist: sqlalchemy (==2.0.48)
 Requires-Dist: tabulate (==0.9.0)
 Requires-Dist: validators (==0.34.0)
+Requires-Dist: xmlschema (>=4.3.2,<5.0.0)
 Project-URL: Csvpath.org, https://www.csvpath.org
 Project-URL: Github, https://github.com/csvpath/csvpath.git
 Description-Content-Type: text/markdown

{csvpath-0.0.620 → csvpath-0.0.621}/assets/config/jenkins-local-gcs.ini

@@ -31,7 +31,7 @@ path = cache
 use_cache = no
 
 [functions]
-imports =
+imports = 
 
 [results]
 archive = tmp/gcs/archive
@@ -91,7 +91,7 @@ port = SFTPPLUS_PORT
 admin_username = SFTPPLUS_ADMIN_USERNAME
 admin_password = SFTPPLUS_ADMIN_PASSWORD
 api_url = https://. . . :10020/json
-scripts_dir =
+scripts_dir = 
 execute_timeout = 300
 
 [ckan]
@@ -106,5 +106,5 @@ timeout = 5
 verify = False
 
 [slack]
-webhook_url =
+webhook_url = 
 

{csvpath-0.0.620 → csvpath-0.0.621}/assets/config/jenkins-local-s3.ini

@@ -31,7 +31,7 @@ path = cache
 use_cache = no
 
 [functions]
-imports =
+imports = 
 
 [results]
 archive = tmp/s3/archive
@@ -91,7 +91,7 @@ port = SFTPPLUS_PORT
 admin_username = SFTPPLUS_ADMIN_USERNAME
 admin_password = SFTPPLUS_ADMIN_PASSWORD
 api_url = https://. . . :10020/json
-scripts_dir =
+scripts_dir = 
 execute_timeout = 300
 
 [ckan]
@@ -106,5 +106,5 @@ timeout = 5
 verify = False
 
 [slack]
-webhook_url =
+webhook_url = 
 

{csvpath-0.0.620 → csvpath-0.0.621}/csvpath/csvpath.py

@@ -1163,6 +1163,13 @@ class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
             lines = []
             self.lines = ListLineSpooler(lines=lines)
         for _ in self.next():
+            if _ is None:
+                raise ValueError("Next line cannot be None")
+            #
+            # the line can be a non list if it is not CSV data
+            #
+            elif not isinstance(_, (list, tuple)):
+                _ = [_]
             _ = _[:]
             self.lines.append(_)
             if nexts == -1:
@@ -1311,22 +1318,26 @@ class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
             raise FileException("There is no filename")
         #
         # DataFileReader is abstract. instantiating it results in a concrete subclass.
-        # pylint doesn't like that just because it doesn't see what we're doing.
-        # otoh, is this a bad way to do it? it works fine.
+        # ideally we use it as a context mgr.
         #
+        """
         reader = DataFileReader(  # pylint: disable=E0110
             self.scanner.filename, delimiter=self.delimiter, quotechar=self.quotechar
         )
-        for line in reader.next():
-            self.track_line(line=line)
-            #
-            # some formats embed headers in each line. JSONL headers are the dict keys in
-            # each line that uses a dict
-            #
-            if reader.updates_headers:
-                _ = reader.current_headers
-                self.headers = self.headers if _ is None else _
-            yield line
+        """
+        with DataFileReader(
+            self.scanner.filename, delimiter=self.delimiter, quotechar=self.quotechar
+        ) as reader:
+            for line in reader.next():
+                self.track_line(line=line)
+                #
+                # some formats embed headers in each line. JSONL headers are the dict keys in
+                # each line that uses a dict
+                #
+                if reader.updates_headers:
+                    _ = reader.current_headers
+                    self.headers = self.headers if _ is None else _
+                yield line
         self.finalize()
 
     def finalize(self) -> None:

{csvpath-0.0.620 → csvpath-0.0.621}/csvpath/csvpaths.py

@@ -867,6 +867,7 @@ Cache: {cache}
             file=file,
             run_uuid=run_uuid,
             method="collect_paths",
+            template=template,
             extra_data=extra_data,
         )
         #
@@ -1038,6 +1039,7 @@ Cache: {cache}
             filename=filename,
             run_uuid=run_uuid,
             method="fast_forward_paths",
+            template=template,
             extra_data=extra_data,
         )
         #
@@ -1157,6 +1159,7 @@ Cache: {cache}
             filename=filename,
             run_uuid=run_uuid,
             method="next_paths",
+            template=template,
             extra_data=extra_data,
         )
         #
@@ -1652,6 +1655,7 @@ Cache: {cache}
             filename=filename,
             run_uuid=run_uuid,
             method="next_paths",
+            template=template,
             extra_data=extra_data,
         )
         #

{csvpath-0.0.620 → csvpath-0.0.621}/csvpath/managers/files/file_manager.py

@@ -199,6 +199,31 @@ class FileManager:
                 return _["reference"]
         return None
 
+    @property
+    def files_root_manifest_path(self) -> str:
+        """@private"""
+        r = self.csvpaths.config.get(section="inputs", name="files")
+        p = Nos(r).join("manifest.json")
+        nos = Nos(r)
+        if not nos.dir_exists():
+            nos.makedirs()
+        nos.path = p
+        if not nos.exists():
+            with DataFileWriter(path=p) as file:
+                file.write("[]")
+        return p
+
+    @property
+    def files_root_manifest(self) -> list:
+        """the Named-File Arrivals Manifest -- a single global ledger
+        tracking every named-file arrival across every name, distinct
+        from get_manifest()'s per-name manifest. mirrors PathsManager's
+        paths_root_manifest. used by FilesReferenceFinder3 to resolve
+        "*" root_major references (bare :manifest(), optionally with a
+        pointer to select one entry by ordinal)."""
+        with DataFileReader(self.files_root_manifest_path) as reader:
+            return json.load(reader.source)
+
     def get_manifest(self, name: NamedFileName) -> json:
         if name is None:
             raise ValueError("Paths name cannot be None")
@@ -519,6 +544,7 @@ class FileManager:
         recurse=True,
         regex: str = None,
     ) -> list[str]:
+        self.csvpaths.logger.info("Starting add_named_files_from_dir for: %s", dirname)
         ret = []
         #
         # legal_name check handled at add_named_file
@@ -542,6 +568,9 @@ class FileManager:
             if regex is not None:
                 m = re.search(regex, p)
                 if not m:
+                    self.csvpaths.logger.info(
+                        "Skipping file: %s doesn't match %s", p, regex
+                    )
                     continue
             #
             #
@@ -560,6 +589,7 @@ class FileManager:
                 # this should be consistent across backends, if ever not, fix it there, not here.
                 #
                 ref = self.add_named_file(name=n, path=p, template=template)
+                self.csvpaths.logger.info("add_named_files_from_dir added: %s", ref)
                 if ref is not None:
                     ret.append(ref)
             else:
@@ -709,7 +739,6 @@ class FileManager:
         http = str(http).strip().lower() in ["on", "yes", "true"]
         local = config.get(section="inputs", name="allow_local_files", default=False)
         local = str(local).strip().lower() in ["on", "yes", "true"]
-        nos = Nos(path)
         self.csvpaths.logger.info(
             "Adding named file %s: checking permission, if needed, to register local and http files",
             name,
@@ -719,6 +748,7 @@ class FileManager:
         # no. the project may be controlled by a context like FlightPath Server, where
         # the user cannot just override certain settings.
         #
+        nos = Nos(path)
         if nos.is_http and http is not True:
             _ = {
                 "name": name,
@@ -780,12 +810,10 @@ class FileManager:
         #
         config = self.describer.get_config(name)
         servers = config.sources
+        self.csvpaths.logger.debug(
+            "Adding named file. Available server configs: %s", servers
+        )
         nos.server_config = servers
-        #
-        #  for _ in servers:
-        #     if _.matches(nos.path):
-        #       nos.do._config = SftpConfig(_)
-        #
         isfile = nos.is_http or nos.isfile()
         self.csvpaths.logger.debug(
             "Adding named file %s: path %s is a file? %s", name, path, isfile
@@ -809,6 +837,7 @@ class FileManager:
             # create folder tree in inputs/named_files/name/filename
             #
             home = self.assure_file_home(name, path, template)
+            self.csvpaths.logger.debug("File home is %s", home)
             file_home = home
             mark = None
             #
@@ -824,6 +853,7 @@ class FileManager:
             if pm > -1:
                 mark = path[pm + 1 :]
                 path = path[0:pm]
+            self.csvpaths.logger.debug("File path, mark: %s, %s", path, mark)
             #
             # we're registering with a mark, so we need to be Xlsx and have a tab with that name.
             #
@@ -976,6 +1006,9 @@ class FileManager:
         copy = pathu.parts(path)[0] == pathu.parts(home)[0]
         copy = copy and nos.is_local
         if copy:
+            self.csvpaths.logger.debug(
+                "Above nos copy: path: %s, name: %s, temp: %s", path, name, temp
+            )
             nos.path = path
             config = self.describer.get_config(name)
             servers = config.sources
@@ -1012,6 +1045,7 @@ class FileManager:
         #
         nos = Nos(path)
         if nos.is_sftp:
+            self.csvpaths.logger.debug("Doing copy_down with sftp path %s", path)
             reader = DataFileReader(path)
             #
             # adding config to the reader allows the reader to look to the config
@@ -1028,6 +1062,7 @@ class FileManager:
             with DataFileWriter(path=temp, mode=mode) as writer:
                 writer.write(lines)
         else:
+            self.csvpaths.logger.debug("Doing copy_down with non-sftp path %s", path)
             with DataFileReader(path) as reader:
                 with DataFileWriter(path=temp, mode=mode) as writer:
                     for line in reader.next_raw():

{csvpath-0.0.620 → csvpath-0.0.621}/csvpath/managers/files/file_registrar.py

@@ -131,6 +131,10 @@ class FileRegistrar(Registrar, Listener):
         if mark is not None:
             mani["mark"] = mark
         mani["template"] = mdata.template or ""
+        mani["manifest_path"] = manifest_path
+        mani["file_name"] = mdata.file_name
+        mani["username"] = mdata.username
+        mani["hostname"] = mdata.hostname
         jdata = self.get_manifest(manifest_path)
         jdata.append(mani)
         self.intermediary.put_json(manifest_path, jdata)

{csvpath-0.0.620 → csvpath-0.0.621}/csvpath/managers/integrations/otlp/otlp_results_listener.py

@@ -60,4 +60,6 @@ class OpenTelemetryResultsListener(OtlpListener):
             cmeta["time_completed"] = mdata.time_completed_string
         if mdata.named_paths_uuid_string:
             cmeta["named_paths_uuid_string"] = mdata.named_paths_uuid_string
+        if mdata.named_paths_fingerprint:
+            cmeta["named_paths_fingerprint"] = mdata.named_paths_fingerprint
         return cmeta

csvpath-0.0.621/csvpath/managers/paths/paths_asset_manager.py

@@ -0,0 +1,78 @@
+from typing import NewType
+
+
+from csvpath.util.nos import Nos
+from csvpath.util.file_writers import DataFileWriter
+from csvpath.util.file_readers import DataFileReader
+from csvpath.util.references.reference_parser import ReferenceParser
+
+NamedPathsName = NewType("NamedPathsName", str)
+
+#
+# TODO: scripts should be handled as assets, not config. move here
+# from the describer.
+#
+
+
+class PathsAssetManager:
+    ASSETS = "assets"
+
+    def __init__(self, paths_manager) -> None:
+        self.paths_manager = paths_manager
+
+    def _name_for_name(self, name: NamedPathsName) -> str:
+        if name.startswith("$"):
+            ref = ReferenceParser(name, csvpaths=self.paths_manager.csvpaths)
+            return ref.root_major
+        return name
+
+    def assets_dir_path(self, name: NamedPathsName) -> str:
+        name = self._name_for_name(name)
+        if not self.paths_manager.has_named_paths(name):
+            raise ValueError(f"No such named-paths group: {name}")
+        home = self.paths_manager.named_paths_home(name)
+        return Nos(home).join(self.ASSETS)
+
+    def assure_assets_dir(self, name: NamedPathsName) -> None:
+        p = self.assets_dir_path
+        nos = Nos(p)
+        if not nos.direxists():
+            Nos(p).makedirs()
+
+    def make_asset_file_path(self, name: NamedPathsName, filename: str) -> str:
+        path = self.assets_dir_path(name)
+        if not Nos(path).dir_exists():
+            Nos(path).makedirs()
+        path = Nos(path).join(filename)
+        return path
+
+    def store_asset_file(
+        self,
+        *,
+        name: NamedPathsName,
+        filename: str,
+        contents: str,
+        mode: str = "w",
+        overwrite: bool = True,
+    ) -> None:
+        path = self.make_asset_file_path(name, filename)
+        if overwrite is False and Nos(path).exists():
+            raise ValueError(f"Cannot overwrite {path}")
+        with DataFileWriter(path=path, mode=mode) as writer:
+            writer.write(contents)
+
+    def get_asset_file(self, name: NamedPathsName, filename: str) -> None:
+        path = self.make_asset_file_path(name, filename)
+        if not Nos(path).exists():
+            return None
+        with DataFileReader(path=path) as reader:
+            return reader.read()
+
+    def delete_asset_file(self, name: NamedPathsName, filename: str) -> None:
+        path = self.make_asset_file_path(name, filename)
+        if Nos(path).exists():
+            return Nos(path).remove()
+
+    def list_asset_files(self, name: NamedPathsName) -> None:
+        path = self.assets_dir_path(name)
+        return Nos(path).listdir(files_only=True)

{csvpath-0.0.620 → csvpath-0.0.621}/csvpath/managers/paths/paths_manager.py

@@ -13,6 +13,7 @@ from csvpath.util.nos import Nos
 from .paths_registrar import PathsRegistrar
 from .paths_metadata import PathsMetadata
 from .paths_describer import NamedPathsDescriber
+from .paths_asset_manager import PathsAssetManager
 from csvpath.util.template_util import TemplateUtility as temu
 from csvpath.matching.util.expression_utility import ExpressionUtility as expu
 
@@ -51,6 +52,17 @@ class PathsManager:
     def describer(self) -> NamedPathsDescriber:
         return NamedPathsDescriber(self)
 
+    @property
+    def asset_manager(self) -> PathsAssetManager:
+        return PathsAssetManager(self)
+
+    @property
+    def registrar(self) -> PathsRegistrar:
+        """@private"""
+        if self._registrar is None:
+            self._registrar = PathsRegistrar(self.csvpaths)
+        return self._registrar
+
     @property
     def last_add_metadata(self) -> PathsMetadata:
         #
@@ -115,18 +127,9 @@ class PathsManager:
         # home = os.path.join(home, "manifest.json")
         return self.registrar.get_manifest(home)
 
-    @property
-    def registrar(self) -> PathsRegistrar:
-        """@private"""
-        if self._registrar is None:
-            self._registrar = PathsRegistrar(self.csvpaths)
-        return self._registrar
-
     def named_paths_home(self, name: NamedPathsName) -> str:
         """@private"""
         home = Nos(self.named_paths_dir).join(name)
-        # home = os.path.join(self.named_paths_dir, name)
-        # nos = self.nos
         nos = Nos(home)
         b = nos.dir_exists()
         if not b:
@@ -148,7 +151,6 @@ class PathsManager:
 
         path = self.named_paths_home(name)
         path = Nos(path).join("manifest.json")
-        # nos = self.nos
         nos = Nos(path)
         if nos.exists():
             with DataFileReader(path) as reader:
@@ -156,6 +158,39 @@ class PathsManager:
                 return m[len(m) - 1]["uuid"]
         raise ValueError(f"No manifest for path named {name}")
 
+    def get_fingerprint_for_name(self, name: NamedPathsName) -> str:
+        """@private -- the most recently registered version's own
+        content fingerprint for named-paths group `name` (the group's
+        own group.csvpaths text, not any run's copy of it) -- mirrors
+        get_named_paths_uuid's own name/reference resolution exactly,
+        just returning "fingerprint" instead of "uuid" off the same last
+        manifest entry. Used by ResultsRegistrar to record which
+        named-paths CONTENT drove a run (Results Run Manifest's own
+        named_paths_fingerprint field), distinct from named_paths_uuid,
+        which only records the group version's registration identity --
+        two groups loaded from identical group.csvpaths text under
+        different names get different uuids but identical fingerprints."""
+        if name is None:
+            raise ValueError("Paths name cannot be None")
+        if name.find("#") > -1:
+            name = name[0 : name.find("#")]
+        if name.startswith("$"):
+            ref = ReferenceParser(name, csvpaths=self.csvpaths)
+            if ref.datatype != ReferenceParser.CSVPATHS:
+                raise ValueError(
+                    "Must be a reference of type {ReferenceParser.CSVPATHS}"
+                )
+            name = ref.root_major
+
+        path = self.named_paths_home(name)
+        path = Nos(path).join("manifest.json")
+        nos = Nos(path)
+        if nos.exists():
+            with DataFileReader(path) as reader:
+                m = json.load(reader.source)
+                return m[len(m) - 1]["fingerprint"]
+        raise ValueError(f"No manifest for path named {name}")
+
     @property
     def named_paths_dir(self) -> str:
         """@private"""
@@ -592,6 +627,10 @@ class PathsManager:
     ####################################################
 
     def store_json_paths_file(self, name: NamedPathsName, jsonpath: str) -> None:
+        #
+        # this method stores the definition.json file. it is not general purpose
+        # for any other json files
+        #
         if name is None:
             raise ValueError("Name cannot be None")
         if name.startswith("$"):
@@ -702,7 +741,7 @@ class PathsManager:
         return self.total_named_paths()
 
     #
-    # ================== internals =====================
+    # ==============================
     #
 
     def _get_named_paths(self, name: NamedPathsName) -> list[Csvpath]:
@@ -764,9 +803,30 @@ class PathsManager:
         return temp
 
     def _group_file_path(self, name: NamedPathsName) -> str:
+        #
+        # TODO: note that in principle name could be a reference. should we allow
+        # that? We would just check for '$' and use ReferenceParser.
+        #
         temp = Nos(self.named_paths_home(name)).join("group.csvpaths")
         return temp
 
+    def group_file_path(self, name: NamedPathsName) -> str:
+        #
+        # supports references as well as names
+        #
+        if name is None:
+            raise ValueError("Name cannot be None")
+        name = str(name).strip()
+        if len(name) == 0:
+            raise ValueError("Name cannot be length 0")
+        if name[0] == "$":
+            name = ReferenceParser(name, csvpaths=self.csvpaths).root_major
+        #
+        # TODO: future state references v3 can return * or a function. that might
+        # not be supported (quite reasonably) but it would be a legal root_major.
+        #
+        return self._group_file_path(name)
+
     def _get_csvpaths_from_file(self, file_path: str) -> list[Csvpath]:
         if self.can_load(file_path) is not True:
             return []

{csvpath-0.0.620 → csvpath-0.0.621}/csvpath/managers/results/result_registrar.py

@@ -148,6 +148,8 @@ class ResultRegistrar(Registrar, Listener):
         m["number_of_files_generated"] = mdata.number_of_files_generated
         m["valid"] = mdata.valid if mdata.valid is not None else ""
         m["completed"] = mdata.completed
+        m["error_count"] = mdata.error_count
+        m["method"] = mdata.method
         m["source_mode_preceding"] = mdata.source_mode_preceding
         if mdata.source_mode_preceding:
             m["preceding_instance_identity"] = mdata.preceding_instance_identity

{csvpath-0.0.620 → csvpath-0.0.621}/csvpath/managers/results/results_manager.py

@@ -1,4 +1,5 @@
 # pylint: disable=C0114
+import json
 import os
 import re
 from uuid import UUID
@@ -10,6 +11,7 @@ from csvpath.util.references.reference_parser import ReferenceParser
 from csvpath.util.references.results_reference_finder_2 import (
     ResultsReferenceFinder2 as ResultsReferenceFinder,
 )
+from csvpath.util.file_readers import DataFileReader
 from csvpath.util.file_writers import DataFileWriter
 from csvpath.util.nos import Nos
 
@@ -61,6 +63,32 @@ class ResultsManager:  # pylint: disable=C0115
         """@private"""
         self._csvpaths = cs
 
+    @property
+    def results_root_manifest_path(self) -> str:
+        """@private"""
+        r = self.csvpaths.config.get(section="results", name="archive")
+        p = Nos(r).join("manifest.json")
+        nos = Nos(r)
+        if not nos.dir_exists():
+            nos.makedirs()
+        nos.path = p
+        if not nos.exists():
+            with DataFileWriter(path=p) as file:
+                file.write("[]")
+        return p
+
+    @property
+    def results_root_manifest(self) -> list:
+        """the Archive Run Manifest -- a single global ledger tracking
+        every run across every named-results group, at the archive
+        root. distinct from any per-group results manifest. mirrors
+        PathsManager's paths_root_manifest. used by
+        ResultsReferenceFinder3 to resolve "*" root_major references
+        (bare :manifest(), optionally with a pointer to select one
+        entry by ordinal)."""
+        with DataFileReader(self.results_root_manifest_path) as reader:
+            return json.load(reader.source)
+
     def complete_run(self, *, run_dir, pathsname, results) -> None:
         """@private"""
         rr = ResultsRegistrar(
@@ -112,6 +140,7 @@ class ResultsManager:  # pylint: disable=C0115
         file: str = None,
         run_uuid: UUID,
         method: str,
+        template: str = None,
         extra_data: Optional[dict[str, str]] = None,
     ) -> ResultsMetadata:
         """@private"""
@@ -158,6 +187,7 @@ class ResultsManager:  # pylint: disable=C0115
         mdata.named_paths_uuid_string = np_uuid
         mdata.named_results_name = pathsname
         mdata.method = method
+        mdata.template = template or ""
         rr.register_start(mdata)
         return mdata
 
@@ -757,10 +787,6 @@ class ResultsManager:  # pylint: disable=C0115
         rs = [None for inst in instances if inst != "manifest.json"]
         for inst in instances:
             if inst.endswith(".json") or inst == "_extra_data":
-                #
-                # exp ^^^^
-                #
-                # if inst == "manifest.json":
                 continue
             r = self.get_named_result_for_instance(
                 name=name, run_dir=path, run=run, instance=inst
@@ -788,13 +814,6 @@ class ResultsManager:  # pylint: disable=C0115
         # it should be deleted.
         #
         _ = ""
-
-        """
-        print(
-            f"resman: get_named_result_for_instance: {name}, {run_dir}, {run}, {instance}"
-        )
-        """
-
         if run_dir.endswith(f"/{instance}") or run_dir.endswith(f"\\{instance}"):
             instance_dir = run_dir
         else:
@@ -823,6 +842,7 @@ class ResultsManager:  # pylint: disable=C0115
             csvpath.match = meta["runtime_data"]["match_part"]
             csvpath.delimiter = meta["runtime_data"]["delimiter"]
             csvpath.quotechar = meta["runtime_data"]["quotechar"]
+            csvpath.is_valid = meta["runtime_data"]["valid"]
         vars = ResultFileReader.vars(instance_dir)
         if vars:
             csvpath.variables = vars