csvpath 0.0.580__tar.gz → 0.0.581__tar.gz

{csvpath-0.0.580 → csvpath-0.0.581}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: csvpath
-Version: 0.0.580
+Version: 0.0.581
 Summary: A data preboarding framework for ingesting, managing, and validating CSV, Excel, and other tabular data files using a Collect, Store, Validate, Publish architecture to create a trusted publisher for downstream data consumers.
 Author: David Kershaw
 Author-email: info@csvpath.org

{csvpath-0.0.580 → csvpath-0.0.581}/csvpath/managers/files/file_manager.py

@@ -387,8 +387,6 @@ class FileManager:
         #
         self.legal_name(name)
         p = Nos(self.named_files_dir).join(name)
-        # p = os.path.join(self.named_files_dir, name)
-        # nos = self.nos
         nos = Nos(p)
         if nos.dir_exists():
             nos.remove()
@@ -458,7 +456,8 @@ class FileManager:
     #
     def add_named_files_from_dir(
         self, dirname: str, *, name=None, template: str = None, recurse=True
-    ):
+    ) -> list[str]:
+        ret = []
         #
         # legal_name handled at add_named_file
         #
@@ -474,7 +473,6 @@ class FileManager:
         dlist = nos.listdir(files_only=True, recurse=recurse)
         for i, _ in enumerate(dlist):
             dlist[i] = f"{nos.join(_)}"
-
         base = dirname
         #
         # collect all full paths that are files and have correct extensions
@@ -499,11 +497,14 @@ class FileManager:
                 else:
                     # path = os.path.join(base, p)
                     path = Nos(base).join(p)
-                self.add_named_file(name=n, path=path, template=template)
+                ref = self.add_named_file(name=n, path=path, template=template)
+                if ref is not None:
+                    ret.append(ref)
             else:
                 self._csvpaths.logger.debug(
                     "%s is not in accept list", Nos(base).join(p)
                 )
+        return ret
 
     def can_load(self, path: str) -> bool:
         #
@@ -533,7 +534,7 @@ class FileManager:
 
     def add_named_file(
         self, *, name: NamedFileName, path: str, template: str = None
-    ) -> None:
+    ) -> str | None:
         self.csvpaths.logger.info("Adding named file %s from %s", name, path)
         if not self.can_load(path):
             #
@@ -607,8 +608,11 @@ class FileManager:
             name_home = self.named_file_home(name)
             rpath, h = self._fingerprint(home)
             self.csvpaths.logger.debug("Fingerprint of %s: %s", path, h)
+            ret = f"${name}.files.{h}"
+            self.csvpaths.logger.debug("Reference to named-file: %s", ret)
             mdata = FileMetadata(self.csvpaths.config)
             mdata.named_file_name = name
+            mdata.named_file_ref = ret
             #
             # we need the declared path, incl. any extra path info, in order
             # to know if we are being pointed at a sub-portion of the data, e.g.
@@ -634,7 +638,6 @@ class FileManager:
             # the fingerprint is the most precise way of referencing a particular
             # named-file version.
             #
-            ret = f"${name}.files.{h}"
             self.csvpaths.logger.debug("Registered %s", ret)
             return ret
         except Exception as ex:

{csvpath-0.0.580 → csvpath-0.0.581}/csvpath/managers/files/file_metadata.py

@@ -22,6 +22,11 @@ class FileMetadata(Metadata):
         self.mark: str = None
         # like csv
         self.type: str = None
-        self.file_size = 0
+        self.file_size: int = 0
         self.template: str = None
-        self.fingerprint = None
+        self.fingerprint: str = None
+        #
+        # named_file_ref is the most specific reference possible it is
+        # in the form $name.files.fingerprint
+        #
+        self.named_file_ref: str = None

{csvpath-0.0.580 → csvpath-0.0.581}/csvpath/managers/files/file_registrar.py

@@ -38,7 +38,6 @@ class FileRegistrar(Registrar, Listener):
         if nos.physical_dirs() and not nos.dir_exists():
             raise InputException(f"Named file home does not exist: {home}")
         mf = Nos(home).join("manifest.json")
-        # mf = os.path.join(home, "manifest.json")
         nos.path = mf
         if not nos.exists():
             self.intermediary.put_json(mf, [])
@@ -124,6 +123,7 @@ class FileRegistrar(Registrar, Listener):
         manifest_path = mdata.manifest_path
         mani = {}
         mani["type"] = t
+        mani["reference"] = mdata.named_file_ref
         mani["file"] = rpath
         mani["file_home"] = mdata.file_home
         mani["fingerprint"] = h

{csvpath-0.0.580 → csvpath-0.0.581}/csvpath/managers/files/files_listener.py

@@ -33,7 +33,6 @@ class FilesListener(Listener):  # Registrar,
     def manifest_path(self) -> str:
         root = self.config.get(section="inputs", name="files")
         mf = Nos(root).join("manifest.json")
-        # mf = os.path.join(root, "manifest.json")
         if not Nos(mf).exists():
             with DataFileWriter(path=mf, mode="w") as writer:
                 writer.append("[]")
@@ -63,6 +62,7 @@ class FilesListener(Listener):  # Registrar,
         mani["named_file_name"] = mdata.named_file_name
         mani["origin_path"] = mdata.origin_path
         mani["fingerprint"] = mdata.fingerprint
+        mani["reference"] = mdata.named_file_ref
         mani["file_path"] = mdata.file_path
         mani["file_home"] = mdata.file_home
         mani["file_name"] = mdata.file_name

{csvpath-0.0.580 → csvpath-0.0.581}/csvpath/managers/paths/paths_manager.py

@@ -412,6 +412,29 @@ class PathsManager:
                 self.csvpaths.logger.debug("Adding %s to %s", _, name)
             s = self._str_from_list(paths)
             group_file_path = self._copy_in(name, s, append=append)
+            #
+            # capture the fingerprint here before/as soon as we write the bytes
+            #
+            # self.registrar._fingerprint(s)
+            #
+            # we should capture the fingerprint here. the way we do it today, we
+            # have Hasher read the file. However, that opens up a race condition. may
+            # not be likely, but we could have a conflict. there are a few problems.
+            # for local files we can use the same Hasher code by writing a temp file
+            # and hashing that. non-local files are harder because we don't know for
+            # 100% certain that the same bytes will result in the same hash if we do
+            # the hashing ourselves vs get it from the backend. given that non-local
+            # backends are inherently performance challenged we would prefer to not
+            # push even the relatively small group files back and forth to check that
+            # they haven't changed. however, since we can see the point in time where
+            # an updated happened to compare it to the point in time when a run
+            # happened we can make a good guess if we have a race and analyze
+            # accordingly. not perfect, but also not a complete information gap, so
+            # this is probably good enough for now.
+            #
+            # h = Hasher().hash(s)
+            #
+
             #
             # the paths have to be reacquired because we might be appending.
             #

{csvpath-0.0.580 → csvpath-0.0.581}/csvpath/managers/paths/paths_registrar.py

@@ -120,7 +120,6 @@ class PathsRegistrar(Registrar, Listener):
     def manifest_path(self, name: str) -> None:
         nhome = self.manager.named_paths_home(name)
         mf = Nos(nhome).join("manifest.json")
-        # mf = os.path.join(nhome, "manifest.json")
         if not Nos(mf).exists():
             self.intermediary.put_json(mf, [])
         return mf
@@ -141,10 +140,13 @@ class PathsRegistrar(Registrar, Listener):
         return sname
 
     def _fingerprint(self, *, name=None, group_file_path=None) -> str:
+        #
+        # in a more perfect world this fingerprint would be generated at the time
+        # we write the group file bytes. as is looks like a fairly large race condition.
+        #
         if group_file_path is None and name is not None:
             home = self.manager.named_paths_home(name)
             group_file_path = Nos(home).join("group.csvpaths")
-            # group_file_path = os.path.join(home, "group.csvpaths")
         elif group_file_path is None and name is None:
             raise InputException(
                 "Either the named-paths name or the path to the group file must be provided"

{csvpath-0.0.580 → csvpath-0.0.581}/csvpath/managers/results/result.py

@@ -198,14 +198,13 @@ class Result(ErrorCollector, Printer, Listener):  # pylint: disable=R0902
     def manifest(self) -> dict:
         p = self.instance_dir
         p = Nos(p).join("manifest.json")
-        # p = os.path.join(p, "manifest.json")
         with DataFileReader(p) as reader:
             return json.load(reader.source)
 
     @property
     def identity_or_index(self) -> str:
         s = self._csvpath.identity
-        if f"{s}".strip() == "":
+        if s is None or f"{s}".strip() == "":
             s = self.run_index
         return s
 

{csvpath-0.0.580 → csvpath-0.0.581}/csvpath/util/line_spooler.py

@@ -2,8 +2,10 @@ import os
 import csv
 import json
 import boto3
+import logging
 from pathlib import Path
 from abc import ABC, abstractmethod
+
 from .exceptions import InputException
 from .file_readers import DataFileReader
 from .file_writers import DataFileWriter
@@ -14,7 +16,7 @@ from .path_util import PathUtility as pathu
 
 class LineSpooler(ABC):
     def __init__(self, myresult) -> None:
-        self.result = myresult if myresult is not None else None
+        self.result = myresult
         self.sink = None
         self._count = 0
         self.closed = False
@@ -60,13 +62,26 @@ class ListLineSpooler(LineSpooler):
 
 
 class CsvLineSpooler(LineSpooler):
-    def __init__(self, myresult) -> None:
+    def __init__(
+        self,
+        myresult,
+        *,
+        path: str = None,
+        logger: logging.Logger = None,
+        delimiter: str = ",",
+        quotechar: str = '"',
+    ) -> None:
         super().__init__(myresult)
-        self._path = None
+        self._path = path
         self.writer = None
+        self._logger = logger
+        self._delimiter = delimiter
+        self._quotechar = quotechar
 
     @property
     def path(self) -> str:
+        if self._path is None:
+            self._instance_data_file_path()
         return self._path
 
     @path.setter
@@ -74,14 +89,40 @@ class CsvLineSpooler(LineSpooler):
         p = pathu.resep(p)
         self._path = p
 
+    @property
+    def logger(self) -> logging.Logger:
+        if self._logger:
+            return self._logger
+        if self.result is not None and self.result.csvpath is not None:
+            return self.result.csvpath.logger
+        return logging.getLogger(self.__class__.__name__)
+
+    @property
+    def delimiter(self) -> str:
+        if self._delimiter is not None:
+            return self._delimiter
+        if self.result is not None and self.result.csvpath is not None:
+            return self.result.csvpath.delimiter
+        else:
+            self.logger.error("No delimiter available")
+
+    @property
+    def quotechar(self) -> str:
+        if self._quotechar is not None:
+            return self._quotechar
+        if self.result is not None and self.result.csvpath is not None:
+            return self.result.csvpath.quotechar
+        else:
+            self.logger.error("No quotechar available")
+
     def __iter__(self):
         return self
 
     def to_list(self) -> list[str]:
-        if self.path is None:
-            self._instance_data_file_path()
-        if Nos(self.path).exists() is False:
-            self.result.csvpath.logger.debug(
+        if not self.path:
+            return []
+        if self.path is not None and Nos(self.path).exists() is False:
+            self.logger.debug(
                 "There is no data.csv at %s. This may or may not be a problem.",
                 self.path,
             )
@@ -90,15 +131,15 @@ class CsvLineSpooler(LineSpooler):
         for line in DataFileReader(
             self.path,
             filetype="csv",
-            delimiter=self.result.csvpath.delimiter,
-            quotechar=self.result.csvpath.quotechar,
+            delimiter=self.delimiter,
+            quotechar=self.quotechar,
         ).next():
             lst.append(line)
         return lst
 
     def __len__(self) -> int:
         if self._count is None or self._count <= 0:
-            if self.result is not None and self.result.instance_dir:
+            if self.result is not None and self.result.instance_dir is not None:
                 d = Nos(self.result.instance_dir).join("meta.json")
                 if Nos(d).exists() is True:
                     with DataFileReader(d) as file:
@@ -108,9 +149,10 @@ class CsvLineSpooler(LineSpooler):
         return self._count
 
     def load_if(self) -> None:
-        p = self._instance_data_file_path()
-        if p is not None:
-            self.sink = self._open_file(p)
+        if self.path is None:
+            ...
+        else:
+            self.sink = self._open_file(self.path)
             self.writer = csv.writer(self.sink)
 
     def _open_file(self, path: str):
@@ -119,10 +161,10 @@ class CsvLineSpooler(LineSpooler):
         return dw.sink
 
     def next(self):
-        if self.path is None:
-            self._instance_data_file_path()
+        if not self.path:
+            ...
         if Nos(self.path).exists() is False:
-            self.result.csvpath.logger.debug(
+            self.logger.debug(
                 "There is no data.csv at %s. This may or may not be a problem.",
                 self.path,
             )
@@ -130,31 +172,35 @@ class CsvLineSpooler(LineSpooler):
         for line in DataFileReader(
             self.path,
             filetype="csv",
-            delimiter=self.result.csvpath.delimiter,
-            quotechar=self.result.csvpath.quotechar,
+            delimiter=self.delimiter,
+            quotechar=self.quotechar,
         ).next():
             yield line
 
     def _warn_if(self) -> None:
-        if self.result is not None and self.result.csvpath:
-            self.result.csvpath.logger.warning(
-                "CsvLineSpooler cannot find instance_data_file_path yet within %s",
+        if self.result is not None and self.result.csvpath is not None:
+            self.logger.warning(
+                "CsvLineSpooler cannot find data file path yet within %s",
                 self.result.run_dir,
             )
+        else:
+            self.logger.warning("No path available")
 
     def _instance_data_file_path(self):
+        if self._path is not None:
+            return
         if self.result is None:
             self._warn_if()
-            return None
+            return
         if self.result.csvpath is None:
             self._warn_if()
-            return None
+            return
         if self.result.csvpath.scanner is None:
             self._warn_if()
-            return None
+            return
         if self.result.csvpath.scanner.filename is None:
             self._warn_if()
-            return None
+            return
         #
         # data file could be not there. we can in principle make sure that doesn't happen.
         # if we did, tho, we would need to be sure we don't create the dir so early that
@@ -162,21 +208,23 @@ class CsvLineSpooler(LineSpooler):
         # one case. leave the concern about the existence of the path aside for now.
         #
         self.path = self.result.data_file_path
-        return self.path
 
     def append(self, line) -> None:
         if not self.writer:
             self.load_if()
         if not self.writer:
-            raise InputException(f"Cannot write to data file for {self.result}")
+            msg = None
+            if self.result:
+                msg = f"Cannot write to data file for {self.result}"
+            else:
+                msg = f"Cannot write to {self.path}"
+            raise InputException(msg)
         self.writer.writerows([line])
         self._count += 1
 
     def bytes_written(self) -> int:
-        p = self._instance_data_file_path()
-        # there may be no file if we're on/before line 0 of the data.csv. that is Ok.
         try:
-            i = FileInfo.info(p)
+            i = FileInfo.info(self.path)
             if i and "bytes" in i:
                 return i["bytes"]
             else:
@@ -194,9 +242,13 @@ class CsvLineSpooler(LineSpooler):
             # drop the sink so no chance for recurse
             self.sink = None
             try:
-                c = self.csvpath if self.csvpath else self.csvpaths
-                c.error_manager.handle_error(source=self, msg=f"{ex}")
+                if self.csvpath:
+                    self.csvpath.error_manager.handle_error(source=self, msg=f"{ex}")
+                elif self.csvpaths:
+                    self.csvpaths.error_manager.handle_error(source=self, msg=f"{ex}")
+                else:
+                    self.logger.error(str(ex))
             except Exception as e:
-                self.result.csvpath.logger.error(
+                self.logger.error(
                     f"Caught {e}. Not raising an exception because closing."
                 )

{csvpath-0.0.580 → csvpath-0.0.581}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "csvpath"
-version = "0.0.580"
+version = "0.0.581"
 description = "A data preboarding framework for ingesting, managing, and validating CSV, Excel, and other tabular data files using a Collect, Store, Validate, Publish architecture to create a trusted publisher for downstream data consumers."
 authors = ["David Kershaw <info@csvpath.org>"]
 readme = "README.md"