csvpath 0.0.558__tar.gz → 0.0.560__tar.gz

{csvpath-0.0.558 → csvpath-0.0.560}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: csvpath
-Version: 0.0.558
+Version: 0.0.560
 Summary: A data preboarding framework for managing and validating CSV, Excel, and other tabular data files using a Collect, Store, Validate, Publish pattern to create a trusted publisher for downstream data consumers.
 Author: David Kershaw
 Author-email: dk107dk@hotmail.com

{csvpath-0.0.558 → csvpath-0.0.560}/assets/config/jenkins-local-azure.ini

@@ -11,8 +11,8 @@ extensions = csvpath, csvpaths
 extensions = csv, tsv, dat, tab, psv, ssv
 
 [errors]
-csvpath = raise, collect, print
-csvpaths = raise, collect, print
+csvpath = print,fail,collect
+csvpaths = print,collect
 use_format = full
 pattern = {time}:{file}:{line}:{paths}:{instance}:{chain}:  {message}
 
@@ -46,7 +46,7 @@ allow_http_files = True
 allow_local_files = True
 
 [listeners]
-groups = 
+groups = default
 sqlite.result = from csvpath.managers.integrations.sqlite.sqlite_result_listener import SqliteResultListener
 sqlite.results = from csvpath.managers.integrations.sqlite.sqlite_results_listener import SqliteResultsListener
 default.file = from csvpath.managers.files.files_listener import FilesListener

{csvpath-0.0.558 → csvpath-0.0.560}/csvpath/csvpath.py

@@ -1338,7 +1338,7 @@ class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
                 self.identity,
             )
             return
-        if self.scanner or self.scanner.filename:
+        if self.scanner and self.scanner.filename:
             filename = self.scanner.filename
         #
         # there are times, e.g. when using Lambda, when it may be better to
@@ -1365,7 +1365,7 @@ class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
                 f"Not using cache to get total lines and headers for {filename}"
             )
             lc = LineCounter(self)
-            lm, headers = lc.get_lines_and_headers(self.scanner.filename)
+            lm, headers = lc.get_lines_and_headers(filename)
             self.line_monitor = lm
             self.headers = headers
 

{csvpath-0.0.558 → csvpath-0.0.560}/csvpath/csvpaths.py

@@ -200,7 +200,7 @@ class CsvPaths(CsvPathsCoordinator, ErrorCollector):
         """ @private """
 
         self.logger.info(
-            "Initialized CsvPaths: {self} in thread: {threading.currentThread()}"
+            f"Initialized CsvPaths: {self} in thread: {threading.current_thread()}"
         )
 
     def _set_managers(self) -> None:
@@ -670,6 +670,7 @@ class CsvPaths(CsvPathsCoordinator, ErrorCollector):
             filename=filename,
             file=file,
             run_uuid=run_uuid,
+            method="collect_paths",
         )
         #
         #
@@ -686,6 +687,7 @@ class CsvPaths(CsvPathsCoordinator, ErrorCollector):
                 run_time=self.current_run_time,
                 run_dir=crt,
                 run_uuid=run_uuid,
+                method="collect_paths",
             )
             # casting a broad net because if "raise" not in the error policy we
             # want to never fail during a run
@@ -813,7 +815,11 @@ class CsvPaths(CsvPathsCoordinator, ErrorCollector):
         # run starts here
         #
         self.run_metadata = self.results_manager.start_run(
-            run_dir=crt, pathsname=pathsname, filename=filename, run_uuid=run_uuid
+            run_dir=crt,
+            pathsname=pathsname,
+            filename=filename,
+            run_uuid=run_uuid,
+            method="fast_forward_paths",
         )
         #
         #
@@ -830,6 +836,7 @@ class CsvPaths(CsvPathsCoordinator, ErrorCollector):
                 run_time=self.current_run_time,
                 run_dir=crt,
                 run_uuid=run_uuid,
+                method="fast_forward_paths",
             )
             try:
                 self._load_csvpath(
@@ -923,7 +930,11 @@ class CsvPaths(CsvPathsCoordinator, ErrorCollector):
         # run starts here
         #
         self.run_metadata = self.results_manager.start_run(
-            run_dir=crt, pathsname=pathsname, filename=filename, run_uuid=run_uuid
+            run_dir=crt,
+            pathsname=pathsname,
+            filename=filename,
+            run_uuid=run_uuid,
+            method="next_paths",
         )
         #
         #
@@ -956,6 +967,7 @@ class CsvPaths(CsvPathsCoordinator, ErrorCollector):
                 run_time=self.current_run_time,
                 run_dir=crt,
                 run_uuid=run_uuid,
+                method="next_paths",
             )
             if self._fail_all:
                 self.logger.warning(
@@ -1043,6 +1055,7 @@ class CsvPaths(CsvPathsCoordinator, ErrorCollector):
                 collect_when_not_matched=collect_when_not_matched,
                 file=file,
                 template=template,
+                method="collect_by_line",
             ):
                 # re: W0612: we need 'line' in order to do the iteration. we have to iterate.
                 lines.append(line)
@@ -1091,6 +1104,7 @@ class CsvPaths(CsvPathsCoordinator, ErrorCollector):
                 collect_when_not_matched=collect_when_not_matched,
                 file=file,
                 template=template,
+                method="fast_forward_by_line",
             ):
                 # re: W0612: we need 'line' in order to do the iteration. we have to iterate.
                 pass
@@ -1117,6 +1131,7 @@ class CsvPaths(CsvPathsCoordinator, ErrorCollector):
         if_all_agree=False,
         collect_when_not_matched=False,
         template: str = None,
+        method: str = None,
     ) -> List[Any]:
         #
         # we're doing a programmatic use when we use next_by_line() so we don't allow
@@ -1147,6 +1162,7 @@ class CsvPaths(CsvPathsCoordinator, ErrorCollector):
         collect_when_not_matched=False,
         template: str = None,
         file: str,
+        method: str = "next_by_line",
     ) -> List[Any]:
         """Does a CsvPath.next() on filename where each row is considered
         by every named path before the next row starts.
@@ -1200,6 +1216,7 @@ class CsvPaths(CsvPathsCoordinator, ErrorCollector):
             filename=filename,
             pathsname=pathsname,
             crt=crt,
+            method=method,
         )
         #
         # setting file into the csvpath is less obviously useful at CsvPaths
@@ -1370,7 +1387,9 @@ class CsvPaths(CsvPathsCoordinator, ErrorCollector):
                 self.error_manager.handle_error(source=self, msg=f"{ex}")
         return csvpath_objects
 
-    def _prep_csvpath_results(self, *, csvpath_objects, filename, pathsname, crt: str):
+    def _prep_csvpath_results(
+        self, *, csvpath_objects, filename, pathsname, crt: str, method: str
+    ):
         """@private"""
         #
         #
@@ -1380,7 +1399,11 @@ class CsvPaths(CsvPathsCoordinator, ErrorCollector):
         # run starts here
         #
         self.run_metadata = self.results_manager.start_run(
-            run_dir=crt, pathsname=pathsname, filename=filename, run_uuid=run_uuid
+            run_dir=crt,
+            pathsname=pathsname,
+            filename=filename,
+            run_uuid=run_uuid,
+            method="next_paths",
         )
         #
         #
@@ -1400,6 +1423,7 @@ class CsvPaths(CsvPathsCoordinator, ErrorCollector):
                     run_dir=crt,
                     by_line=True,
                     run_uuid=run_uuid,
+                    method=method,
                 )
                 csvpath[1] = result
                 #

{csvpath-0.0.558 → csvpath-0.0.560}/csvpath/managers/results/result.py

@@ -39,6 +39,7 @@ class Result(ErrorCollector, Printer, Listener):  # pylint: disable=R0902
         runtime_data: dict = None,
         by_line: bool = False,
         run_uuid: UUID,
+        method: str = None,
     ):
         """@private"""
         ErrorCollector.__init__(self)
@@ -63,6 +64,7 @@ class Result(ErrorCollector, Printer, Listener):  # pylint: disable=R0902
         self._actual_data_file = None
         self._origin_data_file = None
         self._by_line = by_line
+        self._method = method
         #
         # data_file_path is the path to data.csv of this result
         #
@@ -167,6 +169,10 @@ class Result(ErrorCollector, Printer, Listener):  # pylint: disable=R0902
     def by_line(self) -> bool:
         return self._by_line
 
+    @property
+    def method(self) -> bool:
+        return self._method
+
     @property
     def source_mode_preceding(self) -> bool:
         if self._preceding is None:

{csvpath-0.0.558 → csvpath-0.0.560}/csvpath/managers/results/result_metadata.py

@@ -58,10 +58,8 @@ class ResultMetadata(Metadata):
         self.lines_scanned: int = 0
         self.lines_total: int = 0
         self.lines_matched: int = 0
-        #
-        # why do we need this here? the run knows.
-        #
         self.by_line: bool = False
+        self.method: str = None
         #
         # transfer tuples:
         # 1: filename, no extension needed: data | unmatched
@@ -79,6 +77,7 @@ ResultMetadata(
   {self.run_home},{self.instance_home},
   {self.input_data_file},
   {self.file_fingerprints},
+  {self.method},
   {self.valid},{self.completed},{self.files_expected},{self.error_count},
   {self.transfers}
 )"""  # {self.file_count},
@@ -98,6 +97,7 @@ ResultMetadata(
         #
         #
         self.run = m.get("run")
+        self.method = m.get("method")
         self.run_uuid_string = m.get("run_uuid")
         self.run_home = m.get("run_home")
         self.instance_home = m.get("instance_home")

{csvpath-0.0.558 → csvpath-0.0.560}/csvpath/managers/results/results_manager.py

@@ -88,7 +88,14 @@ class ResultsManager:  # pylint: disable=C0115
     # and return that uuid.
     #
     def start_run(
-        self, *, run_dir, pathsname, filename, file: str = None, run_uuid: UUID
+        self,
+        *,
+        run_dir: str,
+        pathsname: str,
+        filename: str,
+        file: str = None,
+        run_uuid: UUID,
+        method: str,
     ) -> ResultsMetadata:
         """@private"""
         rr = ResultsRegistrar(
@@ -121,6 +128,7 @@ class ResultsManager:  # pylint: disable=C0115
         mdata.named_paths_name = pathsname
         mdata.named_paths_uuid_string = np_uuid
         mdata.named_results_name = pathsname
+        mdata.method = method
         rr.register_start(mdata)
         return mdata
 
@@ -217,7 +225,7 @@ class ResultsManager:  # pylint: disable=C0115
     # it doesn't separate printouts from different Result objects. a better way
     # to go might be to iterate the results and pull the printouts you need.
     #
-    def get_printouts(self, name: str, printstream: str = "default") -> dict:
+    def get_printouts(self, name: str, printstream: str = "default") -> list[str]:
         results = self._get_results_list(name)
         ps = []
         for r in results:
@@ -323,6 +331,7 @@ class ResultsManager:  # pylint: disable=C0115
         mdata.identity = result.identity_or_index
         mdata.named_paths_name = result.paths_name
         mdata.named_file_name = result.file_name
+        mdata.method = result.method
         rr = RunRegistrar(self.csvpaths)
         rr.register_start(mdata)
         #
@@ -331,6 +340,10 @@ class ResultsManager:  # pylint: disable=C0115
         # we use the same UUID for both metadata updates because the
         # UUID represents the run, not the metadata object
         #
+        #
+        # collect_paths and collect_by_line expect a data.csv file, even if it has 0-bytes.
+        # we make sure of that here.
+        #
         mdata = ResultMetadata(self.csvpaths.config)
         mdata.uuid = result.uuid
         mdata.run_uuid = result.run_uuid
@@ -341,6 +354,18 @@ class ResultsManager:  # pylint: disable=C0115
         mdata.run = result.run_dir[result.run_dir.rfind(sep) + 1 :]
         mdata.run_home = result.run_dir
         mdata.instance_home = result.instance_dir
+        mdata.method = result.method
+        #
+        # for the two CsvPaths methods that result in data.csv we want to make
+        # sure there is a data.csv, even if it ends up empty. we don't make this
+        # effort for unmatched.csv. perhaps we should but atm seems ok to pass.
+        #
+        if mdata.method in ["collect_paths", "collect_by_line"]:
+            path = Nos(mdata.instance_home).join("data.csv")
+            nos = Nos(path)
+            if not nos.exists():
+                with DataFileWriter(path=path) as file:
+                    file.write("")
         mdata.instance_identity = result.identity_or_index
         mdata.input_data_file = result.file_name
         rs = ResultSerializer(self._csvpaths.config.archive_path)
@@ -534,6 +559,13 @@ class ResultsManager:  # pylint: disable=C0115
                 self.csvpaths.logger.error(msg)
                 if self.csvpaths.ecoms.do_i_raise():
                     raise InputException(msg)
+                return []
+            if len(results.files) > 1:
+                self.csvpaths.logger.warning(
+                    "Referance found multiple runs (%s) when only one was expected and only one will be used: %s",
+                    len(results.files),
+                    name,
+                )
             path = results.files[0]
             run = path[len(self.csvpaths.config.archive_name) :]
             nos = Nos(path)

{csvpath-0.0.558 → csvpath-0.0.560}/csvpath/managers/results/results_metadata.py

@@ -26,6 +26,15 @@ class ResultsMetadata(Metadata):
         self.all_expected_files: bool = None
         self.by_line: bool = False
         self._run_uuid: UUID = None
+        self._method: str = None
+
+    @property
+    def method(self) -> str:
+        return self._method
+
+    @method.setter
+    def method(self, m: str) -> None:
+        self._method = m
 
     @property
     def run_uuid(self) -> UUID:
@@ -101,3 +110,4 @@ class ResultsMetadata(Metadata):
         self.all_valid = m.get("all_valid")
         self.error_count = m.get("error_count")
         self.all_expected_files = m.get("all_expected_files")
+        self.method = m.get("method")

{csvpath-0.0.558 → csvpath-0.0.560}/csvpath/managers/results/results_registrar.py

@@ -122,6 +122,7 @@ class ResultsRegistrar(Registrar, Listener):
         m["hostname"] = mdata.hostname
         m["username"] = mdata.username
         m["ip_address"] = mdata.ip_address
+        m["method"] = mdata.method
         mp = mdata.manifest_path
         m["manifest_path"] = mp
         with DataFileWriter(path=mp) as file:

{csvpath-0.0.558 → csvpath-0.0.560}/csvpath/managers/run/run_metadata.py

@@ -11,6 +11,7 @@ class RunMetadata(Metadata):
         self.named_file_name: str = None
         self.identity: str = None
         self._run_uuid: UUID = None
+        self._method: str = None
 
     @property
     def run_uuid(self) -> UUID:
@@ -29,3 +30,11 @@ class RunMetadata(Metadata):
     @run_uuid_string.setter
     def run_uuid_string(self, u: str) -> None:
         self._run_uuid = UUID(u)
+
+    @property
+    def method(self) -> str:
+        return self._method
+
+    @method.setter
+    def method(self, m: str) -> None:
+        self._method = m

{csvpath-0.0.558 → csvpath-0.0.560}/csvpath/matching/productions/reference.py

@@ -198,7 +198,7 @@ class Reference(Matchable):
             else:
                 ret = v
         else:
-            msg = f"Results exist but the {ref['name']} variable is unknown: {self.my_chain}"
+            msg = f"The {ref['name']} variable is unknown: {self.my_chain}"
             self.matcher.csvpath.error_manager.handle_error(source=self, msg=msg)
             if self.matcher.csvpath.do_i_raise():
                 raise DataException(msg)

{csvpath-0.0.558 → csvpath-0.0.560}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "csvpath"
-version = "0.0.558"
+version = "0.0.560"
 description = "A data preboarding framework for managing and validating CSV, Excel, and other tabular data files using a Collect, Store, Validate, Publish pattern to create a trusted publisher for downstream data consumers."
 authors = ["David Kershaw <dk107dk@hotmail.com>"]
 readme = "README.md"