csvpath 0.0.496__tar.gz → 0.0.498__tar.gz

{csvpath-0.0.496 → csvpath-0.0.498}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: csvpath
-Version: 0.0.496
+Version: 0.0.498
 Summary: A declarative language for validating CSV, Excel, and other tabular data files
 Author: David Kershaw
 Author-email: dk107dk@hotmail.com
@@ -25,16 +25,18 @@ Classifier: Topic :: Utilities
 Provides-Extra: pandas
 Provides-Extra: smartopen
 Requires-Dist: bullet (>=2.2.0,<3.0.0)
-Requires-Dist: inflect (>=7.3.1,<8.0.0)
+Requires-Dist: inflect (>=7.4.0,<8.0.0)
 Requires-Dist: jinja2 (>=3.1.4,<4.0.0)
 Requires-Dist: lark (>=1.2.2,<2.0.0)
+Requires-Dist: marquez-python (>=0.50.0,<0.51.0)
 Requires-Dist: metaphone (>=0.6,<0.7)
-Requires-Dist: pandas (>=2.2.3,<3.0.0) ; extra == "pandas"
+Requires-Dist: openlineage-python (>=1.25.0,<2.0.0)
 Requires-Dist: ply (>=3.11,<4.0)
 Requires-Dist: pylightxl (>=1.61,<2.0)
 Requires-Dist: pytest (>=8.3.3,<9.0.0)
 Requires-Dist: python-dateutil (>=2.9.0.post0,<3.0.0)
-Requires-Dist: smart-open[s3] (>=7.0.5,<8.0.0) ; extra == "smartopen"
+Requires-Dist: pytz (>=2024.2,<2025.0)
+Requires-Dist: requests (>=2.32.3,<3.0.0)
 Requires-Dist: tabulate (>=0.9.0,<0.10.0)
 Project-URL: Csvpath.org, https://www.csvpath.org
 Project-URL: Github, https://github.com/csvpath/csvpath.git

{csvpath-0.0.496 → csvpath-0.0.498}/config/config.ini

@@ -24,8 +24,14 @@ path = cache
 [functions]
 imports = config/functions.imports
 
+[listeners]
+
+[marquez]
+base_url = http://localhost:5000
+
 [results]
 archive = archive
+transfers = transfers
 
 [inputs]
 files = inputs/named_files

{csvpath-0.0.496 → csvpath-0.0.498}/csvpath/csvpath.py

@@ -5,7 +5,7 @@ import csv
 import time
 import os
 import hashlib
-from datetime import datetime
+from datetime import datetime, timezone
 from typing import List, Dict, Any
 from collections.abc import Iterator
 from abc import ABC, abstractmethod
@@ -28,7 +28,7 @@ from .util.exceptions import (
 from .matching.util.exceptions import MatchException
 from csvpath.util.printer import Printer
 from csvpath.util.file_readers import DataFileReader
-from csvpath.managers.line_spooler import LineSpooler, ListLineSpooler
+from csvpath.util.line_spooler import LineSpooler, ListLineSpooler
 from csvpath.modes.mode_controller import ModeController
 
 
@@ -271,7 +271,7 @@ class CsvPath(CsvPathPublic, ErrorCollector, Printer):  # pylint: disable=R0902,
         # _function_times_value collects the time a function spends doing its to_value()
         #
         self._function_times_value = {}
-        self._created_at = datetime.now()
+        self._created_at = datetime.now(timezone.utc)
         self._run_started_at = None
         self._collecting = False
         #
@@ -681,23 +681,6 @@ class CsvPath(CsvPathPublic, ErrorCollector, Printer):  # pylint: disable=R0902,
     def transfers(self) -> list[tuple[str, str]]:
         return self.modes.transfer_mode.transfers
 
-    """
-    def update_arg_validation_mode_if(self) -> None:
-        if self.metadata and "validation-mode" in self.metadata:
-            # sets arg validation reporting. one or more or none of:
-            #  - print
-            #  - log
-            #  - raise
-            #
-            validation_mode = f"{self.metadata['validation-mode']}".strip()
-            if validation_mode:
-                self.set_validation_error_handling(validation_mode)
-                self.logger.info(
-                    "Setting 'validation-mode': %s",
-                    self.metadata["validation-mode"],
-                )
-    """
-
     @property
     def all_expected_files(self) -> list[str]:
         return self.modes.files_mode.all_expected_files
@@ -1099,7 +1082,7 @@ class CsvPath(CsvPathPublic, ErrorCollector, Printer):  # pylint: disable=R0902,
             last_line = self.matcher.line
         self.line_monitor.next_line(last_line=last_line, data=line)
         if self.line_monitor.physical_line_number == 0:
-            self._run_started_at = datetime.now()
+            self._run_started_at = datetime.now(timezone.utc)
 
     def _consider_line(self, line):  # pylint: disable=R0912, R0911
         # re: R0912: this method has already been refactored but maybe
@@ -1234,10 +1217,10 @@ class CsvPath(CsvPathPublic, ErrorCollector, Printer):  # pylint: disable=R0902,
             )
             return -1
         if self.csvpaths:
-            self.line_monitor = self.csvpaths.file_manager.get_new_line_monitor(
+            self.line_monitor = self.csvpaths.file_manager.cacher.get_new_line_monitor(
                 self.scanner.filename
             )
-            self.headers = self.csvpaths.file_manager.get_original_headers(
+            self.headers = self.csvpaths.file_manager.cacher.get_original_headers(
                 self.scanner.filename
             )
         else:

{csvpath-0.0.496 → csvpath-0.0.498}/csvpath/csvpaths.py

@@ -5,18 +5,16 @@ from abc import ABC, abstractmethod
 from typing import List, Any
 import csv
 import traceback
-from datetime import datetime
+from datetime import datetime, timezone
 from .util.error import ErrorHandler, ErrorCollector, Error
 from .util.config import Config
 from .util.log_utility import LogUtility
 from .util.metadata_parser import MetadataParser
 from .util.exceptions import InputException, CsvPathsException
-from .util.csvpaths_registrar import CsvPathsRegistrar, CsvPathsFilesystemRegistrar
-from .managers.paths_manager import PathsManager
-from .managers.file_manager import FileManager
-from .managers.results_manager import ResultsManager
-from .managers.results_registrar import ResultsRegistrar
-from .managers.result import Result
+from .managers.paths.paths_manager import PathsManager
+from .managers.files.file_manager import FileManager
+from .managers.results.results_manager import ResultsManager
+from .managers.results.result import Result
 from . import CsvPath
 
 
@@ -38,7 +36,7 @@ class CsvPathsPublic(ABC):
 
     @abstractmethod
     def collect_paths(self, *, pathsname, filename) -> None:  # pragma: no cover
-        """Sequentially does a CsvPath.collect() on filename for every named path"""
+        """Sequentially does a CsvPath.collect() on filename for every named path. lines are collected into results."""
 
     @abstractmethod
     def fast_forward_paths(self, *, pathsname, filename) -> None:  # pragma: no cover
@@ -150,8 +148,6 @@ class CsvPaths(CsvPathsPublic, CsvPathsCoordinator, ErrorCollector):
         self._skip_all = False
         self._advance_all = 0
         self._current_run_time = None
-        self._csvpaths_registrars = []
-        self._csvpaths_registrars.append(CsvPathsFilesystemRegistrar(self))
         self._run_time_str = None
 
     def run_time_str(self, pathsname=None) -> str:
@@ -168,7 +164,7 @@ class CsvPaths(CsvPathsPublic, CsvPathsCoordinator, ErrorCollector):
     @property
     def current_run_time(self) -> datetime:
         if self._current_run_time is None:
-            self._current_run_time = datetime.now()
+            self._current_run_time = datetime.now(timezone.utc)
         return self._current_run_time
 
     def clear_run_coordination(self) -> None:
@@ -209,15 +205,6 @@ class CsvPaths(CsvPathsPublic, CsvPathsCoordinator, ErrorCollector):
     def advance_all(self, lines: int) -> None:  # pragma: no cover
         self._advance_all = lines
 
-    @property
-    def csvpaths_registrars(self) -> list[CsvPathsRegistrar]:
-        if self._csvpaths_registrars is None:
-            self._csvpaths_registrars = []
-        return self._csvpaths_registrars
-
-    def add_csvpaths_registrar(self, creg) -> None:
-        self.csvpaths_registrars.append(creg)
-
     @property
     def errors(self) -> List[Error]:  # pylint: disable=C0116
         return self._errors
@@ -253,6 +240,15 @@ class CsvPaths(CsvPathsPublic, CsvPathsCoordinator, ErrorCollector):
         )
         crt = self.run_time_str(pathsname)
         results = []
+        #
+        # run starts here
+        #
+        self.results_manager.start_run(
+            run_dir=crt, pathsname=pathsname, filename=filename
+        )
+        #
+        #
+        #
         for i, path in enumerate(paths):
             csvpath = self.csvpath()
             result = Result(
@@ -266,7 +262,6 @@ class CsvPaths(CsvPathsPublic, CsvPathsCoordinator, ErrorCollector):
             # casting a broad net because if "raise" not in the error policy we
             # want to never fail during a run
             try:
-                self.results_manager.add_named_result(result)
                 self._load_csvpath(
                     csvpath=csvpath,
                     path=path,
@@ -274,6 +269,11 @@ class CsvPaths(CsvPathsPublic, CsvPathsCoordinator, ErrorCollector):
                     pathsname=pathsname,
                     filename=filename,
                 )
+                #
+                # the add has to come after _load_csvpath because we need the identity or index
+                # to be stable and the identity is found in load, if it exists.
+                #
+                self.results_manager.add_named_result(result)
                 lines = result.lines
                 self.logger.debug("Collecting lines using a %s", type(lines))
                 csvpath.collect(lines=lines)
@@ -299,10 +299,12 @@ class CsvPaths(CsvPathsPublic, CsvPathsCoordinator, ErrorCollector):
                     raise e
             self.results_manager.save(result)
             results.append(result)
-        rr = ResultsRegistrar(
-            csvpaths=self, run_dir=crt, pathsname=pathsname, results=results
+        #
+        # run ends here
+        #
+        self.results_manager.complete_run(
+            run_dir=crt, pathsname=pathsname, results=results
         )
-        rr.write_manifest()
         #
         # update/write run manifests here
         #  - validity (are all paths valid)
@@ -316,10 +318,22 @@ class CsvPaths(CsvPathsPublic, CsvPathsCoordinator, ErrorCollector):
         )
 
     def _load_csvpath(
-        self, *, csvpath: CsvPath, path: str, file: str, pathsname: str = None, filename
+        self,
+        *,
+        csvpath: CsvPath,
+        path: str,
+        file: str,
+        pathsname: str = None,
+        filename,
+        by_line: bool = False,
     ) -> None:
         # file is the physical file (+/- if preceding mode) filename is the named-file name
         self.logger.debug("Beginning to load csvpath %s with file %s", path, file)
+        #
+        # by_line==True means we are starting a run that is breadth-first ultimately using
+        # next_by_line(). by_line runs cannot be source-mode preceding and have different
+        # semantics around csvpaths influencing one another.
+        #
         # we strip comments from above the path so we need to extract them first
         path = MetadataParser(self).extract_metadata(instance=csvpath, csvpath=path)
         identity = csvpath.identity
@@ -327,9 +341,24 @@ class CsvPaths(CsvPathsPublic, CsvPathsCoordinator, ErrorCollector):
         # update the settings using the metadata fields we just collected
         csvpath.update_settings_from_metadata()
         #
-        # this next line is triggering a write to the inputs dir
+        # if we have a reference, resolve it. we may not actually use the file
+        # if we're in source-mode: preceding, but that doesn't matter from the
+        # pov of the reference.
+        #
+        if filename.startswith("$"):
+            self.logger.debug(
+                "File name is a reference: %s. Replacing the path passed in with the reffed data file path.",
+                filename,
+            )
+            file = self.results_manager.data_file_for_reference(filename)
+        #
+        #
         #
         if csvpath.data_from_preceding is True:
+            if by_line is True:
+                raise CsvPathsException(
+                    "Breadth-first runs do not support source-mode preceding because each line of data flows through each csvpath in order already"
+                )
             #
             # we are in source-mode: preceding
             # that means we ignore the original data file path and
@@ -338,6 +367,16 @@ class CsvPaths(CsvPathsPublic, CsvPathsCoordinator, ErrorCollector):
             # saved data.
             #
             # find the preceding csvpath in the named-paths
+            #
+            # we may be in a file reference like: $sourcemode.csvpaths.source2:from. if so
+            # we just need the named-paths name.
+            #
+            if pathsname.startswith("$"):
+                self.logger.debug(
+                    "Named-paths name is a reference: %s. Stripping it down to just the actual named-paths name.",
+                    pathsname,
+                )
+                pathsname = pathsname[1 : pathsname.find(".")]
             result = self.results_manager.get_last_named_result(
                 name=pathsname, before=csvpath.identity
             )
@@ -351,24 +390,23 @@ class CsvPaths(CsvPathsPublic, CsvPathsCoordinator, ErrorCollector):
                     csvpath.identity,
                     file,
                 )
+            else:
+                self.logger.warning(
+                    "Cannot find a preceding data file to use for csvpath identified as %s running in source-mode",
+                    csvpath.identity,
+                )
         f = path.find("[")
         self.logger.debug("Csvpath matching part starts at char # %s", f)
         apath = f"${file}{path[f:]}"
         self.logger.info("Parsing csvpath %s", apath)
         csvpath.parse(apath)
         #
-        # ready to run. time to register the run.
+        # ready to run. time to register the run. this is separate from
+        # the run_register.py (ResultsRegister) event
+        #
+        # crt = self.run_time_str(pathsname)
+        # fingerprint = self.file_manager.get_fingerprint_for_name(filename)
         #
-        crt = self.run_time_str(pathsname)
-        fingerprint = self.file_manager.get_fingerprint_for_name(filename)
-        for _ in self.csvpaths_registrars:
-            _.update_manifest(
-                csvpath=csvpath,
-                filepath=file,
-                instancepath=crt,
-                fingerprint=fingerprint,
-                identity=identity,
-            )
         self.logger.debug("Done loading csvpath")
 
     def fast_forward_paths(self, *, pathsname, filename):
@@ -383,6 +421,15 @@ class CsvPaths(CsvPathsPublic, CsvPathsCoordinator, ErrorCollector):
             filename,
         )
         crt = self.run_time_str(pathsname)
+        #
+        # run starts here
+        #
+        self.results_manager.start_run(
+            run_dir=crt, pathsname=pathsname, filename=filename
+        )
+        #
+        #
+        #
         results = []
         for i, path in enumerate(paths):
             csvpath = self.csvpath()
@@ -396,7 +443,6 @@ class CsvPaths(CsvPathsPublic, CsvPathsCoordinator, ErrorCollector):
                 run_dir=crt,
             )
             try:
-                self.results_manager.add_named_result(result)
                 self._load_csvpath(
                     csvpath=csvpath,
                     path=path,
@@ -404,6 +450,11 @@ class CsvPaths(CsvPathsPublic, CsvPathsCoordinator, ErrorCollector):
                     pathsname=pathsname,
                     filename=filename,
                 )
+                #
+                # the add has to come after _load_csvpath because we need the identity or index
+                # to be stable and the identity is found in load, if it exists.
+                #
+                self.results_manager.add_named_result(result)
                 self.logger.info(
                     "Parsed csvpath %s pointed at %s and starting to fast-forward",
                     i,
@@ -425,10 +476,12 @@ class CsvPaths(CsvPathsPublic, CsvPathsCoordinator, ErrorCollector):
                     raise e
             self.results_manager.save(result)
             results.append(result)
-        rr = ResultsRegistrar(
-            csvpaths=self, run_dir=crt, pathsname=pathsname, results=results
+        #
+        # run ends here
+        #
+        self.results_manager.complete_run(
+            run_dir=crt, pathsname=pathsname, results=results
         )
-        rr.write_manifest()
         self.clear_run_coordination()
         self.logger.info(
             "Completed fast_forward_paths %s with %s paths", pathsname, len(paths)
@@ -444,6 +497,15 @@ class CsvPaths(CsvPathsPublic, CsvPathsCoordinator, ErrorCollector):
         self.clean(paths=pathsname)
         self.logger.info("Beginning next_paths with %s paths", len(paths))
         crt = self.run_time_str(pathsname)
+        #
+        # run starts here
+        #
+        self.results_manager.start_run(
+            run_dir=crt, pathsname=pathsname, filename=filename
+        )
+        #
+        #
+        #
         results = []
         for i, path in enumerate(paths):
             if self._skip_all:
@@ -478,7 +540,6 @@ class CsvPaths(CsvPathsPublic, CsvPathsCoordinator, ErrorCollector):
                 )
                 csvpath.is_valid = False
             try:
-                self.results_manager.add_named_result(result)
                 self._load_csvpath(
                     csvpath=csvpath,
                     path=path,
@@ -486,13 +547,22 @@ class CsvPaths(CsvPathsPublic, CsvPathsCoordinator, ErrorCollector):
                     pathsname=pathsname,
                     filename=filename,
                 )
+                #
+                # the add has to come after _load_csvpath because we need the identity or index
+                # to be stable and the identity is found in load, if it exists.
+                #
+                self.results_manager.add_named_result(result)
                 for line in csvpath.next():
-                    line.append(result)
+                    #
+                    # removed dec 1. why was this? it doesn't seem to make sense and
+                    # removing it doesn't break any unit tests. was it a mistake?
+                    #
+                    # line.append(result)
                     if collect:
                         result.append(line)
                         result.unmatched = csvpath.unmatched
                     yield line
-            except Exception as ex:  # pylint: disable=W0718
+            except Exception as ex:
                 ex.trace = traceback.format_exc()
                 ex.source = self
                 try:
@@ -502,12 +572,15 @@ class CsvPaths(CsvPathsPublic, CsvPathsCoordinator, ErrorCollector):
                 except Exception as e:
                     self.results_manager.save(result)
                     raise e
+
             self.results_manager.save(result)
             results.append(result)
-        rr = ResultsRegistrar(
-            csvpaths=self, run_dir=crt, pathsname=pathsname, results=results
+        #
+        # run ends here
+        #
+        self.results_manager.complete_run(
+            run_dir=crt, pathsname=pathsname, results=results
         )
-        rr.write_manifest()
         self.clear_run_coordination()
 
     # =============== breadth first processing ================
@@ -590,6 +663,10 @@ class CsvPaths(CsvPathsPublic, CsvPathsCoordinator, ErrorCollector):
             filename=filename,
             pathsname=pathsname,
         )
+        #
+        # prep has to come after _load_csvpath_objects because we need the identity or
+        # indexes to be stable and the identity is found in the load, if it exists.
+        #
         self._prep_csvpath_results(
             csvpath_objects=csvpath_objects, filename=filename, pathsname=pathsname
         )
@@ -725,13 +802,13 @@ class CsvPaths(CsvPathsPublic, CsvPathsCoordinator, ErrorCollector):
             results.append(result)
             result.unmatched = r[0].unmatched
             self.results_manager.save(result)
-        rr = ResultsRegistrar(
-            csvpaths=self,
-            run_dir=results[0].run_dir,
-            pathsname=pathsname,
-            results=results,
+
+        #
+        # run ends here
+        #
+        self.results_manager.complete_run(
+            run_dir=results[0].run_dir, pathsname=pathsname, results=results
         )
-        rr.write_manifest()
         self.clear_run_coordination()
 
     def _load_csvpath_objects(
@@ -754,9 +831,11 @@ class CsvPaths(CsvPathsPublic, CsvPathsCoordinator, ErrorCollector):
                     file=named_file,
                     filename=filename,
                     pathsname=pathsname,
+                    by_line=True,
                 )
                 if csvpath.data_from_preceding is True:
-                    self.logger.warning(
+                    # this exception raise may be redundant, but I'm leaving it for now for good measure.
+                    raise CsvPathsException(
                         "Csvpath identified as {csvpath.identity} is set to use preceding data, but CsvPaths's by_line methods do not permit that"
                     )
                 csvpath_objects.append([csvpath, []])
@@ -773,6 +852,15 @@ class CsvPaths(CsvPathsPublic, CsvPathsCoordinator, ErrorCollector):
 
     def _prep_csvpath_results(self, *, csvpath_objects, filename, pathsname):
         crt = self.run_time_str(pathsname)
+        #
+        # run starts here
+        #
+        self.results_manager.start_run(
+            run_dir=crt, pathsname=pathsname, filename=filename
+        )
+        #
+        #
+        #
         for i, csvpath in enumerate(csvpath_objects):
             try:
                 #
@@ -787,8 +875,13 @@ class CsvPaths(CsvPathsPublic, CsvPathsCoordinator, ErrorCollector):
                     run_index=i,
                     run_time=self.current_run_time,
                     run_dir=crt,
+                    by_line=True,
                 )
                 csvpath[1] = result
+                #
+                # the add has to come after _load_csvpath because we need the identity or index
+                # to be stable and the identity is found in load, if it exists.
+                #
                 self.results_manager.add_named_result(result)
             except Exception as ex:  # pylint: disable=W0718
                 ex.trace = traceback.format_exc()

csvpath-0.0.498/csvpath/managers/files/file_cacher.py

@@ -0,0 +1,50 @@
+import os
+from typing import Dict, List, Tuple
+from csvpath.util.line_counter import LineCounter
+from csvpath.util.line_monitor import LineMonitor
+from csvpath.util.cache import Cache
+from csvpath.util.exceptions import InputException, FileException
+
+
+class FileCacher:
+    def __init__(self, csvpaths=None):
+        self.csvpaths = csvpaths
+        self.cache = Cache(self.csvpaths)
+        self.pathed_lines_and_headers = {}
+
+    def get_new_line_monitor(self, filename: str) -> LineMonitor:
+        if filename not in self.pathed_lines_and_headers:
+            self._find_lines_and_headers(filename)
+        lm = self.pathed_lines_and_headers[filename][0]
+        lm = lm.copy()
+        return lm
+
+    def get_original_headers(self, filename: str) -> List[str]:
+        if filename not in self.pathed_lines_and_headers:
+            self._find_lines_and_headers(filename)
+        return self.pathed_lines_and_headers[filename][1][:]
+
+    def _find_lines_and_headers(self, filename: str) -> None:
+        lm, headers = self._cached_lines_and_headers(filename)
+        if lm is None or headers is None:
+            lc = LineCounter(self.csvpaths)
+            lm, headers = lc.get_lines_and_headers(filename)
+            self._cache_lines_and_headers(filename, lm, headers)
+        self.pathed_lines_and_headers[filename] = (lm, headers)
+
+    def _cached_lines_and_headers(self, filename: str) -> Tuple[LineMonitor, List[str]]:
+        lm = LineMonitor()
+        json = self.cache.cached_text(filename, "json")
+        if json is not None and not json.strip() == "":
+            lm.load(json)
+        else:
+            return (None, None)
+        headers = self.cache.cached_text(filename, "csv")
+        return (lm, headers)
+
+    def _cache_lines_and_headers(
+        self, filename, lm: LineMonitor, headers: List[str]
+    ) -> None:
+        jstr = lm.dump()
+        self.cache.cache_text(filename, "json", jstr)
+        self.cache.cache_text(filename, "csv", ",".join(headers))

csvpath-0.0.498/csvpath/managers/files/file_listener_ol.py

@@ -0,0 +1,27 @@
+from openlineage.client import OpenLineageClient
+
+from ..metadata import Metadata
+from ..listener import Listener
+from ..ol.event import EventBuilder
+from ..ol.job import JobBuilder
+
+
+class OpenLineageFileListener(Listener):
+    def __init__(self, config=None):
+        super().__init__(config)
+        self.ol_client = None
+
+    def metadata_update(self, mdata: Metadata) -> None:
+        if self.ol_client is None:
+            client_url = self.config._get("marquez", "base_url")
+            if client_url is None:
+                print(
+                    "WARNING: OpenLineage listeners are live but there is no Marquez API URL"
+                )
+                return
+                # client_url = "http://localhost:5000"
+            self.ol_client = OpenLineageClient(url=client_url)
+
+        es = EventBuilder().build(mdata)
+        for e in es:
+            self.ol_client.emit(e)