csvpath 0.0.494__tar.gz → 0.0.495__tar.gz

{csvpath-0.0.494 → csvpath-0.0.495}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: csvpath
-Version: 0.0.494
+Version: 0.0.495
 Summary: A declarative language for validating CSV, Excel, and other tabular data files
 Author: David Kershaw
 Author-email: dk107dk@hotmail.com

{csvpath-0.0.494 → csvpath-0.0.495}/config/config.ini

@@ -27,3 +27,8 @@ imports = config/functions.imports
 [results]
 archive = archive
 
+[inputs]
+files = inputs/named_files
+csvpaths = inputs/named_paths
+on_unmatched_file_fingerprints = halt
+

{csvpath-0.0.494 → csvpath-0.0.495}/csvpath/csvpath.py

@@ -233,6 +233,14 @@ class CsvPath(CsvPathPublic, ErrorCollector, Printer):  # pylint: disable=R0902,
         self._errors: List[Error] = None
         self._error_collector = None
         #
+        # transfers from transfer-mode: ((data | unmatched):var-name)(,*)
+        # this setting tells CsvPaths to copy resulting data.csv and/or
+        # unmatched.csv to one or more target locations below the config.ini's
+        # transfer directory. the name "data" or "unmatched" is paired with
+        # a var name that indicates the path to write the indicated file.
+        #
+        self._transfers = None
+        #
         # saves the scan and match parts of paths for reference. mainly helpful
         # for testing the CsvPath library itself; not used end users. the run
         # name becomes the file name of the saved path parts.
@@ -308,7 +316,7 @@ class CsvPath(CsvPathPublic, ErrorCollector, Printer):  # pylint: disable=R0902,
         self._function_times_value = {}
         self._created_at = datetime.now()
         self._run_started_at = None
-
+        self._all_expected_files = []
         self._collecting = False
         self._unmatched = None
         self._unmatched_available = False
@@ -364,11 +372,11 @@ class CsvPath(CsvPathPublic, ErrorCollector, Printer):  # pylint: disable=R0902,
         return self._run_started_at
 
     @property
-    def run_mode(self) -> bool:
+    def will_run(self) -> bool:
         return self._run_mode
 
-    @run_mode.setter
-    def run_mode(self, mode) -> None:
+    @will_run.setter
+    def will_run(self, mode) -> None:
         self._run_mode = mode
 
     def _up_function_time_match(self, c, t) -> None:
@@ -449,6 +457,7 @@ class CsvPath(CsvPathPublic, ErrorCollector, Printer):  # pylint: disable=R0902,
         is preferred over name. E.g.:
         ~ name: my path description: an example ~
         """
+        # this ordering is relied on in Result and possibly elsewhere
         if not self.metadata:
             return ""
         if "id" in self.metadata:
@@ -471,7 +480,7 @@ class CsvPath(CsvPathPublic, ErrorCollector, Printer):  # pylint: disable=R0902,
             self._config = Config()
         return self._config
 
-    def has_errors(self) -> bool:  # pylint: disable=C0116
+    def has_errors(self) -> bool:
         if self.errors and len(self.errors) > 0:
             return True
         if self.error_collector:
@@ -720,6 +729,7 @@ class CsvPath(CsvPathPublic, ErrorCollector, Printer):  # pylint: disable=R0902,
         #   - run-mode: no-run | run
         #   - unmatched-mode: no-keep | keep
         #   - source-mode: preceding | origin
+        #   - files-mode: all | no-data | no-unmatched | no-printouts | data | unmatched | errors | meta | vars | printouts
         #
         self.update_logic_mode_if()
         self.update_run_mode_if()
@@ -729,6 +739,87 @@ class CsvPath(CsvPathPublic, ErrorCollector, Printer):  # pylint: disable=R0902,
         self.update_arg_validation_mode_if()
         self.update_unmatched_mode_if()
         self.update_data_from_preceding_if()
+        self.update_expected_files_if()
+        self.update_transfer_mode_if()
+
+    # =====================
+
+    @property
+    def transfer_mode(self) -> str:
+        return self.metadata.get("transfer-mode")
+
+    @property
+    def source_mode(self) -> str:
+        return self.metadata.get("source-mode")
+
+    @property
+    def files_mode(self) -> str:
+        return self.metadata.get("files-mode")
+
+    @property
+    def validation_mode(self) -> str:
+        return self.metadata.get("validation-mode")
+
+    @property
+    def run_mode(self) -> str:
+        return self.metadata.get("run-mode")
+
+    @property
+    def logic_mode(self) -> str:
+        return "AND" if self.AND else "OR"
+
+    @property
+    def return_mode(self) -> str:
+        return self.metadata.get("return-mode")
+
+    @property
+    def explain_mode(self) -> str:
+        return self.metadata.get("explain-mode")
+
+    @property
+    def print_mode(self) -> str:
+        return self.metadata.get("print-mode")
+
+    @property
+    def unmatched_mode(self) -> str:
+        return self.metadata.get("unmatched-mode")
+
+    # =====================
+
+    @property
+    def transfers(self) -> list[tuple[str, str]]:
+        return self._transfers
+
+    def update_transfer_mode_if(self) -> None:
+        m = self.transfer_mode
+        if m is not None:
+            self._transfers = []
+            for t in m.split(","):
+                i = t.find(">")
+                if i == -1:
+                    raise InputException(
+                        "Transfer mode directive must be in the form file > location"
+                    )
+                file = t[0:i].strip()
+                location = t[i + 1 :].strip()
+                self._transfers.append((file, location))
+
+    def update_expected_files_if(self) -> None:
+        fm = self.files_mode
+        if fm is None:
+            return []
+        fs = [s for s in fm.split(",")]
+        _ = []
+        for s in fs:
+            s = s.strip()
+            if s not in ["data", "unmatched", "vars", "errors", "meta", "printouts"]:
+                self.logger.warning(
+                    "Unknown file-mode token %s. Mode tokens may be separated by , and will be trimmed.",
+                    s,
+                )
+                continue
+            _.append(s)
+        self.all_expected_files = _
 
     def update_data_from_preceding_if(self) -> None:
         if self.metadata and "source-mode" in self.metadata:
@@ -758,9 +849,9 @@ class CsvPath(CsvPathPublic, ErrorCollector, Printer):  # pylint: disable=R0902,
     def update_run_mode_if(self) -> None:
         if self.metadata and "run-mode" in self.metadata:
             if f"{self.metadata['run-mode']}".strip() == "no-run":
-                self.run_mode = False
+                self.will_run = False
             elif f"{self.metadata['run-mode']}".strip() == "run":
-                self.run_mode = True
+                self.will_run = True
             else:
                 self.logger.warning(
                     "Incorrect metadata field value 'run-mode': %s",
@@ -824,6 +915,14 @@ class CsvPath(CsvPathPublic, ErrorCollector, Printer):  # pylint: disable=R0902,
                     self.metadata["print-mode"],
                 )
 
+    @property
+    def all_expected_files(self) -> list[str]:
+        return self._all_expected_files
+
+    @all_expected_files.setter
+    def all_expected_files(self, efs: list[str]) -> None:
+        self._all_expected_files = efs
+
     def _pick_named_path(self, name, *, specific=None) -> str:
         if not self.csvpaths:
             raise CsvPathsException("No CsvPaths object available")
@@ -972,6 +1071,14 @@ class CsvPath(CsvPathPublic, ErrorCollector, Printer):  # pylint: disable=R0902,
     def is_valid(self, tf: bool) -> None:
         self._is_valid = tf
 
+    @property
+    def completed(self) -> bool:
+        if not self.scanner or not self.line_monitor:
+            return False
+        if self.scanner.is_last(self.line_monitor.physical_line_number):
+            return True
+        return False
+
     @property
     def from_line(self):  # pragma: no cover pylint: disable=C0116
         if self.scanner is None:
@@ -1066,12 +1173,18 @@ class CsvPath(CsvPathPublic, ErrorCollector, Printer):  # pylint: disable=R0902,
                 nexts -= 1
             else:
                 break
-        # we don't want to hold on to lines more than needed.
-        # we do want to return them if we're not spooling. the
+        # we don't want to hold on to data more than needed. but
+        # we do want to return data if we're not spooling. the
         # way we do that is to keep the local var available with the
         # list and/or the spooler. the caller needs to be aware of
         # both possibilities, but both offer __len__ and append.
-        self.lines = None
+        #
+        # we keep the self.lines if it is not a list because that
+        # makes it available to the runtime data collector so we can
+        # see the line count in the metadata, saving opening a
+        # potentially large data.csv to find out how many lines.
+        if isinstance(self.lines, list):
+            self.lines = None
         return lines
 
     def fast_forward(self, csvpath=None) -> None:
@@ -1090,7 +1203,7 @@ class CsvPath(CsvPathPublic, ErrorCollector, Printer):  # pylint: disable=R0902,
         if self.scanner is None and csvpath is not None:
             self.parse(csvpath)
         start = time.time()
-        if self.run_mode is True:
+        if self.will_run is True:
             for line in self._next_line():
                 b = self._consider_line(line)
                 if b:

{csvpath-0.0.494 → csvpath-0.0.495}/csvpath/csvpaths.py

@@ -10,10 +10,12 @@ from .util.error import ErrorHandler, ErrorCollector, Error
 from .util.config import Config
 from .util.log_utility import LogUtility
 from .util.metadata_parser import MetadataParser
-from .util.exceptions import InputException
+from .util.exceptions import InputException, CsvPathsException
+from .util.csvpaths_registrar import CsvPathsRegistrar, CsvPathsFilesystemRegistrar
 from .managers.paths_manager import PathsManager
 from .managers.file_manager import FileManager
 from .managers.results_manager import ResultsManager
+from .managers.results_registrar import ResultsRegistrar
 from .managers.result import Result
 from . import CsvPath
 
@@ -148,6 +150,20 @@ class CsvPaths(CsvPathsPublic, CsvPathsCoordinator, ErrorCollector):
         self._skip_all = False
         self._advance_all = 0
         self._current_run_time = None
+        self._csvpaths_registrars = []
+        self._csvpaths_registrars.append(CsvPathsFilesystemRegistrar(self))
+        self._run_time_str = None
+
+    def run_time_str(self, pathsname=None) -> str:
+        if self._run_time_str is None and pathsname is None:
+            raise CsvPathsException(
+                "Cannot have None in both run_time_str and pathsname"
+            )
+        if self._run_time_str is None:
+            self._run_time_str = self.results_manager.get_run_time_str(
+                pathsname, self.current_run_time
+            )
+        return self._run_time_str
 
     @property
     def current_run_time(self) -> datetime:
@@ -193,6 +209,15 @@ class CsvPaths(CsvPathsPublic, CsvPathsCoordinator, ErrorCollector):
     def advance_all(self, lines: int) -> None:  # pragma: no cover
         self._advance_all = lines
 
+    @property
+    def csvpaths_registrars(self) -> list[CsvPathsRegistrar]:
+        if self._csvpaths_registrars is None:
+            self._csvpaths_registrars = []
+        return self._csvpaths_registrars
+
+    def add_csvpaths_registrar(self, creg) -> None:
+        self.csvpaths_registrars.append(creg)
+
     @property
     def errors(self) -> List[Error]:  # pylint: disable=C0116
         return self._errors
@@ -216,13 +241,18 @@ class CsvPaths(CsvPathsPublic, CsvPathsCoordinator, ErrorCollector):
 
     def collect_paths(self, *, pathsname, filename) -> None:
         paths = self.paths_manager.get_named_paths(pathsname)
+        if paths is None:
+            raise InputException(f"No named-paths found for {pathsname}")
         file = self.file_manager.get_named_file(filename)
+        if file is None:
+            raise InputException(f"No named-file found for {filename}")
         self.logger.info("Prepping %s and %s", filename, pathsname)
         self.clean(paths=pathsname)
         self.logger.info(
             "Beginning collect_paths %s with %s paths", pathsname, len(paths)
         )
-        crt = self.results_manager.get_run_time_str(pathsname, self.current_run_time)
+        crt = self.run_time_str(pathsname)
+        results = []
         for i, path in enumerate(paths):
             csvpath = self.csvpath()
             result = Result(
@@ -237,29 +267,25 @@ class CsvPaths(CsvPathsPublic, CsvPathsCoordinator, ErrorCollector):
             # want to never fail during a run
             try:
                 self.results_manager.add_named_result(result)
-                self._load_csvpath(csvpath, path=path, file=file, pathsname=pathsname)
+                self._load_csvpath(
+                    csvpath=csvpath,
+                    path=path,
+                    file=file,
+                    pathsname=pathsname,
+                    filename=filename,
+                )
                 lines = result.lines
+                self.logger.debug("Collecting lines using a %s", type(lines))
                 csvpath.collect(lines=lines)
-                # lines = csvpath.collect()
                 if lines is None:
                     self.logger.error(  # pragma: no cover
                         "Unexpected None for lines after collect_paths: file: %s, match: %s",
                         file,
                         csvpath.match,
                     )
-                """
                 #
-                # this seems to have not been very useful and with LineSpooler it seems
-                # to not be helpful at all.
+                # this is obviously not a good idea for very large files!
                 #
-                if len(lines) == 0:
-                    self.logger.warning(  # pragma: no cover
-                        "No lines collected in collect_paths: file: %s match: %s",
-                        file,
-                        csvpath.match,
-                    )
-                """
-                # result.lines = lines
                 result.unmatched = csvpath.unmatched
             except Exception as ex:  # pylint: disable=W0718
                 ex.trace = traceback.format_exc()
@@ -272,18 +298,32 @@ class CsvPaths(CsvPathsPublic, CsvPathsCoordinator, ErrorCollector):
                     self.results_manager.save(result)
                     raise e
             self.results_manager.save(result)
+            results.append(result)
+        rr = ResultsRegistrar(
+            csvpaths=self, run_dir=crt, pathsname=pathsname, results=results
+        )
+        rr.write_manifest()
+        #
+        # update/write run manifests here
+        #  - validity (are all paths valid)
+        #  - paths-completeness (did they all run and complete)
+        #  - method (collect, fast_forward, next)
+        #  - timestamp
+        #
         self.clear_run_coordination()
         self.logger.info(
             "Completed collect_paths %s with %s paths", pathsname, len(paths)
         )
 
     def _load_csvpath(
-        self, csvpath: CsvPath, path: str, file: str, pathsname: str = None
+        self, *, csvpath: CsvPath, path: str, file: str, pathsname: str = None, filename
     ) -> None:
+        # file is the physical file (+/- if preceding mode) filename is the named-file name
         self.logger.debug("Beginning to load csvpath %s with file %s", path, file)
         # we strip comments from above the path so we need to extract them first
         path = MetadataParser(self).extract_metadata(instance=csvpath, csvpath=path)
-        self.logger.debug("Csvpath after metadata extract: %s", path)
+        identity = csvpath.identity
+        self.logger.debug("Csvpath %s after metadata extract: %s", identity, path)
         # update the settings using the metadata fields we just collected
         csvpath.update_settings_from_metadata()
         #
@@ -316,6 +356,20 @@ class CsvPaths(CsvPathsPublic, CsvPathsCoordinator, ErrorCollector):
         apath = f"${file}{path[f:]}"
         self.logger.info("Parsing csvpath %s", apath)
         csvpath.parse(apath)
+        #
+        # ready to run. time to register the run.
+        #
+        crt = self.run_time_str(pathsname)
+        fingerprint = self.file_manager.get_fingerprint_for_name(filename)
+        for _ in self.csvpaths_registrars:
+            _.update_manifest(
+                csvpath=csvpath,
+                filepath=file,
+                instancepath=crt,
+                fingerprint=fingerprint,
+                identity=identity,
+            )
+        self.logger.debug("Done loading csvpath")
 
     def fast_forward_paths(self, *, pathsname, filename):
         paths = self.paths_manager.get_named_paths(pathsname)
@@ -328,7 +382,8 @@ class CsvPaths(CsvPathsPublic, CsvPathsCoordinator, ErrorCollector):
             len(paths),
             filename,
         )
-        crt = self.results_manager.get_run_time_str(pathsname, self.current_run_time)
+        crt = self.run_time_str(pathsname)
+        results = []
         for i, path in enumerate(paths):
             csvpath = self.csvpath()
             self.logger.debug("Beginning to FF CsvPath instance: %s", csvpath)
@@ -342,7 +397,13 @@ class CsvPaths(CsvPathsPublic, CsvPathsCoordinator, ErrorCollector):
             )
             try:
                 self.results_manager.add_named_result(result)
-                self._load_csvpath(csvpath, path=path, file=file, pathsname=pathsname)
+                self._load_csvpath(
+                    csvpath=csvpath,
+                    path=path,
+                    file=file,
+                    pathsname=pathsname,
+                    filename=filename,
+                )
                 self.logger.info(
                     "Parsed csvpath %s pointed at %s and starting to fast-forward",
                     i,
@@ -363,6 +424,11 @@ class CsvPaths(CsvPathsPublic, CsvPathsCoordinator, ErrorCollector):
                     self.results_manager.save(result)
                     raise e
             self.results_manager.save(result)
+            results.append(result)
+        rr = ResultsRegistrar(
+            csvpaths=self, run_dir=crt, pathsname=pathsname, results=results
+        )
+        rr.write_manifest()
         self.clear_run_coordination()
         self.logger.info(
             "Completed fast_forward_paths %s with %s paths", pathsname, len(paths)
@@ -377,7 +443,8 @@ class CsvPaths(CsvPathsPublic, CsvPathsCoordinator, ErrorCollector):
         self.logger.info("Prepping %s and %s", filename, pathsname)
         self.clean(paths=pathsname)
         self.logger.info("Beginning next_paths with %s paths", len(paths))
-        crt = self.results_manager.get_run_time_str(pathsname, self.current_run_time)
+        crt = self.run_time_str(pathsname)
+        results = []
         for i, path in enumerate(paths):
             if self._skip_all:
                 skip_err = "Found the skip-all signal set. skip_all() is"
@@ -412,7 +479,13 @@ class CsvPaths(CsvPathsPublic, CsvPathsCoordinator, ErrorCollector):
                 csvpath.is_valid = False
             try:
                 self.results_manager.add_named_result(result)
-                self._load_csvpath(csvpath, path=path, file=file, pathsname=pathsname)
+                self._load_csvpath(
+                    csvpath=csvpath,
+                    path=path,
+                    file=file,
+                    pathsname=pathsname,
+                    filename=filename,
+                )
                 for line in csvpath.next():
                     line.append(result)
                     if collect:
@@ -430,6 +503,11 @@ class CsvPaths(CsvPathsPublic, CsvPathsCoordinator, ErrorCollector):
                     self.results_manager.save(result)
                     raise e
             self.results_manager.save(result)
+            results.append(result)
+        rr = ResultsRegistrar(
+            csvpaths=self, run_dir=crt, pathsname=pathsname, results=results
+        )
+        rr.write_manifest()
         self.clear_run_coordination()
 
     # =============== breadth first processing ================
@@ -505,10 +583,13 @@ class CsvPaths(CsvPathsPublic, CsvPathsCoordinator, ErrorCollector):
             raise InputException(
                 f"Pathsname '{pathsname}' must name a list of csvpaths"
             )
+
         csvpath_objects = self._load_csvpath_objects(
             paths=paths,
             named_file=fn,
             collect_when_not_matched=collect_when_not_matched,
+            filename=filename,
+            pathsname=pathsname,
         )
         self._prep_csvpath_results(
             csvpath_objects=csvpath_objects, filename=filename, pathsname=pathsname
@@ -639,21 +720,42 @@ class CsvPaths(CsvPathsPublic, CsvPathsCoordinator, ErrorCollector):
                 yield line
             if sum(stopped_count) == len(csvpath_objects):
                 break
+        results = []
         for r in csvpath_objects:
             result = r[1]
+            results.append(result)
             result.unmatched = r[0].unmatched
             self.results_manager.save(result)
+        rr = ResultsRegistrar(
+            csvpaths=self,
+            run_dir=results[0].run_dir,
+            pathsname=pathsname,
+            results=results,
+        )
+        rr.write_manifest()
         self.clear_run_coordination()
 
     def _load_csvpath_objects(
-        self, *, paths: List[str], named_file: str, collect_when_not_matched=False
+        self,
+        *,
+        paths: List[str],
+        named_file: str,
+        collect_when_not_matched=False,
+        filename,
+        pathsname,
     ):
         csvpath_objects = []
         for path in paths:
             csvpath = self.csvpath()
             csvpath.collect_when_not_matched = collect_when_not_matched
             try:
-                self._load_csvpath(csvpath, path=path, file=named_file)
+                self._load_csvpath(
+                    csvpath=csvpath,
+                    path=path,
+                    file=named_file,
+                    filename=filename,
+                    pathsname=pathsname,
+                )
                 if csvpath.data_from_preceding is True:
                     self.logger.warning(
                         "Csvpath identified as {csvpath.identity} is set to use preceding data, but CsvPaths's by_line methods do not permit that"
@@ -671,7 +773,7 @@ class CsvPaths(CsvPathsPublic, CsvPathsCoordinator, ErrorCollector):
         return csvpath_objects
 
     def _prep_csvpath_results(self, *, csvpath_objects, filename, pathsname):
-        crt = self.results_manager.get_run_time_str(pathsname, self.current_run_time)
+        crt = self.run_time_str(pathsname)
         for i, csvpath in enumerate(csvpath_objects):
             try:
                 #

{csvpath-0.0.494 → csvpath-0.0.495}/csvpath/managers/file_manager.py

@@ -13,7 +13,7 @@ from ..util.error import ErrorHandler
 from ..util.cache import Cache
 from ..util.file_readers import DataFileReader
 from ..util.reference_parser import ReferenceParser
-from ..util.exceptions import InputException
+from ..util.exceptions import InputException, FileException
 from .file_registrar import FileRegistrar
 
 
@@ -218,6 +218,18 @@ class FileManager(CsvPathsFileManager):  # pylint: disable=C0115
             path, filetype=filetype, delimiter=delimiter, quotechar=quotechar
         )
 
+    def get_fingerprint_for_name(self, name) -> str:
+        if name.startswith("$"):
+            # atm, we don't give fingerprints for references doing rewind/replay
+            return ""
+        mpath = self.registrar.manifest_path(name)
+        man = self.registrar.get_manifest(mpath)
+        if man is None or len(man) == 0:
+            raise FileException(
+                f"No fingerprint available for named-file name: {name} at manifest path: {mpath}: manifest: {man}"
+            )
+        return man[len(man) - 1]["fingerprint"]
+
     def _name_from_name_part(self, name):
         i = name.rfind(".")
         if i == -1:

{csvpath-0.0.494 → csvpath-0.0.495}/csvpath/managers/file_registrar.py

@@ -119,7 +119,7 @@ class FileRegistrar:
         jdata = self.get_manifest(manifestpath)
         jdata.append(mdata)
         with open(manifestpath, "w", encoding="utf-8") as file:
-            json.dump(jdata, file)
+            json.dump(jdata, file, indent=2)
 
     def register_named_file(self, *, name: str, path: str) -> None:
         # does the source file exist?

{csvpath-0.0.494 → csvpath-0.0.495}/csvpath/managers/line_spooler.py

@@ -69,6 +69,14 @@ class CsvLineSpooler(LineSpooler):
             self.writer = csv.writer(self.sink)
 
     def next(self):
+        if self.path is None:
+            self._instance_data_file_path()
+        if os.path.exists(self.path) is False:
+            self.result.csvpath.logger.debug(
+                "There is no data.csv at %s. This may or may not be a problem.",
+                self.path,
+            )
+            return
         for line in DataFileReader(
             self.path,
             filetype="csv",

{csvpath-0.0.494 → csvpath-0.0.495}/csvpath/managers/paths_registrar.py

@@ -99,7 +99,7 @@ class PathsRegistrar:
             mdata["time"] = f"{datetime.now()}"
             jdata.append(mdata)
             with open(manifestpath, "w", encoding="utf-8") as file:
-                json.dump(jdata, file)
+                json.dump(jdata, file, indent=2)
 
     def assure_manifest(self, name: str) -> None:
         nhome = self.named_paths_home(name)

{csvpath-0.0.494 → csvpath-0.0.495}/csvpath/managers/result.py

@@ -9,6 +9,7 @@ from ..util.printer import Printer
 from .. import CsvPath
 from .result_serializer import ResultSerializer
 from .line_spooler import LineSpooler, CsvLineSpooler
+from ..util.exceptions import CsvPathsException
 
 
 class Result(ErrorCollector, Printer):  # pylint: disable=R0902
@@ -48,6 +49,21 @@ class Result(ErrorCollector, Printer):  # pylint: disable=R0902
         self._run_dir = run_dir
         self._unmatched = None
         self._data_file_path = None
+        if (
+            csvpath.metadata is None
+            or csvpath.identity is None
+            or csvpath.identity == ""
+        ):
+            if csvpath.metadata is None:
+                raise CsvPathsException(
+                    "Metadata cannot be None. Check order of operations."
+                )
+            #
+            # "NAME" is the least favored identifier. if we parse metadata after setting this
+            # identity and the csvpath uses any of the other five identifiers it will take
+            # precedence over this index. if the csvpath uses NAME it will overwrite.
+            #
+            csvpath.metadata["NAME"] = self.run_index
 
     @property
     def run_time(self) -> datetime:
@@ -139,14 +155,19 @@ class Result(ErrorCollector, Printer):  # pylint: disable=R0902
         self._lines = ls
 
     def append(self, line: List[Any]) -> None:
-        if self._lines is None:
-            self._lines = []
-        self._lines.append(line)
+        self.lines.append(line)
+        # orig
+        # if self._lines is None:
+        #    self._lines = []
+        # self._lines.append(line)
 
     def __len__(self) -> int:
-        if self._lines is None:
-            self._lines = []  # pragma: no cover
-        return len(self._lines)
+        if isinstance(self.lines, list):
+            return len(self._lines)
+        i = 0
+        for _ in self.lines.next():
+            i += 1
+        return i
 
     @property
     def unmatched(self) -> list[list[Any]]:
@@ -183,8 +204,7 @@ class Result(ErrorCollector, Printer):  # pylint: disable=R0902
     def collect_error(self, error: Error) -> None:  # pylint: disable=C0116
         self._errors.append(error)
 
-    @property
-    def has_errors(self) -> bool:  # pylint: disable=C0116
+    def has_errors(self) -> bool:
         return self.errors_count > 0
 
     @property