csvpath 0.0.608__tar.gz → 0.0.612__tar.gz

{csvpath-0.0.608 → csvpath-0.0.612}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: csvpath
-Version: 0.0.608
+Version: 0.0.612
 Summary: CsvPath Framework is a data preboarding automation library for receiving, validating, and tracking CSV, Excel, JSONL and other tabular data files before they can corrupt downstream data consumers.
 License-File: LICENSE
 Author: David Kershaw

{csvpath-0.0.608 → csvpath-0.0.612}/assets/config/config.ini

@@ -35,6 +35,7 @@ use_cache = yes
 
 [results]
 archive = tmp/local/archive
+transfers = tmp/local/transfers
 
 [inputs]
 files = tmp/local/inputs/named_files

{csvpath-0.0.608 → csvpath-0.0.612}/assets/config/jenkins-local-azure.ini

@@ -35,6 +35,7 @@ imports =
 
 [results]
 archive = tmp/azure/archive
+transfers = tmp/azure/transfers
 
 [inputs]
 files = azure://csvpath/mac/inputs/named_files

{csvpath-0.0.608 → csvpath-0.0.612}/assets/config/jenkins-local-filesystem-mysql.ini

@@ -29,6 +29,7 @@ imports =
 
 [results]
 archive = tmp/mysql/archive
+transfers = tmp/mysql/transfers
 
 [inputs]
 files = tmp/mysql/inputs/named_files

{csvpath-0.0.608 → csvpath-0.0.612}/assets/config/jenkins-local-gcs.ini

@@ -31,10 +31,11 @@ path = cache
 use_cache = no
 
 [functions]
-imports = 
+imports =
 
 [results]
 archive = tmp/gcs/archive
+transfers = tmp/gcs/transfers
 
 [inputs]
 files = gs://csvpath-testing-1/mac/inputs/named_files
@@ -90,7 +91,7 @@ port = SFTPPLUS_PORT
 admin_username = SFTPPLUS_ADMIN_USERNAME
 admin_password = SFTPPLUS_ADMIN_PASSWORD
 api_url = https://. . . :10020/json
-scripts_dir = 
+scripts_dir =
 execute_timeout = 300
 
 [ckan]
@@ -105,5 +106,5 @@ timeout = 5
 verify = False
 
 [slack]
-webhook_url = 
+webhook_url =
 

{csvpath-0.0.608 → csvpath-0.0.612}/assets/config/jenkins-local-s3.ini

@@ -31,10 +31,11 @@ path = cache
 use_cache = no
 
 [functions]
-imports = 
+imports =
 
 [results]
 archive = tmp/s3/archive
+transfers = tmp/s3/transfers
 
 [inputs]
 files = s3://csvpath-example-1/mac/inputs/named_files
@@ -90,7 +91,7 @@ port = SFTPPLUS_PORT
 admin_username = SFTPPLUS_ADMIN_USERNAME
 admin_password = SFTPPLUS_ADMIN_PASSWORD
 api_url = https://. . . :10020/json
-scripts_dir = 
+scripts_dir =
 execute_timeout = 300
 
 [ckan]
@@ -105,5 +106,5 @@ timeout = 5
 verify = False
 
 [slack]
-webhook_url = 
+webhook_url =
 

{csvpath-0.0.608 → csvpath-0.0.612}/assets/config/jenkins-local-sftp.ini

@@ -25,10 +25,11 @@ path = cache
 use_cache = no
 
 [functions]
-imports = 
+imports =
 
 [results]
 archive = tmp/sftp/archive
+transfers = tmp/sftp/transfers
 
 [inputs]
 files = sftp://{SFTP_SERVER}:{SFTP_PORT}/mac/inputs/named_files
@@ -94,7 +95,7 @@ port = SFTPPLUS_PORT
 admin_username = SFTPPLUS_ADMIN_USERNAME
 admin_password = SFTPPLUS_ADMIN_PASSWORD
 api_url = https://. . . :10020/json
-scripts_dir = 
+scripts_dir =
 execute_timeout = 300
 
 [ckan]
@@ -109,5 +110,5 @@ timeout = 5
 verify = False
 
 [slack]
-webhook_url = 
+webhook_url =
 

{csvpath-0.0.608 → csvpath-0.0.612}/assets/config/jenkins-windows-azure.ini

@@ -30,6 +30,7 @@ imports =
 
 [results]
 archive = tmp\azure\archive
+transfers = tmp\azure\transfers
 
 [inputs]
 files = azure://csvpath/windows/inputs/named_files

{csvpath-0.0.608 → csvpath-0.0.612}/assets/config/jenkins-windows-gcs.ini

@@ -29,6 +29,7 @@ imports =
 
 [results]
 archive = tmp\gcs\archive
+transfers = tmp\gcs\transfers
 
 [inputs]
 files = gs://csvpath-testing-1/windows/inputs/named_files

{csvpath-0.0.608 → csvpath-0.0.612}/assets/config/jenkins-windows-local.ini

@@ -30,6 +30,7 @@ imports =
 
 [results]
 archive = tmp\local\archive
+transfers = tmp\local\transfers
 
 [inputs]
 files = tmp\local\inputs\named_files

{csvpath-0.0.608 → csvpath-0.0.612}/assets/config/jenkins-windows-s3.ini

@@ -35,6 +35,7 @@ imports =
 
 [results]
 archive = tmp\s3\archive
+transfers = tmp\s3\transfers
 
 [inputs]
 files = s3://csvpath-example-1/windows/inputs/named_files

{csvpath-0.0.608 → csvpath-0.0.612}/assets/config/jenkins-windows-sftp.ini

@@ -38,6 +38,7 @@ imports =
 
 [results]
 archive = tmp\sftp\archive
+transfers = tmp\sftp\transfers
 
 [inputs]
 files = sftp://{SFTP_SERVER}:{SFTP_PORT}/windows/inputs/named_files

{csvpath-0.0.608 → csvpath-0.0.612}/config/config.ini

@@ -35,10 +35,11 @@ path = cache
 use_cache = yes
 
 [functions]
-imports = 
+imports =
 
 [results]
 archive = tmp/local/archive
+transfers = tmp/transfers
 
 [inputs]
 files = tmp/local/inputs/named_files
@@ -105,7 +106,7 @@ port = SFTPPLUS_PORT
 admin_username = SFTPPLUS_ADMIN_USERNAME
 admin_password = SFTPPLUS_ADMIN_PASSWORD
 api_url = https://. . . :10020/json
-scripts_dir = 
+scripts_dir =
 execute_timeout = 300
 
 [ckan]
@@ -120,7 +121,7 @@ timeout = 5
 verify = False
 
 [slack]
-webhook_url = 
+webhook_url =
 
 [scripts]
 run_scripts = yes

{csvpath-0.0.608 → csvpath-0.0.612}/csvpath/csvpath.py

@@ -1,14 +1,13 @@
-""" CsvPath is the main class for the library. most of the magic
-    happens either here or in individual functions. """  # pylint: disable=C0302
+"""CsvPath is the main class for the library. most of the magic
+happens either here or in individual functions."""  # pylint: disable=C0302
 
 import time
 import os
 import hashlib
 import traceback
-from datetime import datetime, timezone
+from datetime import datetime
 from typing import List, Dict, Any, Optional, Callable
 from collections.abc import Iterator
-from abc import ABC, abstractmethod
 from .util.config import Config
 from .util.line_monitor import LineMonitor
 from .util.log_utility import LogUtility as lout
@@ -238,7 +237,7 @@ class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
         #
         self._save_scan_dir = None
         self._save_match_dir = None
-        self._run_name = None
+        self._run_dir = None
         #
         # metadata is collected from "outer" csvpath comments. outer comments
         # separate from the comments within the match part of the csvpath.
@@ -282,6 +281,14 @@ class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
         self._unmatched = None
         self._cacher = None
 
+    @property
+    def run_dir(self) -> str:
+        return self._run_dir
+
+    @run_dir.setter
+    def run_dir(self, d: str) -> None:
+        self._run_dir = d
+
     @property
     def logger(self):
         if self._logger is None:
@@ -293,6 +300,8 @@ class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
         self._logger = ler
 
     def __del__(self) -> None:
+        self.csvpaths = None
+        self.matcher = None
         try:
             # in a test on windows 0.0.570 we see self has no error_manager attribute
             # that is surprising since there is one ^^^^. no idea. this test is cheap tho.
@@ -303,6 +312,7 @@ class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
             ):
                 self.error_manager.error_metrics.provider.shutdown()
                 self.error_manager.error_metrics = None
+                self.error_manager = None
             lout.release_logger(self)
         except Exception:
             print(traceback.format_exc())
@@ -586,6 +596,10 @@ class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
         """@private"""
         return self.modes.validation_mode.collect_validation_errors
 
+    @property
+    def consolidate_printouts(self) -> bool:
+        return self.modes.print_mode.consolidate_printouts
+
     def add_printer(self, printer) -> None:  # pylint: disable=C0116
         """@private"""
         if printer not in self.printers:
@@ -739,7 +753,8 @@ class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
         #   - run-mode: no-run | run
         #   - unmatched-mode: no-keep | keep
         #   - source-mode: preceding | origin
-        #   - files-mode: all | no-data | no-unmatched | no-printouts | data | unmatched | errors | meta | vars | printouts
+        #   - files-mode: all | no-data | no-unmatched | no-printouts | data
+        #                     | unmatched | errors | meta | vars | printouts
         #
         self.modes.update()
         #
@@ -981,16 +996,16 @@ class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
         return scan, matches
 
     def _save_parts_if(self, scan, match):
-        if self._save_scan_dir and self._run_name:
+        if self._save_scan_dir and self._run_dir:
             with open(
-                os.path.join(self._save_scan_dir, f"{self._run_name}.txt"),
+                os.path.join(self._save_scan_dir, f"{self._run_dir}.txt"),
                 "w",
                 encoding="utf-8",
             ) as f:
                 f.write(scan)
-        if self._save_match_dir and self._run_name:
+        if self._save_match_dir and self._run_dir:
             with open(
-                os.path.join(self._save_match_dir, f"{self._run_name}.txt"),
+                os.path.join(self._save_match_dir, f"{self._run_dir}.txt"),
                 "w",
                 encoding="utf-8",
             ) as f:
@@ -1484,9 +1499,7 @@ class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
             self.get_total_lines_and_headers()
         return self.line_monitor.physical_end_line_number
 
-    def get_total_lines_and_headers(
-        self, *, filename: str = None
-    ) -> None:  # pylint: disable=C0116
+    def get_total_lines_and_headers(self, *, filename: str = None) -> None:  # pylint: disable=C0116
         #
         # filename is an option for certain needs but in the usual case
         # we expect scanner.filename, and if both are available for some
@@ -1515,10 +1528,8 @@ class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
                 f"Using cache to get total lines and headers for {filename}"
             )
             cacher = self.cacher
-            print(f"cassff: {cacher}: {filename}")
             self.line_monitor = cacher.get_new_line_monitor(filename)
             self.headers = cacher.get_original_headers(filename)
-            print(f"cassff: {self.line_monitor}: {self.headers}")
         else:
             self.logger.debug(
                 f"Not using cache to get total lines and headers for {filename}"

{csvpath-0.0.608 → csvpath-0.0.612}/csvpath/csvpaths.py

@@ -443,6 +443,7 @@ Cache: {cache}
             path.logger = None
         for _ in self.dynamic_csvpath_error_listeners:
             path.error_manager.add_internal_listener(_)
+        path.run_dir = self._last_run_dir
         return path
 
     def stop_all(self) -> None:  # pragma: no cover
@@ -526,6 +527,17 @@ Cache: {cache}
         """@private
         at this time we do not recommend reusing CsvPaths, but it is doable
         you should clean before reuse unless you want to accumulate results."""
+        #
+        # adding this Config() refresh breaks several tests, indicating -- i think --
+        # mainly that 1) we have some iffy tests that don't act like real use cases
+        # and 2) we sorta say that CsvPaths shouldn't be reused, but sorta act like
+        # we solved that so it can be, but aren't fully safe yet or at least haven't
+        # proved it yet.
+        #
+        # regardless, for now, don't reup the Config.
+        #
+        # self._config = Config()
+        #
         self._set_managers()
         self.clear_run_coordination()
         self._errors = []

{csvpath-0.0.608 → csvpath-0.0.612}/csvpath/managers/files/file_describer.py

@@ -5,12 +5,12 @@ from csvpath.util.nos import Nos
 from csvpath.util.file_writers import DataFileWriter
 from csvpath.util.file_readers import DataFileReader
 from csvpath.util.references.reference_parser import ReferenceParser
+from .file_descriptor import ServerConfig, Config
 
 NamedFileName = NewType("NamedFileName", str)
 
 
 class NamedFileDescriber:
-
     README = "README.md"
     DEFAULT_README = "# Named-File Documentation\n\n"
     JSON_FILE = "definition.json"
@@ -30,6 +30,12 @@ class NamedFileDescriber:
 
     # ========== JSON ============
 
+    def get_config(self, name: NamedFileName) -> Config:
+        return Config.model_validate(self.get_json(name))
+
+    def store_config(self, name: NamedFileName, config: Config) -> None:
+        self.store_json(name, config.model_dump(exclude_none=True))
+
     def store_json(self, name: NamedFileName, j: dict) -> None:
         if name is None:
             raise ValueError("Name cannot be None")
@@ -136,3 +142,35 @@ class NamedFileDescriber:
         else:
             j[self.ON_ARRIVAL] = on
         self.store_json(name, j)
+
+    # ========== sources ============
+
+    def add_server(
+        self, name: NamedFileName, servername: str, server: ServerConfig
+    ) -> None:
+        t = self.get_sources(name)
+        if t is None:
+            t = {}
+        t[servername] = server
+        self.store_sources(name, t)
+
+    def remove_server(self, name: NamedFileName, servername: str) -> None:
+        t = self.get_sources(name)
+        if t is None or servername not in t:
+            raise ValueError(f"{name} does not have server {servername}")
+        del t[servername]
+        self.store_sources(name, t)
+
+    def store_sources(self, name: NamedFileName, t: dict[str, ServerConfig]) -> None:
+        if t is None:
+            t = {}
+        config = self.get_config(name)
+        config.sources = t
+        self.store_config(name, config)
+
+    def get_sources(self, name: NamedFileName) -> dict[str, ServerConfig]:
+        config = self.get_config(name)
+        t = config.sources
+        if t is None:
+            return {}
+        return t

csvpath-0.0.612/csvpath/managers/files/file_descriptor.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+from typing import Optional
+from pydantic import BaseModel, Field
+
+from csvpath.managers.server_config import ServerConfig
+
+"""
+class ServerConfig(BaseModel):
+    address: str = Field(description="Hostname or IP address of the server.")
+    port: int = Field(description="Port number; coerce from string if needed.", default=None)
+    username: str = Field(description="Login username (may be an env-var name).")
+    password: str = Field(description="Login password (may be an env-var name).")
+
+    @field_validator("port", mode="before")
+    @classmethod
+    def coerce_port(cls, v: object) -> int:
+        try:
+            return int(v)
+        except (TypeError, ValueError) as e:
+            raise ValueError(
+                f"port must be an integer or a numeric string, got {v!r}"
+            ) from e
+
+    model_config = {"extra": "forbid"}
+"""
+
+
+class OnArrival(BaseModel):
+    named_paths_group: Optional[str] = Field(
+        default=None,
+        description="The named-paths group to associate with this file.",
+    )
+    run_method: Optional[str] = Field(
+        default=None,
+        description="The method to invoke on arrival (e.g. 'collect_paths').",
+    )
+
+    model_config = {"extra": "allow"}
+
+
+class Config(BaseModel):
+    """
+    contents of a named-file's definition.json.
+    e.g.:
+        {
+            "template": ":0/:3/my/:filename",
+            "on_arrival": {
+                "named_paths_group": "order validations",
+                "run_method": "collect_paths"
+            },
+            sources: {
+                "a": {
+                    "address":"localhost",
+                    "port":22,
+                    "username":"foo",
+                    "password":"bar"
+                }
+            }
+        }
+    """
+
+    template: Optional[str] = Field(
+        default=None,
+        description="Path template string for this named file.",
+    )
+    on_arrival: Optional[OnArrival] = None
+    sources: Optional[dict[str, ServerConfig]] = None
+
+    model_config = {"extra": "allow"}