csvpath 0.0.507__tar.gz → 0.0.508__tar.gz

{csvpath-0.0.507 → csvpath-0.0.508}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: csvpath
-Version: 0.0.507
+Version: 0.0.508
 Summary: An edge governance framework for managing and validating CSV, Excel, and other tabular data files
 Author: David Kershaw
 Author-email: dk107dk@hotmail.com
@@ -23,7 +23,6 @@ Classifier: Topic :: Software Development :: Testing
 Classifier: Topic :: Text Processing
 Classifier: Topic :: Utilities
 Provides-Extra: pandas
-Provides-Extra: smartopen
 Requires-Dist: boto3 (>=1.35.91,<2.0.0)
 Requires-Dist: bullet (>=2.2.0,<3.0.0)
 Requires-Dist: ckanapi (>=4.8,<5.0)
@@ -40,7 +39,7 @@ Requires-Dist: pytest (>=8.3.3,<9.0.0)
 Requires-Dist: python-dateutil (>=2.9.0.post0,<3.0.0)
 Requires-Dist: pytz (>=2024.2,<2025.0)
 Requires-Dist: requests (>=2.32.3,<3.0.0)
-Requires-Dist: smart-open (>=7.1.0,<8.0.0) ; extra == "smartopen"
+Requires-Dist: smart-open (>=7.1.0,<8.0.0)
 Requires-Dist: tabulate (>=0.9.0,<0.10.0)
 Project-URL: Csvpath.org, https://www.csvpath.org
 Project-URL: Github, https://github.com/csvpath/csvpath.git

{csvpath-0.0.507 → csvpath-0.0.508}/csvpath/csvpath.py

@@ -1204,7 +1204,16 @@ class CsvPath(CsvPathPublic, ErrorCollector, Printer):  # pylint: disable=R0902,
                 "Csvpath identified as %s has no filename. Since we could be error handling an exception is not raised.",
                 self.identity,
             )
-        if self.csvpaths:
+        #
+        # there are times, e.g. when using Lambda, when it may be better to
+        # not use a cache. in the case of Lambda the reason is to avoid working
+        # around the read-only filesystem.
+        #
+        use_cache = self.csvpaths is not None
+        if use_cache:
+            uc = self.csvpaths.config.get(section="cache", name="use_cache")
+            use_cache = uc is None or uc.strip().lower() != "no"
+        if self.csvpaths and use_cache is True:
             self.line_monitor = self.csvpaths.file_manager.cacher.get_new_line_monitor(
                 self.scanner.filename
             )

{csvpath-0.0.507 → csvpath-0.0.508}/csvpath/managers/integrations/sftpplus/sftpplus_listener.py

@@ -162,7 +162,7 @@ class SftpPlusListener(Listener, threading.Thread):
                 # land the file at the UUID so that if anything weird we'll only ever
                 # interfere with ourselves.
                 #
-                remote_path = f"{self._account_name}/{self.metadata.uuid_string}.txt"
+                remote_path = f"{self.metadata.uuid_string}.json"
                 self.csvpaths.logger.info("Putting %s to %s", file, remote_path)
                 sftp.putfo(file, remote_path)
                 sftp.close()

{csvpath-0.0.507 → csvpath-0.0.508}/csvpath/managers/integrations/sftpplus/transfer_creator.py

@@ -227,17 +227,16 @@ class SftpPlusTransferCreator:
         return f"{self.csvpaths.config.get(section='sftpplus', name='python_cmd')} "
 
     def _generate_and_place_scripts(self, msg: dict) -> str:
-        path = self._execute_before_script
-        print(f"transfer script path is: {path}")
+        before_script = self._execute_before_script
+        print(f"transfer script path is: {before_script}")
         s = f"""
 #
 # THIS FILE IS GENERATED AT RUNTIME. DO NOT EDIT IT.
 #
-# add named_file_name, filename here (and named_paths_name?)
-{self.python_cmd} {path} "$1"
+{self.python_cmd} {self._execute_before_python_main} "$1"
         """
         print(f"_generate_and_place_scripts: python runner script: {s}")
-        with open(path, "w", encoding="utf-8") as file:
+        with open(before_script, "w", encoding="utf-8") as file:
             file.write(s)
         #
         # do we need to +x the script?
@@ -276,5 +275,5 @@ if __name__ == "__main__":
     h.named_paths_name = "{named_paths_name}"
     h.process_arrival()
 """
-            with open(path, "w", encoding="utf-8") as file:
+            with open(self._execute_before_python_main, "w", encoding="utf-8") as file:
                 file.write(s)

{csvpath-0.0.507 → csvpath-0.0.508}/csvpath/managers/results/result_serializer.py

@@ -7,6 +7,7 @@ from csvpath import CsvPath
 from csvpath.matching.util.runtime_data_collector import RuntimeDataCollector
 from csvpath.util.line_spooler import LineSpooler
 from csvpath.util.file_writers import DataFileWriter
+from csvpath.util.nos import Nos
 
 Simpledata = NewType("Simpledata", Union[None | str | int | float | bool])
 Listdata = NewType("Listdata", list[None | str | int | float | bool])
@@ -158,8 +159,8 @@ class ResultSerializer:
     def get_run_dir(self, *, paths_name, run_time):
         paths_name = self._deref_paths_name(paths_name)
         run_dir = os.path.join(self.base_dir, paths_name)
-        if not os.path.exists(run_dir):
-            os.makedirs(run_dir, exist_ok=True)
+        if not Nos(run_dir).dir_exists():
+            Nos(run_dir).makedirs()
         if not isinstance(run_time, str):
             run_time = self.get_run_dir_name_from_datetime(run_time)
         run_dir = os.path.join(run_dir, f"{run_time}")
@@ -169,10 +170,10 @@ class ResultSerializer:
         # atm. a server process would namespace each CsvPaths instance
         # to prevent conflicts. if there is a conflict the two runs would
         # overwrite each other. this prevents that.
-        if os.path.exists(run_dir):
+        if Nos(run_dir).dir_exists():
             i = 0
             adir = f"{run_dir}.{i}"
-            while os.path.exists(adir):
+            while Nos(adir).dir_exists():
                 i += 1
                 adir = f"{run_dir}.{i}"
             run_dir = adir
@@ -180,5 +181,6 @@ class ResultSerializer:
 
     def get_instance_dir(self, run_dir, identity) -> str:
         run_dir = os.path.join(run_dir, identity)
-        os.makedirs(run_dir, exist_ok=True)
+        if not Nos(run_dir).exists():
+            Nos(run_dir).makedirs()
         return run_dir

csvpath-0.0.508/csvpath/util/box.py

@@ -0,0 +1,18 @@
+#
+# just a box to put things in. initial use is
+# sharing a boto3 client under "boto_client".
+#
+class Box:
+    STUFF = {}
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, exc_traceback):
+        Box.STUFF = {}
+
+    def add(self, key: str, value) -> None:
+        Box.STUFF[key] = value
+
+    def get(self, key: str):
+        return Box.STUFF.get(key)

{csvpath-0.0.507 → csvpath-0.0.508}/csvpath/util/config.py

@@ -254,6 +254,8 @@ on_unmatched_file_fingerprints = halt
         if self.load:
             if self.archive_path is None or self.archive_path.strip() == "":
                 self.archive_path = "archive"
+            if self.archive_path.strip().lower().startswith("s3://"):
+                return
             if not path.exists(self.archive_path):
                 os.makedirs(self.archive_path)
 
@@ -268,6 +270,8 @@ on_unmatched_file_fingerprints = halt
         if self.load:
             if self.inputs_files_path is None or self.inputs_files_path.strip() == "":
                 self.inputs_files_path = "inputs/named_files"
+            if self.inputs_files_path.strip().lower().startswith("s3://"):
+                return
             if not path.exists(self.inputs_files_path):
                 os.makedirs(self.inputs_files_path)
 
@@ -278,10 +282,15 @@ on_unmatched_file_fingerprints = halt
                 or self.inputs_csvpaths_path.strip() == ""
             ):
                 self.inputs_csvpaths_path = "inputs/named_paths"
+            if self.inputs_csvpaths_path.strip().lower().startswith("s3://"):
+                return
             if not path.exists(self.inputs_csvpaths_path):
                 os.makedirs(self.inputs_csvpaths_path)
 
     def _assure_cache_path(self) -> None:
+        uc = self.get(section="cache", name="use_cache")
+        if uc and uc.strip().lower() == "no":
+            return
         if self.load:
             if self.cache_dir_path is None or self.cache_dir_path.strip() == "":
                 self.cache_dir_path = "cache"

{csvpath-0.0.507 → csvpath-0.0.508}/csvpath/util/s3/s3_data_reader.py

@@ -7,6 +7,7 @@ from smart_open import open
 from ..file_readers import CsvDataReader
 from .s3_utils import S3Utils
 from .s3_fingerprinter import S3Fingerprinter
+from csvpath.util.box import Box
 
 
 #
@@ -15,18 +16,22 @@ from .s3_fingerprinter import S3Fingerprinter
 class S3DataReader(CsvDataReader):
     def load_if(self) -> None:
         if self.source is None:
-            session = boto3.Session(
-                aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
-                aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
-            )
-            try:
-                self.source = open(
-                    self._path, "r", transport_params={"client": session.client("s3")}
+            client = Box.STUFF.get("boto_client")
+            if client is None:
+                session = boto3.Session(
+                    aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
+                    aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
                 )
+                client = session.client("s3")
+            try:
+                self.source = open(self._path, "r", transport_params={"client": client})
             except DeprecationWarning:
                 ...
 
     def next(self) -> list[str]:
+        #
+        # TODO: check if smart-open is in play and open w/o client is correct
+        #
         with open(uri=self._path, mode="r") as file:
             reader = csv.reader(
                 file, delimiter=self._delimiter, quotechar=self._quotechar

{csvpath-0.0.507 → csvpath-0.0.508}/csvpath/util/s3/s3_data_writer.py

@@ -4,25 +4,28 @@ import os
 import boto3
 from smart_open import open
 from ..file_writers import DataFileWriter
+from csvpath.util.box import Box
 
 
 class S3DataWriter(DataFileWriter):
     def load_if(self) -> None:
         if self.sink is None:
-            session = boto3.Session(
-                aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
-                aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
-            )
-            import warnings
-
-            warnings.filterwarnings(
-                action="ignore", message=r"datetime.datetime.utcnow"
-            )
+            client = Box.STUFF.get("boto_client")
+            if client is None:
+                session = boto3.Session(
+                    aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
+                    aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
+                )
+                import warnings
 
+                warnings.filterwarnings(
+                    action="ignore", message=r"datetime.datetime.utcnow"
+                )
+                client = session.client("s3")
             self.sink = open(
                 self._path,
                 self._mode,
-                transport_params={"client": session.client("s3")},
+                transport_params={"client": client},
             )
 
     def write(self, data) -> None:
@@ -30,13 +33,14 @@ class S3DataWriter(DataFileWriter):
         as a context manager for this method. for multiple write
         calls to the same file handle use append().
         """
-        session = boto3.Session(
-            aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
-            aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
-        )
-        with open(
-            self._path, "wb", transport_params={"client": session.client("s3")}
-        ) as file:
+        client = Box.STUFF.get("boto_client")
+        if client is None:
+            session = boto3.Session(
+                aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
+                aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
+            )
+            client = session.client("s3")
+        with open(self._path, "wb", transport_params={"client": client}) as file:
             file.write(data.encode("utf-8"))
 
     def file_info(self) -> dict[str, str | int | float]:

{csvpath-0.0.507 → csvpath-0.0.508}/csvpath/util/s3/s3_xlsx_data_reader.py

@@ -5,6 +5,7 @@ import boto3
 from smart_open import open
 from ..file_readers import XlsxDataReader
 from .s3_fingerprinter import S3Fingerprinter
+from csvpath.util.box import Box
 
 
 class S3XlsxDataReader(XlsxDataReader):
@@ -18,13 +19,16 @@ class S3XlsxDataReader(XlsxDataReader):
 
     def load_if(self) -> None:
         if self.source is None:
-            session = boto3.Session(
-                aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
-                aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
-            )
+            client = Box.STUFF.get("boto_client")
+            if client is None:
+                session = boto3.Session(
+                    aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
+                    aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
+                )
+                client = session.client("s3")
             try:
                 self.source = open(
-                    self._path, "rb", transport_params={"client": session.client("s3")}
+                    self._path, "rb", transport_params={"client": client}
                 )
             except DeprecationWarning:
                 ...

{csvpath-0.0.507 → csvpath-0.0.508}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "csvpath"
-version = "0.0.507"
+version = "0.0.508"
 description = "An edge governance framework for managing and validating CSV, Excel, and other tabular data files"
 authors = ["David Kershaw <dk107dk@hotmail.com>"]
 readme = "README.md"
@@ -21,8 +21,6 @@ include = ["config", "logs", "docs"]
 
 [tool.pytest.ini_options]
 #testpaths = ["tests"]
-#norecursedirs = ["old"]
-#filterwarnings = ["ignore::DeprecationWarning"]
 
 [tool.poetry.urls]
 "Github" = "https://github.com/csvpath/csvpath.git"
@@ -48,13 +46,13 @@ bullet = "^2.2.0"
 pytz = "^2024.2"
 openlineage-python = "^1.25.0"
 ckanapi = "^4.8"
-boto3 = "^1.35.91"
 smart-open = "^7.1.0"
 paramiko = "^3.5.0"
+boto3 = "^1.35.91"
 
 [tool.poetry.extras]
 pandas = ["pandas"]
-smartopen = ["smart-open"]
+#smartopen = ["smart-open"]
 
 [tool.poetry.group.dev.dependencies]
 flake8 = "^7.1.0"