csvpath 0.0.508__tar.gz → 0.0.509__tar.gz

{csvpath-0.0.508 → csvpath-0.0.509}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: csvpath
-Version: 0.0.508
+Version: 0.0.509
 Summary: An edge governance framework for managing and validating CSV, Excel, and other tabular data files
 Author: David Kershaw
 Author-email: dk107dk@hotmail.com

{csvpath-0.0.508 → csvpath-0.0.509}/csvpath/managers/files/file_manager.py

@@ -36,7 +36,37 @@ class FileManager:
     # named-file homes are a dir like: inputs/named_files/March-2024/March-2024.csv
     #
     def named_file_home(self, name: str) -> str:
-        home = os.path.join(self.named_files_dir, name)
+        #
+        # not a named-file name
+        #
+        if name.find("/") > -1:
+            #
+            # this is definitely not what we should be returning. but it is what
+            # works in the new world of remote and fully-qualified local paths.
+            # for now, going with it. the previous implementation was wonky too,
+            # in a different and not visible way, but not good, so this is a step
+            # up in multiple ways.
+            #
+            return ""
+        #
+        # added
+        #
+        home = None
+        if name.startswith("/"):
+            home = name
+        else:
+            #
+            # done add
+            #
+            home = os.path.join(self.named_files_dir, name)
+        #
+        # added
+        #
+        if Nos(home).isfile():
+            home = home[0 : home.rfind(Nos(home).sep)]
+        #
+        # done add
+        #
         return home
 
     def assure_named_file_home(self, name: str) -> str:

{csvpath-0.0.508 → csvpath-0.0.509}/csvpath/managers/integrations/sftp/sftp_sender.py

@@ -7,6 +7,7 @@ from csvpath.managers.results.result import Result
 from csvpath.managers.listener import Listener
 from csvpath.util.nos import Nos
 from csvpath.util.var_utility import VarUtility
+from csvpath.util.file_readers import DataFileReader
 
 
 #
@@ -105,10 +106,24 @@ class SftpSender(Listener, threading.Thread):
                     raise ValueError("File name {file} is not in {files}")
                 if to is None:
                     raise ValueError("File name {file} has no destination")
-                path = f"/Users/davidkershaw/dev/csvpath/{self.metadata.run_home}{sep}{self.result.identity_or_index}{sep}{file}"
+                path = f"{self.metadata.run_home}{sep}{self.result.identity_or_index}{sep}{file}"
                 remote_path = f"{self._target_path}/{to}"
                 self.csvpaths.logger.info("Putting %s to %s", path, remote_path)
-                sftp.put(path, remote_path)
+                #
+                # need to use data reader and stream. if we're
+                # sftping from s3 or something this is going to
+                # be slow
+                #
+                # sftp.put(path, remote_path)
+                try:
+                    sftp.stat(self._target_path)
+                except FileNotFoundError:
+                    sftp.mkdir(self._target_path)
+                with DataFileReader(path) as reader:
+                    flo = reader.source
+                    f = sftp.open(remote_path, "wb")
+                    f.write(flo.read())
+                    f.close()
             #
             # send the original file if we need to. this will always be the normative
             # original, w/o regard to source-mode or by_line chaining or to rewind.

{csvpath-0.0.508 → csvpath-0.0.509}/csvpath/managers/results/result_serializer.py

@@ -24,6 +24,9 @@ class ResultSerializer:
     def save_result(self, result) -> None:
         self.result = result
         runtime_data = {}
+        result.csvpath.csvpaths.logger.debug(
+            "Saving result of %s.%s", result.paths_name, result.identity_or_index
+        )
         RuntimeDataCollector.collect(result.csvpath, runtime_data, local=True)
         runtime_data["run_index"] = result.run_index
         es = []

{csvpath-0.0.508 → csvpath-0.0.509}/csvpath/managers/results/results_registrar.py

@@ -119,16 +119,14 @@ class ResultsRegistrar(Registrar, Listener):
     def _size(self, path) -> str:
         try:
             fi = FileInfo.info(path)
-            return fi["bytes"] if fi and "bytes" in fi else -1
+            return fi["bytes"]
         except FileNotFoundError:
             return 0
 
     def _last_change(self, path) -> str:
         try:
             fi = FileInfo.info(path)
-            if fi and "last_mod" in fi:
-                return fi["last_mod"]
-            return None
+            return fi["last_mod"]
         except FileNotFoundError:
             return -1
 

csvpath-0.0.509/csvpath/util/file_info.py

@@ -0,0 +1,49 @@
+import os
+
+
+class FileInfo:
+    @classmethod
+    def info(cls, path) -> dict[str, str | int | float]:
+        if path is None:
+            raise ValueError("Path cannot be None")
+        try:
+            if path.find("://") > -1:
+                return cls._remote(path)
+            return cls._local(path)
+        except Exception:
+            # this shouldn't happen, but it also shouldn't be the
+            # main source of friction. If we have a wrong path or
+            # an asset we don't know how to get info on, we'll know it
+            # above and can handle it there in a more meaningful
+            # context.
+            return cls._empty()
+
+    @classmethod
+    def _remote(cls, path):
+        return cls._empty()
+
+    @classmethod
+    def _empty(cls) -> dict:
+        return {
+            "mode": "",
+            "device": "",
+            "bytes": -1,
+            "created": None,
+            "last_read": None,
+            "last_mod": None,
+            "flags": None,
+        }
+
+    @classmethod
+    def _local(cls, path):
+        s = os.stat(path)
+        meta = {
+            "mode": s.st_mode,
+            "device": s.st_dev,
+            "bytes": s.st_size,
+            "created": s.st_ctime,
+            "last_read": s.st_atime,
+            "last_mod": s.st_mtime,
+            "flags": s.st_flags,
+        }
+        return meta

{csvpath-0.0.508 → csvpath-0.0.509}/csvpath/util/nos.py

@@ -139,6 +139,12 @@ class S3Do:
     def isfile(self) -> bool:
         bucket, key = S3Utils.path_to_parts(self.path)
         client = boto3.client("s3")
+        #
+        # boto3 uses a deprecated feature. pytest doesn't like it. this is a quick fix.
+        #
+        import warnings
+
+        warnings.filterwarnings(action="ignore", message=r"datetime.datetime.utcnow")
         try:
             client.head_object(Bucket=bucket, Key=key)
         except ClientError as e:

{csvpath-0.0.508 → csvpath-0.0.509}/csvpath/util/s3/s3_data_reader.py

@@ -18,11 +18,14 @@ class S3DataReader(CsvDataReader):
         if self.source is None:
             client = Box.STUFF.get("boto_client")
             if client is None:
+                client = S3Utils.make_client()
+                """
                 session = boto3.Session(
                     aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
                     aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
                 )
                 client = session.client("s3")
+                """
             try:
                 self.source = open(self._path, "r", transport_params={"client": client})
             except DeprecationWarning:

{csvpath-0.0.508 → csvpath-0.0.509}/csvpath/util/s3/s3_data_writer.py

@@ -5,6 +5,7 @@ import boto3
 from smart_open import open
 from ..file_writers import DataFileWriter
 from csvpath.util.box import Box
+from csvpath.util.s3.s3_utils import S3Utils
 
 
 class S3DataWriter(DataFileWriter):
@@ -12,16 +13,18 @@ class S3DataWriter(DataFileWriter):
         if self.sink is None:
             client = Box.STUFF.get("boto_client")
             if client is None:
+                client = S3Utils.make_client()
+                """
                 session = boto3.Session(
                     aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
                     aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
                 )
                 import warnings
-
                 warnings.filterwarnings(
                     action="ignore", message=r"datetime.datetime.utcnow"
                 )
                 client = session.client("s3")
+                """
             self.sink = open(
                 self._path,
                 self._mode,
@@ -35,11 +38,14 @@ class S3DataWriter(DataFileWriter):
         """
         client = Box.STUFF.get("boto_client")
         if client is None:
+            client = S3Utils.make_client()
+            """
             session = boto3.Session(
                 aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
                 aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
             )
             client = session.client("s3")
+            """
         with open(self._path, "wb", transport_params={"client": client}) as file:
             file.write(data.encode("utf-8"))
 

{csvpath-0.0.508 → csvpath-0.0.509}/csvpath/util/s3/s3_utils.py

@@ -1,9 +1,22 @@
+import os
 import boto3
 import uuid
 from botocore.exceptions import ClientError
 
 
 class S3Utils:
+    @classmethod
+    def make_client(cls):
+        import warnings
+
+        warnings.filterwarnings(action="ignore", message=r"datetime.datetime.utcnow")
+        session = boto3.Session(
+            aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
+            aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
+        )
+        client = session.client("s3")
+        return client
+
     @classmethod
     def path_to_parts(self, path) -> tuple[str, str]:
         if path.startswith("s3://"):

{csvpath-0.0.508 → csvpath-0.0.509}/csvpath/util/s3/s3_xlsx_data_reader.py

@@ -6,6 +6,7 @@ from smart_open import open
 from ..file_readers import XlsxDataReader
 from .s3_fingerprinter import S3Fingerprinter
 from csvpath.util.box import Box
+from csvpath.util.s3.s3_utils import S3Utils
 
 
 class S3XlsxDataReader(XlsxDataReader):
@@ -21,11 +22,14 @@ class S3XlsxDataReader(XlsxDataReader):
         if self.source is None:
             client = Box.STUFF.get("boto_client")
             if client is None:
+                client = S3Utils.make_client()
+                """
                 session = boto3.Session(
                     aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
                     aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
                 )
                 client = session.client("s3")
+                """
             try:
                 self.source = open(
                     self._path, "rb", transport_params={"client": client}

{csvpath-0.0.508 → csvpath-0.0.509}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "csvpath"
-version = "0.0.508"
+version = "0.0.509"
 description = "An edge governance framework for managing and validating CSV, Excel, and other tabular data files"
 authors = ["David Kershaw <dk107dk@hotmail.com>"]
 readme = "README.md"

csvpath-0.0.508/csvpath/util/file_info.py

@@ -1,29 +0,0 @@
-import os
-
-
-class FileInfo:
-    @classmethod
-    def info(cls, path) -> dict[str, str | int | float]:
-        if path is None:
-            raise ValueError("Path cannot be None")
-        if path.find("://") > -1:
-            return cls._remote(path)
-        return cls._local(path)
-
-    @classmethod
-    def _remote(cls, path):
-        return {}
-
-    @classmethod
-    def _local(cls, path):
-        s = os.stat(path)
-        meta = {
-            "mode": s.st_mode,
-            "device": s.st_dev,
-            "bytes": s.st_size,
-            "created": s.st_ctime,
-            "last_read": s.st_atime,
-            "last_mod": s.st_mtime,
-            "flags": s.st_flags,
-        }
-        return meta