csvpath 0.0.527__tar.gz → 0.0.528__tar.gz

{csvpath-0.0.527 → csvpath-0.0.528}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: csvpath
-Version: 0.0.527
+Version: 0.0.528
 Summary: A data preboarding framework for managing and validating CSV, Excel, and other tabular data files using a Collect, Store, Validate, Publish pattern to create a trusted publisher for downstream data consumers.
 Author: David Kershaw
 Author-email: dk107dk@hotmail.com

{csvpath-0.0.527 → csvpath-0.0.528}/csvpath/csvpath.py

@@ -1314,10 +1314,10 @@ class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
             uc = self.csvpaths.config.get(section="cache", name="use_cache")
             use_cache = uc is None or uc.strip().lower() != "no"
         if self.csvpaths and use_cache is True:
-            self.line_monitor = self.csvpaths.file_manager.cacher.get_new_line_monitor(
+            self.line_monitor = self.csvpaths.file_manager.lines_and_headers_cacher.get_new_line_monitor(
                 self.scanner.filename
             )
-            self.headers = self.csvpaths.file_manager.cacher.get_original_headers(
+            self.headers = self.csvpaths.file_manager.lines_and_headers_cacher.get_original_headers(
                 self.scanner.filename
             )
         else:

{csvpath-0.0.527 → csvpath-0.0.528}/csvpath/managers/files/file_manager.py

@@ -10,7 +10,7 @@ from csvpath.util.exceptions import InputException, FileException
 from csvpath.util.nos import Nos
 from csvpath.util.box import Box
 from .file_registrar import FileRegistrar
-from .file_cacher import FileCacher
+from .lines_and_headers_cacher import LinesAndHeadersCacher
 from .file_metadata import FileMetadata
 
 NamedFileName = NewType("NamedFileName", str)
@@ -25,7 +25,10 @@ class FileManager:
         self._csvpaths = csvpaths
         self.registrar = FileRegistrar(csvpaths)
         """@private"""
-        self.cacher = FileCacher(csvpaths)
+        #
+        # used by csvpath direct access
+        #
+        self.lines_and_headers_cacher = LinesAndHeadersCacher(csvpaths)
         """@private"""
         self._nos = None
 
@@ -81,18 +84,27 @@ class FileManager:
             nos = self.nos
             nos.path = path
             if nos.exists():
+                m = self.registrar.get_manifest(path)
+                return m["uuid"]
+                """
                 with DataFileReader(path) as reader:
                     m = json.load(reader.source)
                     return m["uuid"]
+                """
         else:
             path = self.named_file_home(name)
             path = os.path.join(path, "manifest.json")
             nos = self.nos
             nos.path = path
             if nos.exists():
+                m = self.registrar.get_manifest(path)
+                return m[len(m) - 1]["uuid"]
+
+                """
                 with DataFileReader(path) as reader:
                     m = json.load(reader.source)
                     return m[len(m) - 1]["uuid"]
+                """
         raise ValueError(f"No manifest for file named {name}")
 
     #
@@ -114,34 +126,11 @@ class FileManager:
             # up in multiple ways.
             #
             return ""
-        #
-        # added
-        #
-        home = None
-        #
-        # this will never happen with the addition of the / test above. :/
-        #
-        """
-        if name.startswith("/"):
-            home = name
-        else:
-            #
-            # done add
-            #
-            home = os.path.join(self.named_files_dir, name)
-        """
         home = os.path.join(self.named_files_dir, name)
-        #
-        # added
-        #
-        # nos = Nos(home)
         nos = self.nos
         nos.path = home
         if nos.isfile():
             home = home[0 : home.rfind(nos.sep)]
-        #
-        # done add
-        #
         return home
 
     def assure_named_file_home(self, name: str) -> str:
@@ -149,9 +138,7 @@ class FileManager:
         home = self.named_file_home(name)
         nos = self.nos
         nos.path = home
-        # nos = Nos(home)
         if not nos.exists():
-            # if not os.path.exists(home):
             nos.makedirs()
         return home
 
@@ -194,8 +181,6 @@ class FileManager:
             nos.path = os.path.join(b, n)
             if not nos.isfile():
                 ns.append(n)
-
-        # ns = [n for n in Nos(b).listdir() if not Nos(os.path.join(b, n)).isfile()]
         return ns
 
     def name_exists(self, name: NamedFileName) -> bool:

{csvpath-0.0.527 → csvpath-0.0.528}/csvpath/managers/files/file_registrar.py

@@ -2,12 +2,14 @@ import os
 import json
 from datetime import datetime
 from csvpath.util.exceptions import InputException, FileException
-from csvpath.util.file_readers import DataFileReader
-from csvpath.util.file_writers import DataFileWriter
+
+# from csvpath.util.file_readers import DataFileReader
+# from csvpath.util.file_writers import DataFileWriter
 from csvpath.util.nos import Nos
 from csvpath.managers.registrar import Registrar
 from csvpath.managers.listener import Listener
 from csvpath.managers.metadata import Metadata
+from csvpath.util.intermediary import Intermediary
 
 
 class FileRegistrar(Registrar, Listener):
@@ -21,6 +23,7 @@ class FileRegistrar(Registrar, Listener):
         self.csvpaths = csvpaths
         self.config = csvpaths.config
         self.type_name = "file"
+        self.intermediary = Intermediary(csvpaths)
 
     def get_fingerprint(self, home) -> str:
         mpath = self.manifest_path(home)
@@ -38,16 +41,26 @@ class FileRegistrar(Registrar, Listener):
         mf = os.path.join(home, "manifest.json")
         nos.path = mf
         if not nos.exists():
+            self.intermediary.put_json(mf, [])
+            """
             with DataFileWriter(path=mf, mode="w") as writer:
                 writer.append("[]")
+            """
         return mf
 
     def get_manifest(self, mpath) -> list:
+        j = self.intermediary.get_json(mpath)
+        if j is None:
+            j = []
+            self.intermediary.put_json(mpath, j)
+        return j
+        """
         try:
             with DataFileReader(mpath) as reader:
                 return json.load(reader.source)
         except FileNotFoundError:
             return []
+        """
 
     def metadata_update(self, mdata: Metadata) -> None:
         path = mdata.origin_path
@@ -68,8 +81,12 @@ class FileRegistrar(Registrar, Listener):
             mani["mark"] = mark
         jdata = self.get_manifest(manifest_path)
         jdata.append(mani)
+        # self.intermediary.clear(manifest_path)
+        self.intermediary.put_json(manifest_path, jdata)
+        """
         with DataFileWriter(path=manifest_path, mode="w") as writer:
             json.dump(jdata, writer.sink, indent=2)
+        """
         #
         # drop update into an all-inputs/files record here?
         #
@@ -146,6 +163,22 @@ class FileRegistrar(Registrar, Listener):
 
     def registered_file(self, home: str) -> str:
         mpath = self.manifest_path(home)
+        mdata = self.get_manifest(mpath)
+        if mdata is None or len(mdata) == 0:
+            raise InputException(f"Manifest for {home} at {mpath} is empty")
+        m = mdata[len(mdata) - 1]
+        if "file" not in m:
+            raise ValueError(
+                "File path cannot be None. Check your config file and named-files."
+            )
+        path = m["file"]
+        mark = None
+        if "mark" in m:
+            mark = m["mark"]
+        if mark is not None:
+            path = f"{path}#{mark}"
+        return path
+        """
         with DataFileReader(mpath) as reader:
             mdata = json.load(reader.source)
             if mdata is None or len(mdata) == 0:
@@ -162,3 +195,4 @@ class FileRegistrar(Registrar, Listener):
             if mark is not None:
                 path = f"{path}#{mark}"
             return path
+        """

{csvpath-0.0.527 → csvpath-0.0.528}/csvpath/managers/files/files_listener.py

@@ -8,6 +8,10 @@ from csvpath.util.nos import Nos
 from csvpath.managers.listener import Listener
 from csvpath.managers.metadata import Metadata
 
+#
+# TODO: not using intermediary
+#
+
 
 class FilesListener(Listener):  # Registrar,
     """@private

csvpath-0.0.527/csvpath/managers/files/file_cacher.py → csvpath-0.0.528/csvpath/managers/files/lines_and_headers_cacher.py

@@ -6,7 +6,7 @@ from csvpath.util.cache import Cache
 from csvpath.util.exceptions import InputException, FileException
 
 
-class FileCacher:
+class LinesAndHeadersCacher:
     """@private"""
 
     def __init__(self, csvpaths=None):

{csvpath-0.0.527 → csvpath-0.0.528}/csvpath/managers/paths/paths_registrar.py

@@ -2,12 +2,14 @@ import os
 import json
 from csvpath.util.exceptions import InputException
 from csvpath.util.file_readers import DataFileReader
-from csvpath.util.file_writers import DataFileWriter
+
+# from csvpath.util.file_writers import DataFileWriter
 from csvpath.util.nos import Nos
 from .paths_metadata import PathsMetadata
 from ..listener import Listener
 from ..metadata import Metadata
 from ..registrar import Registrar
+from csvpath.util.intermediary import Intermediary
 
 
 class PathsRegistrar(Registrar, Listener):
@@ -19,6 +21,7 @@ class PathsRegistrar(Registrar, Listener):
         Listener.__init__(self, csvpaths.config)
         self._manager = None
         self.type_name = "paths"
+        self.intermediary = Intermediary(csvpaths)
 
     @property
     def manager(self):
@@ -27,9 +30,13 @@ class PathsRegistrar(Registrar, Listener):
         return self._manager
 
     def get_manifest(self, mpath) -> list:
+        j = self.intermediary.get_json(mpath)
+        return j
+        """
         with DataFileReader(mpath) as file:
             j = json.load(file.source)
             return j
+        """
 
     def register_complete(self, mdata: Metadata) -> None:
         mdata.manifest_path = self.manifest_path(name=mdata.named_paths_name)
@@ -102,8 +109,11 @@ class PathsRegistrar(Registrar, Listener):
             m["uuid"] = mdata.uuid_string
             m["manifest_path"] = mdata.manifest_path
             jdata.append(m)
+            self.intermediary.put_json(mdata.manifest_path, jdata)
+            """
             with DataFileWriter(path=mdata.manifest_path) as file:
                 json.dump(jdata, file.sink, indent=2)
+            """
         else:
             #
             # leave as info so nobody has to dig to see why no update
@@ -117,8 +127,11 @@ class PathsRegistrar(Registrar, Listener):
         nhome = self.manager.named_paths_home(name)
         mf = os.path.join(nhome, "manifest.json")
         if not Nos(mf).exists():
+            self.intermediary.put_json(mf, [])
+            """
             with DataFileWriter(path=mf) as file:
                 file.append("[]")
+            """
         return mf
 
     def _most_recent_fingerprint(self, manifest_path: str) -> str:

csvpath-0.0.528/csvpath/util/intermediary.py

@@ -0,0 +1,84 @@
+# pylint: disable=C0114
+import os
+import json
+from csvpath.util.file_readers import DataFileReader
+from csvpath.util.file_writers import DataFileWriter
+
+
+class Intermediary:
+    HIT_COUNT = 0
+    MISS_COUNT = 0
+    WRITE_COUNT = 0
+
+    def __init__(self, csvpaths) -> None:
+        self._csvpaths = csvpaths
+
+    def get_json(self, path):
+        ...
+
+    def put_json(self, path, j) -> None:
+        ...
+
+    def clear(self, path) -> None:
+        ...
+
+    def __new__(cls, csvpaths):
+        if cls == Intermediary:
+            uc = csvpaths.config.get(section="cache", name="use_cache")
+            if uc and uc.strip().lower() == "no":
+                return NoCacheIntermediary(csvpaths)
+            return CacheIntermediary(csvpaths)
+        else:
+            instance = super().__new__(cls)
+            return instance
+
+
+class CacheIntermediary:
+    def __init__(self, csvpaths) -> None:
+        self._csvpaths = csvpaths
+        self._cache = {}
+
+    def get_json(self, path):
+        if path in self._cache:
+            Intermediary.HIT_COUNT += 1
+            # print(f"Intermediary.hit: {path}: h: {Intermediary.HIT_COUNT}, m: {Intermediary.MISS_COUNT}, w: {Intermediary.WRITE_COUNT}")
+            return self._cache[path]
+        try:
+            Intermediary.MISS_COUNT += 1
+            # print(f"Intermediary.miss: {path}: h: {Intermediary.HIT_COUNT}, m: {Intermediary.MISS_COUNT}, w: {Intermediary.WRITE_COUNT}")
+            with DataFileReader(path) as reader:
+                j = json.load(reader.source)
+                self._cache[path] = j
+                return j
+        except FileNotFoundError:
+            self.put_json(path, [])
+            return []
+
+    def put_json(self, path, j) -> None:
+        self._cache[path] = j
+        Intermediary.WRITE_COUNT += 1
+        # print(f"Intermediary.write: {path}: h: {Intermediary.HIT_COUNT}, m: {Intermediary.MISS_COUNT}, w: {Intermediary.WRITE_COUNT}")
+        with DataFileWriter(path=path, mode="w") as writer:
+            json.dump(j, writer.sink, indent=2)
+
+    def clear(self, path) -> None:
+        del self._cache[path]
+
+
+class NoCacheIntermediary:
+    def __init__(self, csvpaths) -> None:
+        self._csvpaths = csvpaths
+
+    def get_json(self, path):
+        try:
+            with DataFileReader(path) as reader:
+                return json.load(reader.source)
+        except FileNotFoundError:
+            return []
+
+    def put_json(self, path, j) -> None:
+        with DataFileWriter(path=path, mode="w") as writer:
+            json.dump(j, writer.sink, indent=2)
+
+    def clear(self, path) -> None:
+        ...

{csvpath-0.0.527 → csvpath-0.0.528}/csvpath/util/sftp/sftp_data_reader.py

@@ -15,10 +15,17 @@ from .sftp_nos import SftpDo
 
 
 class SftpDataReader(CsvDataReader):
+    # LOAD = 0
+
     def load_if(self) -> None:
         if self.source is None:
             config = Box.STUFF.get(Box.CSVPATHS_CONFIG)
             c = SftpConfig(config)
+            # SftpDataReader.LOAD += 1
+            # print(f"SftpDataReader: load_if: loding: {SftpDataReader.LOAD}")
+            # from csvpath.util.log_utility import LogUtility
+            # LogUtility.log_brief_trace()
+
             self.source = open(
                 self._path,
                 "r",

{csvpath-0.0.527 → csvpath-0.0.528}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "csvpath"
-version = "0.0.527"
+version = "0.0.528"
 description = "A data preboarding framework for managing and validating CSV, Excel, and other tabular data files using a Collect, Store, Validate, Publish pattern to create a trusted publisher for downstream data consumers."
 authors = ["David Kershaw <dk107dk@hotmail.com>"]
 readme = "README.md"