csvpath 0.0.490__tar.gz → 0.0.492__tar.gz

{csvpath-0.0.490 → csvpath-0.0.492}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: csvpath
-Version: 0.0.490
+Version: 0.0.492
 Summary: A declarative language for validating CSV, Excel, and other tabular data files
 Author: David Kershaw
 Author-email: dk107dk@hotmail.com
@@ -23,6 +23,7 @@ Classifier: Topic :: Text Processing
 Classifier: Topic :: Utilities
 Provides-Extra: pandas
 Provides-Extra: smartopen
+Requires-Dist: bullet (>=2.2.0,<3.0.0)
 Requires-Dist: inflect (>=7.3.1,<8.0.0)
 Requires-Dist: jinja2 (>=3.1.4,<4.0.0)
 Requires-Dist: lark (>=1.2.2,<2.0.0)

csvpath-0.0.492/csvpath/cli/__init__.py

@@ -0,0 +1,5 @@
+""" this is the main entry point for the CsvPath library """
+
+from csvpath.cli.cli import Cli
+
+__all__ = ["Cli"]

csvpath-0.0.492/csvpath/cli/cli.py

@@ -0,0 +1,197 @@
+from csvpath import CsvPath, CsvPaths
+import sys
+import os
+import time
+from bullet import Bullet
+from .drill_down import DrillDown
+
+
+class Cli:
+    def __init__(self):
+        self._csvpaths = CsvPaths()
+
+    def clear(self):
+        print(chr(27) + "[2J")
+
+    def pause(self):
+        time.sleep(1.2)
+
+    def short_pause(self):
+        time.sleep(0.5)
+
+    ITALIC = "\033[3m"
+    SIDEBAR_COLOR = "\033[36m"
+    REVERT = "\033[0m"
+    STOP_HERE = f"{SIDEBAR_COLOR}{ITALIC}... done picking dir{REVERT}"
+    CANCEL = f"{SIDEBAR_COLOR}{ITALIC}... cancel{REVERT}"
+
+    def _return_to_cont(self):
+        print(
+            f"\n{Cli.SIDEBAR_COLOR}{Cli.ITALIC}... Hit return to continue{Cli.REVERT}\n"
+        )
+        self._input("")
+
+    def _response(self, text: str) -> None:
+        sys.stdout.write(f"\u001b[30;1m{text}{Cli.REVERT}\n")
+
+    def _action(self, text: str) -> None:
+        sys.stdout.write(f"\033[36m{text}{Cli.REVERT}\n")
+
+    def _input(self, prompt: str) -> str:
+        try:
+            response = input(f"{prompt}\033[93m")
+            sys.stdout.write(Cli.REVERT)
+            return response.strip()
+        except KeyboardInterrupt:
+            return "cancel"
+
+    def end(self) -> None:
+        print(chr(27) + "[2J")
+
+    def ask(self, choices: list[str]) -> str:
+        self.clear()
+        b = Bullet(bullet=" > ", choices=choices)
+        t = b.launch()
+        return t
+
+    def loop(self):
+        while True:
+            t = None
+            try:
+                self.clear()
+                b = Bullet(
+                    bullet=" > ",
+                    choices=[
+                        "named-files",
+                        "named-paths",
+                        "named-results",
+                        "run",
+                        "quit",
+                    ],
+                )
+                t = b.launch()
+            except KeyboardInterrupt:
+                self.end()
+                return
+            t = self._do(t)
+            if t == "quit":
+                self.end()
+                return
+
+    def _do(self, t: str) -> str | None:
+        try:
+            if t == "quit":
+                return t
+            if t == "run":
+                self.run()
+            if t == "named-files":
+                self._files()
+            if t == "named-paths":
+                self._paths()
+            if t == "named-results":
+                self._results()
+        except KeyboardInterrupt:
+            return "quit"
+        except Exception as e:
+            print(f"Error: {e}")
+            self._return_to_cont()
+
+    def _files(self) -> None:
+        choices = ["add named-file", "list named-files", "cancel"]
+        t = self.ask(choices)
+        if t == "add named-file":
+            DrillDown(self).name_file()
+        if t == "list named-files":
+            self.list_named_files()
+
+    def _paths(self) -> None:
+        choices = ["add named-paths", "list named-paths", "cancel"]
+        t = self.ask(choices)
+        if t == "add named-paths":
+            DrillDown(self).name_paths()
+        if t == "list named-paths":
+            self.list_named_paths()
+        if t == "cancel":
+            return
+
+    def _results(self) -> None:
+        choices = ["open named-result", "list named-results", "cancel"]
+        t = self.ask(choices)
+        if t == "open named-result":
+            self.open_named_result()
+        if t == "list named-results":
+            self.list_named_results()
+
+    def list_named_results(self):
+        self.clear()
+        names = self._csvpaths.results_manager.list_named_results()
+        print(f"{len(names)} named-results names:")
+        for n in names:
+            self._response(f"   {n}")
+        self._return_to_cont()
+
+    def open_named_result(self):
+        self.clear()
+        try:
+            names = self._csvpaths.results_manager.list_named_results()
+            print(f"{len(names)} named-results names:")
+            for n in names:
+                self._response(f"   {n}")
+            t = self._input("Open which? ")
+            t = f"{self._csvpaths.config.archive_path}{os.sep}{t}"
+            self._action(f"Opening results at {t}...")
+            self.short_pause()
+            c = f"open {t}"
+            os.system(c)
+        except Exception as e:
+            print(e)
+
+    def list_named_paths(self):
+        self.clear()
+        names = self._csvpaths.paths_manager.named_paths_names
+        names.sort()
+        print(f"{len(names)} named-paths names:")
+        for n in names:
+            self._response(f"   {n}")
+        self._return_to_cont()
+
+    def list_named_files(self):
+        self.clear()
+        names = self._csvpaths.file_manager.named_file_names
+        names.sort()
+        print(f"{len(names)} named-file names:")
+        for n in names:
+            self._response(f"   {n}")
+        self._return_to_cont()
+
+    def run(self):
+        self.clear()
+        print("What named-file? ")
+        files = self._csvpaths.file_manager.named_file_names
+        cli = Bullet(bullet=" > ", choices=files)
+        file = cli.launch()
+        self.clear()
+        print("What named-paths? ")
+        allpaths = self._csvpaths.paths_manager.named_paths_names
+        cli = Bullet(bullet=" > ", choices=allpaths)
+        paths = cli.launch()
+        self.clear()
+        print("What method? ")
+        cli = Bullet(bullet=" > ", choices=["collect", "fast forward"])
+        method = cli.launch()
+        self.clear()
+        self._action(f"Running {paths} against {file} using {method}\n")
+        self.pause()
+        if method == "collect":
+            self._csvpaths.collect_paths(filename=file, pathsname=paths)
+        else:
+            self._csvpaths.fast_forward_paths(filename=file, pathsname=paths)
+
+
+def run():
+    cli = Cli()
+    cli.loop()
+
+
+if __name__ == "__main__":
+    run()

csvpath-0.0.492/csvpath/cli/drill_down.py

@@ -0,0 +1,185 @@
+from csvpath import CsvPath, CsvPaths
+import sys
+import os
+from bullet import Bullet
+
+
+class DrillDown:
+    def __init__(self, cli):
+        self._cli = cli
+
+    # ============================
+    # File
+    # ============================
+
+    def name_file(self):
+        #
+        # get name
+        #
+        self._cli.clear()
+        name = self._cli._input("Named-file name? ")
+        #
+        # get path
+        #
+        t = self._get_add_type()
+        p = self._get_path(t, self._cli._csvpaths.config.csv_file_extensions)
+        if p is False:
+            return
+        #
+        # do the add
+        #
+        self._cli.clear()
+        self._cli._action(f"Adding: {p}\n")
+        self._cli.pause()
+        if t == "file":
+            self._cli._csvpaths.file_manager.add_named_file(name=name, path=p)
+        elif t == "dir":
+            self._cli._csvpaths.file_manager.add_named_files_from_dir(dirname=p)
+        else:
+            self._cli._csvpaths.file_manager.set_named_files_from_json(
+                name=name, filename=p
+            )
+
+    # ============================
+    # Paths
+    # ============================
+
+    def name_paths(self):
+        #
+        # get name
+        #
+        self._cli.clear()
+        name = self._cli._input("Named-paths name? ")
+        #
+        # get path
+        #
+        t = self._get_add_type()
+        exts = self._cli._csvpaths.config.csvpath_file_extensions
+        p = self._get_path(t, exts)
+        if p is False:
+            return
+        #
+        # do the add
+        #
+        self._cli.clear()
+        self._cli._action(f"Adding: {p}\n")
+        self._cli.pause()
+        if t == "file":
+            self._cli._csvpaths.paths_manager.add_named_paths(name=name, from_file=p)
+        elif t == "dir":
+            self._cli._csvpaths.paths_manager.add_named_paths(name=name, from_dir=p)
+        else:
+            self._cli._csvpaths.paths_manager.add_named_paths(name=name, from_json=p)
+
+    # ============================
+    # Utilities
+    # ============================
+
+    def _get_path(self, t: str, extensions: list[str]) -> str:
+        dir_only = t == "dir"
+        p = "."
+        if t == "json":
+            extensions.append("")
+            extensions.append("json")
+        elif t == "file":
+            extensions.append("")
+        while p is not None and p != "" and not os.path.isfile(p):
+            self._cli.clear()
+            self._cli._action(f"{p}\n")
+            p = self._drill_down(
+                path=p,
+                json=True if t == "json" else False,
+                extensions=extensions,
+                dir_only=dir_only,
+            )
+            if isinstance(p, tuple) and p[1] is True:
+                p = p[0]
+                break
+            if isinstance(p, tuple) and p[1] is False:
+                p = False
+                break
+        return p
+
+    def _get_add_type(self) -> str:
+        self._cli.clear()
+        t = None
+        cli = Bullet(bullet=" > ", choices=["dir", "file", "json"])
+        t = cli.launch()
+        return t
+
+    def _drill_down(self, *, path, extensions, json=False, dir_only=False) -> str:
+        names = os.listdir(path)
+        names = self._filter_hidden(names)
+        if dir_only:
+            names = self._filter_dirs_only(path, names)
+            names.sort()
+        else:
+            names = self._filter_extensions(path, names, extensions)
+            names.sort()
+        names = self._decorate(path, names)
+        cli = Bullet(bullet=" > ", choices=names)
+        t = cli.launch()
+        if t == self._cli.STOP_HERE:
+            return (path, True)
+        if t == self._cli.CANCEL:
+            return (path, False)
+        if t.startswith("📂 ") or t.startswith("📄 "):
+            t = t[2:]
+        return os.path.join(path, t)
+
+    def _decorate(self, path, names) -> list[str]:
+        ns = []
+        for n in names:
+            if n == self._cli.STOP_HERE:
+                pass
+            elif os.path.isfile(os.path.join(path, n)):
+                n = f"📄 {n}"
+            else:
+                n = f"📂 {n}"
+            ns.append(n)
+        ns.append(self._cli.STOP_HERE)
+        ns.append(self._cli.CANCEL)
+        return ns
+
+    def _filter_hidden(self, names) -> list[str]:
+        if len(names) == 0:
+            return []
+        names = [n for n in names if not n[0] == "."]
+        return names
+
+    def _filter_files_only(self, path, names) -> list[str]:
+        if len(names) == 0:
+            return []
+        ns = []
+        for n in names:
+            if os.path.isfile(os.path.join(path, n)):
+                ns.append(n)
+        return ns
+
+    def _filter_dirs_only(self, path, names) -> list[str]:
+        if len(names) == 0:
+            return []
+        ns = []
+        for n in names:
+            if not os.path.isfile(os.path.join(path, n)):
+                ns.append(n)
+        return ns
+
+    def _filter_extensions(self, path, names, extensions) -> list[str]:
+        if len(names) == 0:
+            return []
+        if len(extensions) == 0:
+            return []
+        ns = []
+        for n in names:
+            ext = self._ext_if(n)
+            if ext in extensions:
+                ns.append(n)
+        return ns
+
+    def _ext_if(self, name) -> str:
+        i = name.rfind(".")
+        if i == -1:
+            return ""
+        ext = name[i + 1 :]
+        return ext

{csvpath-0.0.490 → csvpath-0.0.492}/csvpath/managers/csvpaths_manager.py

@@ -2,6 +2,7 @@
 from typing import Dict, List
 import os
 import json
+import shutil
 from typing import NewType
 from json import JSONDecodeError
 from abc import ABC, abstractmethod
@@ -82,17 +83,23 @@ class CsvPathsManager(ABC):
         pass  # pragma: no cover
 
     @abstractmethod
-    def number_of_named_paths(self) -> bool:  # pylint: disable=C0116
-        pass  # pragma: no cover
+    def total_named_paths(self) -> int:
+        pass
+
+    @abstractmethod
+    def number_of_named_paths(self, name: str) -> int:
+        pass
+
+    @property
+    @abstractmethod
+    def named_paths_names(self) -> list[str]:
+        pass
 
 
 class PathsManager(CsvPathsManager):  # pylint: disable=C0115, C0116
     MARKER: str = "---- CSVPATH ----"
 
     def __init__(self, *, csvpaths, named_paths=None):
-        if named_paths is None:
-            named_paths = {}
-        self.named_paths = named_paths
         self.csvpaths = csvpaths
         self._registrar = PathsRegistrar(self.csvpaths.config)
 
@@ -103,10 +110,11 @@ class PathsManager(CsvPathsManager):  # pylint: disable=C0115, C0116
     def set_named_paths(self, np: Dict[str, List[str]]) -> None:
         for name in np:
             if not isinstance(np[name], list):
-                ie = InputException("Named-path names must key a list of csvpath")
+                ie = InputException(
+                    f"Named-paths name {name} must key a list of csvpath"
+                )
                 ErrorHandler(csvpaths=self.csvpaths).handle_error(ie)
                 return
-        self.named_paths = np
         for k, v in np.items():
             self.add_named_paths(name=k, paths=v)
         self.csvpaths.logger.info(
@@ -139,6 +147,10 @@ class PathsManager(CsvPathsManager):  # pylint: disable=C0115, C0116
 
     def add_named_paths_from_file(self, *, name: str, file_path: str) -> None:
         self.csvpaths.logger.debug("Reading csvpaths file at %s", file_path)
+        _ = self._get_csvpaths_from_file(file_path)
+        self.add_named_paths(name=name, paths=_)
+
+    def _get_csvpaths_from_file(self, file_path: str) -> list[str]:
         with open(file_path, "r", encoding="utf-8") as f:
             cp = f.read()
             _ = [
@@ -147,7 +159,7 @@ class PathsManager(CsvPathsManager):  # pylint: disable=C0115, C0116
                 if apath.strip() != ""
             ]
             self.csvpaths.logger.debug("Found %s csvpaths in file", len(_))
-            self.add_named_paths(name=name, paths=_)
+            return _
 
     def add_named_paths_from_json(self, file_path: str) -> None:
         try:
@@ -158,8 +170,12 @@ class PathsManager(CsvPathsManager):  # pylint: disable=C0115, C0116
                 for k in j:
                     self.registrar.store_json_paths_file(k, file_path)
                     v = j[k]
+                    paths = []
                     for f in v:
-                        self.add_named_paths_from_file(name=k, file_path=f)
+                        _ = self._get_csvpaths_from_file(f)
+                        paths += _
+                    self.add_named_paths(name=k, paths=paths)
+
         except (OSError, ValueError, TypeError, JSONDecodeError) as ex:
             self.csvpaths.logger.error(f"Error: cannot load {file_path}: {ex}")
             ErrorHandler(csvpaths=self.csvpaths).handle_error(ex)
@@ -188,34 +204,10 @@ class PathsManager(CsvPathsManager):  # pylint: disable=C0115, C0116
             )
             ErrorHandler(csvpaths=self.csvpaths).handle_error(ie)
         self.csvpaths.logger.debug("Adding csvpaths to named-paths group %s", name)
-        #
-        # assure the name-paths name folder
-        # create an-all-in-one file
-        # assure a manifest
-        # write the fingerprint and timestamp in manifest
-        # return path to all-in-one file
-        #
-        if name in self.named_paths:
-            for p in paths:
-                if p in self.named_paths[name]:
-                    self.csvpaths.logger.debug(
-                        "csvpaths %s already exists in named-paths group %s", p, name
-                    )
-                    pass
-                else:
-                    self.csvpaths.logger.debug("Adding %s to %s", p, name)
-                    self.named_paths[name].append(p)
-                    self.registrar.register_named_paths(
-                        name=name, paths=self.named_paths[name]
-                    )
-        else:
-            for _ in paths:
-                self.csvpaths.logger.debug("Adding %s to %s", _, name)
-            self.registrar.register_named_paths(name=name, paths=paths)
-            self.named_paths[name] = paths
+        for _ in paths:
+            self.csvpaths.logger.debug("Adding %s to %s", _, name)
+        self.registrar.register_named_paths(name=name, paths=paths)
 
-    #
-    # ========================
     #
     # adding ref handling for the form: $many.csvpaths.food
     # which is equiv to: many#food
@@ -234,16 +226,15 @@ class PathsManager(CsvPathsManager):  # pylint: disable=C0115, C0116
             identity = ref.name_one
         else:
             npn, identity = self._paths_name_path(name)
-        self.load_named_paths_name_if(name)
+        if identity is None and self.has_named_paths(npn):
+            ret = self._get_named_paths(npn)
+        elif identity is not None and identity.find(":") == -1:
+            ret = [self._find_one(npn, identity)]
         #
         # we need to be able to grab paths up to and starting from like this:
         #   $many.csvpaths.food:to
         #   $many.csvpaths.food:from
         #
-        if identity is None and npn in self.named_paths:
-            ret = self.named_paths[npn]
-        elif identity is not None and identity.find(":") == -1:
-            ret = [self._find_one(npn, identity)]
         elif identity is not None:
             i = identity.find(":")
             directive = identity[i:]
@@ -256,28 +247,23 @@ class PathsManager(CsvPathsManager):  # pylint: disable=C0115, C0116
                 raise InputException(
                     f"Reference directive must be :to or :from, not {directive}"
                 )
-        #
-        # if not in self.named_paths we would return None here
-        # need to instead look to the named_paths dir to see what
-        # has been registered
-        #
         return ret
 
-    def load_named_paths_name_if(self, name: str) -> None:
-        if name not in self.named_paths:
-            #
-            # see if we have it in the central dir
-            #
-            path = self.registrar.named_paths_home(name)
-            s = ""
-            if os.path.exists(path):
-                with open(
-                    os.path.join(path, "group.csvpaths"), "r", encoding="utf-8"
-                ) as file:
-                    s = file.read()
-            cs = s.split("---- CSVPATH ----")
-            cs = [s for s in cs if s.strip() != ""]
-            self.named_paths[name] = cs
+    def _get_named_paths(self, name: str) -> None:
+        if not self.has_named_paths(name):
+            return None
+        s = ""
+        path = self.registrar.named_paths_home(name)
+        grp = os.path.join(path, "group.csvpaths")
+        if os.path.exists(grp):
+            with open(grp, "r", encoding="utf-8") as file:
+                s = file.read()
+        cs = s.split("---- CSVPATH ----")
+        cs = [s for s in cs if s.strip() != ""]
+        # if someone put a new group.csvpaths file by hand we want to
+        # capture its fingerprint for future reference.
+        self.registrar.update_manifest_if(name=name)
+        return cs
 
     def _paths_name_path(self, pathsname) -> tuple[NamedPathsName, Identity]:
         specificpath = None
@@ -326,23 +312,32 @@ class PathsManager(CsvPathsManager):  # pylint: disable=C0115, C0116
             f"Path identified as '{identity}' must be in the group identitied as '{npn}'"
         )
 
-    #
-    #
-    # ========================
-    #
-    #
+    @property
+    def named_paths_names(self) -> list[str]:
+        path = self.registrar.named_paths_dir
+        names = [n for n in os.listdir(path) if not n.startswith(".")]
+        return names
 
     def remove_named_paths(self, name: str) -> None:
-        if name in self.named_paths:
-            del self.named_paths[name]
-        else:
-            raise InputException("{name} not found")
+        if not self.has_named_paths(name):
+            raise InputException(f"Named-paths name {name} not found")
+        home = self.registrar.named_paths_home(name)
+        shutil.rmtree(home)
+
+    def remove_all_named_paths(self) -> None:
+        names = self.named_paths_names
+        for name in names:
+            self.remove_named_paths(name)
 
     def has_named_paths(self, name: str) -> bool:
-        return self.get_named_paths(name)
+        path = os.path.join(self.registrar.named_paths_dir, name)
+        return os.path.exists(path)
+
+    def number_of_named_paths(self, name: str) -> int:
+        return len(self._get_named_paths(name))
 
-    def number_of_named_paths(self) -> bool:
-        return len(self.named_paths)  # pragma: no cover
+    def total_named_paths(self) -> bool:
+        return len(self.named_paths_names)  # pragma: no cover
 
     def _name_from_name_part(self, name):
         i = name.rfind(".")

{csvpath-0.0.490 → csvpath-0.0.492}/csvpath/managers/file_manager.py

@@ -56,6 +56,16 @@ class CsvPathsFileManager(ABC):
     def get_original_headers(self, filename: str) -> List[str]:
         """gets the cached headers"""
 
+    @property
+    @abstractmethod
+    def named_files_count(self) -> int:
+        pass
+
+    @property
+    @abstractmethod
+    def named_file_names(self) -> list:
+        pass
+
 
 class FileManager(CsvPathsFileManager):  # pylint: disable=C0115
     def __init__(self, *, named_files: Dict[str, str] = None, csvpaths=None):

{csvpath-0.0.490 → csvpath-0.0.492}/csvpath/managers/file_registrar.py

@@ -157,7 +157,6 @@ class FileRegistrar:
         #
         # append the metadata
         #
-        print(f"registarar: rpath: {rpath}, path:{path}, mpath: {mpath}")
         self.update_manifest(
             manifestpath=mpath, regpath=rpath, sourcepath=path, fingerprint=h, mark=mark
         )