csvpath 0.0.511__tar.gz → 0.0.513__tar.gz

{csvpath-0.0.511 → csvpath-0.0.513}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: csvpath
-Version: 0.0.511
+Version: 0.0.513
 Summary: An edge governance framework for managing and validating CSV, Excel, and other tabular data files
 Author: David Kershaw
 Author-email: dk107dk@hotmail.com
@@ -24,8 +24,8 @@ Classifier: Topic :: Text Processing
 Classifier: Topic :: Utilities
 Provides-Extra: pandas
 Requires-Dist: boto3 (>=1.35.91,<2.0.0)
-Requires-Dist: bullet (>=2.2.0,<3.0.0)
 Requires-Dist: ckanapi (>=4.8,<5.0)
+Requires-Dist: email-validator (>=2.2.0,<3.0.0)
 Requires-Dist: inflect (>=7.4.0,<8.0.0)
 Requires-Dist: jinja2 (>=3.1.4,<4.0.0)
 Requires-Dist: lark (>=1.2.2,<2.0.0)
@@ -35,6 +35,7 @@ Requires-Dist: openlineage-python (>=1.25.0,<2.0.0)
 Requires-Dist: opentelemetry-distro[otlp] (>=0.50b0,<0.51)
 Requires-Dist: paramiko (>=3.5.0,<4.0.0)
 Requires-Dist: ply (>=3.11,<4.0)
+Requires-Dist: prompt-toolkit (>=3.0.50,<4.0.0)
 Requires-Dist: pylightxl (>=1.61,<2.0)
 Requires-Dist: pytest (>=8.3.3,<9.0.0)
 Requires-Dist: python-dateutil (>=2.9.0.post0,<3.0.0)
@@ -42,6 +43,7 @@ Requires-Dist: pytz (>=2024.2,<2025.0)
 Requires-Dist: requests (>=2.32.3,<3.0.0)
 Requires-Dist: smart-open (>=7.1.0,<8.0.0)
 Requires-Dist: tabulate (>=0.9.0,<0.10.0)
+Requires-Dist: validators (>=0.34.0,<0.35.0)
 Project-URL: Csvpath.org, https://www.csvpath.org
 Project-URL: Github, https://github.com/csvpath/csvpath.git
 Description-Content-Type: text/markdown
@@ -72,6 +74,9 @@ CsvPath can stream lineage events to an OpenLineage server, such as the open sou
 <a href='https://peppy-sprite-186812.netlify.app/' >
 <img target='_blank' src="https://github.com/csvpath/csvpath/blob/main/docs/images/marquez-logo-sm.png" alt="Marquez Server"/></a>
 
+DataOps demands observability! Pipe CsvPath events through OpenTelemetry to your APM or observability platform. Read about <a href='https://www.csvpath.org/getting-started/integrations/getting-started-with-csvpath-+-opentelemetry' target="_blank">how to get started here</a>, with an example using Grafana.
+<br/><img target='_blank' src="https://github.com/csvpath/csvpath/blob/main/docs/images/opentelemetry.png" alt="OpenTelemetry Logo"/>
+
 Need to publish validated datasets to a CKAN data portal? <a href="https://www.csvpath.org/getting-started/getting-started-with-csvpath-+-ckan" target="_blank">Read about how CsvPath is integrated with CKAN</a>.
 <a href="https://ckan.org/" target="_blank"><img target='_blank' src="https://github.com/csvpath/csvpath/blob/main/docs/images/ckan-logo-sm.png" alt="CKAN Data Portal"/></a>
 
@@ -125,7 +130,7 @@ CsvPath's goal is to make simple validations almost trivial and more complex sit
 <a name="install"></a>
 # Install
 
-<a href='https://pypi.org/project/csvpath/'>CsvPath is available on PyPi</a>. Install with
+<a href='https://pypi.org/project/csvpath/'>CsvPath is available on PyPi</a>. It has been tested on 3.10, 3.11 and 3.13. Install with
 ```
     pip install csvpath
 ```

{csvpath-0.0.511 → csvpath-0.0.513}/README.md

@@ -24,6 +24,9 @@ CsvPath can stream lineage events to an OpenLineage server, such as the open sou
 <a href='https://peppy-sprite-186812.netlify.app/' >
 <img target='_blank' src="https://github.com/csvpath/csvpath/blob/main/docs/images/marquez-logo-sm.png" alt="Marquez Server"/></a>
 
+DataOps demands observability! Pipe CsvPath events through OpenTelemetry to your APM or observability platform. Read about <a href='https://www.csvpath.org/getting-started/integrations/getting-started-with-csvpath-+-opentelemetry' target="_blank">how to get started here</a>, with an example using Grafana.
+<br/><img target='_blank' src="https://github.com/csvpath/csvpath/blob/main/docs/images/opentelemetry.png" alt="OpenTelemetry Logo"/>
+
 Need to publish validated datasets to a CKAN data portal? <a href="https://www.csvpath.org/getting-started/getting-started-with-csvpath-+-ckan" target="_blank">Read about how CsvPath is integrated with CKAN</a>.
 <a href="https://ckan.org/" target="_blank"><img target='_blank' src="https://github.com/csvpath/csvpath/blob/main/docs/images/ckan-logo-sm.png" alt="CKAN Data Portal"/></a>
 
@@ -77,7 +80,7 @@ CsvPath's goal is to make simple validations almost trivial and more complex sit
 <a name="install"></a>
 # Install
 
-<a href='https://pypi.org/project/csvpath/'>CsvPath is available on PyPi</a>. Install with
+<a href='https://pypi.org/project/csvpath/'>CsvPath is available on PyPi</a>. It has been tested on 3.10, 3.11 and 3.13. Install with
 ```
     pip install csvpath
 ```

{csvpath-0.0.511 → csvpath-0.0.513}/config/config.ini

@@ -30,6 +30,7 @@ groups =
 
 otlp.result = from csvpath.managers.integrations.otlp.otlp_result_listener import OpenTelemetryResultListener
 otlp.results = from csvpath.managers.integrations.otlp.otlp_results_listener import OpenTelemetryResultsListener
+otlp.errors = from csvpath.managers.integrations.otlp.otlp_error_listener import OpenTelemetryErrorListener
 
 # add sftpplus to the list of groups above to automate registration and named-paths group runs on file arrival at an SFTPPlus server
 sftpplus.paths = from csvpath.managers.integrations.sftpplus.sftpplus_listener import SftpPlusListener

{csvpath-0.0.511 → csvpath-0.0.513}/csvpath/cli/cli.py

@@ -2,9 +2,10 @@ import sys
 import os
 import time
 import traceback
-from bullet import Bullet
 from csvpath import CsvPaths
 from .drill_down import DrillDown
+from .select import Select
+from .error import Error
 
 
 class Cli:
@@ -24,7 +25,9 @@ class Cli:
     SIDEBAR_COLOR = "\033[36m"
     REVERT = "\033[0m"
     STOP_HERE = f"{SIDEBAR_COLOR}{ITALIC}... done picking dir{REVERT}"
+    STOP_HERE2 = "👍 pick this dir"
     CANCEL = f"{SIDEBAR_COLOR}{ITALIC}... cancel{REVERT}"
+    CANCEL2 = "← cancel"
 
     def _return_to_cont(self):
         print(
@@ -51,26 +54,28 @@ class Cli:
 
     def ask(self, choices: list[str]) -> str:
         self.clear()
-        b = Bullet(bullet=" > ", choices=choices)
-        t = b.launch()
+        if choices[len(choices) - 1] == Cli.CANCEL:
+            choices[len(choices) - 1] = Cli.CANCEL2
+        if choices[len(choices) - 2] == Cli.STOP_HERE:
+            choices[len(choices) - 2] = Cli.STOP_HERE2
+        cs = [(s, s) for s in choices]
+        t = Select().ask(title="", values=cs, cancel_value="CANCEL")
+        self.clear()
         return t
 
     def loop(self):
         while True:
             t = None
             try:
-                self.clear()
-                b = Bullet(
-                    bullet=" > ",
-                    choices=[
-                        "named-files",
-                        "named-paths",
-                        "named-results",
-                        "run",
-                        "quit",
-                    ],
-                )
-                t = b.launch()
+                choices = [
+                    "named-files",
+                    "named-paths",
+                    "named-results",
+                    "run",
+                    "config",
+                    "quit",
+                ]
+                t = self.ask(choices)
             except KeyboardInterrupt:
                 self.end()
                 return
@@ -91,12 +96,17 @@ class Cli:
                 self._paths()
             if t == "named-results":
                 self._results()
+            if t == "config":
+                self._config()
         except KeyboardInterrupt:
             return "quit"
         except Exception:
             print(traceback.format_exc())
             self._return_to_cont()
 
+    def _config(self) -> None:
+        Error(self).show()
+
     def _files(self) -> None:
         choices = ["add named-file", "list named-files", "cancel"]
         t = self.ask(choices)
@@ -135,8 +145,7 @@ class Cli:
             names = self.csvpaths.results_manager.list_named_results()
             print(f"{len(names)} named-results names:")
             names.append(self.CANCEL)
-            cli = Bullet(bullet=" > ", choices=names)
-            t = cli.launch()
+            t = self.ask(names)
             if t == self.CANCEL:
                 return
             t = f"{self.csvpaths.config.archive_path}{os.sep}{t}"
@@ -169,17 +178,15 @@ class Cli:
         self.clear()
         print("What named-file? ")
         files = self.csvpaths.file_manager.named_file_names
-        cli = Bullet(bullet=" > ", choices=files)
-        file = cli.launch()
+        file = self.ask(files)
         self.clear()
         print("What named-paths? ")
         allpaths = self.csvpaths.paths_manager.named_paths_names
-        cli = Bullet(bullet=" > ", choices=allpaths)
-        paths = cli.launch()
+        paths = self.ask(allpaths)
         self.clear()
+        choices = ["collect", "fast-forward"]
         print("What method? ")
-        cli = Bullet(bullet=" > ", choices=["collect", "fast-forward"])
-        method = cli.launch()
+        method = self.ask(choices)
         self.clear()
         self.action(f"Running {paths} against {file} using {method}\n")
         self.pause()

{csvpath-0.0.511 → csvpath-0.0.513}/csvpath/cli/drill_down.py

@@ -1,5 +1,6 @@
 import os
-from bullet import Bullet
+import traceback
+from .error import Error
 
 
 class DrillDown:
@@ -29,14 +30,30 @@ class DrillDown:
         self._cli.clear()
         self._cli.action(f"Adding: {p}\n")
         self._cli.pause()
-        if t == "file":
-            self._cli.csvpaths.file_manager.add_named_file(name=name, path=p)
-        elif t == "dir":
-            self._cli.csvpaths.file_manager.add_named_files_from_dir(dirname=p)
-        else:
-            self._cli.csvpaths.file_manager.set_named_files_from_json(
-                name=name, filename=p
-            )
+        try:
+            if t == "file":
+                self._cli.csvpaths.file_manager.add_named_file(name=name, path=p)
+            elif t == "dir":
+                self._cli.csvpaths.file_manager.add_named_files_from_dir(dirname=p)
+            else:
+                self._cli.csvpaths.file_manager.set_named_files_from_json(filename=p)
+        except Exception:
+            cfg = None
+            while cfg in [None, "c", "e"]:
+                print("There was an error.")
+                print("Click 'e' and return to print the stack trace. ")
+                print("Click 'c' and return to change config options. ")
+                print("Click return to continue. ")
+                cfg = input("")
+                if cfg == "c":
+                    Error(self).show()
+                elif cfg == "e":
+                    self._cli.clear()
+                    print(traceback.format_exc())
+                    input("\n\nClick return to continue")
+                else:
+                    return
+                self._cli.clear()
 
     # ============================
     # Paths
@@ -62,12 +79,34 @@ class DrillDown:
         self._cli.clear()
         self._cli.action(f"Adding: {p}\n")
         self._cli.pause()
-        if t == "file":
-            self._cli.csvpaths.paths_manager.add_named_paths(name=name, from_file=p)
-        elif t == "dir":
-            self._cli.csvpaths.paths_manager.add_named_paths(name=name, from_dir=p)
-        else:
-            self._cli.csvpaths.paths_manager.add_named_paths(name=name, from_json=p)
+        try:
+            if t == "file":
+                self._cli.csvpaths.paths_manager.add_named_paths_from_file(
+                    name=name, file_path=p
+                )
+            elif t == "dir":
+                self._cli.csvpaths.paths_manager.add_named_paths_from_dir(
+                    name=name, directory=p
+                )
+            else:
+                self._cli.csvpaths.paths_manager.add_named_paths_from_json(file_path=p)
+        except Exception:
+            cfg = None
+            while cfg in [None, "c", "e"]:
+                print("There was an error.")
+                print("Click 'e' and return to print the stack trace. ")
+                print("Click 'c' and return to change config options. ")
+                print("Click return to continue. ")
+                cfg = input("")
+                if cfg == "c":
+                    Error(self).show()
+                elif cfg == "e":
+                    self._cli.clear()
+                    print(traceback.format_exc())
+                    input("\n\nClick return to continue")
+                else:
+                    return
+                self._cli.clear()
 
     # ============================
     # Utilities
@@ -100,9 +139,9 @@ class DrillDown:
 
     def _get_add_type(self) -> str:
         self._cli.clear()
+        choices = ["dir", "file", "json"]
         t = None
-        cli = Bullet(bullet=" > ", choices=["dir", "file", "json"])
-        t = cli.launch()
+        t = self._cli.ask(choices)
         return t
 
     def _drill_down(self, *, path, extensions, json=False, dir_only=False) -> str:
@@ -115,11 +154,10 @@ class DrillDown:
             names = self._filter_extensions(path, names, extensions)
             names.sort()
         names = self._decorate(path, names, select_dir=dir_only)
-        cli = Bullet(bullet=" > ", choices=names)
-        t = cli.launch()
-        if t == self._cli.STOP_HERE:
+        t = self._cli.ask(names)
+        if t in [self._cli.STOP_HERE, self._cli.STOP_HERE2]:
             return (path, True)
-        if t == self._cli.CANCEL:
+        if t in [self._cli.CANCEL, self._cli.CANCEL2]:
             return (path, False)
         if t.startswith("📂 ") or t.startswith("📄 "):
             t = t[2:]
@@ -128,7 +166,7 @@ class DrillDown:
     def _decorate(self, path, names, select_dir=False) -> list[str]:
         ns = []
         for n in names:
-            if n == self._cli.STOP_HERE:
+            if n in [self._cli.STOP_HERE, self._cli.STOP_HERE2]:
                 pass
             elif os.path.isfile(os.path.join(path, n)):
                 n = f"📄 {n}"

csvpath-0.0.513/csvpath/cli/error.py

@@ -0,0 +1,61 @@
+from prompt_toolkit.shortcuts import message_dialog
+from prompt_toolkit.shortcuts import checkboxlist_dialog
+
+
+class Error:
+    def __init__(self, holder):
+        self._holder = holder
+        if hasattr(holder, "_cli"):
+            self._cli = self._holder._cli
+        else:
+            self._cli = holder
+        self._paths = self._cli.csvpaths
+        self._config = self._paths.config
+
+    def show(self):
+        log_loc = self._config.get(section="logging", name="log_file")
+        cfg_loc = self._config.config_path
+
+        raising = "suppress" if self.is_raising() else "raise"
+        raising_msg = f"Change error policy to {raising} exceptions."
+
+        debugging = "INFO" if self.is_debugging() else "DEBUG"
+        debugging_msg = f"Set logging to {debugging}. (Your log file  is at {log_loc})."
+
+        results = checkboxlist_dialog(
+            title="Config Settings",
+            text=f"These settings are in your config file at {cfg_loc}. Changing them may help you debug. \nNote that surpressing errors is effective only when running CsvPath expressions, not when loading files.\n",
+            values=[
+                ("flip_debug", debugging_msg),
+                ("flip_raise", raising_msg),
+            ],
+        ).run()
+
+        if results and "flip_raise" in results:
+            if self.is_raising():
+                self._config.add_to_config("errors", "csvpaths", "print")
+                self._config.add_to_config("errors", "csvpath", "print")
+            else:
+                self._config.add_to_config("errors", "csvpaths", "print, raise")
+                self._config.add_to_config("errors", "csvpath", "print, raise")
+
+        if results and "flip_debug" in results:
+            if self.is_debugging():
+                self._config.add_to_config("logging", "csvpath", "info")
+                self._config.add_to_config("logging", "csvpaths", "info")
+            else:
+                self._config.add_to_config("logging", "csvpath", "debug")
+                self._config.add_to_config("logging", "csvpaths", "debug")
+        if results:
+            self._config.save_config()
+            self._config.reload()
+            self._cli.csvpaths._set_managers()
+
+    def is_debugging(self) -> bool:
+        psd = self._config.get(section="logging", name="csvpaths")
+        pd = self._config.get(section="logging", name="csvpath")
+        return psd is True and pd is True
+
+    def is_raising(self) -> bool:
+        policy = self._config.get(section="errors", name="csvpath")
+        return self._paths.ecoms.do_i_raise() and "raise" in policy

csvpath-0.0.513/csvpath/cli/select.py

@@ -0,0 +1,68 @@
+# from __future__ import unicode_literals
+from typing import Sequence, Tuple, Optional, TypeVar
+from prompt_toolkit.application import Application
+from prompt_toolkit.key_binding.defaults import load_key_bindings
+from prompt_toolkit.key_binding.key_bindings import KeyBindings, merge_key_bindings
+from prompt_toolkit.layout import Layout
+from prompt_toolkit.layout.containers import HSplit
+from prompt_toolkit.widgets import RadioList, Label
+from prompt_toolkit.keys import Keys
+from prompt_toolkit.formatted_text import AnyFormattedText
+from prompt_toolkit.styles import BaseStyle
+
+
+T = TypeVar("T")
+
+
+class Select:
+    def ask(
+        self,
+        title: str = "",
+        values: Sequence[Tuple[T, AnyFormattedText]] = None,
+        default: Optional[T] = None,
+        cancel_value: Optional[T] = None,
+        style: Optional[BaseStyle] = None,
+        async_: bool = False,
+    ) -> T:
+        #
+        # this class sourced from https://github.com/prompt-toolkit/python-prompt-toolkit/issues/756
+        #
+        radio_list = RadioList(values, default)
+        #
+        # Remove the enter key binding so that we can augment it
+        #
+        radio_list.control.key_bindings.remove("enter")
+        bindings = KeyBindings()
+        #
+        # Replace the enter key binding to select the value and also submit it
+        #
+
+        @bindings.add("enter")
+        def exit_with_value(event):
+            #
+            # enter exits the select, returning the highlighted value
+            #
+            radio_list._handle_enter()
+            event.app.exit(result=radio_list.current_value)
+
+        @bindings.add("c-c")
+        def backup_exit_with_value(event):
+            #
+            # ctrl-c exits the user interface with the cancel_value
+            #
+            event.app.exit(result=cancel_value)
+
+        #
+        # Create and run the mini inline application
+        #
+        application = Application(
+            layout=Layout(HSplit([Label(title), radio_list])),
+            key_bindings=merge_key_bindings([load_key_bindings(), bindings]),
+            mouse_support=True,
+            style=style,
+            full_screen=False,
+        )
+        if async_:
+            return application.run_async()
+        else:
+            return application.run()