csvpath 0.0.560__tar.gz → 0.0.561__tar.gz

{csvpath-0.0.560 → csvpath-0.0.561}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: csvpath
-Version: 0.0.560
+Version: 0.0.561
 Summary: A data preboarding framework for managing and validating CSV, Excel, and other tabular data files using a Collect, Store, Validate, Publish pattern to create a trusted publisher for downstream data consumers.
 Author: David Kershaw
 Author-email: dk107dk@hotmail.com
@@ -34,6 +34,7 @@ Requires-Dist: metaphone (>=0.6,<0.7)
 Requires-Dist: mysqlclient (>=2.2.7,<3.0.0)
 Requires-Dist: openlineage-python (>=1.25.0,<2.0.0)
 Requires-Dist: opentelemetry-distro[otlp] (>=0.50b0,<0.51)
+Requires-Dist: pandas (>=2.2.2,<3.0.0) ; extra == "pandas"
 Requires-Dist: paramiko (>=3.5.0,<4.0.0)
 Requires-Dist: pdoc (>=15.0.1,<16.0.0)
 Requires-Dist: prompt-toolkit (>=3.0.50,<4.0.0)

{csvpath-0.0.560 → csvpath-0.0.561}/config/config.ini

@@ -17,15 +17,15 @@ use_format = full
 pattern = {time}:{file}:{line}:{paths}:{instance}:{chain}:  {message}
 
 [logging]
-csvpath = info
-csvpaths = info
+csvpath = debug
+csvpaths = debug
 log_file = logs/csvpath.log
 log_files_to_keep = 100
 log_file_size = 52428800
 handler = file
 
 [config]
-path = assets/config/config.ini
+path = config/config.ini
 allow_var_sub = True
 var_sub_source = env
 
@@ -104,7 +104,7 @@ port = SFTPPLUS_PORT
 admin_username = SFTPPLUS_ADMIN_USERNAME
 admin_password = SFTPPLUS_ADMIN_PASSWORD
 api_url = https://localhost:10020/json
-scripts_dir = 
+scripts_dir =
 execute_timeout = 300
 
 [ckan]
@@ -119,7 +119,7 @@ timeout = 5
 verify = False
 
 [slack]
-webhook_url = 
+webhook_url =
 
 [scripts]
 run_scripts = yes

csvpath-0.0.561/config/env.json

@@ -0,0 +1 @@
+{}

{csvpath-0.0.560 → csvpath-0.0.561}/csvpath/csvpaths.py

@@ -160,6 +160,7 @@ class CsvPaths(CsvPathsCoordinator, ErrorCollector):
         """ @private """
         self._config._assure_logs_path()
         self.logger = LogUtility.logger(self)
+        self.info_dump()
         """ @private """
         self._errors = []
         # coordinator attributes
@@ -203,6 +204,17 @@ class CsvPaths(CsvPathsCoordinator, ErrorCollector):
             f"Initialized CsvPaths: {self} in thread: {threading.current_thread()}"
         )
 
+    def info_dump(self) -> None:
+        self.logger.info(
+            "Initated logging on log path: %s",
+            self.config.get(section="logging", name="log_file"),
+        )
+        self.logger.info("Config file is at: %s", self.config.configpath)
+        intgs = self.config.get(section="listeners", name="groups")
+        self.logger.debug("Active integrations:")
+        for _ in intgs:
+            self.logger.debug("  - %s", _)
+
     def _set_managers(self) -> None:
         self.paths_manager = PathsManager(csvpaths=self)
         self.file_manager = FileManager(csvpaths=self)

{csvpath-0.0.560 → csvpath-0.0.561}/csvpath/managers/integrations/otlp/metrics.py

@@ -64,39 +64,57 @@ class Metrics:
     @property
     def provider(self) -> LoggerProvider:
         if self._provider is None:
-            # Add resource information
-            resource = Resource.create(
-                {"service.name": "CsvPath", "service.version": "1.0.0"}
-            )
-            self._provider = LoggerProvider(resource=resource)
-            set_logger_provider(self._provider)
-            #
-            # these were working values for openobserve. I don't think we need
-            # OTEL_EXPORTER_OTLP_PROTOCOL  since we are programmatically instantiating
-            # the protobuf
-            #
-            # OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
-            # OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:5080/api/default
-            # OTEL_EXPORTER_OTLP_HEADERS=Authorization=Basic ZGsxMDdka0Bob3RtYWlsLmNvbTpoYW5nemhvdQ==,stream-name=FlightPath
-            #
-            # certificate_file=None,
-            # client_key_file=None,
-            # client_certificate_file=None,
-            # timeout=None,
-            # compression=None,
-            # session=None
-            #
-            # in CsvPath and FlightPath Data these can come from regular env vars
-            # but in FlightPath Server they must come from var_sub_source=config/env.json
-            # because we plan to allow projects to push data to their own choice of OTLP
-            # platform. FlightPath Data will have to provide an API for setting env.json
-            # and assistence in copying its own env vars and the OS env vars to env.json
-            # on the server.
-            #
-            endpoint = self._get(name="OTEL_EXPORTER_OTLP_ENDPOINT")
-            headers = self._get(name="OTEL_EXPORTER_OTLP_HEADERS")
-            exporter = OTLPLogExporter(endpoint=endpoint, headers=headers)
-            self._provider.add_log_record_processor(BatchLogRecordProcessor(exporter))
+            try:
+                # Add resource information
+                resource = Resource.create(
+                    {"service.name": "CsvPath", "service.version": "1.0.0"}
+                )
+                self._provider = LoggerProvider(resource=resource)
+                set_logger_provider(self._provider)
+                #
+                # these were working values for a local openobserve.
+                #
+                # OTEL_EXPORTER_OTLP_ENDPOINT=http://0.0.0.0:5080/api/default/v1/logs
+                # OTEL_EXPORTER_OTLP_HEADERS=Authorization=Basic ZGsxMDdka0Bob3RtYWlsLmNvbTpoYW5nemhvdQ==,stream-name=flightpath
+                #
+                # certificate_file=None,
+                # client_key_file=None,
+                # client_certificate_file=None,
+                # timeout=None,
+                # compression=None,
+                # session=None
+                #
+                # in CsvPath and FlightPath Data these can come from regular env vars
+                # but in FlightPath Server they must come from var_sub_source=config/env.json
+                # because we plan to allow projects to push data to their own choice of OTLP
+                # platform. FlightPath Data will have to provide an API for setting env.json
+                # and assistence in copying its own env vars and the OS env vars to env.json
+                # on the server.
+                #
+                endpoint = self._get(name="OTEL_EXPORTER_OTLP_LOGS_ENDPOINT")
+                if endpoint is None or endpoint == "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT":
+                    endpoint = self._get(name="OTEL_EXPORTER_OTLP_ENDPOINT")
+                if endpoint is None or endpoint == "OTEL_EXPORTER_OTLP_ENDPOINT":
+                    raise ValueError(
+                        "You must pass either OTEL_EXPORTER_OTLP_ENDPOINT or OTEL_EXPORTER_OTLP_LOGS_ENDPOINT, the latter preferred"
+                    )
+                headers = self._get(name="OTEL_EXPORTER_OTLP_HEADERS")
+                if headers is None or headers == "OTEL_EXPORTER_OTLP_HEADERS":
+                    raise ValueError("OTEL_EXPORTER_OTLP_HEADERS cannot be None")
+                headers = headers.split(",")
+                d = {}
+                for _ in headers:
+                    k = _[0 : _.find("=")]
+                    v = _[_.find("=") + 1 :]
+                    d[k] = v
+                exporter = OTLPLogExporter(endpoint=endpoint, headers=d)
+                self._provider.add_log_record_processor(
+                    BatchLogRecordProcessor(exporter)
+                )
+            except Exception as ex:
+                if self._csvpaths:
+                    self._csvpaths.logger.error("Cannot configure OTLP")
+                    self._csvpaths.logger.error(ex)
         return self._provider
 
     def logger(self, project: str = "csvpath") -> Logger:
@@ -114,4 +132,5 @@ class Metrics:
             # Prevent propagation to avoid duplicate logs
             logger.propagate = False
             Metrics.LOGGERS[project] = logger
+            # logging.basicConfig(level=logging.DEBUG)
         return logger

{csvpath-0.0.560 → csvpath-0.0.561}/csvpath/managers/integrations/otlp/otlp_paths_listener.py

@@ -21,6 +21,13 @@ class OpenTelemetryPathsListener(OtlpListener):
             )
         self.assure_metrics()
         try:
+            ids = []
+            #
+            # otlp doesn't like Nones. should there not be any?
+            #
+            for _ in mdata.named_paths_identities:
+                if _:
+                    ids.append(_)
             etype = "named-paths group load"
             extra = {
                 "event_type": etype,
@@ -29,7 +36,7 @@ class OpenTelemetryPathsListener(OtlpListener):
                 "named_paths_home": mdata.named_paths_home,
                 "group_file_path": mdata.group_file_path,
                 "named_paths_count": mdata.named_paths_count,
-                "named_paths_identities": mdata.named_paths_identities,
+                "named_paths_identities": ids,
                 "source_path": mdata.source_path,
                 "template": mdata.template if mdata.template else "",
                 **self.core_meta(mdata),

{csvpath-0.0.560 → csvpath-0.0.561}/csvpath/managers/integrations/otlp/otlp_results_listener.py

@@ -49,6 +49,7 @@ class OpenTelemetryResultsListener(OtlpListener):
             self.csvpaths.__class__.METRICS.logger().debug(
                 "Csvpath completed", extra=extra
             )
+            self.csvpaths.logger.info("Csvpath shipped log entry to OTLP integration")
         except Exception as ex:
             print(traceback.format_exc())
             self.csvpaths.logger.error(ex)

{csvpath-0.0.560 → csvpath-0.0.561}/csvpath/managers/registrar.py

@@ -51,13 +51,18 @@ class Registrar(ABC):
         # notable exception of errors, errors are not thrown in non-CsvPaths code.
         #
         if self.csvpaths is not None:
+            self.csvpaths.logger.info("Distributing updates to listeners")
             try:
                 self.load_additional_listeners(self.type_name, listeners)
             except Exception as ex:
                 print(traceback.format_exc())
                 if self.csvpaths:
-                    self.csvpaths.logger(f"Error in loading listeners: {ex}")
+                    self.csvpaths.logger.error(f"Error in loading listeners: {ex}")
         for lst in listeners:
+            if self.csvpaths:
+                self.csvpaths.logger.debug(
+                    "Updating listener %s with metadata %s", lst, mdata
+                )
             try:
                 lst.metadata_update(mdata)
             except Exception as ex:
@@ -75,27 +80,27 @@ class Registrar(ABC):
         """
         if self.csvpaths:
             ss = self.csvpaths.config.additional_listeners(listener_type_name)
+            self.csvpaths.logger.info("Loading additional listener type(s) %s", ss)
             if ss and not isinstance(ss, list):
                 ss = [ss]
             if ss and len(ss) > 0:
                 for lst in ss:
-                    try:
-                        self.load_additional_listener(lst, listeners)
-                    except Exception as ex:
-                        print(traceback.format_exc())
-                        self.csvpaths.logger.error(
-                            f"Failed to load listener {lst}: {ex}"
-                        )
+                    self.load_additional_listener(lst, listeners)
 
     def load_additional_listener(self, load_cmd: str, listeners: list) -> None:
-        loader = ClassLoader()
-        alistener = loader.load(load_cmd)
-        if alistener is not None:
-            if hasattr(alistener, "csvpaths"):
-                setattr(alistener, "csvpaths", self.csvpaths)
-            if hasattr(alistener, "result"):
-                setattr(alistener, "result", self.result)
-            if hasattr(self, "csvpath") and hasattr(alistener, "csvpath"):
-                alistener.csvpath = self.csvpath
-            alistener.config = self.csvpaths.config
-            listeners.append(alistener)
+        self.csvpaths.logger.info("Loading additional listener %s", load_cmd)
+        try:
+            loader = ClassLoader()
+            alistener = loader.load(load_cmd)
+            if alistener is not None:
+                if hasattr(alistener, "csvpaths"):
+                    setattr(alistener, "csvpaths", self.csvpaths)
+                if hasattr(alistener, "result"):
+                    setattr(alistener, "result", self.result)
+                if hasattr(self, "csvpath") and hasattr(alistener, "csvpath"):
+                    alistener.csvpath = self.csvpath
+                alistener.config = self.csvpaths.config
+                listeners.append(alistener)
+        except Exception as e:
+            print(traceback.format_exc())
+            self.csvpaths.logger.error(e)

{csvpath-0.0.560 → csvpath-0.0.561}/csvpath/matching/functions/validity/line.py

@@ -248,7 +248,8 @@ class Line(MatchDecider):
         #
         if t and t.name != self.matcher.csvpath.headers[i] and t.name != f"{i}":
             ii = i + 1
-            msg = f"The {ExpressionUtility._numeric_string(ii)} item, {t}, does not match the current header"
+
+            msg = f"The {ExpressionUtility._numeric_string(ii)} item, {t}, does not match the current header '{self.matcher.csvpath.headers[i]}'"
             errors.append(msg)
         return True
 
@@ -264,6 +265,6 @@ class Line(MatchDecider):
         #
         if t and t.name != self.matcher.csvpath.headers[i] and t.name != f"{i}":
             ii = i + 1
-            msg = f"The {ExpressionUtility._numeric_string(ii)} item, {t}, does not match the current header"
+            msg = f"The {ExpressionUtility._numeric_string(ii)} item, {t}, does not match the current header '{self.matcher.csvpath.headers[i]}'"
             errors.append(msg)
         return True