csvpath 0.0.617__tar.gz → 0.0.620__tar.gz

{csvpath-0.0.617 → csvpath-0.0.620}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: csvpath
-Version: 0.0.617
+Version: 0.0.620
 Summary: CsvPath Framework is a data preboarding automation library for receiving, validating, and tracking CSV, Excel, JSONL and other tabular data files before they can corrupt downstream data consumers.
 License-File: LICENSE
 Author: David Kershaw
@@ -113,8 +113,7 @@ If you need help getting started, there are lots of ways to reach us.
 - Email support@csvpath.org
 - Or reach out to one of our [sponsors, below](#sponsors).
 
-![PyPI - Python Version](https://img.shields.io/pypi/pyversions/csvpath?logoColor=green&color=green) ![GitHub commit activity](https://img.shields.io/github/commit-activity/m/dk107dk/csvpath) ![PyPI - Version](https://img.shields.io/pypi/v/csvpath)
-
+![PyPI - Python Version](https://img.shields.io/pypi/pyversions/csvpath?logoColor=yellow&color=yellow) ![GitHub commit activity](https://img.shields.io/github/commit-activity/m/dk107dk/csvpath) ![PyPI - Version](https://img.shields.io/pypi/v/csvpath) [![PyPI Downloads](https://static.pepy.tech/personalized-badge/csvpath?period=total&units=INTERNATIONAL_SYSTEM&left_color=BLACK&right_color=GREEN&left_text=downloads)](https://pepy.tech/projects/csvpath)
 
 # Contents
 

{csvpath-0.0.617 → csvpath-0.0.620}/README.md

@@ -48,8 +48,7 @@ If you need help getting started, there are lots of ways to reach us.
 - Email support@csvpath.org
 - Or reach out to one of our [sponsors, below](#sponsors).
 
-![PyPI - Python Version](https://img.shields.io/pypi/pyversions/csvpath?logoColor=green&color=green) ![GitHub commit activity](https://img.shields.io/github/commit-activity/m/dk107dk/csvpath) ![PyPI - Version](https://img.shields.io/pypi/v/csvpath)
-
+![PyPI - Python Version](https://img.shields.io/pypi/pyversions/csvpath?logoColor=yellow&color=yellow) ![GitHub commit activity](https://img.shields.io/github/commit-activity/m/dk107dk/csvpath) ![PyPI - Version](https://img.shields.io/pypi/v/csvpath) [![PyPI Downloads](https://static.pepy.tech/personalized-badge/csvpath?period=total&units=INTERNATIONAL_SYSTEM&left_color=BLACK&right_color=GREEN&left_text=downloads)](https://pepy.tech/projects/csvpath)
 
 # Contents
 

{csvpath-0.0.617 → csvpath-0.0.620}/assets/config/jenkins-windows-local.ini

@@ -26,7 +26,7 @@ path = cache
 use_cache = no
 
 [functions]
-imports =
+imports = 
 
 [results]
 archive = tmp\local\archive
@@ -86,7 +86,7 @@ port = SFTPPLUS_PORT
 admin_username = SFTPPLUS_ADMIN_USERNAME
 admin_password = SFTPPLUS_ADMIN_PASSWORD
 api_url = https://. . . :10020/json
-scripts_dir =
+scripts_dir = 
 execute_timeout = 300
 
 [ckan]
@@ -101,7 +101,7 @@ timeout = 5
 verify = False
 
 [slack]
-webhook_url =
+webhook_url = 
 
 [scripts]
 run_scripts = yes

{csvpath-0.0.617 → csvpath-0.0.620}/assets/gocd/windows/azure.bat

@@ -26,9 +26,18 @@ set OTEL_RESOURCE_ATTRIBUTES=""
 
 echo %CSVPATH_CONFIG_PATH%
 call c:\dev\win-exports.bat
-cmd.exe /C c:\Users\python\.local\bin\poetry.exe install
-cmd.exe /C c:\Users\python\.local\bin\poetry.exe run pytest
 
+IF NOT EXIST "%USERPROFILE%\.local\bin\poetry.exe" (
+    echo ERROR: The Poetry binary was not found.
+    EXIT /B 1
+)
+
+cmd.exe /C "%USERPROFILE%\.local\bin\poetry.exe" install
+cmd.exe /C "%USERPROFILE%\.local\bin\poetry.exe" run pytest
+if %ERRORLEVEL% neq 0 (
+    echo [ERROR] The pytest failed with code %ERRORLEVEL%.
+    exit /b %ERRORLEVEL%
+)
 
 net stop w32time
 w32tm /unregister

{csvpath-0.0.617 → csvpath-0.0.620}/assets/gocd/windows/gcs.bat

@@ -25,9 +25,13 @@ set OTEL_SERVICE_NAME=""
 set OTEL_RESOURCE_ATTRIBUTES=""
 
 call c:\dev\win-exports.bat
-cmd.exe /C c:\Users\python\.local\bin\poetry.exe install
-cmd.exe /C c:\Users\python\.local\bin\poetry.exe run pytest
 
+cmd.exe /C "%USERPROFILE%\.local\bin\poetry.exe" install
+cmd.exe /C "%USERPROFILE%\.local\bin\poetry.exe" run pytest
+IF %ERRORLEVEL% NEQ 0 (
+    echo "Tests failed. Failing the build..."
+    exit /b %ERRORLEVEL%
+)
 
 net stop w32time
 w32tm /unregister

{csvpath-0.0.617 → csvpath-0.0.620}/assets/gocd/windows/local.bat

@@ -49,8 +49,13 @@ echo(
 ::
 :: do the build
 ::
-cmd.exe /C c:\Users\python\.local\bin\poetry.exe install
-cmd.exe /C c:\Users\python\.local\bin\poetry.exe run pytest
+cmd.exe /C "%USERPROFILE%\.local\bin\poetry.exe" install
+cmd.exe /C "%USERPROFILE%\.local\bin\poetry.exe" run pytest
+if %ERRORLEVEL% neq 0 (
+    echo [ERROR] The pytest failed with code %ERRORLEVEL%.
+    exit /b %ERRORLEVEL%
+)
+
 
 
 net stop w32time

{csvpath-0.0.617 → csvpath-0.0.620}/assets/gocd/windows/s3.bat

@@ -27,8 +27,14 @@ set OTEL_RESOURCE_ATTRIBUTES=""
 
 
 call c:\dev\win-exports.bat
-cmd.exe /C c:\Users\python\.local\bin\poetry.exe install
-cmd.exe /C c:\Users\python\.local\bin\poetry.exe run pytest
+
+cmd.exe /C "%USERPROFILE%\.local\bin\poetry.exe" install
+cmd.exe /C "%USERPROFILE%\.local\bin\poetry.exe" run pytest
+if %ERRORLEVEL% neq 0 (
+    echo [ERROR] The pytest failed with code %ERRORLEVEL%.
+    exit /b %ERRORLEVEL%
+)
+
 
 
 net stop w32time

{csvpath-0.0.617 → csvpath-0.0.620}/assets/gocd/windows/sftp.bat

@@ -25,8 +25,14 @@ set OTEL_SERVICE_NAME=""
 set OTEL_RESOURCE_ATTRIBUTES=""
 
 call c:\dev\win-exports.bat
-cmd.exe /C c:\Users\python\.local\bin\poetry.exe install
-cmd.exe /C c:\Users\python\.local\bin\poetry.exe run pytest
+
+cmd.exe /C "%USERPROFILE%\.local\bin\poetry.exe" install
+cmd.exe /C "%USERPROFILE%\.local\bin\poetry.exe" run pytest
+if %ERRORLEVEL% neq 0 (
+    echo [ERROR] The pytest failed with code %ERRORLEVEL%.
+    exit /b %ERRORLEVEL%
+)
+
 
 
 net stop w32time

{csvpath-0.0.617 → csvpath-0.0.620}/csvpath/csvpaths.py

@@ -4,7 +4,6 @@ of the CsvPath library. it makes it easier to scale your CSV quality control."""
 import os
 import traceback
 import atexit
-import threading
 
 from uuid import uuid4
 from abc import ABC, abstractmethod
@@ -226,9 +225,17 @@ class CsvPaths(CsvPathsCoordinator, ErrorCollector):
         #
         #
         #
-        self.logger.info(
-            f"Initialized CsvPaths: {self} in thread: {threading.current_thread()}"
-        )
+        # this log line didn't add especial value. but it did set the logger
+        # to a certain log file before we had a chance to do a cd-and-reload
+        # to switch into a particular project. the fix isn't just to remove the
+        # line, it's also to make the logger travel with the config used. but
+        # removing the log line is easy enough and we don't lose much. And the
+        # real fix was already in set_config_path_and_reload() -- it was just
+        # releasing/renewing the logger.
+        #
+        # self.logger.info(
+        #    f"Initialized CsvPaths: {self} in thread: {threading.current_thread()}"
+        # )
 
     @property
     def logger(self):

{csvpath-0.0.617 → csvpath-0.0.620}/csvpath/managers/files/file_describer.py

@@ -1,4 +1,5 @@
 import json
+import traceback
 from typing import NewType
 
 from csvpath.util.nos import Nos
@@ -57,16 +58,21 @@ class NamedFileDescriber:
         home = self.file_manager.named_file_home(name)
         path = Nos(home).join(self.JSON_FILE)
         nos = Nos(path)
-        if nos.exists() is False:
-            #
-            # cannot return None (today) and we definitely should not
-            # create+save a new json file. returning None may be better
-            # for today this works.
-            #
-            return {}
-            # self.store_json(name, {})
-        with DataFileReader(path) as file:
-            return json.load(file.source)
+        if nos.exists() is True:
+            with DataFileReader(path) as file:
+                try:
+                    return json.load(file.source)
+                except Exception:
+                    msg = "Error getting named-file discriptor JSON for {name}: {ex}"
+                    self.file_manager.csvpaths.logger.error(msg)
+                    msg = traceback.format_exc()
+                    self.file_manager.csvpaths.logger.debug(msg)
+        #
+        # cannot return None (today; << explain?) and we definitely should not
+        # create+save a new json file. (<< reasoning?) returning None may be
+        # better (<< why?) but for today this works.
+        #
+        return {}
 
     # ========== Templates ============
 

{csvpath-0.0.617 → csvpath-0.0.620}/csvpath/managers/files/file_manager.py

@@ -57,7 +57,6 @@ class FileManager:
 
     @property
     def csvpaths(self):
-        """@private"""
         return self._csvpaths
 
     #
@@ -65,7 +64,6 @@ class FileManager:
     #
     @property
     def named_files_dir(self) -> str:
-        """@private"""
         return self._csvpaths.config.inputs_files_path
 
     #
@@ -75,7 +73,6 @@ class FileManager:
     #
     @property
     def files_root_manifest(self) -> dict:
-        """@private"""
         p = self.files_root_manifest_path
         nos = Nos(p)
         if nos.exists():
@@ -85,7 +82,6 @@ class FileManager:
 
     @property
     def files_root_manifest_path(self) -> dict:
-        """@private"""
         return Nos(self.named_files_dir).join("manifest.json")
 
     def _named_file_ref_to_uuid(self, ref: ReferenceParser) -> str:
@@ -283,6 +279,13 @@ class FileManager:
         home = pathu.resep(home)
         return home
 
+    #
+    # creates a named-file before registering any files.
+    #
+    def assure_named_file(self, name: NamedFileName) -> None:
+        home = self.assure_named_file_home(name)
+        self.registrar.manifest_path(home)
+
     def configure_named_file(self, name: NamedFileName, descriptor: Config) -> None:
         #
         # updates the named-file's descriptor by replacement
@@ -665,7 +668,9 @@ class FileManager:
             self._registration_failed(_)
             #
             return
+        self.csvpaths.logger.info("Adding named file %s: can be loaded", name)
         self.legal_name(name)
+        self.csvpaths.logger.info("Adding named file %s: name is legal", name)
         if path is None or path.strip() == "":
             _ = {
                 "name": name,
@@ -678,7 +683,13 @@ class FileManager:
         if template is None:
             template = self.describer.get_template(name)
         if template is not None:
+            self.csvpaths.logger.debug(
+                "Adding named file %s: checking template %s", name, template
+            )
             temu.valid(template, file=True)
+        self.csvpaths.logger.info(
+            "Adding named file %s: template is valid or None: %s", name, template
+        )
         if config is not None:
             #
             # doing config here enables copy_in() to pull down files
@@ -687,6 +698,9 @@ class FileManager:
             # configured in advance, but that doesn't have to be the
             # case.
             #
+            self.csvpaths.logger.info(
+                "Adding named file %s: configuring named-file", name
+            )
             self.configure_named_file(name, config)
         path = pathu.resep(path)
         self.csvpaths.logger.debug("[%s] Path after resep %s", registration_uuid, path)
@@ -696,6 +710,10 @@ class FileManager:
         local = config.get(section="inputs", name="allow_local_files", default=False)
         local = str(local).strip().lower() in ["on", "yes", "true"]
         nos = Nos(path)
+        self.csvpaths.logger.info(
+            "Adding named file %s: checking permission, if needed, to register local and http files",
+            name,
+        )
         #
         # can the descriptor override the project's prohibition on http(s) loads?
         # no. the project may be controlled by a context like FlightPath Server, where
@@ -729,6 +747,9 @@ class FileManager:
             if self.csvpaths.ecoms.do_i_raise():
                 raise FileException(msg)
             return
+        self.csvpaths.logger.info(
+            "Adding named file %s: registration of %s permitted", name, path
+        )
         #
         # we should know enough to redirect if handed a dir path. we'll not recurse
         # because the caller hasn't given us enough info to know if we should.
@@ -737,6 +758,7 @@ class FileManager:
         # method. if using FlightPath Server, the server should allow them to be likewise
         # more specific. this branch is essentially a fallback presumed to be covering for
         # lazy developers and people who are working without better information.
+        #
         if nos.path.find("#") > -1:
             #
             # if we passed in a root minor, a.k.a. "mark", we need to strip
@@ -744,13 +766,35 @@ class FileManager:
             #
             nos = Nos(nos.path[0 : nos.path.find("#")])
         #
-        # note for now we treat any http as a file
+        # note for now we treat any http as a file.
+        #
+        # also note that this Nos must be savvy to any ServerConfig that may
+        # be available. it should use the possibly cached backend, if that
+        # matches, but if not, it must search any ServerConfig to configure
+        # itself to do operations on one of those servers.
+        #
+        # server config can be passed to a non-sftp nos. there won't be any
+        # complaints, but it won't do anything. atm, the judgement call is that
+        # we save an if-test by allowing any nos to receive server config and
+        # accepting the cost of searching for any server config in json.
+        #
+        config = self.describer.get_config(name)
+        servers = config.sources
+        nos.server_config = servers
+        #
+        #  for _ in servers:
+        #     if _.matches(nos.path):
+        #       nos.do._config = SftpConfig(_)
         #
         isfile = nos.is_http or nos.isfile()
+        self.csvpaths.logger.debug(
+            "Adding named file %s: path %s is a file? %s", name, path, isfile
+        )
         if not isfile:
             return self.add_named_files_from_dir(
                 path, name=name, template=template, recurse=False
             )
+        self.csvpaths.logger.debug("Adding named file %s: continuing with file", name)
         try:
             self.csvpaths.logger.debug(
                 "[%s] Ready to register %s", registration_uuid, path
@@ -896,6 +940,9 @@ class FileManager:
         mdata.template = data.get("template") or ""
         mdata.uuid_string = data.get("registration_uuid")
         mdata.status = "Registration failed"
+        self.csvpaths.logger.warning(
+            "[%s] Registration failed for %s", mdata.named_file_name, data
+        )
         lst.metadata_update(mdata)
 
     def _validate_xlsx_mark(self, path: str, mark: str) -> None:
@@ -927,8 +974,12 @@ class FileManager:
         fname = self._clean_file_name(fname)
         temp = f"{home}{sep}{fname}"
         copy = pathu.parts(path)[0] == pathu.parts(home)[0]
+        copy = copy and nos.is_local
         if copy:
             nos.path = path
+            config = self.describer.get_config(name)
+            servers = config.sources
+            nos.server_config = servers
             nos.copy(temp)
         else:
             self._copy_down(name=name, path=path, temp=temp, mode="wb")

{csvpath-0.0.617 → csvpath-0.0.620}/csvpath/managers/files/file_registrar.py

@@ -154,15 +154,19 @@ class FileRegistrar(Registrar, Listener):
             raise InputException(
                 f"File mgr and registrar marks should match: {mdata.mark}, {mark}"
             )
-        if (
-            # Nos doesn't handle http files. they are special--inbound only.
-            not path.startswith("http:")
-            and not path.startswith("https:")
-            and not Nos(path).exists()
-        ):
-            # if not path.startswith("s3:") and not os.path.exists(path):
-            #
-            raise InputException(f"Path {path} does not exist")
+        opath = Nos(path)
+        #
+        # if we're registering a non-local file and it's an sftp file it may be
+        # on an sftp server that is configured in the named-file description. so
+        # we setup nos to be able to check, just in case.
+        #
+        config = self.csvpaths.file_manager.describer.get_config(mdata.named_file_name)
+        servers = config.sources
+        opath.server_config = servers
+
+        if not opath.is_http:
+            if not opath.exists():
+                raise InputException(f"Path {opath.path} does not exist")
         #
         # if the fingerprint already exists we don't store the file again.
         # we rename the file to the fingerprint. from this point the registrar

{csvpath-0.0.617 → csvpath-0.0.620}/csvpath/managers/integrations/webhook/webhook_listener.py

@@ -128,7 +128,7 @@ class WebhookListener(Listener, threading.Thread):
                 url=url, payload=payload, headers=headers, timeout=self.timeout
             )
             # x = requests.post(url, json=payload, headers=headers, timeout=self.timeout)
-            if x and x.status_code != 200:
+            if x and x.status_code >= 200 and x.status_code < 300:
                 if self.csvpaths is not None:
                     self.csvpaths.logger.warning(
                         "WebhookListener received status code %s from %s",

{csvpath-0.0.617 → csvpath-0.0.620}/csvpath/managers/results/transfers_manager.py

@@ -6,6 +6,7 @@ from csvpath.managers.paths.paths_descriptor import Transfer, Transfers
 
 from csvpath.util.nos import Nos
 from csvpath.util.path_util import PathUtility as pathu
+from csvpath.util.var_utility import VarUtility as vaut
 
 
 class TransfersManager:
@@ -299,6 +300,11 @@ class TransfersManager:
             msg = f"Variable {t} is invalid: {f}"
             self.csvpaths.error_manager.handle_error(source=self, msg=msg)
             raise InputException(msg)
+
+        #
+        # make sure we have vars interpolated
+        #
+        f = vaut.parse_var_value(self.csvpaths.config, t, f)
         #
         # if we're shipping to another backend, we don't need to change the target location
         #

{csvpath-0.0.617 → csvpath-0.0.620}/csvpath/matching/functions/sql/select.py

@@ -79,6 +79,7 @@ class Select(ValueProducer):
     ##################
 
     def _query(self, *, table: str, column: str, where: str, conn: str) -> bool:
+        engine = None
         try:
             engine = Db.get_engine(conn=conn)
             return self._do_query(
@@ -86,6 +87,12 @@ class Select(ValueProducer):
             )
         except Exception:
             print(traceback.format_exc())
+        finally:
+            if engine is not None:
+                try:
+                    engine.dispose()
+                except Exception:
+                    ...
 
     def _do_query(self, *, table: str, column: str, where: str, engine: Engine) -> list:
         table = sa.table(table, sa.column(column))

{csvpath-0.0.617 → csvpath-0.0.620}/csvpath/matching/functions/sql/sql_in.py

@@ -1,5 +1,5 @@
 from typing import Any
-from sqlalchemy import create_engine, Engine
+from sqlalchemy import Engine
 import sqlalchemy as sa
 import traceback
 
@@ -11,7 +11,6 @@ from csvpath.matching.functions.args import Args
 
 
 class SqlIn(ValueProducer, MatchDecider):
-
     def __init__(self, matcher: Any, name: str, child: Matchable = None) -> None:
         super().__init__(matcher, name=name, child=child)
         self._cached = None
@@ -69,38 +68,56 @@ class SqlIn(ValueProducer, MatchDecider):
         conn = self._value_four(skip=skip)
         cache = self._value_five(skip=skip)
         cache = False if cache is None else cache
-        self.value = self._query(table=table, column=column, value=v, conn=conn, cache=cache)
+        self.value = self._query(
+            table=table, column=column, value=v, conn=conn, cache=cache
+        )
 
     def _decide_match(self, skip=None) -> None:
         self.match = self.to_value(skip=skip)
 
-##################
+    ##################
 
-    def _query(self, *, table:str, column:str, value:Any, cache:bool, conn:str) -> bool:
+    def _query(
+        self, *, table: str, column: str, value: Any, cache: bool, conn: str
+    ) -> bool:
+        engine = None
         try:
             engine = Db.get_engine(conn=conn)
-            return self._do_query(table=table, column=column, value=value, cache=cache, engine=engine)
-        except Exception as ex:
+            return self._do_query(
+                table=table, column=column, value=value, cache=cache, engine=engine
+            )
+        except Exception:
             print(traceback.format_exc())
-
-    def _do_query(self, *, table:str, column:str, value:Any, cache:bool, engine:Engine) -> bool:
+        finally:
+            if engine is not None:
+                try:
+                    engine.dispose()
+                except Exception:
+                    ...
+
+    def _do_query(
+        self, *, table: str, column: str, value: Any, cache: bool, engine: Engine
+    ) -> bool:
         if cache is True:
             if self._cached is None:
                 self._do_cache(value=value, table=table, column=column, engine=engine)
             return value in self._cached
         else:
-            return self._do_lookup(value=value, table=table, column=column, engine=engine)
+            return self._do_lookup(
+                value=value, table=table, column=column, engine=engine
+            )
 
-    def _do_cache(self, *, value:Any, table:str, column:str, engine:Engine) -> bool:
+    def _do_cache(self, *, value: Any, table: str, column: str, engine: Engine) -> bool:
         table = sa.table(table, sa.column(column))
         sql = sa.select(sa.column(column)).select_from(table)
 
-
         with engine.connect() as conn:
             rows = conn.execute(sql).fetchall()
             self._cached = {row[0]: None for row in rows}
 
-    def _do_lookup(self, *, value:Any, table:str, column:str, engine:Engine) -> bool:
+    def _do_lookup(
+        self, *, value: Any, table: str, column: str, engine: Engine
+    ) -> bool:
         table = sa.table(table, sa.column(column))
         query = (
             sa.select(sa.column(column))
@@ -111,5 +128,3 @@ class SqlIn(ValueProducer, MatchDecider):
         with engine.connect() as conn:
             result = conn.execute(query).fetchone()
             return result is not None
-
-

{csvpath-0.0.617 → csvpath-0.0.620}/csvpath/matching/functions/strings/format.py

@@ -1,7 +1,6 @@
 from typing import Any
 from csvpath.matching.productions import Term, Variable, Header, Reference
 from csvpath.matching.functions.function import Function
-from csvpath.matching.util.exceptions import ChildrenException, MatchException
 from ..args import Args
 from ..function_focus import ValueProducer
 
@@ -21,7 +20,8 @@ class Format(ValueProducer):
         if self.name == "interpolate":
             s = self.wrap(
                 """Interpolates a complete Pythonic formatting string with one replacement value.
-                Use {{ and }} to demarcate your replacement pattern."""
+                Use {{ and }} to demarcate your replacement pattern. For example, to set a variable
+                to a value as a string, use: @a = interpolate(uuid(), "{}")"""
             )
             self.description.append(s)