csvpath 0.0.617__tar.gz → 0.0.619__tar.gz

{csvpath-0.0.617 → csvpath-0.0.619}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: csvpath
-Version: 0.0.617
+Version: 0.0.619
 Summary: CsvPath Framework is a data preboarding automation library for receiving, validating, and tracking CSV, Excel, JSONL and other tabular data files before they can corrupt downstream data consumers.
 License-File: LICENSE
 Author: David Kershaw

{csvpath-0.0.617 → csvpath-0.0.619}/assets/config/jenkins-windows-local.ini

@@ -26,7 +26,7 @@ path = cache
 use_cache = no
 
 [functions]
-imports =
+imports = 
 
 [results]
 archive = tmp\local\archive
@@ -86,7 +86,7 @@ port = SFTPPLUS_PORT
 admin_username = SFTPPLUS_ADMIN_USERNAME
 admin_password = SFTPPLUS_ADMIN_PASSWORD
 api_url = https://. . . :10020/json
-scripts_dir =
+scripts_dir = 
 execute_timeout = 300
 
 [ckan]
@@ -101,7 +101,7 @@ timeout = 5
 verify = False
 
 [slack]
-webhook_url =
+webhook_url = 
 
 [scripts]
 run_scripts = yes

{csvpath-0.0.617 → csvpath-0.0.619}/assets/gocd/windows/azure.bat

@@ -26,9 +26,18 @@ set OTEL_RESOURCE_ATTRIBUTES=""
 
 echo %CSVPATH_CONFIG_PATH%
 call c:\dev\win-exports.bat
-cmd.exe /C c:\Users\python\.local\bin\poetry.exe install
-cmd.exe /C c:\Users\python\.local\bin\poetry.exe run pytest
 
+IF NOT EXIST "%USERPROFILE%\.local\bin\poetry.exe" (
+    echo ERROR: The Poetry binary was not found.
+    EXIT /B 1
+)
+
+cmd.exe /C "%USERPROFILE%\.local\bin\poetry.exe" install
+cmd.exe /C "%USERPROFILE%\.local\bin\poetry.exe" run pytest
+if %ERRORLEVEL% neq 0 (
+    echo [ERROR] The pytest failed with code %ERRORLEVEL%.
+    exit /b %ERRORLEVEL%
+)
 
 net stop w32time
 w32tm /unregister

{csvpath-0.0.617 → csvpath-0.0.619}/assets/gocd/windows/gcs.bat

@@ -25,9 +25,13 @@ set OTEL_SERVICE_NAME=""
 set OTEL_RESOURCE_ATTRIBUTES=""
 
 call c:\dev\win-exports.bat
-cmd.exe /C c:\Users\python\.local\bin\poetry.exe install
-cmd.exe /C c:\Users\python\.local\bin\poetry.exe run pytest
 
+cmd.exe /C "%USERPROFILE%\.local\bin\poetry.exe" install
+cmd.exe /C "%USERPROFILE%\.local\bin\poetry.exe" run pytest
+IF %ERRORLEVEL% NEQ 0 (
+    echo "Tests failed. Failing the build..."
+    exit /b %ERRORLEVEL%
+)
 
 net stop w32time
 w32tm /unregister

{csvpath-0.0.617 → csvpath-0.0.619}/assets/gocd/windows/local.bat

@@ -49,8 +49,13 @@ echo(
 ::
 :: do the build
 ::
-cmd.exe /C c:\Users\python\.local\bin\poetry.exe install
-cmd.exe /C c:\Users\python\.local\bin\poetry.exe run pytest
+cmd.exe /C "%USERPROFILE%\.local\bin\poetry.exe" install
+cmd.exe /C "%USERPROFILE%\.local\bin\poetry.exe" run pytest
+if %ERRORLEVEL% neq 0 (
+    echo [ERROR] The pytest failed with code %ERRORLEVEL%.
+    exit /b %ERRORLEVEL%
+)
+
 
 
 net stop w32time

{csvpath-0.0.617 → csvpath-0.0.619}/assets/gocd/windows/s3.bat

@@ -27,8 +27,14 @@ set OTEL_RESOURCE_ATTRIBUTES=""
 
 
 call c:\dev\win-exports.bat
-cmd.exe /C c:\Users\python\.local\bin\poetry.exe install
-cmd.exe /C c:\Users\python\.local\bin\poetry.exe run pytest
+
+cmd.exe /C "%USERPROFILE%\.local\bin\poetry.exe" install
+cmd.exe /C "%USERPROFILE%\.local\bin\poetry.exe" run pytest
+if %ERRORLEVEL% neq 0 (
+    echo [ERROR] The pytest failed with code %ERRORLEVEL%.
+    exit /b %ERRORLEVEL%
+)
+
 
 
 net stop w32time

{csvpath-0.0.617 → csvpath-0.0.619}/assets/gocd/windows/sftp.bat

@@ -25,8 +25,14 @@ set OTEL_SERVICE_NAME=""
 set OTEL_RESOURCE_ATTRIBUTES=""
 
 call c:\dev\win-exports.bat
-cmd.exe /C c:\Users\python\.local\bin\poetry.exe install
-cmd.exe /C c:\Users\python\.local\bin\poetry.exe run pytest
+
+cmd.exe /C "%USERPROFILE%\.local\bin\poetry.exe" install
+cmd.exe /C "%USERPROFILE%\.local\bin\poetry.exe" run pytest
+if %ERRORLEVEL% neq 0 (
+    echo [ERROR] The pytest failed with code %ERRORLEVEL%.
+    exit /b %ERRORLEVEL%
+)
+
 
 
 net stop w32time

{csvpath-0.0.617 → csvpath-0.0.619}/csvpath/csvpaths.py

@@ -4,7 +4,6 @@ of the CsvPath library. it makes it easier to scale your CSV quality control."""
 import os
 import traceback
 import atexit
-import threading
 
 from uuid import uuid4
 from abc import ABC, abstractmethod
@@ -226,9 +225,17 @@ class CsvPaths(CsvPathsCoordinator, ErrorCollector):
         #
         #
         #
-        self.logger.info(
-            f"Initialized CsvPaths: {self} in thread: {threading.current_thread()}"
-        )
+        # this log line didn't add especial value. but it did set the logger
+        # to a certain log file before we had a chance to do a cd-and-reload
+        # to switch into a particular project. the fix isn't just to remove the
+        # line, it's also to make the logger travel with the config used. but
+        # removing the log line is easy enough and we don't lose much. And the
+        # real fix was already in set_config_path_and_reload() -- it was just
+        # releasing/renewing the logger.
+        #
+        # self.logger.info(
+        #    f"Initialized CsvPaths: {self} in thread: {threading.current_thread()}"
+        # )
 
     @property
     def logger(self):

{csvpath-0.0.617 → csvpath-0.0.619}/csvpath/managers/files/file_manager.py

@@ -283,6 +283,13 @@ class FileManager:
         home = pathu.resep(home)
         return home
 
+    #
+    # creates a named-file before registering any files.
+    #
+    def assure_named_file(self, name: NamedFileName) -> None:
+        home = self.assure_named_file_home(name)
+        self.registrar.manifest_path(home)
+
     def configure_named_file(self, name: NamedFileName, descriptor: Config) -> None:
         #
         # updates the named-file's descriptor by replacement
@@ -665,7 +672,9 @@ class FileManager:
             self._registration_failed(_)
             #
             return
+        self.csvpaths.logger.info("Adding named file %s: can be loaded", name)
         self.legal_name(name)
+        self.csvpaths.logger.info("Adding named file %s: name is legal", name)
         if path is None or path.strip() == "":
             _ = {
                 "name": name,
@@ -678,7 +687,13 @@ class FileManager:
         if template is None:
             template = self.describer.get_template(name)
         if template is not None:
+            self.csvpaths.logger.debug(
+                "Adding named file %s: checking template %s", name, template
+            )
             temu.valid(template, file=True)
+        self.csvpaths.logger.info(
+            "Adding named file %s: template is valid or None: %s", name, template
+        )
         if config is not None:
             #
             # doing config here enables copy_in() to pull down files
@@ -687,6 +702,9 @@ class FileManager:
             # configured in advance, but that doesn't have to be the
             # case.
             #
+            self.csvpaths.logger.info(
+                "Adding named file %s: configuring named-file", name
+            )
             self.configure_named_file(name, config)
         path = pathu.resep(path)
         self.csvpaths.logger.debug("[%s] Path after resep %s", registration_uuid, path)
@@ -696,6 +714,10 @@ class FileManager:
         local = config.get(section="inputs", name="allow_local_files", default=False)
         local = str(local).strip().lower() in ["on", "yes", "true"]
         nos = Nos(path)
+        self.csvpaths.logger.info(
+            "Adding named file %s: checking permission, if needed, to register local and http files",
+            name,
+        )
         #
         # can the descriptor override the project's prohibition on http(s) loads?
         # no. the project may be controlled by a context like FlightPath Server, where
@@ -729,6 +751,9 @@ class FileManager:
             if self.csvpaths.ecoms.do_i_raise():
                 raise FileException(msg)
             return
+        self.csvpaths.logger.info(
+            "Adding named file %s: registration of %s permitted", name, path
+        )
         #
         # we should know enough to redirect if handed a dir path. we'll not recurse
         # because the caller hasn't given us enough info to know if we should.
@@ -737,6 +762,7 @@ class FileManager:
         # method. if using FlightPath Server, the server should allow them to be likewise
         # more specific. this branch is essentially a fallback presumed to be covering for
         # lazy developers and people who are working without better information.
+        #
         if nos.path.find("#") > -1:
             #
             # if we passed in a root minor, a.k.a. "mark", we need to strip
@@ -744,13 +770,35 @@ class FileManager:
             #
             nos = Nos(nos.path[0 : nos.path.find("#")])
         #
-        # note for now we treat any http as a file
+        # note for now we treat any http as a file.
+        #
+        # also note that this Nos must be savvy to any ServerConfig that may
+        # be available. it should use the possibly cached backend, if that
+        # matches, but if not, it must search any ServerConfig to configure
+        # itself to do operations on one of those servers.
+        #
+        # server config can be passed to a non-sftp nos. there won't be any
+        # complaints, but it won't do anything. atm, the judgement call is that
+        # we save an if-test by allowing any nos to receive server config and
+        # accepting the cost of searching for any server config in json.
+        #
+        config = self.describer.get_config(name)
+        servers = config.sources
+        nos.server_config = servers
+        #
+        #  for _ in servers:
+        #     if _.matches(nos.path):
+        #       nos.do._config = SftpConfig(_)
         #
         isfile = nos.is_http or nos.isfile()
+        self.csvpaths.logger.debug(
+            "Adding named file %s: path %s is a file? %s", name, path, isfile
+        )
         if not isfile:
             return self.add_named_files_from_dir(
                 path, name=name, template=template, recurse=False
             )
+        self.csvpaths.logger.debug("Adding named file %s: continuing with file", name)
         try:
             self.csvpaths.logger.debug(
                 "[%s] Ready to register %s", registration_uuid, path
@@ -896,6 +944,9 @@ class FileManager:
         mdata.template = data.get("template") or ""
         mdata.uuid_string = data.get("registration_uuid")
         mdata.status = "Registration failed"
+        self.csvpaths.logger.warn(
+            "[%s] Registration failed for %s", mdata.named_file_name, data
+        )
         lst.metadata_update(mdata)
 
     def _validate_xlsx_mark(self, path: str, mark: str) -> None:
@@ -927,8 +978,12 @@ class FileManager:
         fname = self._clean_file_name(fname)
         temp = f"{home}{sep}{fname}"
         copy = pathu.parts(path)[0] == pathu.parts(home)[0]
+        copy = copy and nos.is_local
         if copy:
             nos.path = path
+            config = self.describer.get_config(name)
+            servers = config.sources
+            nos.server_config = servers
             nos.copy(temp)
         else:
             self._copy_down(name=name, path=path, temp=temp, mode="wb")

{csvpath-0.0.617 → csvpath-0.0.619}/csvpath/managers/files/file_registrar.py

@@ -154,15 +154,19 @@ class FileRegistrar(Registrar, Listener):
             raise InputException(
                 f"File mgr and registrar marks should match: {mdata.mark}, {mark}"
             )
-        if (
-            # Nos doesn't handle http files. they are special--inbound only.
-            not path.startswith("http:")
-            and not path.startswith("https:")
-            and not Nos(path).exists()
-        ):
-            # if not path.startswith("s3:") and not os.path.exists(path):
-            #
-            raise InputException(f"Path {path} does not exist")
+        opath = Nos(path)
+        #
+        # if we're registering a non-local file and it's an sftp file it may be
+        # on an sftp server that is configured in the named-file description. so
+        # we setup nos to be able to check, just in case.
+        #
+        config = self.csvpaths.file_manager.describer.get_config(mdata.named_file_name)
+        servers = config.sources
+        opath.server_config = servers
+
+        if not opath.is_http:
+            if not opath.exists():
+                raise InputException(f"Path {opath.path} does not exist")
         #
         # if the fingerprint already exists we don't store the file again.
         # we rename the file to the fingerprint. from this point the registrar

{csvpath-0.0.617 → csvpath-0.0.619}/csvpath/managers/results/transfers_manager.py

@@ -6,6 +6,7 @@ from csvpath.managers.paths.paths_descriptor import Transfer, Transfers
 
 from csvpath.util.nos import Nos
 from csvpath.util.path_util import PathUtility as pathu
+from csvpath.util.var_utility import VarUtility as vaut
 
 
 class TransfersManager:
@@ -299,6 +300,11 @@ class TransfersManager:
             msg = f"Variable {t} is invalid: {f}"
             self.csvpaths.error_manager.handle_error(source=self, msg=msg)
             raise InputException(msg)
+
+        #
+        # make sure we have vars interpolated
+        #
+        f = vaut.parse_var_value(self.csvpaths.config, t, f)
         #
         # if we're shipping to another backend, we don't need to change the target location
         #

{csvpath-0.0.617 → csvpath-0.0.619}/csvpath/matching/functions/sql/select.py

@@ -79,6 +79,7 @@ class Select(ValueProducer):
     ##################
 
     def _query(self, *, table: str, column: str, where: str, conn: str) -> bool:
+        engine = None
         try:
             engine = Db.get_engine(conn=conn)
             return self._do_query(
@@ -86,6 +87,12 @@ class Select(ValueProducer):
             )
         except Exception:
             print(traceback.format_exc())
+        finally:
+            if engine is not None:
+                try:
+                    engine.dispose()
+                except Exception:
+                    ...
 
     def _do_query(self, *, table: str, column: str, where: str, engine: Engine) -> list:
         table = sa.table(table, sa.column(column))

{csvpath-0.0.617 → csvpath-0.0.619}/csvpath/matching/functions/sql/sql_in.py

@@ -1,5 +1,5 @@
 from typing import Any
-from sqlalchemy import create_engine, Engine
+from sqlalchemy import Engine
 import sqlalchemy as sa
 import traceback
 
@@ -11,7 +11,6 @@ from csvpath.matching.functions.args import Args
 
 
 class SqlIn(ValueProducer, MatchDecider):
-
     def __init__(self, matcher: Any, name: str, child: Matchable = None) -> None:
         super().__init__(matcher, name=name, child=child)
         self._cached = None
@@ -69,38 +68,56 @@ class SqlIn(ValueProducer, MatchDecider):
         conn = self._value_four(skip=skip)
         cache = self._value_five(skip=skip)
         cache = False if cache is None else cache
-        self.value = self._query(table=table, column=column, value=v, conn=conn, cache=cache)
+        self.value = self._query(
+            table=table, column=column, value=v, conn=conn, cache=cache
+        )
 
     def _decide_match(self, skip=None) -> None:
         self.match = self.to_value(skip=skip)
 
-##################
+    ##################
 
-    def _query(self, *, table:str, column:str, value:Any, cache:bool, conn:str) -> bool:
+    def _query(
+        self, *, table: str, column: str, value: Any, cache: bool, conn: str
+    ) -> bool:
+        engine = None
         try:
             engine = Db.get_engine(conn=conn)
-            return self._do_query(table=table, column=column, value=value, cache=cache, engine=engine)
-        except Exception as ex:
+            return self._do_query(
+                table=table, column=column, value=value, cache=cache, engine=engine
+            )
+        except Exception:
             print(traceback.format_exc())
-
-    def _do_query(self, *, table:str, column:str, value:Any, cache:bool, engine:Engine) -> bool:
+        finally:
+            if engine is not None:
+                try:
+                    engine.dispose()
+                except Exception:
+                    ...
+
+    def _do_query(
+        self, *, table: str, column: str, value: Any, cache: bool, engine: Engine
+    ) -> bool:
         if cache is True:
             if self._cached is None:
                 self._do_cache(value=value, table=table, column=column, engine=engine)
             return value in self._cached
         else:
-            return self._do_lookup(value=value, table=table, column=column, engine=engine)
+            return self._do_lookup(
+                value=value, table=table, column=column, engine=engine
+            )
 
-    def _do_cache(self, *, value:Any, table:str, column:str, engine:Engine) -> bool:
+    def _do_cache(self, *, value: Any, table: str, column: str, engine: Engine) -> bool:
         table = sa.table(table, sa.column(column))
         sql = sa.select(sa.column(column)).select_from(table)
 
-
         with engine.connect() as conn:
             rows = conn.execute(sql).fetchall()
             self._cached = {row[0]: None for row in rows}
 
-    def _do_lookup(self, *, value:Any, table:str, column:str, engine:Engine) -> bool:
+    def _do_lookup(
+        self, *, value: Any, table: str, column: str, engine: Engine
+    ) -> bool:
         table = sa.table(table, sa.column(column))
         query = (
             sa.select(sa.column(column))
@@ -111,5 +128,3 @@ class SqlIn(ValueProducer, MatchDecider):
         with engine.connect() as conn:
             result = conn.execute(query).fetchone()
             return result is not None
-
-

{csvpath-0.0.617 → csvpath-0.0.619}/csvpath/util/nos.py

@@ -6,6 +6,9 @@ from .config import Config
 from .path_util import PathUtility as pathu
 from .class_loader import ClassLoader
 from typing import Self
+from csvpath.util.sftp.sftp_config import SftpConfig
+from csvpath.util.sftp.sftp_server_creds import SftpServerCreds
+from csvpath.managers.server_config import ServerConfig
 
 
 class Nos:
@@ -14,6 +17,7 @@ class Nos:
         self._do = None
         self._config = config
         self.path = path
+        self._servers = None
 
     def __str__(self) -> str:
         return f"{type(self)}: do: {self.do}, path: {self.path}"
@@ -92,6 +96,8 @@ class Nos:
 
     @property
     def location(self) -> str:
+        #
+        # pathu.location(path) is now available. prefer that going forward.
         #
         # returns None if a local path. there may be some gray areas around mounted drives?
         # otherwise, returns the first path part after the protocol. port is included, if any.
@@ -106,6 +112,9 @@ class Nos:
 
     @property
     def location_and_port(self) -> tuple[str, int]:
+        #
+        # pathu.location_and_port(path) is now available. prefer that going forward.
+        #
         location = self.location
         if location is None:
             return None
@@ -134,6 +143,82 @@ class Nos:
             return path[7:]
         return path
 
+    #
+    # SFTP only:
+    # server configs are set on named-files and named-paths for non-local file
+    # registration and post run transfers, respectively. if we have a matching
+    # config, we will use it, not the backend, even if the backend is configured
+    # for sftp.
+    #
+    # ServerConfig connections/clients are not cached/reused like the regular
+    # SFTP backend is.
+    #
+    # also see DataFileReader and DataFileWriter's ServerConfig logic.
+    #
+    @property
+    def server_config(self) -> list[ServerConfig]:
+        return self._servers
+
+    @server_config.setter
+    def server_config(self, servers: list[ServerConfig]) -> None:
+        #
+        # when we set server config on a nos we're enabling the SFTP not only
+        # from the cached backend location, but also uncached creds stored in a
+        # named-file or named-paths definition json.
+        #
+        # for an example see: TestUtilNos.test_nos_server_config_1
+        # or see FileManager.add_named_file() around like 750:
+        #
+        #        config = self.describer.get_config(name)
+        #        servers = config.sources
+        #        nos.server_config = servers
+        #
+        # if there is a match, that is what is used, if no match, it should
+        # fall back to the backend sftp config, if any.
+        #
+        if servers is None:
+            return
+        if not self.is_sftp:
+            #
+            # do we need to worry about people adding server config to non-SFTP
+            # Nos? there isn't much practical advantage and we can simplify use
+            # just a bit by making a non-sftp a no-op.
+            #
+            return
+        if len(servers) == 0:
+            return
+        self._servers = servers
+        server, port = self.location_and_port
+        for k, v in servers.items():
+            #
+            # there is a lot of use of _ (underscored) methods that are supposed
+            # to be hiding below here. that could and probably should change, but
+            # simply making the methods not _ would not better guarantee that we
+            # be careful with the way Nos, DataFileReader, DataFileWriter and their
+            # implementation classes are handled.
+            #
+            if v.address == server and v.port == port:
+                instance = ClassLoader.load(
+                    "from csvpath.util.sftp.sftp_nos import SftpDo",
+                    args=[self.path, False],
+                )
+                self._do = instance
+                self._do.server_config = v
+                #
+                # for getting Sftp server creds we need a csvpath.util.Config.
+                # it isn't certain we'll have one ourselves; however, the SftpDo
+                # will look for one in Box or create one for us.
+                #
+                if self._config is None:
+                    self._config = self._do._csvpath_config()
+                c = SftpConfig(self._config)
+                c.username, c.password = SftpServerCreds.server_credentials(self)
+                c.server = server
+                c.port = port
+                c._do_load()
+                self._do._config = c
+                break
+
     @property
     def do(self):
         if self.path is not None and self._do is None:

{csvpath-0.0.617 → csvpath-0.0.619}/csvpath/util/path_util.py

@@ -79,6 +79,9 @@ class PathUtility:
 
     @classmethod
     def stripp(cls, apath: str) -> str:
+        #
+        # removes protocol + location, incl. port.
+        #
         i = apath.find("://")
         j = -1
         if i > -1:
@@ -88,6 +91,38 @@ class PathUtility:
                 apath = apath[j + 1 :]
         return apath
 
+    @classmethod
+    def location(cls, path: str):
+        #
+        # find the location of a non-local path. i.e. in sftp://mylocation:22/a file.txt
+        # the return would be mylocation:22
+        #
+        i = path.find("://")
+        if i == -1:
+            return None
+        path = path[i + 3 :]
+        i = path.find("/")
+        if i == -1:
+            return path
+        return path[0:i]
+
+    @classmethod
+    def location_and_port(cls, path: str) -> tuple[str, int]:
+        #
+        # find the location of a non-local path. \
+        #
+        # in sftp://mylocation:22/a file.txt  the return would be (mylocation,22)
+        # in sftp://mylocation/a file.txt  the return would be (mylocation,None)
+        #
+        location = cls.location(path)
+        if location is None:
+            return None
+        i = location.find(":")
+        if i == -1:
+            return (location, None)
+        port = location[i + 1 :]
+        return location[0:i], port
+
     @classmethod
     def equal(cls, pathone: str, pathtwo: str, stripp=False) -> bool:
         #