csvpath 0.0.533__tar.gz → 0.0.534__tar.gz

{csvpath-0.0.533 → csvpath-0.0.534}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: csvpath
-Version: 0.0.533
+Version: 0.0.534
 Summary: A data preboarding framework for managing and validating CSV, Excel, and other tabular data files using a Collect, Store, Validate, Publish pattern to create a trusted publisher for downstream data consumers.
 Author: David Kershaw
 Author-email: dk107dk@hotmail.com
@@ -27,22 +27,25 @@ Requires-Dist: ckanapi (>=4.8,<5.0)
 Requires-Dist: email-validator (>=2.2.0,<3.0.0)
 Requires-Dist: google-cloud-storage (>=3.1.0,<4.0.0)
 Requires-Dist: inflect (>=7.4.0,<8.0.0)
-Requires-Dist: jinja2 (>=3.1.4,<4.0.0)
+Requires-Dist: jinja2 (>=3.1.6,<4.0.0)
 Requires-Dist: lark (>=1.2.2,<2.0.0)
 Requires-Dist: marquez-python (>=0.50.0,<0.51.0)
 Requires-Dist: metaphone (>=0.6,<0.7)
+Requires-Dist: mysqlclient (>=2.2.7,<3.0.0)
 Requires-Dist: openlineage-python (>=1.25.0,<2.0.0)
 Requires-Dist: opentelemetry-distro[otlp] (>=0.50b0,<0.51)
 Requires-Dist: paramiko (>=3.5.0,<4.0.0)
 Requires-Dist: pdoc (>=15.0.1,<16.0.0)
 Requires-Dist: ply (>=3.11,<4.0)
 Requires-Dist: prompt-toolkit (>=3.0.50,<4.0.0)
+Requires-Dist: psycopg2-binary (>=2.9.10,<3.0.0)
 Requires-Dist: pylightxl (>=1.61,<2.0)
 Requires-Dist: pytest (>=8.3.3,<9.0.0)
 Requires-Dist: python-dateutil (>=2.9.0.post0,<3.0.0)
 Requires-Dist: pytz (>=2024.2,<2025.0)
 Requires-Dist: requests (>=2.32.3,<3.0.0)
 Requires-Dist: smart-open (>=7.1.0,<8.0.0)
+Requires-Dist: sqlalchemy (>=2.0.38,<3.0.0)
 Requires-Dist: tabulate (>=0.9.0,<0.10.0)
 Requires-Dist: validators (>=0.34.0,<0.35.0)
 Project-URL: Csvpath.org, https://www.csvpath.org

{csvpath-0.0.533 → csvpath-0.0.534/assets}/config/config.ini

@@ -33,9 +33,9 @@ archive = archive
 
 [inputs]
 files = inputs/named_files
+csvpaths = inputs/named_paths
 #csvpaths = s3://csvpath-example-1/inputs/named_paths
 #files = sftp://172.17.0.3:10022/inputs/named_files
-
 on_unmatched_file_fingerprints = halt
 
 #
@@ -46,7 +46,13 @@ on_unmatched_file_fingerprints = halt
 groups =
 #slack, marquez, ckan, sftp, sftpplus, otlp, default, sqlite
 
-# add sqlite to capture staging, loads, and results in a local sqlite db
+# add sql to capture staging, loads, and results in mysql, postgres, ms sql server, or sqlite
+sql.file = from csvpath.managers.integrations.sql.sql_file_listener import SqlFileListener
+sql.paths = from csvpath.managers.integrations.sql.sql_paths_listener import SqlPathsListener
+sql.result = from csvpath.managers.integrations.sql.sql_result_listener import SqlResultListener
+sql.results = from csvpath.managers.integrations.sql.sql_results_listener import SqlResultsListener
+
+# add sqlite to capture staging, loads, and results in a local sqlite file
 sqlite.result = from csvpath.managers.integrations.sqlite.sqlite_result_listener import SqliteResultListener
 sqlite.results = from csvpath.managers.integrations.sqlite.sqlite_results_listener import SqliteResultsListener
 
@@ -99,6 +105,11 @@ password = tinpenny
 [sqlite]
 db = archive/csvpath.db
 
+[sql]
+# sqlite, mysql, postgres, or sql_server
+dialect = sqlite
+connection_string = sqlite:///archive/csvpath-sqlite.db
+
 [sftpplus]
 # these are only needed by the csvpath writer
 mailbox_user = MAILBOX_USER

csvpath-0.0.534/assets/config/jenkins-local-filesystem-mysql.ini

@@ -0,0 +1,138 @@
+[csvpath_files]
+extensions = txt, csvpath, csvpaths
+
+[csv_files]
+extensions = txt, csv, tsv, dat, tab, psv, ssv
+
+[errors]
+csvpath = raise, collect, stop, fail, print
+csvpaths = raise, collect
+use_format = full
+pattern = {time}:{file}:{line}:{paths}:{instance}:{chain}:  {message}
+
+[logging]
+csvpath = debug
+csvpaths = debug
+log_file = logs/csvpath.log
+log_files_to_keep = 100
+log_file_size = 52428800
+
+[config]
+path = assets/config/jenkins-local-filesystem.ini
+
+[cache]
+path = cache
+use_cache = no
+
+[functions]
+imports = config/functions.imports
+
+[results]
+archive = archive
+
+[inputs]
+files = inputs/named_files
+csvpaths = inputs/named_paths
+on_unmatched_file_fingerprints = halt
+
+#
+# integrations activation
+#
+
+[listeners]
+groups = sql
+#slack, marquez, ckan, sftp, sftpplus, otlp, default, sqlite
+
+sql.file = from csvpath.managers.integrations.sql.sql_file_listener import SqlFileListener
+sql.paths = from csvpath.managers.integrations.sql.sql_paths_listener import SqlPathsListener
+sql.result = from csvpath.managers.integrations.sql.sql_result_listener import SqlResultListener
+sql.results = from csvpath.managers.integrations.sql.sql_results_listener import SqlResultsListener
+
+# add sqlite to capture staging, loads, and results in a local sqlite db
+sqlite.result = from csvpath.managers.integrations.sqlite.sqlite_result_listener import SqliteResultListener
+sqlite.results = from csvpath.managers.integrations.sqlite.sqlite_results_listener import SqliteResultsListener
+
+# these simple default listeners track all files staged and all paths loaded in
+# central manifests. unlike other manifest writers, if they have multiple
+# concurrent users they have potential a race condition. users that do not
+# share inputs directories do not need to worry about this. you should vet
+# using them against your use case; potential utility vs. the modest but non-0
+# risk of lost updates. they can be disabled with little loss of function.
+# alternatively a database-backed version may be more suitable.
+default.file = from csvpath.managers.files.files_listener import FilesListener
+default.paths = from csvpath.managers.paths.paths_listener import PathsListener
+
+# add otlp to support any OpenTelemetry backend
+otlp.result = from csvpath.managers.integrations.otlp.otlp_result_listener import OpenTelemetryResultListener
+otlp.results = from csvpath.managers.integrations.otlp.otlp_results_listener import OpenTelemetryResultsListener
+otlp.errors = from csvpath.managers.integrations.otlp.otlp_error_listener import OpenTelemetryErrorListener
+
+# add sftpplus to the list of groups above to automate registration and named-paths group runs on file arrival at an SFTPPlus server
+sftpplus.paths = from csvpath.managers.integrations.sftpplus.sftpplus_listener import SftpPlusListener
+
+# add sftp to the list of groups above to push results to an sftp account
+sftp.results = from csvpath.managers.integrations.sftp.sftp_listener import SftpListener
+
+# add ckan to the list of groups above for alerts to slack webhooks
+ckan.results = from csvpath.managers.integrations.ckan.ckan_listener import CkanListener
+
+#add marquez to the list of groups above for OpenLineage events to a Marquez server
+marquez.file = from csvpath.managers.integrations.ol.file_listener_ol import OpenLineageFileListener
+marquez.paths = from csvpath.managers.integrations.ol.paths_listener_ol import OpenLineagePathsListener
+marquez.result = from csvpath.managers.integrations.ol.result_listener_ol import OpenLineageResultListener
+marquez.results = from csvpath.managers.integrations.ol.results_listener_ol import OpenLineageResultsListener
+
+# add slack to the list of groups above for alerts to slack webhooks
+slack.file = from csvpath.managers.integrations.slack.sender import SlackSender
+slack.paths = from csvpath.managers.integrations.slack.sender import SlackSender
+slack.result = from csvpath.managers.integrations.slack.sender import SlackSender
+slack.results = from csvpath.managers.integrations.slack.sender import SlackSender
+
+#
+# integrations setup
+#
+
+[sftp]
+server = 192.168.67.2
+port = 10022
+username = tinpenny
+password = tinpenny
+
+[sql]
+dialect = mysql
+connection_string = mysql://csvpath:hangzhou@127.0.0.1/csvpath
+
+[sqlite]
+db = archive/csvpath.db
+
+[sftpplus]
+# these are only needed by the csvpath writer
+mailbox_user = MAILBOX_USER
+mailbox_password = MAILBOX_PASSWORD
+server = SFTPPLUS_SERVER
+port = SFTPPLUS_PORT
+# these are only needed on the server
+admin_username = SFTPPLUS_ADMIN_USERNAME
+admin_password = SFTPPLUS_ADMIN_PASSWORD
+api_url = https://localhost:10020/json
+scripts_dir =
+execute_timeout = 300
+
+[ckan]
+server = http://localhost:80
+api_token = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJqdGkiOiI3akJwc1ZuSkVrZm1aNnBtVTJfTW5CNlJXZ211YjdOOHVXZ1l1cUFDa0Q4IiwiaWF0IjoxNzM0NzE4NDQ3fQ.QXWXoJoSxVES4NwXYBteYUD7enX9D5T2htmETLGFzrs
+
+[marquez]
+base_url = http://localhost:5000
+endpoint = api/v1/lineage
+api_key = "none"
+timeout = 5
+verify = False
+
+[slack]
+# add your main webhook here. to set webhooks on a csvpath-by-csvpath basis add
+# on-valid-slack: webhook-minus-'https://' and/or
+# on-invalid-slack: webhook-minus-'https://'
+webhook_url =
+
+

{csvpath-0.0.533 → csvpath-0.0.534}/assets/config/jenkins-local-gcs.ini

@@ -18,7 +18,7 @@ log_files_to_keep = 100
 log_file_size = 52428800
 
 [config]
-path = assets/config/jenkins-local-azure.ini
+path = assets/config/jenkins-local-gcs.ini
 
 [cache]
 path = cache

{csvpath-0.0.533/assets → csvpath-0.0.534}/config/config.ini

@@ -33,9 +33,9 @@ archive = archive
 
 [inputs]
 files = inputs/named_files
+csvpaths = inputs/named_paths
 #csvpaths = s3://csvpath-example-1/inputs/named_paths
 #files = sftp://172.17.0.3:10022/inputs/named_files
-
 on_unmatched_file_fingerprints = halt
 
 #
@@ -46,7 +46,13 @@ on_unmatched_file_fingerprints = halt
 groups =
 #slack, marquez, ckan, sftp, sftpplus, otlp, default, sqlite
 
-# add sqlite to capture staging, loads, and results in a local sqlite db
+# add sql to capture staging, loads, and results in mysql, postgres, ms sql server, or sqlite
+sql.file = from csvpath.managers.integrations.sql.sql_file_listener import SqlFileListener
+sql.paths = from csvpath.managers.integrations.sql.sql_paths_listener import SqlPathsListener
+sql.result = from csvpath.managers.integrations.sql.sql_result_listener import SqlResultListener
+sql.results = from csvpath.managers.integrations.sql.sql_results_listener import SqlResultsListener
+
+# add sqlite to capture staging, loads, and results in a local sqlite file
 sqlite.result = from csvpath.managers.integrations.sqlite.sqlite_result_listener import SqliteResultListener
 sqlite.results = from csvpath.managers.integrations.sqlite.sqlite_results_listener import SqliteResultsListener
 
@@ -99,6 +105,11 @@ password = tinpenny
 [sqlite]
 db = archive/csvpath.db
 
+[sql]
+# sqlite, mysql, postgres, or sql_server
+dialect = sqlite
+connection_string = sqlite:///archive/csvpath-sqlite.db
+
 [sftpplus]
 # these are only needed by the csvpath writer
 mailbox_user = MAILBOX_USER

{csvpath-0.0.533 → csvpath-0.0.534}/csvpath/managers/files/file_manager.py

@@ -302,6 +302,9 @@ class FileManager:
         mdata.file_name = file_home[file_home.rfind(nos.sep) + 1 :]
         mdata.name_home = name_home
         mdata.mark = mark
+        #
+        # TODO: add file_size. move FileInfo into Nos. for now it is 0.
+        #
         self.registrar.register_complete(mdata)
 
     def _clean_file_name(self, fname: str) -> str:

{csvpath-0.0.533 → csvpath-0.0.534}/csvpath/managers/files/file_metadata.py

@@ -22,3 +22,4 @@ class FileMetadata(Metadata):
         self.mark = None
         # like csv
         self.type = None
+        self.file_size = 0

{csvpath-0.0.533 → csvpath-0.0.534}/csvpath/managers/files/file_registrar.py

@@ -49,6 +49,10 @@ class FileRegistrar(Registrar, Listener):
         return j
 
     def patch_named_file(self, *, name, patch, index=-1) -> None:
+        #
+        # TODO: distribute an metadata event to give metadata stores a
+        # chance to update their info re: the file
+        #
         home = self.csvpaths.file_manager.named_file_home(name)
         mp = self.manifest_path(home)
         mani = self.get_manifest(mp)
@@ -140,10 +144,12 @@ class FileRegistrar(Registrar, Listener):
                 f"File mgr and registrar marks should match: {mdata.mark}, {mark}"
             )
         if (
+            # TODO: s3 can do nos.exists
             not path.startswith("s3:")
+            # Nos doesn't handle http files. they are special--inbound only.
             and not path.startswith("http:")
             and not path.startswith("https:")
-            # and not azure?
+            # and not azure? gcp? should be handled by nos anyway.
             and not Nos(path).exists()
         ):
             # if not path.startswith("s3:") and not os.path.exists(path):

csvpath-0.0.534/csvpath/managers/integrations/sql/engine.py

@@ -0,0 +1,24 @@
+from sqlalchemy import create_engine, Engine
+from csvpath.util.config import Config
+
+
+class Db:
+    @classmethod
+    def get(self, config: Config) -> Engine:
+        dialect = config.get(section="sql", name="dialect")
+        c_str = config.get(section="sql", name="connection_string")
+        if dialect == "sqlite":
+            # sqlite:///example.db
+            engine = create_engine(c_str)
+        elif dialect == "postgres":
+            # postgresql+psycopg2://user:password@localhost/dbname
+            engine = create_engine(c_str)
+        elif dialect == "mysql":
+            # mysql+pymysql://user:password@localhost/dbname
+            engine = create_engine(c_str)
+        elif dialect == "sql_server":
+            # mssql+pyodbc://user:password@localhost/dbname
+            engine = create_engine(c_str)
+        else:
+            raise ValueError("Unknown RDBMS dialect %s", dialect)
+        return engine

csvpath-0.0.534/csvpath/managers/integrations/sql/sql_file_listener.py

@@ -0,0 +1,85 @@
+from sqlalchemy import Table
+from sqlalchemy.dialects.postgresql import insert as pg_insert
+from sqlalchemy.dialects.mysql import insert as mysql_insert
+from sqlalchemy.dialects.sqlite import insert as sqlite_insert
+from csvpath.managers.metadata import Metadata
+from .sql_listener import SqlListener
+
+
+class SqlFileListener(SqlListener):
+    def __init__(self, config=None):
+        SqlListener.__init__(self, config=config)
+        self._named_file = None
+
+    @property
+    def named_file(self) -> Table:
+        if self._named_file is None:
+            self._named_file = self.tables.named_file
+        return self._named_file
+
+    def metadata_update(self, mdata: Metadata) -> None:
+        if not self.csvpaths:
+            raise RuntimeError("CsvPaths cannot be None")
+        named_file_data = {
+            "uuid": mdata.uuid_string,
+            "at": mdata.time,
+            "named_file_name": mdata.named_file_name,
+            "origin_path": mdata.origin_path,
+            "name_home": mdata.name_home,
+            "file_home": mdata.file_home,
+            "file_path": mdata.file_path,
+            "file_name": mdata.file_name,
+            "mark": mdata.mark,
+            "type": mdata.type,
+            "file_size": mdata.file_size,
+            "ip_address": mdata.ip_address,
+            "hostname": mdata.hostname,
+            "username": mdata.username,
+            "files_root": mdata.named_files_root,
+            "base_path": mdata.base_path,
+            "manifest_path": mdata.manifest_path,
+        }
+        self._upsert_named_file(named_file_data)
+
+    def _upsert_named_file(self, named_file_data: dict, *, dispose: bool = True):
+        with self.engine.connect() as conn:
+            dialect = conn.dialect.name
+            self.csvpaths.logger.info("Inserting named-file metadata into %s", dialect)
+            stmt = None
+            if dialect in ["postgresql", "sqlite"]:
+                ist = pg_insert if dialect == "postgresql" else sqlite_insert
+                stmt = (
+                    ist(self.named_file)
+                    .values(named_file_data)
+                    .on_conflict_do_update(
+                        index_elements=["uuid"], set_=self._set(named_file_data)
+                    )
+                )
+            elif dialect == "mysql":
+                stmt = (
+                    mysql_insert(self.named_file)
+                    .values(named_file_data)
+                    .on_duplicate_key_update(self._set(named_file_data))
+                )
+            elif dialect == "mssql":
+                raise NotImplementedError("SQL Server support is not yet implemented.")
+            else:
+                raise ValueError(f"Unsupported database dialect: {dialect}")
+            conn.execute(stmt)
+            conn.commit()
+        if dispose is True:
+            self.engine.dispose()
+
+    def _set(self, named_file_data) -> dict:
+        return {
+            "files_root": named_file_data["files_root"],
+            "file_home": named_file_data["file_home"],
+            "file_path": named_file_data["file_path"],
+            "file_name": named_file_data["file_name"],
+            "mark": named_file_data["mark"],
+            "type": named_file_data["type"],
+            "file_size": named_file_data["file_size"],
+            "ip_address": named_file_data["ip_address"],
+            "hostname": named_file_data["hostname"],
+            "username": named_file_data["username"],
+        }

csvpath-0.0.534/csvpath/managers/integrations/sql/sql_listener.py

@@ -0,0 +1,29 @@
+from sqlalchemy import Engine
+from csvpath.managers.listener import Listener
+from .engine import Db
+from .tables import Tables
+
+
+class SqlListener(Listener):
+    def __init__(self, config=None):
+        Listener.__init__(self, config=config)
+        self.csvpaths = None
+        self._tables = None
+        self._engine = None
+
+    @property
+    def tables(self) -> Tables:
+        if self._tables is None:
+            self._tables = Tables(self.config, engine=self.engine)
+            self._tables.assure_tables()
+        return self._tables
+
+    @property
+    def engine(self) -> Engine:
+        if self._engine is None:
+            self._engine = Db.get(self.config)
+        return self._engine
+
+    @engine.setter
+    def engine(self, engine: Engine):
+        self._engine = engine

csvpath-0.0.534/csvpath/managers/integrations/sql/sql_paths_listener.py

@@ -0,0 +1,78 @@
+from sqlalchemy import Table
+from sqlalchemy.dialects.postgresql import insert as pg_insert
+from sqlalchemy.dialects.mysql import insert as mysql_insert
+from sqlalchemy.dialects.sqlite import insert as sqlite_insert
+from csvpath.managers.metadata import Metadata
+from .sql_listener import SqlListener
+
+
+class SqlPathsListener(SqlListener):
+    def __init__(self, config=None):
+        SqlListener.__init__(self, config=config)
+        self._named_paths = None
+
+    @property
+    def named_paths(self) -> Table:
+        if self._named_paths is None:
+            self._named_paths = self.tables.named_paths
+        return self._named_paths
+
+    def metadata_update(self, mdata: Metadata) -> None:
+        if not self.csvpaths:
+            raise RuntimeError("CsvPaths cannot be None")
+        named_paths_data = {
+            "uuid": mdata.uuid_string,
+            "at": mdata.time,
+            "paths_root": mdata.named_paths_root,
+            "paths_name": mdata.named_paths_name,
+            "paths_home": mdata.named_paths_home,
+            "group_file_path": mdata.group_file_path,
+            "paths_count": mdata.named_paths_count,
+            "ip_address": mdata.ip_address,
+            "hostname": mdata.hostname,
+            "username": mdata.username,
+            "base_path": mdata.base_path,
+            "manifest_path": mdata.manifest_path,
+        }
+        self._upsert_named_paths(named_paths_data)
+
+    def _upsert_named_paths(self, named_paths_data: dict, *, dispose: bool = True):
+        with self.engine.connect() as conn:
+            dialect = conn.dialect.name
+            self.csvpaths.logger.info("Inserting named-paths metadata into %s", dialect)
+            stmt = None
+            if dialect in ["postgresql", "sqlite"]:
+                ist = pg_insert if dialect == "postgresql" else sqlite_insert
+                stmt = (
+                    ist(self.named_paths)
+                    .values(named_paths_data)
+                    .on_conflict_do_update(
+                        index_elements=["uuid"], set_=self._set(named_paths_data)
+                    )
+                )
+            elif dialect == "mysql":
+                stmt = (
+                    mysql_insert(self.named_paths)
+                    .values(named_paths_data)
+                    .on_duplicate_key_update(self._set(named_paths_data))
+                )
+            elif dialect == "mssql":
+                raise NotImplementedError("SQL Server support is not yet implemented.")
+            else:
+                raise ValueError(f"Unsupported database dialect: {dialect}")
+            conn.execute(stmt)
+            conn.commit()
+        if dispose is True:
+            self.engine.dispose()
+
+    def _set(self, named_paths_data) -> dict:
+        return {
+            "paths_root": named_paths_data["paths_name"],
+            "paths_name": named_paths_data["paths_name"],
+            "paths_home": named_paths_data["paths_home"],
+            "group_file_path": named_paths_data["group_file_path"],
+            "paths_count": named_paths_data["paths_count"],
+            "ip_address": named_paths_data["ip_address"],
+            "hostname": named_paths_data["hostname"],
+            "username": named_paths_data["username"],
+        }

csvpath-0.0.534/csvpath/managers/integrations/sql/sql_result_listener.py

@@ -0,0 +1,100 @@
+from sqlalchemy import Table
+from sqlalchemy.dialects.postgresql import insert as pg_insert
+from sqlalchemy.dialects.mysql import insert as mysql_insert
+from sqlalchemy.dialects.sqlite import insert as sqlite_insert
+from csvpath.managers.metadata import Metadata
+from .sql_listener import SqlListener
+
+
+class SqlResultListener(SqlListener):
+    def __init__(self, config=None):
+        SqlListener.__init__(self, config=config)
+        self._instance_run = None
+
+    @property
+    def instance_run(self) -> Table:
+        if self._instance_run is None:
+            self._instance_run = self.tables.instance_run
+        return self._instance_run
+
+    def metadata_update(self, mdata: Metadata) -> None:
+        if not self.csvpaths:
+            raise RuntimeError("CsvPaths cannot be None")
+        instance_run_data = {
+            "uuid": mdata.uuid_string,
+            "at": mdata.time,
+            "group_run_uuid": mdata.named_paths_uuid_string,
+            "instance_identity": mdata.instance_identity,
+            "instance_index": mdata.instance_index,
+            "instance_home": mdata.instance_home,
+            "preceding_instance_identity": mdata.preceding_instance_identity,
+            "actual_data_file": mdata.actual_data_file,
+            "source_mode_preceding": "Y"
+            if (
+                mdata.source_mode_preceding is True
+                or mdata.source_mode_preceding == "Y"
+            )
+            else "N",
+            "valid": "Y" if (mdata.valid is True or mdata.valid == "Y") else "N",
+            "completed": "Y"
+            if (mdata.completed is True or mdata.completed == "Y")
+            else "N",
+            "files_expected": "Y"
+            if (mdata.files_expected is True or mdata.files_expected == "Y")
+            else "N",
+            "error_count": mdata.error_count if mdata.error_count else 0,
+            "number_of_files_expected": mdata.number_of_files_expected,
+            "number_of_files_generated": mdata.number_of_files_generated,
+            "lines_scanned": mdata.lines_scanned,
+            "lines_total": mdata.lines_total,
+            "lines_matched": mdata.lines_matched,
+            "manifest_path": mdata.manifest_path,
+        }
+        self._upsert_instance_run(instance_run_data)
+
+    def _upsert_instance_run(self, instance_run_data, *, dispose=True):
+        with self.engine.connect() as conn:
+            dialect = conn.dialect.name
+            self.csvpaths.logger.info("Inserting run result metadata into %s", dialect)
+            s = self._set(instance_run_data)
+            if dialect in ["postgresql", "sqlite"]:
+                ist = pg_insert if dialect == "postgresql" else sqlite_insert
+                stmt = (
+                    ist(self.instance_run)
+                    .values(instance_run_data)
+                    .on_conflict_do_update(index_elements=["uuid"], set_=s)
+                )
+            elif dialect == "mysql":
+                stmt = (
+                    mysql_insert(self.instance_run)
+                    .values(instance_run_data)
+                    .on_duplicate_key_update(self._set(instance_run_data))
+                )
+            elif dialect == "mssql":
+                raise NotImplementedError("SQL Server support is not yet implemented.")
+            else:
+                raise ValueError(f"Unsupported database dialect: {dialect}")
+            conn.execute(stmt)
+            conn.commit()
+        if dispose is True:
+            self.engine.dispose()
+
+    def _set(self, instance_run_data: dict) -> dict:
+        return {
+            "valid": "Y"
+            if (instance_run_data["valid"] is True or instance_run_data["valid"] == "Y")
+            else "N",
+            "completed": "Y"
+            if (
+                instance_run_data["completed"] is True
+                or instance_run_data["completed"] == "Y"
+            )
+            else "N",
+            "error_count": instance_run_data["error_count"],
+            "number_of_files_expected": instance_run_data["number_of_files_expected"],
+            "number_of_files_generated": instance_run_data["number_of_files_generated"],
+            "files_expected": instance_run_data["files_expected"],
+            "lines_scanned": instance_run_data["lines_scanned"],
+            "lines_total": instance_run_data["lines_total"],
+            "lines_matched": instance_run_data["lines_matched"],
+        }