thds.tabularasa 0.14.0__py3-none-any.whl → 0.14.2__py3-none-any.whl

thds/tabularasa/__main__.py

@@ -1,5 +1,4 @@
 import logging
-import os
 import shutil
 import subprocess
 import sys
@@ -14,7 +13,7 @@ from typing import Dict, Iterable, Iterator, List, NamedTuple, Optional, Set, Tu
 import networkx as nx
 import pkg_resources
 
-from thds.core import parallel
+from thds.core import link, parallel
 from thds.tabularasa.data_dependencies.adls import (
     ADLSFileIntegrityError,
     ADLSFileSystem,
@@ -730,7 +729,7 @@ class ReferenceDataManager:
             file_path = self.data_path_for(table)
             if file_path.exists():
                 self.logger.warning(f"Removing built file for table {table.name} at {file_path}")
-                os.remove(file_path)
+                file_path.unlink()
             else:
                 self.logger.info(f"No file found for table {table.name}; nothing to remove")
         try:
@@ -763,7 +762,7 @@ class ReferenceDataManager:
         for table in tables_to_update:
             table_name = table.name
             table_path = self.data_path_for(table)
-            if os.path.exists(table_path):
+            if table_path.exists():
                 md5 = hash_file(table_path)
                 old_md5 = table.md5
                 if old_md5 is None:
@@ -847,10 +846,10 @@ class ReferenceDataManager:
                 local_cache_path = paths[0].local_path
                 if sync_data.local_file_exists:
                     self.logger.warning(f"Removing existing file {sync_data.local_path}")
-                    os.remove(sync_data.local_path)
+                    sync_data.local_path.unlink()
                 self.logger.info(f"Linking downloaded file to local build file {sync_data.local_path}")
                 sync_data.local_path.parent.mkdir(parents=True, exist_ok=True)
-                os.link(local_cache_path, sync_data.local_path)
+                link.link(local_cache_path, sync_data.local_path)
             return True
 
     def sync_blob_store(

thds/tabularasa/data_dependencies/sqlite.py

@@ -222,7 +222,9 @@ def populate_sqlite_db(
     table_predicate: Callable[[Table], bool] = is_build_time_package_table,
     data_path_overrides: Optional[Mapping[str, Path]] = None,
 ):
-    """Populate a sqlite database with data for a set of tables from a `reference_data.schema.Schema`
+    """Populate a sqlite database with data for a set of tables from a `reference_data.schema.Schema`.
+    Note that this can safely be called concurrently in multiple processes on the same database file; a file lock
+    is acquired on the database file and only released when the data insertion is complete.
 
     :param schema: the `reference_data.schema.Schema` object defining the data to be inserted
     :param db_package: name of the package where the database file is stored, if any. In case `None` is
@@ -259,6 +261,9 @@ def populate_sqlite_db(
     # gather all tables before executing any I/O
     insert_tables = [table for table in schema.filter_tables(table_predicate) if table.has_indexes]
 
+    if not insert_tables:
+        return
+
     with bulk_write_connection(db_path, db_package, close=True) as con:
         for table in insert_tables:
             table_filename: Optional[str]

thds/tabularasa/loaders/sqlite_util.py

@@ -22,7 +22,6 @@ from thds.tabularasa.sqlite3_compat import sqlite3
 
 DEFAULT_ATTR_SQLITE_CACHE_SIZE = 100_000
 DEFAULT_MMAP_BYTES = int(os.environ.get("TABULA_RASA_DEFAULT_MMAP_BYTES", 8_589_934_592))  # 8 GB
-DISABLE_WAL_MODE = bool(os.environ.get("REF_D_DISABLE_SQLITE_WAL_MODE", False))
 
 PARAMETERIZABLE_BUILTINS = sys.version_info >= (3, 9)
 
@@ -159,11 +158,6 @@ def set_bulk_write_mode(con: sqlite3.Connection) -> sqlite3.Connection:
     logger.debug("Setting pragmas for bulk write optimization")
     # https://www.sqlite.org/pragma.html#pragma_synchronous
     _log_exec_sql(logger, con, "PRAGMA synchronous = 0")  # OFF
-    # https://www.sqlite.org/pragma.html#pragma_journal_mode
-    if not DISABLE_WAL_MODE:
-        _log_exec_sql(logger, con, "PRAGMA journal_mode = WAL")
-    # https://www.sqlite.org/pragma.html#pragma_locking_mode
-    _log_exec_sql(logger, con, "PRAGMA locking_mode = EXCLUSIVE")
 
     return con
 
@@ -171,16 +165,7 @@ def set_bulk_write_mode(con: sqlite3.Connection) -> sqlite3.Connection:
 def unset_bulk_write_mode(con: sqlite3.Connection) -> sqlite3.Connection:
     logger = logging.getLogger(__name__)
     logger.debug("Setting pragmas for bulk write optimization")
-    # https://www.sqlite.org/pragma.html#pragma_journal_mode
-    # resetting this to the default. This is a property of the database, rather than the connection.
-    # the other settings are connection-specific.
-    # according to the docs, the WAL journal mode should be disabled before the locking mode is restored,
-    # else any attempt to do so is a no-op.
-    _log_exec_sql(logger, con, "PRAGMA journal_mode = DELETE")
-    # https://www.sqlite.org/pragma.html#pragma_synchronous
     _log_exec_sql(logger, con, "PRAGMA synchronous = 2")  # FULL (default)
-    # https://www.sqlite.org/pragma.html#pragma_locking_mode
-    _log_exec_sql(logger, con, "PRAGMA locking_mode = NORMAL")
 
     return con
 
@@ -191,9 +176,10 @@ def bulk_write_connection(
 ) -> ty.Generator[sqlite3.Connection, None, None]:
     """Context manager to set/unset bulk write mode on a sqlite connection. Sets pragmas for efficient bulk writes,
     such as loosening synchronous and locking modes. If `close` is True, the connection will be closed on exit.
-    Since setting the pragmas may mutate the database file, and since by design this context manager exists to enable
-    bulk writes which intentionally mutate the database, if a `db_path` (and optionally `db_package`, if specified as
-    package data) is given, we also acquire a file lock on the database file on entry and release it on exit.
+    To avoid bulk insert routines being run by other processes concurrently, we also acquire a file lock on the
+    database file on entry and release it on exit. Other processes attempting to perform bulk writes to the same file
+    will block until the lock is released. In the case of tabularasa init-sqlite, the semantics then imply that those
+    workers will perform no writes at all, since metadata will indicate that the data in the file is up-to-date.
     """
     db_path_ = to_local_path(db_path, db_package).absolute()
     lock_path = db_path_.with_suffix(".lock")

{thds_tabularasa-0.14.0.dist-info → thds_tabularasa-0.14.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: thds.tabularasa
-Version: 0.14.0
+Version: 0.14.2
 Summary: Trilliant Health reference data build system.
 Author-email: Trilliant Health <info@trillianthealth.com>
 Project-URL: Repository, https://github.com/TrilliantHealth/ds-monorepo

{thds_tabularasa-0.14.0.dist-info → thds_tabularasa-0.14.2.dist-info}/RECORD

@@ -1,5 +1,5 @@
 thds/tabularasa/__init__.py,sha256=jc6w1WD868MQ2t4wkRNYvRssojwXvPDcNyC8V5gwbl0,169
-thds/tabularasa/__main__.py,sha256=Ryfd7YogTE_qFjp8IJA-KTeTXXD9INS5GJGmdPVvWBw,47791
+thds/tabularasa/__main__.py,sha256=DlaUfXu03tbBVucRuMw7354LeBs8d5tRCuAprZs0XYs,47778
 thds/tabularasa/compat.py,sha256=j0313TPIXtkbfvRI0AH4if8GLrjQSrDJ9heayCIl9w8,1037
 thds/tabularasa/git_util.py,sha256=fBFhaCPi_5W2BpG2B3WiPcAWJvuVI_pG47rt73wLO6E,1388
 thds/tabularasa/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -9,7 +9,7 @@ thds/tabularasa/to_sqlite.py,sha256=5lcEUh38MNebxAJdLp2XGWOP_WQDIADtL1fyhOvi9UU,
 thds/tabularasa/data_dependencies/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 thds/tabularasa/data_dependencies/adls.py,sha256=vJAuc5Key-vO1N6DGo5dj9fIx_4hMALAVC17qhvkT7Y,3257
 thds/tabularasa/data_dependencies/build.py,sha256=6iYgw93sOF2Nlnb6WSmA9NvPNwOf_Yyi2wXUQpRVkJM,23382
-thds/tabularasa/data_dependencies/sqlite.py,sha256=eweuLdoxyGlG-PvQUANarlMe_mmZgA5cxuMbOYxcpsQ,12576
+thds/tabularasa/data_dependencies/sqlite.py,sha256=sMP_NInBEDoH5SScIRYxtOvcPUi9WXfE3_jCoOBduGo,12825
 thds/tabularasa/data_dependencies/tabular.py,sha256=oq9wFse235ikLEv8Zvol59ptRRojZbkbzXJyQeFfC9o,6529
 thds/tabularasa/data_dependencies/util.py,sha256=FQ9G1nIpqKh00z2lXOt0Y2R1mLQsEb-BC6Tka1z2egc,8489
 thds/tabularasa/diff/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -19,7 +19,7 @@ thds/tabularasa/diff/summary.py,sha256=gENtDwhSrDYeN-8fWr6Ug2zgdp584b0pZF9UBYzKF
 thds/tabularasa/loaders/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 thds/tabularasa/loaders/lazy_adls.py,sha256=jrWy5tTKDQfWEv6aHQ3UJhFzLrDPOlSGsArv9zcl1g8,1375
 thds/tabularasa/loaders/parquet_util.py,sha256=u75j3PkMSakO2zfq4zksWzXLYnaO--WizAgXTcSpXRY,13354
-thds/tabularasa/loaders/sqlite_util.py,sha256=GLP271SF-GVQwxaaB0p5kCG0eGay6j4mWabL00a9pz0,11954
+thds/tabularasa/loaders/sqlite_util.py,sha256=3Gi1Y4iTVCD9FXqylQw1eyFwVuplQUrjY1J0SC5FFWg,11099
 thds/tabularasa/loaders/util.py,sha256=XmsGkDdL6O8R6B4667Iqi5HoRgq0YMs6LP3VvPIqPVU,21369
 thds/tabularasa/schema/__init__.py,sha256=bowvNXrrDrWB3TAmwDxCeEAvVEe9z7iRfqRaNg1Qmo4,440
 thds/tabularasa/schema/constraints.py,sha256=V2vh01BhYR8OVQvgdujqSi0l_fMJvFKYSlBvWExZFG0,9744
@@ -41,8 +41,8 @@ thds/tabularasa/schema/compilation/sqlite.py,sha256=wSrSlVCYeuTpOf9AOHAnp6gJHkjH
 thds/tabularasa/schema/compilation/util.py,sha256=YXFe1_yoBobED010hstKIoq-dwLHo6SBv1v1IAw6AYU,3886
 thds/tabularasa/testing/__init__.py,sha256=XoLzB-DotxFw9KHt2vfH72k7pyAAFI2bW-qqq6nww1g,85
 thds/tabularasa/testing/mock_sqlite.py,sha256=xoV4w_GaDgtZf17iUux2-LA6Va1XRJdC2FU34dysh0o,4769
-thds_tabularasa-0.14.0.dist-info/METADATA,sha256=fzbOzf8zgv-IBEcUN_6stkhIkhccgbrmJBi_jbGdkS4,26786
-thds_tabularasa-0.14.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-thds_tabularasa-0.14.0.dist-info/entry_points.txt,sha256=PX4ShRonjv6lMsVjrGu8RkFzpyyvgM9EnZlNfMomd9k,61
-thds_tabularasa-0.14.0.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
-thds_tabularasa-0.14.0.dist-info/RECORD,,
+thds_tabularasa-0.14.2.dist-info/METADATA,sha256=H48Ilbbjrag4tb1s1IhIlh5tNbUnwjXdaQ3zSC9rejg,26786
+thds_tabularasa-0.14.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+thds_tabularasa-0.14.2.dist-info/entry_points.txt,sha256=PX4ShRonjv6lMsVjrGu8RkFzpyyvgM9EnZlNfMomd9k,61
+thds_tabularasa-0.14.2.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
+thds_tabularasa-0.14.2.dist-info/RECORD,,