acryl-datahub 1.2.0.10rc8__py3-none-any.whl → 1.2.0.11rc2__py3-none-any.whl

datahub/secret/environment_secret_store.py

@@ -0,0 +1,29 @@
+import os
+from typing import Dict, List, Union
+
+from datahub.secret.secret_store import SecretStore
+
+
+# Simple SecretStore implementation that fetches Secret values from the local environment.
+class EnvironmentSecretStore(SecretStore):
+    def __init__(self, config):
+        pass
+
+    def close(self) -> None:
+        return
+
+    def get_secret_values(self, secret_names: List[str]) -> Dict[str, Union[str, None]]:
+        values = {}
+        for secret_name in secret_names:
+            values[secret_name] = os.getenv(secret_name)
+        return values
+
+    def get_secret_value(self, secret_name: str) -> Union[str, None]:
+        return os.getenv(secret_name)
+
+    def get_id(self) -> str:
+        return "env"
+
+    @classmethod
+    def create(cls, config: Dict) -> "EnvironmentSecretStore":
+        return cls(config)

datahub/secret/file_secret_store.py

@@ -0,0 +1,49 @@
+import logging
+import os
+from typing import Any, Dict, List, Union
+
+from pydantic import BaseModel
+
+from datahub.secret.secret_store import SecretStore
+
+logger = logging.getLogger(__name__)
+
+
+class FileSecretStoreConfig(BaseModel):
+    basedir: str = "/mnt/secrets"
+    max_length: int = 1024768
+
+
+# Simple SecretStore implementation that fetches Secret values from the local files.
+class FileSecretStore(SecretStore):
+    def __init__(self, config: FileSecretStoreConfig):
+        self.config = config
+
+    def get_secret_values(self, secret_names: List[str]) -> Dict[str, Union[str, None]]:
+        values = {}
+        for secret_name in secret_names:
+            values[secret_name] = self.get_secret_value(secret_name)
+        return values
+
+    def get_secret_value(self, secret_name: str) -> Union[str, None]:
+        secret_path = os.path.join(self.config.basedir, secret_name)
+        if os.path.exists(secret_path):
+            with open(secret_path, "r") as f:
+                secret_value = f.read(self.config.max_length + 1)
+                if len(secret_value) > self.config.max_length:
+                    logger.warning(
+                        f"Secret {secret_name} is longer than {self.config.max_length} and will be truncated."
+                    )
+                return secret_value[: self.config.max_length].rstrip()
+        return None
+
+    def get_id(self) -> str:
+        return "file"
+
+    def close(self) -> None:
+        return
+
+    @classmethod
+    def create(cls, config: Any) -> "FileSecretStore":
+        config = FileSecretStoreConfig.parse_obj(config)
+        return cls(config)

datahub/utilities/file_backed_collections.py

@@ -33,13 +33,12 @@ from datahub.utilities.sentinels import Unset, unset
 
 logger: logging.Logger = logging.getLogger(__name__)
 
-OVERRIDE_SQLITE_VERSION_REQUIREMENT_STR = (
-    os.environ.get("OVERRIDE_SQLITE_VERSION_REQ") or ""
-)
-OVERRIDE_SQLITE_VERSION_REQUIREMENT = (
-    OVERRIDE_SQLITE_VERSION_REQUIREMENT_STR
-    and OVERRIDE_SQLITE_VERSION_REQUIREMENT_STR.lower() != "false"
-)
+
+def _get_sqlite_version_override() -> bool:
+    """Check if SQLite version requirement should be overridden at runtime."""
+    override_str = os.environ.get("OVERRIDE_SQLITE_VERSION_REQ") or ""
+    return bool(override_str and override_str.lower() != "false")
+
 
 _DEFAULT_FILE_NAME = "sqlite.db"
 _DEFAULT_TABLE_NAME = "data"
@@ -231,7 +230,7 @@ class FileBackedDict(MutableMapping[str, _VT], Closeable, Generic[_VT]):
             # We use the ON CONFLICT clause to implement UPSERTs with sqlite.
             # This was added in 3.24.0 from 2018-06-04.
             # See https://www.sqlite.org/lang_conflict.html
-            if OVERRIDE_SQLITE_VERSION_REQUIREMENT:
+            if _get_sqlite_version_override():
                 self._use_sqlite_on_conflict = False
             else:
                 raise RuntimeError("SQLite version 3.24.0 or later is required")