dsw-config 4.27.0__py3-none-any.whl

dsw/config/__init__.py

@@ -0,0 +1,4 @@
+from .parser import DSWConfigParser, MissingConfigurationError
+
+
+__all__ = ['DSWConfigParser', 'MissingConfigurationError']

dsw/config/build_info.py

@@ -0,0 +1,17 @@
+# Generated file
+# - do not overwrite
+# - do not include in git
+from collections import namedtuple
+
+BuildInfo = namedtuple(
+    'BuildInfo',
+    ['version', 'built_at', 'sha', 'branch', 'tag'],
+)
+
+BUILD_INFO = BuildInfo(
+    version='v4.27.0~8ec71bd',
+    built_at='2026-02-03 08:44:06Z',
+    sha='8ec71bd85dfbea66adedb6590f7d76ae5143bbaa',
+    branch='HEAD',
+    tag='v4.27.0',
+)

dsw/config/keys.py

@@ -0,0 +1,268 @@
+# pylint: disable=too-few-public-methods
+import collections
+import typing
+
+
+T = typing.TypeVar('T')
+
+
+def cast_bool(value: typing.Any) -> bool:
+    return bool(value)
+
+
+def cast_optional_bool(value: typing.Any) -> bool | None:
+    if value is None:
+        return None
+    return bool(value)
+
+
+def cast_int(value: typing.Any) -> int:
+    return int(value)
+
+
+def cast_optional_int(value: typing.Any) -> int | None:
+    if value is None:
+        return None
+    return int(value)
+
+
+def cast_float(value: typing.Any) -> float:
+    return float(value)
+
+
+def cast_optional_float(value: typing.Any) -> float | None:
+    if value is None:
+        return None
+    return float(value)
+
+
+def cast_str(value: typing.Any) -> str:
+    return str(value)
+
+
+def cast_optional_str(value: typing.Any) -> str | None:
+    if value is None:
+        return None
+    return str(value)
+
+
+def cast_optional_dict(value: typing.Any) -> dict | None:
+    if not isinstance(value, dict):
+        return None
+    return value
+
+
+class ConfigKey[T]:
+
+    def __init__(self, *, yaml_path: list[str],
+                 cast: typing.Callable[[typing.Any], T],
+                 var_names=None, default=None, required=False):
+        self.yaml_path = yaml_path
+        self.var_names = var_names or []  # type: list[str]
+        self.default = default
+        self.required = required
+        self.cast = cast
+
+    def __str__(self):
+        return f'ConfigKey: {".".join(self.yaml_path)}'
+
+
+class ConfigKeysMeta(type):
+
+    @classmethod
+    # pylint: disable-next=unused-argument
+    def __prepare__(mcs, name: str, bases: tuple,
+                    /, **kwargs) -> collections.abc.MutableMapping[str, object]:
+        return collections.OrderedDict()
+
+    def __init__(cls, name, bases, namespace):
+        cls._config_keys = []
+        for attr in namespace:
+            if attr.startswith('_'):
+                continue
+            value = getattr(cls, attr)
+            if isinstance(value, ConfigKey):
+                cls._config_keys.append(value)
+            keys = getattr(value, '_config_keys', None)
+            if keys is not None and isinstance(keys, list):
+                cls._config_keys.extend(keys)
+        super().__init__(name, bases, namespace)
+
+    def __iter__(cls):
+        return iter(cls._config_keys)
+
+
+class ConfigKeysContainer(metaclass=ConfigKeysMeta):
+    pass
+
+
+class _GeneralKeys(ConfigKeysContainer):
+    environment = ConfigKey(
+        yaml_path=['general', 'environment'],
+        var_names=['GENERAL_ENVIRONMENT'],
+        default='Production',
+        cast=cast_str,
+    )
+    client_url = ConfigKey(
+        yaml_path=['general', 'clientUrl'],
+        var_names=['GENERAL_CLIENT_URL'],
+        default='http://localhost:8080',
+        cast=cast_str,
+    )
+    secret = ConfigKey(
+        yaml_path=['general', 'secret'],
+        var_names=['GENERAL_SECRET'],
+        default='',
+        cast=cast_str,
+    )
+
+
+class _LoggingKeys(ConfigKeysContainer):
+    level = ConfigKey(
+        yaml_path=['logging', 'level'],
+        var_names=['LOGGING_ENVIRONMENT'],
+        default='INFO',
+        cast=cast_str,
+    )
+    global_level = ConfigKey(
+        yaml_path=['logging', 'globalLevel'],
+        var_names=['LOGGING_CLIENT_URL'],
+        default='WARNING',
+        cast=cast_str,
+    )
+    format = ConfigKey(
+        yaml_path=['logging', 'format'],
+        var_names=['LOGGING_FORMAT'],
+        default='%(asctime)s | %(levelname)8s | %(name)s: [T:%(traceId)s] %(message)s',
+        cast=cast_str,
+    )
+    dict_config = ConfigKey(
+        yaml_path=['logging', 'dict_config'],
+        var_names=[],
+        default=None,
+        cast=cast_optional_dict,
+    )
+
+
+class _CloudKeys(ConfigKeysContainer):
+    enabled = ConfigKey(
+        yaml_path=['cloud', 'enabled'],
+        var_names=['CLOUD_ENABLED'],
+        default=False,
+        cast=cast_bool,
+    )
+
+
+class _SentryKeys(ConfigKeysContainer):
+    enabled = ConfigKey(
+        yaml_path=['sentry', 'enabled'],
+        var_names=['SENTRY_ENABLED'],
+        default=False,
+        cast=cast_bool,
+    )
+    worker_dsn = ConfigKey(
+        yaml_path=['sentry', 'workersDsn'],
+        var_names=['SENTRY_WORKER_DSN', 'SENTRY_DSN'],
+        default='',
+        cast=cast_str,
+    )
+    traces_sample_rate = ConfigKey(
+        yaml_path=['sentry', 'tracesSampleRate'],
+        var_names=['SENTRY_TRACES_SAMPLE_RATE'],
+        default='',
+        cast=cast_str,
+    )
+    max_breadcrumbs = ConfigKey(
+        yaml_path=['sentry', 'maxBreadcrumbs'],
+        var_names=['SENTRY_MAX_BREADCRUMBS'],
+        default='',
+        cast=cast_str,
+    )
+    environment = ConfigKey(
+        yaml_path=['sentry', 'environment'],
+        var_names=['SENTRY_ENVIRONMENT'],
+        default='production',
+        cast=cast_str,
+    )
+
+
+class _DatabaseKeys(ConfigKeysContainer):
+    connection_string = ConfigKey(
+        yaml_path=['database', 'connectionString'],
+        var_names=['DATABASE_CONNECTION_STRING'],
+        default='postgresql://postgres:postgres@postgres:5432/engine-wizard',
+        cast=cast_str,
+    )
+    connection_timeout = ConfigKey(
+        yaml_path=['database', 'connectionTimeout'],
+        var_names=['DATABASE_CONNECTION_TIMEOUT'],
+        default=30000,
+        cast=cast_int,
+    )
+    queue_timeout = ConfigKey(
+        yaml_path=['database', 'queueTimeout'],
+        var_names=['DATABASE_QUEUE_TIMEOUT'],
+        default=180,
+        cast=cast_int,
+    )
+
+
+class _S3Keys(ConfigKeysContainer):
+    url = ConfigKey(
+        yaml_path=['s3', 'url'],
+        var_names=['S3_URL'],
+        default='http://minio:9000',
+        cast=cast_str,
+    )
+    bucket = ConfigKey(
+        yaml_path=['s3', 'bucket'],
+        var_names=['S3_BUCKET'],
+        default='engine-wizard',
+        cast=cast_str,
+    )
+    region = ConfigKey(
+        yaml_path=['s3', 'region'],
+        var_names=['S3_REGION'],
+        default='eu-central-1',
+        cast=cast_str,
+    )
+    username = ConfigKey(
+        yaml_path=['s3', 'username'],
+        var_names=['S3_USERNAME'],
+        default='minio',
+        cast=cast_str,
+    )
+    password = ConfigKey(
+        yaml_path=['s3', 'password'],
+        var_names=['S3_PASSWORD'],
+        default='minioPassword',
+        cast=cast_str,
+    )
+
+
+class _AWSKeys(ConfigKeysContainer):
+    access_key_id = ConfigKey(
+        yaml_path=['aws', 'awsAccessKeyId'],
+        var_names=['AWS_AWS_ACCESS_KEY_ID'],
+        cast=cast_optional_str,
+    )
+    secret_access_key = ConfigKey(
+        yaml_path=['aws', 'awsSecretAccessKey'],
+        var_names=['AWS_AWS_SECRET_ACCESS_KEY'],
+        cast=cast_optional_str,
+    )
+    region = ConfigKey(
+        yaml_path=['aws', 'awsRegion'],
+        var_names=['AWS_AWS_REGION'],
+        cast=cast_optional_str,
+    )
+
+
+class ConfigKeys(ConfigKeysContainer):
+    aws = _AWSKeys
+    cloud = _CloudKeys
+    database = _DatabaseKeys
+    general = _GeneralKeys
+    logging = _LoggingKeys
+    s3 = _S3Keys
+    sentry = _SentryKeys

dsw/config/logging.py

@@ -0,0 +1,51 @@
+import logging
+import logging.config
+import sys
+
+
+class DSWLogFilter(logging.Filter):
+
+    def __init__(self, extras=None):
+        super().__init__()
+        self.extras = extras or {'traceId': ''}
+
+    def set_extra(self, key: str, value: str):
+        self.extras[key] = value
+
+    def filter(self, record: logging.LogRecord):
+        record.__dict__.update(self.extras)
+        return True
+
+
+LOG_FILTER = DSWLogFilter()
+
+
+class DSWLogger(logging.Logger):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.addFilter(LOG_FILTER)
+
+
+def prepare_logging(logging_cfg):
+    # pylint: disable-next=no-member
+    logger_dict = logging.root.manager.loggerDict
+    if logging_cfg.dict_config is not None:
+        logging.config.dictConfig(logging_cfg.dict_config)
+    else:
+        logging.basicConfig(
+            stream=sys.stdout,
+            level=logging_cfg.global_level,
+            format=logging_cfg.message_format,
+        )
+        dsw_loggers = (logging.getLogger(name) for name in logger_dict
+                       if name.lower().startswith('dsw'))
+        for logger in dsw_loggers:
+            logger.setLevel(logging_cfg.level)
+    # Set for all existing loggers
+    logging.getLogger().addFilter(filter=LOG_FILTER)
+    loggers = (logging.getLogger(name) for name in logger_dict)
+    for logger in loggers:
+        logger.addFilter(filter=LOG_FILTER)
+    # Set for any future loggers
+    logging.setLoggerClass(DSWLogger)

dsw/config/model.py

@@ -0,0 +1,82 @@
+import dataclasses
+
+from .logging import LOG_FILTER, prepare_logging
+
+
+def _config_to_string(config: object):
+    lines = [f'{type(config).__name__}']
+    fields = (f for f in config.__dict__ if not f.startswith('_'))
+    for field in fields:
+        v = str(getattr(config, field))
+        t = type(getattr(config, field)).__name__
+        lines.append(f'- {field} = {v} [{t}]')
+    return '\n'.join(lines)
+
+
+class ConfigModel:
+
+    def __str__(self):
+        return _config_to_string(self)
+
+
+@dataclasses.dataclass
+class GeneralConfig(ConfigModel):
+    environment: str
+    client_url: str
+    secret: str
+
+
+@dataclasses.dataclass
+class SentryConfig(ConfigModel):
+    enabled: bool
+    workers_dsn: str | None
+    traces_sample_rate: float | None
+    max_breadcrumbs: int | None
+    environment: str
+
+
+@dataclasses.dataclass
+class DatabaseConfig(ConfigModel):
+    connection_string: str
+    connection_timeout: int
+    queue_timeout: int
+
+
+@dataclasses.dataclass
+class S3Config(ConfigModel):
+    url: str
+    username: str
+    password: str
+    bucket: str
+    region: str
+
+
+@dataclasses.dataclass
+class LoggingConfig(ConfigModel):
+    level: str
+    global_level: str
+    message_format: str
+    dict_config: dict | None = None
+
+    def apply(self):
+        prepare_logging(self)
+
+    @staticmethod
+    def set_logging_extra(key: str, value: str):
+        LOG_FILTER.set_extra(key, value)
+
+
+@dataclasses.dataclass
+class AWSConfig(ConfigModel):
+    access_key_id: str | None
+    secret_access_key: str | None
+    region: str | None
+
+    @property
+    def has_credentials(self) -> bool:
+        return self.access_key_id is not None and self.secret_access_key is not None
+
+
+@dataclasses.dataclass
+class CloudConfig(ConfigModel):
+    multi_tenant: bool

dsw/config/parser.py

@@ -0,0 +1,137 @@
+import os
+import typing
+
+import yaml
+
+from . import model
+from .keys import ConfigKey, ConfigKeys
+
+
+class MissingConfigurationError(Exception):
+
+    def __init__(self, missing: list[str]):
+        self.missing = missing
+
+
+class DSWConfigParser:
+
+    def __init__(self, keys=ConfigKeys):
+        self.cfg = {}
+        self.keys = keys
+
+    @staticmethod
+    def can_read(content: str):
+        try:
+            yaml.safe_load(content)
+            return True
+        except Exception:
+            return False
+
+    def read_file(self, fp: typing.IO):
+        self.cfg = yaml.safe_load(fp) or self.cfg
+
+    def read_string(self, content: str):
+        self.cfg = yaml.safe_load(content) or self.cfg
+
+    def has_value_for_path(self, yaml_path: list[str]):
+        x = self.cfg
+        for p in yaml_path:
+            if not hasattr(x, 'keys') or p not in x:
+                return False
+            x = x[p]
+        return True
+
+    @staticmethod
+    def _prefix_var(var_name: str) -> str:
+        return f'DSW_{var_name}'
+
+    def has_value_for_key(self, key: ConfigKey):
+        if self.has_value_for_path(key.yaml_path):
+            return True
+        for var_name in key.var_names:
+            if var_name in os.environ or self._prefix_var(var_name) in os.environ:
+                return True
+        return False
+
+    def get_or_default(self, key: ConfigKey):
+        x: typing.Any = self.cfg
+        for p in key.yaml_path:
+            if not hasattr(x, 'keys') or p not in x:
+                return key.default
+            x = x[p]
+        return x
+
+    def get(self, key: ConfigKey):
+        for var_name in key.var_names:
+            if var_name in os.environ:
+                return key.cast(os.environ[var_name])
+            if self._prefix_var(var_name) in os.environ:
+                return key.cast(os.environ[self._prefix_var(var_name)])
+        return key.cast(self.get_or_default(key))
+
+    def validate(self):
+        missing = [
+            '.'.join(key.yaml_path) for key in self.keys
+            if key.required and not self.has_value_for_key(key)
+        ]
+        if len(missing) > 0:
+            raise MissingConfigurationError(missing)
+
+    @property
+    def db(self) -> model.DatabaseConfig:
+        return model.DatabaseConfig(
+            connection_string=self.get(self.keys.database.connection_string),
+            connection_timeout=self.get(self.keys.database.connection_timeout),
+            queue_timeout=self.get(self.keys.database.queue_timeout),
+        )
+
+    @property
+    def s3(self) -> model.S3Config:
+        return model.S3Config(
+            url=self.get(self.keys.s3.url),
+            username=self.get(self.keys.s3.username),
+            password=self.get(self.keys.s3.password),
+            bucket=self.get(self.keys.s3.bucket),
+            region=self.get(self.keys.s3.region),
+        )
+
+    @property
+    def logging(self) -> model.LoggingConfig:
+        return model.LoggingConfig(
+            level=self.get(self.keys.logging.level),
+            global_level=self.get(self.keys.logging.global_level),
+            message_format=self.get(self.keys.logging.format),
+            dict_config=self.get(self.keys.logging.dict_config),
+        )
+
+    @property
+    def cloud(self) -> model.CloudConfig:
+        return model.CloudConfig(
+            multi_tenant=self.get(self.keys.cloud.enabled),
+        )
+
+    @property
+    def sentry(self) -> model.SentryConfig:
+        return model.SentryConfig(
+            enabled=self.get(self.keys.sentry.enabled),
+            workers_dsn=self.get(self.keys.sentry.worker_dsn),
+            traces_sample_rate=self.get(self.keys.sentry.traces_sample_rate),
+            max_breadcrumbs=self.get(self.keys.sentry.max_breadcrumbs),
+            environment=self.get(self.keys.sentry.environment),
+        )
+
+    @property
+    def general(self) -> model.GeneralConfig:
+        return model.GeneralConfig(
+            environment=self.get(self.keys.general.environment),
+            client_url=self.get(self.keys.general.client_url),
+            secret=self.get(self.keys.general.secret),
+        )
+
+    @property
+    def aws(self) -> model.AWSConfig:
+        return model.AWSConfig(
+            access_key_id=self.get(self.keys.aws.access_key_id),
+            secret_access_key=self.get(self.keys.aws.secret_access_key),
+            region=self.get(self.keys.aws.region),
+        )

dsw/config/sentry.py

@@ -0,0 +1,60 @@
+import logging
+import typing
+
+import sentry_sdk
+from sentry_sdk.integrations.logging import LoggingIntegration
+from sentry_sdk.types import Event, Hint
+
+from .model import SentryConfig
+
+
+EventProcessor = typing.Callable[[Event, Hint], Event | None]
+
+
+class SentryReporter:
+    report = False
+    filters = []  # type: list[EventProcessor]
+
+    @classmethod
+    def initialize(cls, *, config: SentryConfig, prog_name: str, release: str,
+                   breadcrumb_level: int | None = logging.INFO,
+                   event_level: int | None = logging.ERROR):
+        cls.report = config.enabled and config.workers_dsn is not None
+        if cls.report:
+            def before_send(event, hint):
+                for f in cls.filters:
+                    if not f(event, hint):
+                        return None
+                return event
+
+            sentry_sdk.init(
+                dsn=config.workers_dsn,
+                traces_sample_rate=config.traces_sample_rate or 1.0,
+                max_breadcrumbs=config.max_breadcrumbs or sentry_sdk.consts.DEFAULT_MAX_BREADCRUMBS,
+                release=release,
+                environment=config.environment,
+                before_send=before_send,
+                default_integrations=False,
+                integrations=[
+                    LoggingIntegration(
+                        level=breadcrumb_level,
+                        event_level=event_level,
+                    ),
+                ],
+            )
+            sentry_sdk.set_tag('component', prog_name)
+
+    @classmethod
+    def capture_exception(cls, *args, **kwargs):
+        if cls.report:
+            sentry_sdk.capture_exception(*args, **kwargs)
+
+    @classmethod
+    def capture_message(cls, *args, **kwargs):
+        if cls.report:
+            sentry_sdk.capture_message(*args, **kwargs)
+
+    @classmethod
+    def set_tags(cls, **tags):
+        if cls.report:
+            sentry_sdk.set_tags(tags)

dsw_config-4.27.0.dist-info/METADATA

@@ -0,0 +1,43 @@
+Metadata-Version: 2.3
+Name: dsw-config
+Version: 4.27.0
+Summary: Library for DSW config manipulation
+Keywords: dsw,config,yaml,parser
+Author: Marek Suchánek
+Author-email: Marek Suchánek <marek.suchanek@ds-wizard.org>
+License: Apache License 2.0
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Text Processing
+Classifier: Topic :: Utilities
+Requires-Dist: pyyaml
+Requires-Dist: sentry-sdk
+Requires-Python: >=3.12, <4
+Project-URL: Homepage, https://ds-wizard.org
+Project-URL: Repository, https://github.com/ds-wizard/engine-tools
+Project-URL: Documentation, https://guide.ds-wizard.org
+Project-URL: Issues, https://github.com/ds-wizard/ds-wizard/issues
+Description-Content-Type: text/markdown
+
+# Data Stewardship Wizard: Config
+
+[![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/ds-wizard/engine-tools)](https://github.com/ds-wizard/engine-tools/releases)
+[![PyPI](https://img.shields.io/pypi/v/dsw-config)](https://pypi.org/project/dsw-config/)
+[![LICENSE](https://img.shields.io/github/license/ds-wizard/engine-tools)](LICENSE)
+[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/4975/badge)](https://bestpractices.coreinfrastructure.org/projects/4975)
+[![Python Version](https://img.shields.io/badge/Python-%E2%89%A5%203.7-blue)](https://python.org)
+
+*Library for working with DSW configuration*
+
+## Usage
+
+Currently, this library is intended for internal use of DSW tooling only. 
+Enhancements for use in custom scripts are planned for future development.
+
+## License
+
+This project is licensed under the Apache License v2.0 - see the
+[LICENSE](LICENSE) file for more details.

dsw_config-4.27.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+dsw/config/__init__.py,sha256=xHLvInMTcWvmRwZJIM6ZvhSIKE3J35MgejsM7-6ALoc,124
+dsw/config/build_info.py,sha256=q1VZ-03xJkIRGFaYomTfIG49Of9XRQnbk34em4NhKeE,381
+dsw/config/keys.py,sha256=KdWsKoqlw4UjPA-cHGH6D3NvA54k4sPDORMXthpPAmc,6952
+dsw/config/logging.py,sha256=OMoOlQsBaWkLAxAPdtDZiK05e_-hgEhqyiGhmGBONoQ,1480
+dsw/config/model.py,sha256=j_5IuCgDsbxKorYX29gaoh_ChWI1pgR6-9jrRY2Q_-8,1753
+dsw/config/parser.py,sha256=RKKSMiuYwycdc-IVTrRpjKg5PJuzyfa2QdMTQqFE4XI,4363
+dsw/config/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dsw/config/sentry.py,sha256=2Jo0FWvGweso5dagLVYltS3aRXIdCPMlJWc22-H3L7c,1930
+dsw_config-4.27.0.dist-info/WHEEL,sha256=fAguSjoiATBe7TNBkJwOjyL1Tt4wwiaQGtNtjRPNMQA,80
+dsw_config-4.27.0.dist-info/METADATA,sha256=U8UWJhRcQZMHLTAkZTK4vQZbrGV3MVHPH8dBWqHSSI0,1866
+dsw_config-4.27.0.dist-info/RECORD,,

dsw_config-4.27.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: uv 0.9.28
+Root-Is-Purelib: true
+Tag: py3-none-any