limitsdb 0.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- limitsdb/__init__.py +0 -0
- limitsdb/cli/__init__.py +0 -0
- limitsdb/cli/ldb_crypt.py +44 -0
- limitsdb/cli/ldb_impl.py +207 -0
- limitsdb/cli/ldb_init.py +60 -0
- limitsdb/cli/ldb_run.py +53 -0
- limitsdb/core/__init__.py +0 -0
- limitsdb/core/ldb_config_loader.py +198 -0
- limitsdb/core/ldb_crypto.py +91 -0
- limitsdb/core/ldb_errors.py +30 -0
- limitsdb/core/ldb_ilm_config.py +228 -0
- limitsdb/core/ldb_logger.py +57 -0
- limitsdb/core/ldb_meta.py +31 -0
- limitsdb/core/ldb_params_config.py +459 -0
- limitsdb/core/ldb_runner.py +810 -0
- limitsdb/core/ldb_status.py +8 -0
- limitsdb/core/ldb_utils.py +332 -0
- limitsdb/db/__init__.py +0 -0
- limitsdb/db/ldb_engine_loader.py +16 -0
- limitsdb/db/ldb_engines.py +347 -0
- limitsdb/db/oracle/__init__.py +0 -0
- limitsdb/db/oracle/ldb_conf.sql +151 -0
- limitsdb/db/oracle/ldb_engine_impl.py +1505 -0
- limitsdb/resources/__init__.py +0 -0
- limitsdb/resources/ilm.example.yml +71 -0
- limitsdb-0.5.0.dist-info/METADATA +586 -0
- limitsdb-0.5.0.dist-info/RECORD +32 -0
- limitsdb-0.5.0.dist-info/WHEEL +4 -0
- limitsdb-0.5.0.dist-info/entry_points.txt +6 -0
- limitsdb-0.5.0.dist-info/licenses/LICENSE +118 -0
- limitsdb-0.5.0.dist-info/licenses/LICENSE-DUAL.md +59 -0
- limitsdb-0.5.0.dist-info/licenses/NOTICE +12 -0
limitsdb/__init__.py
ADDED
|
File without changes
|
limitsdb/cli/__init__.py
ADDED
|
File without changes
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""Command line interface for encrypting secret configuration values."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
|
|
7
|
+
from limitsdb.core.ldb_crypto import load_or_create_key
|
|
8
|
+
from limitsdb.core.ldb_errors import LimitsDbError
|
|
9
|
+
from limitsdb.core.ldb_logger import configure_logger, get_logger
|
|
10
|
+
from limitsdb.core.ldb_utils import encrypt_secrets_in_place
|
|
11
|
+
|
|
12
|
+
configure_logger(level="WARNING")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _parse_args() -> argparse.Namespace:
|
|
16
|
+
parser = argparse.ArgumentParser(
|
|
17
|
+
description="Encrypt cleartext secrets in secrets.json",
|
|
18
|
+
)
|
|
19
|
+
parser.add_argument("--schema", required=True, help="Schema name (folder under schemas/)")
|
|
20
|
+
parser.add_argument("--profile", help="Profile name (e.g., dev, prod)")
|
|
21
|
+
parser.add_argument("--config-dir", help="Configuration root (overrides autodiscovery)")
|
|
22
|
+
return parser.parse_args()
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def run_cli() -> None:
|
|
26
|
+
"""Entry point for the ``ldb-crypt`` command."""
|
|
27
|
+
logger = get_logger("crypt")
|
|
28
|
+
try:
|
|
29
|
+
args = _parse_args()
|
|
30
|
+
load_or_create_key()
|
|
31
|
+
updated = encrypt_secrets_in_place(
|
|
32
|
+
schema=args.schema, profile=getattr(args, "profile", None), config_root=getattr(args, "config_dir", None)
|
|
33
|
+
)
|
|
34
|
+
if updated:
|
|
35
|
+
logger.info("Secrets updated: %s", updated)
|
|
36
|
+
else:
|
|
37
|
+
logger.info("No secrets to update (file missing or already encrypted).")
|
|
38
|
+
except (LimitsDbError, ValueError) as exc:
|
|
39
|
+
logger.error("%s", exc)
|
|
40
|
+
raise SystemExit(1) from exc
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
if __name__ == "__main__": # pragma: no cover
|
|
44
|
+
run_cli()
|
limitsdb/cli/ldb_impl.py
ADDED
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
"""Bootstrap database objects required by LimitsDb."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
|
|
7
|
+
from limitsdb.core.ldb_crypto import load_or_create_key
|
|
8
|
+
from limitsdb.core.ldb_errors import LimitsDbError
|
|
9
|
+
from limitsdb.core.ldb_logger import configure_logger, get_logger, reconfigure_logger
|
|
10
|
+
from limitsdb.core.ldb_params_config import Config, build_config
|
|
11
|
+
from limitsdb.core.ldb_utils import encrypt_secrets_in_place
|
|
12
|
+
from limitsdb.db.ldb_engine_loader import get_db_engine
|
|
13
|
+
from limitsdb.db.ldb_engines import (
|
|
14
|
+
ColumnDefinition,
|
|
15
|
+
DatabaseLinkDefinition,
|
|
16
|
+
RoleDefinition,
|
|
17
|
+
SequenceDefinition,
|
|
18
|
+
TableDefinition,
|
|
19
|
+
UserDefinition,
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
configure_logger(level="WARNING")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _parse_args() -> argparse.Namespace:
|
|
26
|
+
parser = argparse.ArgumentParser(
|
|
27
|
+
description="Ensure LimitsDb control objects exist in the database.",
|
|
28
|
+
)
|
|
29
|
+
parser.add_argument("--schema", required=True, help="Schema name (folder under schemas/)")
|
|
30
|
+
parser.add_argument("--profile", help="Profile name (e.g., dev, prod)")
|
|
31
|
+
parser.add_argument("--config-dir", help="Configuration root (overrides autodiscovery)")
|
|
32
|
+
parser.add_argument("--config-file", help="Additional configuration overlay")
|
|
33
|
+
parser.add_argument(
|
|
34
|
+
"--set", action="append", default=[], help="Overrides like key=value; supports dotted keys for nesting"
|
|
35
|
+
)
|
|
36
|
+
parser.add_argument(
|
|
37
|
+
"--log-level", default="INFO", choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], help="Logging level"
|
|
38
|
+
)
|
|
39
|
+
return parser.parse_args()
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _build_control_tables(owner: str) -> list[TableDefinition]:
|
|
43
|
+
return [
|
|
44
|
+
TableDefinition(
|
|
45
|
+
owner=owner,
|
|
46
|
+
name="LDB_CTL",
|
|
47
|
+
columns=(
|
|
48
|
+
ColumnDefinition("CTL_OWNER", "string", length=50, nullable=False),
|
|
49
|
+
ColumnDefinition("CTL_TABLE_NAME", "string", length=50, nullable=False),
|
|
50
|
+
ColumnDefinition("CTL_PROCESS_DATE", "date"),
|
|
51
|
+
ColumnDefinition("CTL_ACTION", "string", length=11, nullable=False),
|
|
52
|
+
ColumnDefinition("CTL_STATUS", "string", length=10, nullable=False),
|
|
53
|
+
ColumnDefinition("CTL_PROCESS_START", "date"),
|
|
54
|
+
ColumnDefinition("CTL_PROCESS_END", "date"),
|
|
55
|
+
ColumnDefinition("CTL_ROWS_PROCESSED", "number", precision=18, scale=0),
|
|
56
|
+
ColumnDefinition("CTL_PLSQL", "clob"),
|
|
57
|
+
),
|
|
58
|
+
primary_key=("CTL_OWNER", "CTL_TABLE_NAME"),
|
|
59
|
+
),
|
|
60
|
+
TableDefinition(
|
|
61
|
+
owner=owner,
|
|
62
|
+
name="LDB_LOG",
|
|
63
|
+
columns=(
|
|
64
|
+
ColumnDefinition("LOG_ID", "number", precision=18, scale=0, nullable=False),
|
|
65
|
+
ColumnDefinition("LOG_OWNER", "string", length=50, nullable=False),
|
|
66
|
+
ColumnDefinition("LOG_TABLE_NAME", "string", length=50, nullable=False),
|
|
67
|
+
ColumnDefinition("LOG_PROCESS_DATE", "date", nullable=False),
|
|
68
|
+
ColumnDefinition("LOG_ACTION", "string", length=11, nullable=False),
|
|
69
|
+
ColumnDefinition("LOG_STATUS", "string", length=10, nullable=False),
|
|
70
|
+
ColumnDefinition("LOG_PROCESS_START", "date", nullable=False),
|
|
71
|
+
ColumnDefinition("LOG_PROCESS_END", "date"),
|
|
72
|
+
ColumnDefinition("LOG_MESSAGE", "string", length=4000),
|
|
73
|
+
ColumnDefinition("LOG_ROWS_PROCESSED", "number", precision=18, scale=0),
|
|
74
|
+
ColumnDefinition("LOG_PLSQL", "clob"),
|
|
75
|
+
),
|
|
76
|
+
primary_key=("LOG_ID",),
|
|
77
|
+
),
|
|
78
|
+
]
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _build_sequences(owner: str) -> list[SequenceDefinition]:
|
|
82
|
+
return [
|
|
83
|
+
SequenceDefinition(owner=owner, name="LDB_LOG_ID"),
|
|
84
|
+
]
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def run_cli() -> None:
|
|
88
|
+
logger = get_logger("impl")
|
|
89
|
+
try:
|
|
90
|
+
args = _parse_args()
|
|
91
|
+
reconfigure_logger(level=args.log_level)
|
|
92
|
+
|
|
93
|
+
load_or_create_key()
|
|
94
|
+
encrypt_secrets_in_place(
|
|
95
|
+
schema=args.schema,
|
|
96
|
+
profile=getattr(args, "profile", None),
|
|
97
|
+
config_root=getattr(args, "config_dir", None),
|
|
98
|
+
)
|
|
99
|
+
cfg_dict = build_config({}, args)
|
|
100
|
+
config = Config.from_dict(cfg_dict)
|
|
101
|
+
connections = config.connections
|
|
102
|
+
administration = config.administration
|
|
103
|
+
|
|
104
|
+
if not administration.source.username or not administration.source.password:
|
|
105
|
+
raise ValueError("admin_source_username and admin_source_password are required")
|
|
106
|
+
if not administration.history.username or not administration.history.password:
|
|
107
|
+
raise ValueError("admin_history_username and admin_history_password are required")
|
|
108
|
+
if not connections.source.username or not connections.source.password:
|
|
109
|
+
raise ValueError("source_username and source_password are required")
|
|
110
|
+
if not connections.history.username or not connections.history.password:
|
|
111
|
+
raise ValueError("history_username and history_password are required")
|
|
112
|
+
if not connections.history.dsn:
|
|
113
|
+
raise ValueError("history_dsn is required to create the database link")
|
|
114
|
+
if not connections.source.dsn:
|
|
115
|
+
raise ValueError("source_dsn is required to create the database link")
|
|
116
|
+
|
|
117
|
+
engine = get_db_engine(connections.db_engine)
|
|
118
|
+
except (LimitsDbError, ValueError) as exc:
|
|
119
|
+
logger.error("%s", exc)
|
|
120
|
+
raise SystemExit(1) from exc
|
|
121
|
+
|
|
122
|
+
source_roles = [RoleDefinition(name=administration.source_role_name)]
|
|
123
|
+
source_privileges = getattr(engine, "REQUIRED_SYSTEM_PRIVILEGES", ())
|
|
124
|
+
history_privileges = getattr(engine, "REQUIRED_SYSTEM_PRIVILEGES", ())
|
|
125
|
+
|
|
126
|
+
source_user = UserDefinition(
|
|
127
|
+
name=connections.source.username,
|
|
128
|
+
password=connections.source.password,
|
|
129
|
+
default_tablespace=administration.source_default_tablespace or None,
|
|
130
|
+
roles=(administration.source_role_name,),
|
|
131
|
+
roles_with_admin_option=(administration.source_role_name,),
|
|
132
|
+
system_privileges=source_privileges,
|
|
133
|
+
)
|
|
134
|
+
history_roles = [RoleDefinition(name=administration.history_role_name)]
|
|
135
|
+
history_user = UserDefinition(
|
|
136
|
+
name=connections.history.username,
|
|
137
|
+
password=connections.history.password,
|
|
138
|
+
default_tablespace=administration.history_default_tablespace or None,
|
|
139
|
+
roles=(administration.history_role_name,),
|
|
140
|
+
roles_with_admin_option=(administration.history_role_name,),
|
|
141
|
+
system_privileges=history_privileges,
|
|
142
|
+
)
|
|
143
|
+
source_tables = _build_control_tables(connections.source.username)
|
|
144
|
+
source_sequences = _build_sequences(connections.source.username)
|
|
145
|
+
history_tables = _build_control_tables(connections.history.username)
|
|
146
|
+
history_sequences = _build_sequences(connections.history.username)
|
|
147
|
+
source_db_link = DatabaseLinkDefinition(
|
|
148
|
+
name=administration.source_to_history_dblink_name,
|
|
149
|
+
username=connections.history.username,
|
|
150
|
+
password=connections.history.password,
|
|
151
|
+
dsn=connections.history.dsn,
|
|
152
|
+
)
|
|
153
|
+
history_db_link = DatabaseLinkDefinition(
|
|
154
|
+
name=administration.history_to_source_dblink_name,
|
|
155
|
+
username=connections.source.username,
|
|
156
|
+
password=connections.source.password,
|
|
157
|
+
dsn=connections.source.dsn,
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
summary: dict[str, list[str]] = {}
|
|
161
|
+
|
|
162
|
+
source_admin_conn = engine.get_connection(config, admin=True, env="SOURCE")
|
|
163
|
+
try:
|
|
164
|
+
summary["source_roles"] = engine.ensure_roles(source_admin_conn, source_roles)
|
|
165
|
+
summary["source_users"] = engine.ensure_users(source_admin_conn, [source_user])
|
|
166
|
+
summary["source_tables"] = engine.ensure_tables(source_admin_conn, source_tables)
|
|
167
|
+
summary["source_sequences"] = engine.ensure_sequences(source_admin_conn, source_sequences)
|
|
168
|
+
summary["source_supporting_objects"] = engine.ensure_supporting_objects(
|
|
169
|
+
source_admin_conn, connections.source.username
|
|
170
|
+
)
|
|
171
|
+
finally:
|
|
172
|
+
engine.close_connection(source_admin_conn)
|
|
173
|
+
|
|
174
|
+
history_admin_conn = engine.get_connection(config, admin=True, env="HISTORY")
|
|
175
|
+
try:
|
|
176
|
+
summary["history_roles"] = engine.ensure_roles(history_admin_conn, history_roles)
|
|
177
|
+
summary["history_users"] = engine.ensure_users(history_admin_conn, [history_user])
|
|
178
|
+
summary["history_tables"] = engine.ensure_tables(history_admin_conn, history_tables)
|
|
179
|
+
summary["history_sequences"] = engine.ensure_sequences(history_admin_conn, history_sequences)
|
|
180
|
+
summary["history_supporting_objects"] = engine.ensure_supporting_objects(
|
|
181
|
+
history_admin_conn, connections.history.username
|
|
182
|
+
)
|
|
183
|
+
finally:
|
|
184
|
+
engine.close_connection(history_admin_conn)
|
|
185
|
+
|
|
186
|
+
source_user_conn = engine.get_connection(config, admin=False, env="SOURCE")
|
|
187
|
+
try:
|
|
188
|
+
summary["source_database_links"] = engine.ensure_database_links(source_user_conn, [source_db_link])
|
|
189
|
+
finally:
|
|
190
|
+
engine.close_connection(source_user_conn)
|
|
191
|
+
|
|
192
|
+
history_user_conn = engine.get_connection(config, admin=False, env="HISTORY")
|
|
193
|
+
try:
|
|
194
|
+
summary["history_database_links"] = engine.ensure_database_links(history_user_conn, [history_db_link])
|
|
195
|
+
finally:
|
|
196
|
+
engine.close_connection(history_user_conn)
|
|
197
|
+
|
|
198
|
+
anything_created = any(summary.values())
|
|
199
|
+
for key, values in summary.items():
|
|
200
|
+
if values:
|
|
201
|
+
logger.info("Created %s: %s", key.replace("_", " "), ", ".join(values))
|
|
202
|
+
if not anything_created:
|
|
203
|
+
logger.info("All LimitsDb database objects are already present.")
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
if __name__ == "__main__": # pragma: no cover
|
|
207
|
+
run_cli()
|
limitsdb/cli/ldb_init.py
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"""Command line interface for bootstrapping schema/profile configurations."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
|
|
7
|
+
from limitsdb.core.ldb_crypto import load_or_create_key
|
|
8
|
+
from limitsdb.core.ldb_logger import configure_logger, get_logger, reconfigure_logger
|
|
9
|
+
from limitsdb.core.ldb_utils import init_schema
|
|
10
|
+
|
|
11
|
+
configure_logger(level="WARNING")
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _parse_args() -> argparse.Namespace:
|
|
15
|
+
parser = argparse.ArgumentParser(
|
|
16
|
+
description="Initialize schema/profile config files",
|
|
17
|
+
)
|
|
18
|
+
parser.add_argument("--schema", required=True, help="Schema name (folder under schemas/)")
|
|
19
|
+
parser.add_argument("--profile", help="Profile name (e.g., dev, prod)")
|
|
20
|
+
parser.add_argument("--config-dir", help="Configuration root (overrides autodiscovery)")
|
|
21
|
+
parser.add_argument("--overwrite", action="store_true", help="Overwrite existing files")
|
|
22
|
+
parser.add_argument("--no-examples", action="store_true", help="Do not copy example ILM templates")
|
|
23
|
+
parser.add_argument(
|
|
24
|
+
"--log-level",
|
|
25
|
+
default="INFO",
|
|
26
|
+
help="Logging level (e.g., DEBUG, INFO, WARNING)",
|
|
27
|
+
choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
|
|
28
|
+
)
|
|
29
|
+
return parser.parse_args()
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def run_cli() -> None:
|
|
33
|
+
"""Entry point for the ``ldb-init`` command."""
|
|
34
|
+
logger = get_logger("init")
|
|
35
|
+
try:
|
|
36
|
+
args = _parse_args()
|
|
37
|
+
reconfigure_logger(level=args.log_level)
|
|
38
|
+
# Ensure key exists (idempotent); secrets will be auto-encrypted if present.
|
|
39
|
+
load_or_create_key()
|
|
40
|
+
logger.info("Creating configuration files with defaults and help")
|
|
41
|
+
cfg_path, ilm_path, secrets_path, ilm_examples_path = init_schema(
|
|
42
|
+
schema=args.schema,
|
|
43
|
+
profile=getattr(args, "profile", None),
|
|
44
|
+
config_root=getattr(args, "config_dir", None),
|
|
45
|
+
overwrite=bool(args.overwrite),
|
|
46
|
+
with_examples=not bool(args.no_examples),
|
|
47
|
+
auto_encrypt=True,
|
|
48
|
+
)
|
|
49
|
+
logger.info("Initialized: %s", cfg_path)
|
|
50
|
+
logger.info("Initialized: %s", ilm_path)
|
|
51
|
+
logger.info("Initialized: %s", secrets_path)
|
|
52
|
+
if ilm_examples_path:
|
|
53
|
+
logger.info("Initialized: %s", ilm_examples_path)
|
|
54
|
+
except ValueError as exc:
|
|
55
|
+
logger.error("%s", exc)
|
|
56
|
+
raise SystemExit(1) from exc
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
if __name__ == "__main__": # pragma: no cover
|
|
60
|
+
run_cli()
|
limitsdb/cli/ldb_run.py
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""Command line interface to execute ILM runs."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from limitsdb.core.ldb_crypto import load_or_create_key
|
|
8
|
+
from limitsdb.core.ldb_errors import LimitsDbError
|
|
9
|
+
from limitsdb.core.ldb_logger import configure_logger, get_logger, reconfigure_logger
|
|
10
|
+
from limitsdb.core.ldb_params_config import Config, build_config, parse_args
|
|
11
|
+
from limitsdb.core.ldb_runner import ldb_run
|
|
12
|
+
from limitsdb.core.ldb_utils import encrypt_secrets_in_place, secret_keys_from_config
|
|
13
|
+
|
|
14
|
+
# Boot the logger early with a conservative level; we'll reconfigure after loading config.
|
|
15
|
+
configure_logger(level="WARNING")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _mask_secrets(config_dict: dict[str, Any]) -> dict[str, Any]:
|
|
19
|
+
masked = dict(config_dict)
|
|
20
|
+
for key in secret_keys_from_config():
|
|
21
|
+
if masked.get(key):
|
|
22
|
+
masked[key] = "****"
|
|
23
|
+
return masked
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def run_cli() -> None:
|
|
27
|
+
"""Entry point for the ``ldb-run`` command."""
|
|
28
|
+
logger = get_logger("run")
|
|
29
|
+
try:
|
|
30
|
+
args = parse_args()
|
|
31
|
+
if hasattr(args, "log_level"):
|
|
32
|
+
reconfigure_logger(level=args.log_level)
|
|
33
|
+
# Ensure encryption key exists and auto-encrypt secrets if user left cleartext.
|
|
34
|
+
load_or_create_key()
|
|
35
|
+
encrypt_secrets_in_place(
|
|
36
|
+
schema=args.schema,
|
|
37
|
+
profile=getattr(args, "profile", None),
|
|
38
|
+
config_root=getattr(args, "config_dir", None),
|
|
39
|
+
)
|
|
40
|
+
# Build final config and run
|
|
41
|
+
cfg_dict = build_config({}, args)
|
|
42
|
+
config = Config.from_dict(cfg_dict)
|
|
43
|
+
reconfigure_logger(level=config.execution.log_level)
|
|
44
|
+
logger.debug("Effective config: %s", _mask_secrets(config.to_dict()))
|
|
45
|
+
ldb_run(config)
|
|
46
|
+
except (LimitsDbError, ValueError, KeyError) as exc:
|
|
47
|
+
logger.error("%s", exc)
|
|
48
|
+
logger.error(exc, exc_info=True)
|
|
49
|
+
raise SystemExit(1) from exc
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
if __name__ == "__main__": # pragma: no cover
|
|
53
|
+
run_cli()
|
|
File without changes
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
# limitsdb/core/ldb_config_loader.py
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
from collections.abc import Mapping
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any, cast
|
|
9
|
+
|
|
10
|
+
import yaml
|
|
11
|
+
|
|
12
|
+
from limitsdb.core.ldb_crypto import decrypt as _DECRYPT
|
|
13
|
+
from limitsdb.core.ldb_crypto import is_encrypted as _IS_ENC
|
|
14
|
+
from limitsdb.core.ldb_errors import ConfigurationError, SecretError
|
|
15
|
+
from limitsdb.core.ldb_logger import get_logger
|
|
16
|
+
from limitsdb.core.ldb_utils import get_config_roots, resolve_schema_file, secret_keys_from_config
|
|
17
|
+
|
|
18
|
+
logger = get_logger("config_loader")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
# --- utils --------------------------------------------------------------------
|
|
22
|
+
def _deep_merge(a: Mapping[str, Any], b: Mapping[str, Any]) -> dict[str, Any]:
|
|
23
|
+
res: dict[str, Any] = dict(a)
|
|
24
|
+
for k, v in b.items():
|
|
25
|
+
av = res.get(k)
|
|
26
|
+
if isinstance(av, Mapping) and isinstance(v, Mapping):
|
|
27
|
+
res[k] = _deep_merge(cast(Mapping[str, Any], av), cast(Mapping[str, Any], v))
|
|
28
|
+
else:
|
|
29
|
+
res[k] = v
|
|
30
|
+
return res
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _load_yaml(path: Path) -> dict[str, Any]:
|
|
34
|
+
if not path.exists():
|
|
35
|
+
return {}
|
|
36
|
+
try:
|
|
37
|
+
text = path.read_text(encoding="utf-8")
|
|
38
|
+
data: Any = yaml.safe_load(text)
|
|
39
|
+
except (OSError, UnicodeError, yaml.YAMLError) as exc:
|
|
40
|
+
raise ConfigurationError(f"Unable to read YAML configuration at {path}: {exc}") from exc
|
|
41
|
+
if data is None:
|
|
42
|
+
return {}
|
|
43
|
+
if isinstance(data, dict):
|
|
44
|
+
return cast(dict[str, Any], data)
|
|
45
|
+
raise ConfigurationError(f"YAML root must be a mapping (dict) in {path}")
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _load_secrets(path: Path) -> dict[str, Any]:
|
|
49
|
+
try:
|
|
50
|
+
data: Any = json.loads(path.read_text(encoding="utf-8"))
|
|
51
|
+
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
|
|
52
|
+
raise SecretError(f"Unable to read secrets file at {path}: {exc}") from exc
|
|
53
|
+
if not isinstance(data, dict):
|
|
54
|
+
raise SecretError(f"Secrets root must be a mapping (object) in {path}")
|
|
55
|
+
return cast(dict[str, Any], data)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def read_env_overrides(bindings: Mapping[str, str]) -> dict[str, Any]:
|
|
59
|
+
"""Read only environment variables explicitly registered by Config metadata."""
|
|
60
|
+
out: dict[str, Any] = {}
|
|
61
|
+
for env_name, config_key in bindings.items():
|
|
62
|
+
if env_name not in os.environ:
|
|
63
|
+
continue
|
|
64
|
+
value = os.environ[env_name]
|
|
65
|
+
# basic typing
|
|
66
|
+
if value.lower() in ("true", "false"):
|
|
67
|
+
out[config_key] = value.lower() == "true"
|
|
68
|
+
continue
|
|
69
|
+
try:
|
|
70
|
+
out[config_key] = int(value)
|
|
71
|
+
continue
|
|
72
|
+
except ValueError:
|
|
73
|
+
pass
|
|
74
|
+
try:
|
|
75
|
+
out[config_key] = float(value)
|
|
76
|
+
continue
|
|
77
|
+
except ValueError:
|
|
78
|
+
pass
|
|
79
|
+
out[config_key] = value
|
|
80
|
+
return out
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _automatic_config_paths(*, schema: str, profile: str | None, explicit_config_dir: str | None) -> list[Path]:
|
|
84
|
+
"""Return existing runtime-config layers from lowest to highest priority."""
|
|
85
|
+
filename = f"config.{profile}.yml" if profile else "config.yml"
|
|
86
|
+
if explicit_config_dir:
|
|
87
|
+
roots = [Path(explicit_config_dir)]
|
|
88
|
+
else:
|
|
89
|
+
user_root, system_root = get_config_roots()
|
|
90
|
+
roots = [system_root, user_root]
|
|
91
|
+
return [path for root in roots if (path := root / "schemas" / schema / filename).exists()]
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _apply_dot_set(cfg: dict[str, Any], key: str, value: Any) -> None:
|
|
95
|
+
cur = cfg
|
|
96
|
+
parts = key.split(".")
|
|
97
|
+
for p in parts[:-1]:
|
|
98
|
+
if p not in cur or not isinstance(cur[p], dict):
|
|
99
|
+
cur[p] = {}
|
|
100
|
+
cur = cur[p]
|
|
101
|
+
cur[parts[-1]] = value
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _apply_overrides(cfg: dict[str, Any], kvs: dict[str, Any]) -> dict[str, Any]:
|
|
105
|
+
out = dict(cfg)
|
|
106
|
+
for k, v in kvs.items():
|
|
107
|
+
_apply_dot_set(out, k, v)
|
|
108
|
+
return out
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
# --- loaders ------------------------------------------------------------------
|
|
112
|
+
def load_runtime_config(
|
|
113
|
+
*,
|
|
114
|
+
schema: str,
|
|
115
|
+
profile: str | None,
|
|
116
|
+
cli_sets: dict[str, Any],
|
|
117
|
+
env_bindings: Mapping[str, str] | None = None,
|
|
118
|
+
explicit_config_file: str | None = None,
|
|
119
|
+
explicit_config_dir: str | None = None,
|
|
120
|
+
enforce_encrypted_secrets: bool = True,
|
|
121
|
+
) -> dict[str, Any]:
|
|
122
|
+
"""
|
|
123
|
+
config_file → registered environment variables → --set
|
|
124
|
+
Then inject **decrypted** secrets from schemas/<schema>/secrets*.json.
|
|
125
|
+
If enforce_encrypted_secrets=True, plaintext secrets are rejected.
|
|
126
|
+
"""
|
|
127
|
+
cfg: dict[str, Any] = {}
|
|
128
|
+
# 1-2) System then user config, or the single explicitly selected root.
|
|
129
|
+
for config_path in _automatic_config_paths(schema=schema, profile=profile, explicit_config_dir=explicit_config_dir):
|
|
130
|
+
cfg = _deep_merge(cfg, _load_yaml(config_path))
|
|
131
|
+
# 3) Explicit config file is an overlay, not a replacement for the
|
|
132
|
+
# automatically discovered layers.
|
|
133
|
+
if explicit_config_file:
|
|
134
|
+
config_file = resolve_schema_file(
|
|
135
|
+
schema=schema,
|
|
136
|
+
profile=profile,
|
|
137
|
+
explicit_config_dir=explicit_config_dir,
|
|
138
|
+
explicit_file=explicit_config_file,
|
|
139
|
+
prefix_name="config",
|
|
140
|
+
extension_name="yml",
|
|
141
|
+
description="configuration file",
|
|
142
|
+
)
|
|
143
|
+
assert config_file is not None
|
|
144
|
+
cfg = _deep_merge(cfg, _load_yaml(Path(config_file)))
|
|
145
|
+
# 4) ENV
|
|
146
|
+
env_over = read_env_overrides(env_bindings or {})
|
|
147
|
+
if env_over:
|
|
148
|
+
cfg = _apply_overrides(cfg, env_over)
|
|
149
|
+
# 5) CLI --set
|
|
150
|
+
if cli_sets:
|
|
151
|
+
cfg = _apply_overrides(cfg, cli_sets)
|
|
152
|
+
# 6) Secrets (user > system), strictly enforced
|
|
153
|
+
secrets: dict[str, Any] = {}
|
|
154
|
+
secrets_path = resolve_schema_file(
|
|
155
|
+
schema=schema,
|
|
156
|
+
profile=profile,
|
|
157
|
+
explicit_config_dir=explicit_config_dir,
|
|
158
|
+
explicit_file=None,
|
|
159
|
+
prefix_name="secrets",
|
|
160
|
+
extension_name="json",
|
|
161
|
+
description="secrets file",
|
|
162
|
+
)
|
|
163
|
+
if secrets_path:
|
|
164
|
+
secrets = _load_secrets(Path(secrets_path))
|
|
165
|
+
if secrets:
|
|
166
|
+
for k in secret_keys_from_config():
|
|
167
|
+
val = secrets.get(k)
|
|
168
|
+
if val is None or val == "":
|
|
169
|
+
continue
|
|
170
|
+
if isinstance(val, str) and _IS_ENC(val):
|
|
171
|
+
try:
|
|
172
|
+
cfg[k] = _DECRYPT(val)
|
|
173
|
+
except Exception as exc:
|
|
174
|
+
raise SecretError(f"Unable to decrypt secret '{k}' at {secrets_path}") from exc
|
|
175
|
+
else:
|
|
176
|
+
if enforce_encrypted_secrets:
|
|
177
|
+
where = f" at {secrets_path}" if secrets_path else ""
|
|
178
|
+
raise SecretError(f"Secret '{k}' must be encrypted (enc:v1:aes256gcm:...){where}")
|
|
179
|
+
cfg[k] = val # only if you explicitly disabled enforcement
|
|
180
|
+
# minimal defaults
|
|
181
|
+
return cfg
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def load_ilm_config(*, schema: str, profile: str | None, explicit_config_dir: str | None = None) -> dict[str, Any]:
|
|
185
|
+
"""Load ILM config from schema/profile layers."""
|
|
186
|
+
ilm: dict[str, Any] = {}
|
|
187
|
+
ilm_config_path = resolve_schema_file(
|
|
188
|
+
schema=schema,
|
|
189
|
+
profile=profile,
|
|
190
|
+
explicit_config_dir=explicit_config_dir,
|
|
191
|
+
explicit_file=None,
|
|
192
|
+
prefix_name="ilm",
|
|
193
|
+
extension_name="yml",
|
|
194
|
+
description="ILM configuration file",
|
|
195
|
+
)
|
|
196
|
+
if ilm_config_path:
|
|
197
|
+
ilm = _load_yaml(Path(ilm_config_path))
|
|
198
|
+
return ilm
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import base64
|
|
4
|
+
import binascii
|
|
5
|
+
import os
|
|
6
|
+
import stat
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
|
|
10
|
+
|
|
11
|
+
from limitsdb.core.ldb_errors import SecretError
|
|
12
|
+
|
|
13
|
+
LDB_DIR = Path(os.path.expanduser("~/.limitsdb"))
|
|
14
|
+
KEY_PATH = LDB_DIR / "ldb.key"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _b64e(b: bytes) -> str:
|
|
18
|
+
return base64.urlsafe_b64encode(b).decode("ascii")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _b64d(s: str) -> bytes:
|
|
22
|
+
return base64.urlsafe_b64decode(s.encode("ascii"))
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _ensure_dir() -> None:
|
|
26
|
+
LDB_DIR.mkdir(parents=True, exist_ok=True)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _chmod600(path: Path) -> None:
|
|
30
|
+
try:
|
|
31
|
+
os.chmod(path, stat.S_IRUSR | stat.S_IWUSR)
|
|
32
|
+
except OSError as exc:
|
|
33
|
+
raise SecretError(f"Unable to restrict permissions on key file {path}") from exc
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _generate_aes256_key() -> bytes:
|
|
37
|
+
return AESGCM.generate_key(256)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def load_or_create_key() -> bytes:
|
|
41
|
+
env = os.getenv("LDB_MASTER_KEY")
|
|
42
|
+
if env:
|
|
43
|
+
if all(c in "0123456789abcdefABCDEF" for c in env) and len(env) == 64:
|
|
44
|
+
return bytes.fromhex(env)
|
|
45
|
+
try:
|
|
46
|
+
k = base64.b64decode(env)
|
|
47
|
+
if len(k) == 32:
|
|
48
|
+
return k
|
|
49
|
+
except (binascii.Error, ValueError):
|
|
50
|
+
pass
|
|
51
|
+
if len(env) == 32:
|
|
52
|
+
return env.encode("utf-8")
|
|
53
|
+
raise SecretError("Invalid LDB_MASTER_KEY: use hex(64) / base64(32B) / raw(32B).")
|
|
54
|
+
|
|
55
|
+
_ensure_dir()
|
|
56
|
+
if KEY_PATH.exists():
|
|
57
|
+
data = KEY_PATH.read_bytes()
|
|
58
|
+
try:
|
|
59
|
+
key = base64.b64decode(data)
|
|
60
|
+
except (binascii.Error, ValueError):
|
|
61
|
+
key = data
|
|
62
|
+
if len(key) != 32:
|
|
63
|
+
raise SecretError(f"Invalid key at {KEY_PATH} (expected 32 bytes).")
|
|
64
|
+
return key
|
|
65
|
+
|
|
66
|
+
key = _generate_aes256_key()
|
|
67
|
+
KEY_PATH.write_bytes(base64.b64encode(key))
|
|
68
|
+
_chmod600(KEY_PATH)
|
|
69
|
+
return key
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def encrypt(plaintext: str, key: bytes | None = None) -> str:
|
|
73
|
+
key = key or load_or_create_key()
|
|
74
|
+
aes = AESGCM(key)
|
|
75
|
+
nonce = os.urandom(12)
|
|
76
|
+
ct = aes.encrypt(nonce, plaintext.encode("utf-8"), associated_data=None)
|
|
77
|
+
return f"enc:v1:aes256gcm:{_b64e(nonce)}:{_b64e(ct)}"
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def is_encrypted(val: str) -> bool:
|
|
81
|
+
return val.startswith("enc:v1:aes256gcm:")
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def decrypt(token: str, key: bytes | None = None) -> str:
|
|
85
|
+
if not is_encrypted(token):
|
|
86
|
+
raise SecretError("Attempted to decrypt a non-encrypted token.")
|
|
87
|
+
key = key or load_or_create_key()
|
|
88
|
+
_, _, _, nonce_b64, ct_b64 = token.split(":", 4)
|
|
89
|
+
aes = AESGCM(key)
|
|
90
|
+
pt = aes.decrypt(_b64d(nonce_b64), _b64d(ct_b64), associated_data=None)
|
|
91
|
+
return pt.decode("utf-8")
|