moj-data-loader 2.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,223 @@
1
+ import base64
2
+ import io
3
+ import os
4
+ import tempfile
5
+ from functools import wraps
6
+ from pathlib import Path
7
+ from typing import Callable
8
+
9
+ import gnupg
10
+ from cloudpathlib import S3Path
11
+ from loguru import logger
12
+
13
+
14
+ def with_pgp_decryption(reader_func: Callable) -> Callable:
15
+ """Decorator that handles PGP decryption before calling the reader."""
16
+
17
+ @wraps(reader_func)
18
+ def wrapper(files: list[S3Path | Path], **kwargs):
19
+ # Extract PGP config if present
20
+ pgp_config = kwargs.pop("pgp", {})
21
+
22
+ # Check for PGP configuration
23
+ private_key_env_var = pgp_config.get("private_key_env_var")
24
+ passphrase_env_var = pgp_config.get("passphrase_env_var")
25
+ signing_key_env_var = pgp_config.get("signing_key_env_var")
26
+
27
+ if not private_key_env_var:
28
+ # No PGP config - pass through to original reader
29
+ yield from reader_func(files, **kwargs)
30
+ return
31
+
32
+ logger.info(f"PGP decryption enabled for {len(files)} files")
33
+
34
+ # Create temp directory for decrypted files
35
+ temp_dir = Path(tempfile.mkdtemp(prefix="pgp_decrypted_"))
36
+ decrypted_files = []
37
+ file_mapping = {}
38
+
39
+ try:
40
+ # Decrypt each file to temp location
41
+ for file in files:
42
+ logger.info(f"Decrypting: {file}")
43
+
44
+ # Decrypt using environment variable-based keys
45
+ decrypted_stream = decrypt_pgp_file_from_env(
46
+ file_path=file,
47
+ private_key_env_var=private_key_env_var,
48
+ passphrase_env_var=passphrase_env_var,
49
+ signing_key_env_var=signing_key_env_var,
50
+ )
51
+
52
+ # Write to temp file (preserve original name without .pgp)
53
+ original_name = file.name.replace(".pgp", "")
54
+ temp_file = temp_dir / original_name
55
+
56
+ with open(temp_file, "wb") as f:
57
+ f.write(decrypted_stream.read())
58
+
59
+ decrypted_files.append(temp_file)
60
+ file_mapping[str(temp_file)] = str(file)
61
+
62
+ logger.info(f"Decrypted to: {temp_file}")
63
+
64
+ # Pass file mapping to reader via kwargs
65
+ kwargs["_original_file_paths"] = file_mapping
66
+
67
+ # Call original reader with decrypted files
68
+ yield from reader_func(decrypted_files, **kwargs)
69
+
70
+ finally:
71
+ # Cleanup temp files
72
+ import shutil
73
+
74
+ logger.info(f"Cleaning up temp directory: {temp_dir}")
75
+ shutil.rmtree(temp_dir, ignore_errors=True)
76
+
77
+ return wrapper
78
+
79
+
80
+ def decrypt_pgp_file_from_env(
81
+ file_path: S3Path | Path | str,
82
+ private_key_env_var: str,
83
+ passphrase_env_var: str | None = None,
84
+ signing_key_env_var: str | None = None,
85
+ ) -> io.BytesIO:
86
+ """
87
+ Decrypt a PGP file using keys from environment variables.
88
+
89
+ Args:
90
+ file_path: Path to the encrypted file (S3Path, Path, or string)
91
+ private_key_env_var: Name of env var containing the private key
92
+ passphrase_env_var: Optional name of env var containing passphrase
93
+ signing_key_env_var: Optional name of env var containing signing key
94
+
95
+ Returns:
96
+ BytesIO object containing decrypted data
97
+
98
+ Raises:
99
+ ValueError: If required environment variable is not set or decryption fails
100
+
101
+ Example:
102
+ >>> decrypted = decrypt_pgp_file_from_env(
103
+ ... S3Path("s3://my-bucket/data/file.pgp"),
104
+ ... private_key_env_var="PGP_PRIVATE_KEY",
105
+ ... signing_key_env_var="PGP_SIGNING_KEY",
106
+ ... )
107
+ """
108
+ # Get private key from environment
109
+ private_key_data = os.getenv(private_key_env_var)
110
+
111
+ # Debug logging (without exposing the secret)
112
+ logger.info(f"🔍 Looking for env var: {private_key_env_var}")
113
+ if not private_key_data:
114
+ logger.error(
115
+ f"❌ Environment variable '{private_key_env_var}' is not set or is empty"
116
+ )
117
+ raise ValueError(f"Environment variable '{private_key_env_var}' not set")
118
+
119
+ logger.info(f"✅ Found env var (length: {len(private_key_data)} chars)")
120
+
121
+ # Decode from base64
122
+ try:
123
+ private_key_data = base64.b64decode(private_key_data).decode("utf-8")
124
+ logger.info(
125
+ "✅ Successfully decoded base64 key (decoded length:"
126
+ f" {len(private_key_data)} chars)"
127
+ )
128
+ except Exception as e:
129
+ logger.error(
130
+ f"❌ Failed to decode base64 private key: {type(e).__name__}: {str(e)}"
131
+ )
132
+ raise ValueError(f"Private key must be base64 encoded. Failed to decode: {e}")
133
+
134
+ logger.info(f"🔑 Using private key from env var: {private_key_env_var}")
135
+
136
+ # Get optional passphrase
137
+ passphrase = None
138
+ if passphrase_env_var:
139
+ passphrase = os.getenv(passphrase_env_var)
140
+ if passphrase:
141
+ logger.info(f"🔐 Using passphrase from env var: {passphrase_env_var}")
142
+ else:
143
+ logger.info(
144
+ f"ℹ️ Passphrase env var '{passphrase_env_var}'"
145
+ " not set - proceeding without"
146
+ )
147
+ else:
148
+ logger.info("ℹ️ No passphrase env var configured - proceeding without")
149
+
150
+ # Get optional signing key
151
+ signing_key_data = None
152
+ if signing_key_env_var:
153
+ signing_key_data = os.getenv(signing_key_env_var)
154
+ if signing_key_data:
155
+ logger.info(f"🔏 Using signing key from env var: {signing_key_env_var}")
156
+ else:
157
+ logger.info(
158
+ f"ℹ️ Signing key env var '{signing_key_env_var}'"
159
+ " not set - proceeding without"
160
+ )
161
+ else:
162
+ logger.info("ℹ️ No signing key env var configured")
163
+
164
+ # Setup GPG with env-based keys
165
+ gpg_home = tempfile.mkdtemp()
166
+ gpg = gnupg.GPG(gnupghome=gpg_home)
167
+
168
+ try:
169
+ # Import private key
170
+ import_result = gpg.import_keys(private_key_data)
171
+ logger.info(
172
+ f"🔐 Import attempt complete - count: {import_result.count}, "
173
+ f"fingerprints: {import_result.fingerprints}"
174
+ )
175
+
176
+ if import_result.count == 0:
177
+ logger.error(f"❌ Failed to import key. Results: {import_result.results}")
178
+ raise ValueError("Failed to import private key from environment variable")
179
+
180
+ logger.info(f"✅ Private key imported: {import_result.fingerprints}")
181
+
182
+ # Import signing key if provided
183
+ if signing_key_data:
184
+ signing_result = gpg.import_keys(signing_key_data)
185
+ if signing_result.count > 0:
186
+ logger.info(f"✅ Signing key imported: {signing_result.fingerprints}")
187
+ else:
188
+ logger.warning("⚠️ Signing key import failed")
189
+
190
+ # Read encrypted file
191
+ if isinstance(file_path, S3Path):
192
+ logger.info(f"Reading encrypted file from S3: {file_path}")
193
+ encrypted_data = file_path.read_bytes()
194
+ else:
195
+ file_path = Path(file_path)
196
+ logger.info(f"Reading encrypted local file: {file_path}")
197
+ with open(file_path, "rb") as f:
198
+ encrypted_data = f.read()
199
+
200
+ # Decrypt
201
+ if passphrase:
202
+ decrypted_data = gpg.decrypt(encrypted_data, passphrase=passphrase)
203
+ else:
204
+ decrypted_data = gpg.decrypt(encrypted_data)
205
+
206
+ if not decrypted_data.ok:
207
+ raise ValueError(f"Decryption failed: {decrypted_data.status}")
208
+
209
+ logger.info("✅ File decrypted successfully")
210
+
211
+ # Log signature verification if present
212
+ if hasattr(decrypted_data, "valid") and decrypted_data.valid:
213
+ logger.info(f"✅ Valid signature from: {decrypted_data.username}")
214
+ elif hasattr(decrypted_data, "valid"):
215
+ logger.warning("⚠️ Signature verification failed or not present")
216
+
217
+ return io.BytesIO(decrypted_data.data)
218
+
219
+ finally:
220
+ # Cleanup GPG home directory
221
+ import shutil
222
+
223
+ shutil.rmtree(gpg_home, ignore_errors=True)
@@ -0,0 +1,81 @@
1
+ import datetime
2
+ import time
3
+ from dataclasses import dataclass, field
4
+
5
+ from loguru import logger
6
+
7
+
8
+ @dataclass
9
+ class SummaryLogInstance:
10
+ """
11
+ Class for run summary logs
12
+ """
13
+
14
+ start_time: float = 0
15
+ end_time: float = 0
16
+ duration_sec: int = 0
17
+ duration_str: str = ""
18
+ files_at_source: int = 0
19
+ files_to_load: int = 0
20
+ already_loaded_files: int = 0
21
+ files_loaded:int = 0
22
+
23
+ summary_strings: list[str] = field(default_factory=list)
24
+
25
+ def __enter__(self):
26
+ self.set_start_time()
27
+ pass
28
+
29
+ def __exit__(self, exc_type, exc, tb):
30
+ self.set_end_time()
31
+ self.log_summary()
32
+ pass
33
+
34
+ """
35
+ adds a new string to the optional summary string for debugging
36
+ """
37
+
38
+ def add_summary_string(self, string_):
39
+ self.summary_strings.append(string_)
40
+
41
+ """
42
+ Sets the start time for the run timer
43
+ """
44
+
45
+ def set_start_time(self):
46
+ self.start_time = time.time()
47
+
48
+ """
49
+ Sets the end time for the run timer and starts calculation of run duration
50
+ """
51
+
52
+ def set_end_time(self):
53
+ self.end_time = time.time()
54
+ self._calculate_run_duration()
55
+
56
+ """
57
+ Logs all the tracked stats
58
+ """
59
+
60
+ def log_summary(self):
61
+ logger.info(f"Total Run Duration : {self.duration_str}")
62
+ logger.info(f"Total Files : {self.files_at_source}")
63
+ logger.info(f"Files To Load : {self.files_to_load}")
64
+ logger.info(f"Files Already Loaded : {self.already_loaded_files}")
65
+ logger.info(f"Files Loaded : {self.files_loaded}")
66
+
67
+ if len(self.summary_strings) > 0:
68
+ for string in self.summary_strings:
69
+ logger.info(string)
70
+
71
+ """
72
+ Calculates the duration of the run
73
+ """
74
+
75
+ def _calculate_run_duration(self):
76
+ dur = self.end_time - self.start_time
77
+ self.duration_sec = int(dur)
78
+ self.duration_str = str(datetime.timedelta(seconds=dur))
79
+
80
+
81
+ summary_log = SummaryLogInstance()
@@ -0,0 +1,15 @@
1
+ from cloudpathlib import S3Path
2
+
3
+ from .exceptions import ConfigError
4
+
5
+
6
+ def validate_s3_path_exists(s3_path: S3Path) -> S3Path:
7
+ if not s3_path.exists():
8
+ raise ConfigError(f"Error. S3 Path does not exist: {s3_path}")
9
+ return s3_path
10
+
11
+
12
+ def validate_is_s3_dir(s3_path: S3Path) -> S3Path:
13
+ if not s3_path.is_dir():
14
+ raise ConfigError("Provided directory: {s3_path} is not a valid directory")
15
+ return s3_path
@@ -0,0 +1,124 @@
1
+ Metadata-Version: 2.3
2
+ Name: moj-data-loader
3
+ Version: 2.2.0
4
+ Summary: MoJ Data Loader (MoDaL) provides an easy-to-use, config-driven way to load files from S3 into Athena.
5
+ Requires-Dist: awswrangler>=3.17.0
6
+ Requires-Dist: charset-normalizer>=3.4.9
7
+ Requires-Dist: click>=8.4.2
8
+ Requires-Dist: cloudpathlib>=0.24.0
9
+ Requires-Dist: cryptography>=50.0.0
10
+ Requires-Dist: dlt[athena,pyiceberg]>=1.29.1
11
+ Requires-Dist: duckdb>=1.5.5
12
+ Requires-Dist: fsspec>=2026.7.0
13
+ Requires-Dist: gitpython>=3.1.58
14
+ Requires-Dist: loguru>=0.7.3
15
+ Requires-Dist: msoffcrypto-tool>=5.4.2
16
+ Requires-Dist: openpyxl>=3.1.5
17
+ Requires-Dist: pandas>=3.0.5
18
+ Requires-Dist: pathlib>=1.0.1
19
+ Requires-Dist: psutil>=7.2.2
20
+ Requires-Dist: pyarrow>=24.0.0
21
+ Requires-Dist: pydantic>=2.13.4
22
+ Requires-Dist: pydantic-settings>=2.14.2
23
+ Requires-Dist: pyiceberg>=0.9.0
24
+ Requires-Dist: python-calamine>=0.8.2
25
+ Requires-Dist: python-gnupg>=0.5.6
26
+ Requires-Dist: pyyaml>=6.0.2
27
+ Requires-Dist: setuptools>=83.0.0
28
+ Requires-Dist: sqlalchemy>=2.0.51
29
+ Requires-Dist: tornado>=6.6.1
30
+ Requires-Dist: xlrd>=2.0.2
31
+ Requires-Dist: ruff ; extra == 'linting'
32
+ Requires-Python: >=3.12
33
+ Provides-Extra: linting
34
+ Description-Content-Type: text/markdown
35
+
36
+ # MoJ Data Loader (MoDaL)
37
+ MoJ Data Loader provides an easy to use, config-driven way to load files from S3 into Athena.
38
+
39
+ ## Install in Your Project
40
+
41
+ MoDaL is distributed as the `moj-data-loader` Python package on PyPI. It requires Python 3.12 or newer. Install it in the repository that owns your pipelines; you do not need to clone this repository.
42
+
43
+ For a project managed with [uv](https://docs.astral.sh/uv/), run from your project root:
44
+
45
+ ```bash
46
+ uv add moj-data-loader
47
+ uv run moj-data-loader --help
48
+ ```
49
+
50
+ Alternatively, install it in a virtual environment with pip:
51
+
52
+ ```bash
53
+ python -m venv .venv
54
+ source .venv/bin/activate
55
+ python -m pip install moj-data-loader
56
+ moj-data-loader --help
57
+ ```
58
+
59
+ The package name and CLI command are `moj-data-loader`, not `modal`.
60
+
61
+ ## Use Your Own Pipeline Repository
62
+
63
+ Keep your pipeline YAMLs in your own repository:
64
+
65
+ ```text
66
+ your-project/
67
+ pyproject.toml
68
+ pipelines/
69
+ your_domain/
70
+ your_database/
71
+ your_table.yml
72
+ ```
73
+
74
+ Run from that repository's root:
75
+
76
+ ```bash
77
+ uv run moj-data-loader load your_table --env test
78
+ ```
79
+
80
+ With an activated pip environment, omit `uv run`. Configure AWS credentials and permissions for the source S3 data and Athena destination before loading data.
81
+
82
+ By default, MoDaL searches `./pipelines/`, not the installed package's directory. Use `--configs-dir /path/to/pipelines` or `MODAL_CONFIG_DIR` to select another config root, retaining the `domain/database/table.yml` layout.
83
+
84
+ Existing pipelines in this repository can continue to use the monorepo and its Airflow image during the transition. New pipelines will soon no longer be accepted here: create them in your own repository using the PyPI package, and plan to migrate existing pipelines there too. See the [migration guide](docs/migration_v2.md#move-pipelines-to-your-own-repository).
85
+
86
+ ## Releases
87
+
88
+ The Airflow container workflow runs on pushed tags without `/`. A monorepo tag such as `monorepo-v2.1.25` builds the container without publishing to PyPI; use this for interim pipeline-only releases.
89
+
90
+ For a package release, update the version in `pyproject.toml` and the lockfile, then push a version tag such as `v2.1.25` matching the package version. This publishes to PyPI and also builds the container. A manually dispatched PyPI workflow publishes to TestPyPI only. Configure the GitHub `release` environment and PyPI trusted publisher for production publishing.
91
+
92
+ ## Usage guidance
93
+
94
+ You can find usage guidance in [`docs/user_guide`](docs/user_guide.md)
95
+
96
+ Pipeline YAML specification is documented in [`docs/pipeline_config.md`](docs/pipeline_config.md).
97
+
98
+ ## Migration notes
99
+
100
+ v2.0 is a breaking release.
101
+
102
+ - Supported CLI: `moj-data-loader`
103
+ - Supported environment variables: `MODAL_*`
104
+ - Removed in v2.0: `airflow-file` command alias and `AFFILE_*` environment variables
105
+
106
+ For migration steps and old-to-new mappings, see:
107
+
108
+ - [`docs/migration_v2.md`](docs/migration_v2.md)
109
+ - [`docs/release_notes.md`](docs/release_notes.md)
110
+
111
+ ## Useful links
112
+ - Documentation can be found at the Data and Analytics Engineering internal Pages site [here]: TBC
113
+ - Development of this repo is managed via this [Project Board](https://github.com/orgs/moj-analytical-services/projects/262/views/1)
114
+
115
+ ## Contributing
116
+
117
+ To develop MoDaL itself, clone this repository and run:
118
+
119
+ ```bash
120
+ uv sync
121
+ uv run pytest tests/unit
122
+ ```
123
+
124
+ `uv sync` installs MoDaL in editable mode for development. Users should add the released package to their own project's dependencies instead. Commit your project's dependency manifest and lockfile to keep local, CI, and deployment environments consistent.
@@ -0,0 +1,17 @@
1
+ moj_data_loader/Pipeline.py,sha256=xS08E_FocolEj21lzXHRJap7baAHpIRVfwqtyjiWIc4,24111
2
+ moj_data_loader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
3
+ moj_data_loader/main.py,sha256=gSctxdhpHNLRH4jVIVPopGitOSkClgvxvg5cMXNIsQQ,9495
4
+ moj_data_loader/readers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
5
+ moj_data_loader/readers/registry.py,sha256=jmTZNT1wkDgcoUCJ2GvkcT5Il_jB5-oL1SEGkoo54bU,6005
6
+ moj_data_loader/readers/transforms.py,sha256=pfVEqLil929lD0I7mCulm8JrxVQz3o9O-d91yYYNW30,11252
7
+ moj_data_loader/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
8
+ moj_data_loader/utils/batch.py,sha256=SJtnCbWzmztKh13b-5PeJqRWFFuMud4kWBZGuEt_7HE,6514
9
+ moj_data_loader/utils/exceptions.py,sha256=X7gSYnCArePp125Hn9wZpg2_wM2RNPmrgBGoZgKUb8o,212
10
+ moj_data_loader/utils/files.py,sha256=-fCQreuF5oImUSKce1uFucaSynSzkJriRhdjaQybHFw,1202
11
+ moj_data_loader/utils/pgp.py,sha256=dEIPIy35DrBjHb29L_sMSi8e5LPZQx6NjaXTdi70psc,7957
12
+ moj_data_loader/utils/summary_log.py,sha256=XbaxiAZeG-Pj-qUsY04z22flmRjoUWyr0Zm1qBNqI0g,1950
13
+ moj_data_loader/utils/validators.py,sha256=itsvZ7jsoOb-dBch6Cay9rpFBdoib1cUWExPqNvEQ3o,431
14
+ moj_data_loader-2.2.0.dist-info/WHEEL,sha256=3NO1oYsF7mANFUWVdyEQLM-zmecczWWEmmF5rEbXrWk,80
15
+ moj_data_loader-2.2.0.dist-info/entry_points.txt,sha256=sKPodKqRxJmwTNNI53VBAYn_p1NsrhHuSIDOGPan2Uo,62
16
+ moj_data_loader-2.2.0.dist-info/METADATA,sha256=jmJ1xTyLnFMDxJbHgZSZuqL6qHVBNJnEQPEfkEl8AzA,4731
17
+ moj_data_loader-2.2.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: uv 0.13.6
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,3 @@
1
+ [console_scripts]
2
+ moj-data-loader = moj_data_loader.main:cli
3
+