moj-data-loader 2.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- moj_data_loader/Pipeline.py +659 -0
- moj_data_loader/__init__.py +0 -0
- moj_data_loader/main.py +291 -0
- moj_data_loader/readers/__init__.py +0 -0
- moj_data_loader/readers/registry.py +194 -0
- moj_data_loader/readers/transforms.py +334 -0
- moj_data_loader/utils/__init__.py +0 -0
- moj_data_loader/utils/batch.py +221 -0
- moj_data_loader/utils/exceptions.py +6 -0
- moj_data_loader/utils/files.py +46 -0
- moj_data_loader/utils/pgp.py +223 -0
- moj_data_loader/utils/summary_log.py +81 -0
- moj_data_loader/utils/validators.py +15 -0
- moj_data_loader-2.2.0.dist-info/METADATA +124 -0
- moj_data_loader-2.2.0.dist-info/RECORD +17 -0
- moj_data_loader-2.2.0.dist-info/WHEEL +4 -0
- moj_data_loader-2.2.0.dist-info/entry_points.txt +3 -0
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
import base64
|
|
2
|
+
import io
|
|
3
|
+
import os
|
|
4
|
+
import tempfile
|
|
5
|
+
from functools import wraps
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Callable
|
|
8
|
+
|
|
9
|
+
import gnupg
|
|
10
|
+
from cloudpathlib import S3Path
|
|
11
|
+
from loguru import logger
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def with_pgp_decryption(reader_func: Callable) -> Callable:
|
|
15
|
+
"""Decorator that handles PGP decryption before calling the reader."""
|
|
16
|
+
|
|
17
|
+
@wraps(reader_func)
|
|
18
|
+
def wrapper(files: list[S3Path | Path], **kwargs):
|
|
19
|
+
# Extract PGP config if present
|
|
20
|
+
pgp_config = kwargs.pop("pgp", {})
|
|
21
|
+
|
|
22
|
+
# Check for PGP configuration
|
|
23
|
+
private_key_env_var = pgp_config.get("private_key_env_var")
|
|
24
|
+
passphrase_env_var = pgp_config.get("passphrase_env_var")
|
|
25
|
+
signing_key_env_var = pgp_config.get("signing_key_env_var")
|
|
26
|
+
|
|
27
|
+
if not private_key_env_var:
|
|
28
|
+
# No PGP config - pass through to original reader
|
|
29
|
+
yield from reader_func(files, **kwargs)
|
|
30
|
+
return
|
|
31
|
+
|
|
32
|
+
logger.info(f"PGP decryption enabled for {len(files)} files")
|
|
33
|
+
|
|
34
|
+
# Create temp directory for decrypted files
|
|
35
|
+
temp_dir = Path(tempfile.mkdtemp(prefix="pgp_decrypted_"))
|
|
36
|
+
decrypted_files = []
|
|
37
|
+
file_mapping = {}
|
|
38
|
+
|
|
39
|
+
try:
|
|
40
|
+
# Decrypt each file to temp location
|
|
41
|
+
for file in files:
|
|
42
|
+
logger.info(f"Decrypting: {file}")
|
|
43
|
+
|
|
44
|
+
# Decrypt using environment variable-based keys
|
|
45
|
+
decrypted_stream = decrypt_pgp_file_from_env(
|
|
46
|
+
file_path=file,
|
|
47
|
+
private_key_env_var=private_key_env_var,
|
|
48
|
+
passphrase_env_var=passphrase_env_var,
|
|
49
|
+
signing_key_env_var=signing_key_env_var,
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
# Write to temp file (preserve original name without .pgp)
|
|
53
|
+
original_name = file.name.replace(".pgp", "")
|
|
54
|
+
temp_file = temp_dir / original_name
|
|
55
|
+
|
|
56
|
+
with open(temp_file, "wb") as f:
|
|
57
|
+
f.write(decrypted_stream.read())
|
|
58
|
+
|
|
59
|
+
decrypted_files.append(temp_file)
|
|
60
|
+
file_mapping[str(temp_file)] = str(file)
|
|
61
|
+
|
|
62
|
+
logger.info(f"Decrypted to: {temp_file}")
|
|
63
|
+
|
|
64
|
+
# Pass file mapping to reader via kwargs
|
|
65
|
+
kwargs["_original_file_paths"] = file_mapping
|
|
66
|
+
|
|
67
|
+
# Call original reader with decrypted files
|
|
68
|
+
yield from reader_func(decrypted_files, **kwargs)
|
|
69
|
+
|
|
70
|
+
finally:
|
|
71
|
+
# Cleanup temp files
|
|
72
|
+
import shutil
|
|
73
|
+
|
|
74
|
+
logger.info(f"Cleaning up temp directory: {temp_dir}")
|
|
75
|
+
shutil.rmtree(temp_dir, ignore_errors=True)
|
|
76
|
+
|
|
77
|
+
return wrapper
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def decrypt_pgp_file_from_env(
|
|
81
|
+
file_path: S3Path | Path | str,
|
|
82
|
+
private_key_env_var: str,
|
|
83
|
+
passphrase_env_var: str | None = None,
|
|
84
|
+
signing_key_env_var: str | None = None,
|
|
85
|
+
) -> io.BytesIO:
|
|
86
|
+
"""
|
|
87
|
+
Decrypt a PGP file using keys from environment variables.
|
|
88
|
+
|
|
89
|
+
Args:
|
|
90
|
+
file_path: Path to the encrypted file (S3Path, Path, or string)
|
|
91
|
+
private_key_env_var: Name of env var containing the private key
|
|
92
|
+
passphrase_env_var: Optional name of env var containing passphrase
|
|
93
|
+
signing_key_env_var: Optional name of env var containing signing key
|
|
94
|
+
|
|
95
|
+
Returns:
|
|
96
|
+
BytesIO object containing decrypted data
|
|
97
|
+
|
|
98
|
+
Raises:
|
|
99
|
+
ValueError: If required environment variable is not set or decryption fails
|
|
100
|
+
|
|
101
|
+
Example:
|
|
102
|
+
>>> decrypted = decrypt_pgp_file_from_env(
|
|
103
|
+
... S3Path("s3://my-bucket/data/file.pgp"),
|
|
104
|
+
... private_key_env_var="PGP_PRIVATE_KEY",
|
|
105
|
+
... signing_key_env_var="PGP_SIGNING_KEY",
|
|
106
|
+
... )
|
|
107
|
+
"""
|
|
108
|
+
# Get private key from environment
|
|
109
|
+
private_key_data = os.getenv(private_key_env_var)
|
|
110
|
+
|
|
111
|
+
# Debug logging (without exposing the secret)
|
|
112
|
+
logger.info(f"🔍 Looking for env var: {private_key_env_var}")
|
|
113
|
+
if not private_key_data:
|
|
114
|
+
logger.error(
|
|
115
|
+
f"❌ Environment variable '{private_key_env_var}' is not set or is empty"
|
|
116
|
+
)
|
|
117
|
+
raise ValueError(f"Environment variable '{private_key_env_var}' not set")
|
|
118
|
+
|
|
119
|
+
logger.info(f"✅ Found env var (length: {len(private_key_data)} chars)")
|
|
120
|
+
|
|
121
|
+
# Decode from base64
|
|
122
|
+
try:
|
|
123
|
+
private_key_data = base64.b64decode(private_key_data).decode("utf-8")
|
|
124
|
+
logger.info(
|
|
125
|
+
"✅ Successfully decoded base64 key (decoded length:"
|
|
126
|
+
f" {len(private_key_data)} chars)"
|
|
127
|
+
)
|
|
128
|
+
except Exception as e:
|
|
129
|
+
logger.error(
|
|
130
|
+
f"❌ Failed to decode base64 private key: {type(e).__name__}: {str(e)}"
|
|
131
|
+
)
|
|
132
|
+
raise ValueError(f"Private key must be base64 encoded. Failed to decode: {e}")
|
|
133
|
+
|
|
134
|
+
logger.info(f"🔑 Using private key from env var: {private_key_env_var}")
|
|
135
|
+
|
|
136
|
+
# Get optional passphrase
|
|
137
|
+
passphrase = None
|
|
138
|
+
if passphrase_env_var:
|
|
139
|
+
passphrase = os.getenv(passphrase_env_var)
|
|
140
|
+
if passphrase:
|
|
141
|
+
logger.info(f"🔐 Using passphrase from env var: {passphrase_env_var}")
|
|
142
|
+
else:
|
|
143
|
+
logger.info(
|
|
144
|
+
f"ℹ️ Passphrase env var '{passphrase_env_var}'"
|
|
145
|
+
" not set - proceeding without"
|
|
146
|
+
)
|
|
147
|
+
else:
|
|
148
|
+
logger.info("ℹ️ No passphrase env var configured - proceeding without")
|
|
149
|
+
|
|
150
|
+
# Get optional signing key
|
|
151
|
+
signing_key_data = None
|
|
152
|
+
if signing_key_env_var:
|
|
153
|
+
signing_key_data = os.getenv(signing_key_env_var)
|
|
154
|
+
if signing_key_data:
|
|
155
|
+
logger.info(f"🔏 Using signing key from env var: {signing_key_env_var}")
|
|
156
|
+
else:
|
|
157
|
+
logger.info(
|
|
158
|
+
f"ℹ️ Signing key env var '{signing_key_env_var}'"
|
|
159
|
+
" not set - proceeding without"
|
|
160
|
+
)
|
|
161
|
+
else:
|
|
162
|
+
logger.info("ℹ️ No signing key env var configured")
|
|
163
|
+
|
|
164
|
+
# Setup GPG with env-based keys
|
|
165
|
+
gpg_home = tempfile.mkdtemp()
|
|
166
|
+
gpg = gnupg.GPG(gnupghome=gpg_home)
|
|
167
|
+
|
|
168
|
+
try:
|
|
169
|
+
# Import private key
|
|
170
|
+
import_result = gpg.import_keys(private_key_data)
|
|
171
|
+
logger.info(
|
|
172
|
+
f"🔐 Import attempt complete - count: {import_result.count}, "
|
|
173
|
+
f"fingerprints: {import_result.fingerprints}"
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
if import_result.count == 0:
|
|
177
|
+
logger.error(f"❌ Failed to import key. Results: {import_result.results}")
|
|
178
|
+
raise ValueError("Failed to import private key from environment variable")
|
|
179
|
+
|
|
180
|
+
logger.info(f"✅ Private key imported: {import_result.fingerprints}")
|
|
181
|
+
|
|
182
|
+
# Import signing key if provided
|
|
183
|
+
if signing_key_data:
|
|
184
|
+
signing_result = gpg.import_keys(signing_key_data)
|
|
185
|
+
if signing_result.count > 0:
|
|
186
|
+
logger.info(f"✅ Signing key imported: {signing_result.fingerprints}")
|
|
187
|
+
else:
|
|
188
|
+
logger.warning("⚠️ Signing key import failed")
|
|
189
|
+
|
|
190
|
+
# Read encrypted file
|
|
191
|
+
if isinstance(file_path, S3Path):
|
|
192
|
+
logger.info(f"Reading encrypted file from S3: {file_path}")
|
|
193
|
+
encrypted_data = file_path.read_bytes()
|
|
194
|
+
else:
|
|
195
|
+
file_path = Path(file_path)
|
|
196
|
+
logger.info(f"Reading encrypted local file: {file_path}")
|
|
197
|
+
with open(file_path, "rb") as f:
|
|
198
|
+
encrypted_data = f.read()
|
|
199
|
+
|
|
200
|
+
# Decrypt
|
|
201
|
+
if passphrase:
|
|
202
|
+
decrypted_data = gpg.decrypt(encrypted_data, passphrase=passphrase)
|
|
203
|
+
else:
|
|
204
|
+
decrypted_data = gpg.decrypt(encrypted_data)
|
|
205
|
+
|
|
206
|
+
if not decrypted_data.ok:
|
|
207
|
+
raise ValueError(f"Decryption failed: {decrypted_data.status}")
|
|
208
|
+
|
|
209
|
+
logger.info("✅ File decrypted successfully")
|
|
210
|
+
|
|
211
|
+
# Log signature verification if present
|
|
212
|
+
if hasattr(decrypted_data, "valid") and decrypted_data.valid:
|
|
213
|
+
logger.info(f"✅ Valid signature from: {decrypted_data.username}")
|
|
214
|
+
elif hasattr(decrypted_data, "valid"):
|
|
215
|
+
logger.warning("⚠️ Signature verification failed or not present")
|
|
216
|
+
|
|
217
|
+
return io.BytesIO(decrypted_data.data)
|
|
218
|
+
|
|
219
|
+
finally:
|
|
220
|
+
# Cleanup GPG home directory
|
|
221
|
+
import shutil
|
|
222
|
+
|
|
223
|
+
shutil.rmtree(gpg_home, ignore_errors=True)
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import datetime
|
|
2
|
+
import time
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
|
+
|
|
5
|
+
from loguru import logger
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@dataclass
|
|
9
|
+
class SummaryLogInstance:
|
|
10
|
+
"""
|
|
11
|
+
Class for run summary logs
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
start_time: float = 0
|
|
15
|
+
end_time: float = 0
|
|
16
|
+
duration_sec: int = 0
|
|
17
|
+
duration_str: str = ""
|
|
18
|
+
files_at_source: int = 0
|
|
19
|
+
files_to_load: int = 0
|
|
20
|
+
already_loaded_files: int = 0
|
|
21
|
+
files_loaded:int = 0
|
|
22
|
+
|
|
23
|
+
summary_strings: list[str] = field(default_factory=list)
|
|
24
|
+
|
|
25
|
+
def __enter__(self):
|
|
26
|
+
self.set_start_time()
|
|
27
|
+
pass
|
|
28
|
+
|
|
29
|
+
def __exit__(self, exc_type, exc, tb):
|
|
30
|
+
self.set_end_time()
|
|
31
|
+
self.log_summary()
|
|
32
|
+
pass
|
|
33
|
+
|
|
34
|
+
"""
|
|
35
|
+
adds a new string to the optional summary string for debugging
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
def add_summary_string(self, string_):
|
|
39
|
+
self.summary_strings.append(string_)
|
|
40
|
+
|
|
41
|
+
"""
|
|
42
|
+
Sets the start time for the run timer
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
def set_start_time(self):
|
|
46
|
+
self.start_time = time.time()
|
|
47
|
+
|
|
48
|
+
"""
|
|
49
|
+
Sets the end time for the run timer and starts calculation of run duration
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
def set_end_time(self):
|
|
53
|
+
self.end_time = time.time()
|
|
54
|
+
self._calculate_run_duration()
|
|
55
|
+
|
|
56
|
+
"""
|
|
57
|
+
Logs all the tracked stats
|
|
58
|
+
"""
|
|
59
|
+
|
|
60
|
+
def log_summary(self):
|
|
61
|
+
logger.info(f"Total Run Duration : {self.duration_str}")
|
|
62
|
+
logger.info(f"Total Files : {self.files_at_source}")
|
|
63
|
+
logger.info(f"Files To Load : {self.files_to_load}")
|
|
64
|
+
logger.info(f"Files Already Loaded : {self.already_loaded_files}")
|
|
65
|
+
logger.info(f"Files Loaded : {self.files_loaded}")
|
|
66
|
+
|
|
67
|
+
if len(self.summary_strings) > 0:
|
|
68
|
+
for string in self.summary_strings:
|
|
69
|
+
logger.info(string)
|
|
70
|
+
|
|
71
|
+
"""
|
|
72
|
+
Calculates the duration of the run
|
|
73
|
+
"""
|
|
74
|
+
|
|
75
|
+
def _calculate_run_duration(self):
|
|
76
|
+
dur = self.end_time - self.start_time
|
|
77
|
+
self.duration_sec = int(dur)
|
|
78
|
+
self.duration_str = str(datetime.timedelta(seconds=dur))
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
summary_log = SummaryLogInstance()
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
from cloudpathlib import S3Path
|
|
2
|
+
|
|
3
|
+
from .exceptions import ConfigError
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def validate_s3_path_exists(s3_path: S3Path) -> S3Path:
|
|
7
|
+
if not s3_path.exists():
|
|
8
|
+
raise ConfigError(f"Error. S3 Path does not exist: {s3_path}")
|
|
9
|
+
return s3_path
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def validate_is_s3_dir(s3_path: S3Path) -> S3Path:
|
|
13
|
+
if not s3_path.is_dir():
|
|
14
|
+
raise ConfigError("Provided directory: {s3_path} is not a valid directory")
|
|
15
|
+
return s3_path
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: moj-data-loader
|
|
3
|
+
Version: 2.2.0
|
|
4
|
+
Summary: MoJ Data Loader (MoDaL) provides an easy-to-use, config-driven way to load files from S3 into Athena.
|
|
5
|
+
Requires-Dist: awswrangler>=3.17.0
|
|
6
|
+
Requires-Dist: charset-normalizer>=3.4.9
|
|
7
|
+
Requires-Dist: click>=8.4.2
|
|
8
|
+
Requires-Dist: cloudpathlib>=0.24.0
|
|
9
|
+
Requires-Dist: cryptography>=50.0.0
|
|
10
|
+
Requires-Dist: dlt[athena,pyiceberg]>=1.29.1
|
|
11
|
+
Requires-Dist: duckdb>=1.5.5
|
|
12
|
+
Requires-Dist: fsspec>=2026.7.0
|
|
13
|
+
Requires-Dist: gitpython>=3.1.58
|
|
14
|
+
Requires-Dist: loguru>=0.7.3
|
|
15
|
+
Requires-Dist: msoffcrypto-tool>=5.4.2
|
|
16
|
+
Requires-Dist: openpyxl>=3.1.5
|
|
17
|
+
Requires-Dist: pandas>=3.0.5
|
|
18
|
+
Requires-Dist: pathlib>=1.0.1
|
|
19
|
+
Requires-Dist: psutil>=7.2.2
|
|
20
|
+
Requires-Dist: pyarrow>=24.0.0
|
|
21
|
+
Requires-Dist: pydantic>=2.13.4
|
|
22
|
+
Requires-Dist: pydantic-settings>=2.14.2
|
|
23
|
+
Requires-Dist: pyiceberg>=0.9.0
|
|
24
|
+
Requires-Dist: python-calamine>=0.8.2
|
|
25
|
+
Requires-Dist: python-gnupg>=0.5.6
|
|
26
|
+
Requires-Dist: pyyaml>=6.0.2
|
|
27
|
+
Requires-Dist: setuptools>=83.0.0
|
|
28
|
+
Requires-Dist: sqlalchemy>=2.0.51
|
|
29
|
+
Requires-Dist: tornado>=6.6.1
|
|
30
|
+
Requires-Dist: xlrd>=2.0.2
|
|
31
|
+
Requires-Dist: ruff ; extra == 'linting'
|
|
32
|
+
Requires-Python: >=3.12
|
|
33
|
+
Provides-Extra: linting
|
|
34
|
+
Description-Content-Type: text/markdown
|
|
35
|
+
|
|
36
|
+
# MoJ Data Loader (MoDaL)
|
|
37
|
+
MoJ Data Loader provides an easy to use, config-driven way to load files from S3 into Athena.
|
|
38
|
+
|
|
39
|
+
## Install in Your Project
|
|
40
|
+
|
|
41
|
+
MoDaL is distributed as the `moj-data-loader` Python package on PyPI. It requires Python 3.12 or newer. Install it in the repository that owns your pipelines; you do not need to clone this repository.
|
|
42
|
+
|
|
43
|
+
For a project managed with [uv](https://docs.astral.sh/uv/), run from your project root:
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
uv add moj-data-loader
|
|
47
|
+
uv run moj-data-loader --help
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
Alternatively, install it in a virtual environment with pip:
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
python -m venv .venv
|
|
54
|
+
source .venv/bin/activate
|
|
55
|
+
python -m pip install moj-data-loader
|
|
56
|
+
moj-data-loader --help
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
The package name and CLI command are `moj-data-loader`, not `modal`.
|
|
60
|
+
|
|
61
|
+
## Use Your Own Pipeline Repository
|
|
62
|
+
|
|
63
|
+
Keep your pipeline YAMLs in your own repository:
|
|
64
|
+
|
|
65
|
+
```text
|
|
66
|
+
your-project/
|
|
67
|
+
pyproject.toml
|
|
68
|
+
pipelines/
|
|
69
|
+
your_domain/
|
|
70
|
+
your_database/
|
|
71
|
+
your_table.yml
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
Run from that repository's root:
|
|
75
|
+
|
|
76
|
+
```bash
|
|
77
|
+
uv run moj-data-loader load your_table --env test
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
With an activated pip environment, omit `uv run`. Configure AWS credentials and permissions for the source S3 data and Athena destination before loading data.
|
|
81
|
+
|
|
82
|
+
By default, MoDaL searches `./pipelines/`, not the installed package's directory. Use `--configs-dir /path/to/pipelines` or `MODAL_CONFIG_DIR` to select another config root, retaining the `domain/database/table.yml` layout.
|
|
83
|
+
|
|
84
|
+
Existing pipelines in this repository can continue to use the monorepo and its Airflow image during the transition. New pipelines will soon no longer be accepted here: create them in your own repository using the PyPI package, and plan to migrate existing pipelines there too. See the [migration guide](docs/migration_v2.md#move-pipelines-to-your-own-repository).
|
|
85
|
+
|
|
86
|
+
## Releases
|
|
87
|
+
|
|
88
|
+
The Airflow container workflow runs on pushed tags without `/`. A monorepo tag such as `monorepo-v2.1.25` builds the container without publishing to PyPI; use this for interim pipeline-only releases.
|
|
89
|
+
|
|
90
|
+
For a package release, update the version in `pyproject.toml` and the lockfile, then push a version tag such as `v2.1.25` matching the package version. This publishes to PyPI and also builds the container. A manually dispatched PyPI workflow publishes to TestPyPI only. Configure the GitHub `release` environment and PyPI trusted publisher for production publishing.
|
|
91
|
+
|
|
92
|
+
## Usage guidance
|
|
93
|
+
|
|
94
|
+
You can find usage guidance in [`docs/user_guide`](docs/user_guide.md)
|
|
95
|
+
|
|
96
|
+
Pipeline YAML specification is documented in [`docs/pipeline_config.md`](docs/pipeline_config.md).
|
|
97
|
+
|
|
98
|
+
## Migration notes
|
|
99
|
+
|
|
100
|
+
v2.0 is a breaking release.
|
|
101
|
+
|
|
102
|
+
- Supported CLI: `moj-data-loader`
|
|
103
|
+
- Supported environment variables: `MODAL_*`
|
|
104
|
+
- Removed in v2.0: `airflow-file` command alias and `AFFILE_*` environment variables
|
|
105
|
+
|
|
106
|
+
For migration steps and old-to-new mappings, see:
|
|
107
|
+
|
|
108
|
+
- [`docs/migration_v2.md`](docs/migration_v2.md)
|
|
109
|
+
- [`docs/release_notes.md`](docs/release_notes.md)
|
|
110
|
+
|
|
111
|
+
## Useful links
|
|
112
|
+
- Documentation can be found at the Data and Analytics Engineering internal Pages site [here]: TBC
|
|
113
|
+
- Development of this repo is managed via this [Project Board](https://github.com/orgs/moj-analytical-services/projects/262/views/1)
|
|
114
|
+
|
|
115
|
+
## Contributing
|
|
116
|
+
|
|
117
|
+
To develop MoDaL itself, clone this repository and run:
|
|
118
|
+
|
|
119
|
+
```bash
|
|
120
|
+
uv sync
|
|
121
|
+
uv run pytest tests/unit
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
`uv sync` installs MoDaL in editable mode for development. Users should add the released package to their own project's dependencies instead. Commit your project's dependency manifest and lockfile to keep local, CI, and deployment environments consistent.
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
moj_data_loader/Pipeline.py,sha256=xS08E_FocolEj21lzXHRJap7baAHpIRVfwqtyjiWIc4,24111
|
|
2
|
+
moj_data_loader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
|
+
moj_data_loader/main.py,sha256=gSctxdhpHNLRH4jVIVPopGitOSkClgvxvg5cMXNIsQQ,9495
|
|
4
|
+
moj_data_loader/readers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
5
|
+
moj_data_loader/readers/registry.py,sha256=jmTZNT1wkDgcoUCJ2GvkcT5Il_jB5-oL1SEGkoo54bU,6005
|
|
6
|
+
moj_data_loader/readers/transforms.py,sha256=pfVEqLil929lD0I7mCulm8JrxVQz3o9O-d91yYYNW30,11252
|
|
7
|
+
moj_data_loader/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
+
moj_data_loader/utils/batch.py,sha256=SJtnCbWzmztKh13b-5PeJqRWFFuMud4kWBZGuEt_7HE,6514
|
|
9
|
+
moj_data_loader/utils/exceptions.py,sha256=X7gSYnCArePp125Hn9wZpg2_wM2RNPmrgBGoZgKUb8o,212
|
|
10
|
+
moj_data_loader/utils/files.py,sha256=-fCQreuF5oImUSKce1uFucaSynSzkJriRhdjaQybHFw,1202
|
|
11
|
+
moj_data_loader/utils/pgp.py,sha256=dEIPIy35DrBjHb29L_sMSi8e5LPZQx6NjaXTdi70psc,7957
|
|
12
|
+
moj_data_loader/utils/summary_log.py,sha256=XbaxiAZeG-Pj-qUsY04z22flmRjoUWyr0Zm1qBNqI0g,1950
|
|
13
|
+
moj_data_loader/utils/validators.py,sha256=itsvZ7jsoOb-dBch6Cay9rpFBdoib1cUWExPqNvEQ3o,431
|
|
14
|
+
moj_data_loader-2.2.0.dist-info/WHEEL,sha256=3NO1oYsF7mANFUWVdyEQLM-zmecczWWEmmF5rEbXrWk,80
|
|
15
|
+
moj_data_loader-2.2.0.dist-info/entry_points.txt,sha256=sKPodKqRxJmwTNNI53VBAYn_p1NsrhHuSIDOGPan2Uo,62
|
|
16
|
+
moj_data_loader-2.2.0.dist-info/METADATA,sha256=jmJ1xTyLnFMDxJbHgZSZuqL6qHVBNJnEQPEfkEl8AzA,4731
|
|
17
|
+
moj_data_loader-2.2.0.dist-info/RECORD,,
|