moj-data-loader 2.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,291 @@
1
+ import os
2
+ import tempfile
3
+
4
+ import click
5
+ from dotenv import load_dotenv
6
+ from loguru import logger
7
+
8
+ from moj_data_loader.Pipeline import Pipeline
9
+ from moj_data_loader.utils.summary_log import summary_log
10
+
11
+ LEGACY_ENV_VAR_MAP = {
12
+ "AFFILE_PIPELINE_NAME": "MODAL_PIPELINE_NAME",
13
+ "AFFILE_ENV": "MODAL_ENV",
14
+ "AFFILE_FULL_REFRESH": "MODAL_FULL_REFRESH",
15
+ "AFFILE_SCHEMA_CONTRACT": "MODAL_SCHEMA_CONTRACT",
16
+ "AFFILE_LOAD_LIMIT": "MODAL_LOAD_LIMIT",
17
+ "AFFILE_ALLOW_PROD": "MODAL_ALLOW_PROD",
18
+ "AFFILE_CONFIG_DIR": "MODAL_CONFIG_DIR",
19
+ "AFFILE_DLT_DATA_DIR": "MODAL_DLT_DATA_DIR",
20
+ "AFFILE_DLT_PROJECT_DIR": "MODAL_DLT_PROJECT_DIR",
21
+ }
22
+
23
+
24
+ def fail_if_legacy_affile_env_vars_present() -> None:
25
+ """Fail fast when removed AFFILE_* env vars are present in v2.0."""
26
+ legacy_in_use = []
27
+ for legacy_name, modal_name in LEGACY_ENV_VAR_MAP.items():
28
+ if os.getenv(legacy_name) is not None:
29
+ legacy_in_use.append(f"{legacy_name} -> {modal_name}")
30
+
31
+ if legacy_in_use:
32
+ raise click.UsageError(
33
+ "Detected removed AFFILE_* environment variables (v2.0 breaking change).\n"
34
+ "Please migrate to MODAL_* equivalents:\n"
35
+ + "\n".join([f"- {entry}" for entry in legacy_in_use])
36
+ + "\nSee docs/migration_v2.md for migration steps."
37
+ )
38
+
39
+
40
+ @click.group()
41
+ def cli():
42
+ """MoJ Data Loader (MoDaL) - Config-driven data pipeline system."""
43
+ fail_if_legacy_affile_env_vars_present()
44
+ pass
45
+
46
+
47
+ @cli.command()
48
+ @click.argument("pipeline_name", envvar="MODAL_PIPELINE_NAME")
49
+ @click.option(
50
+ "--env",
51
+ type=click.Choice(["test", "dev", "prod"]),
52
+ default="test",
53
+ help="Environment to load the pipeline in",
54
+ envvar="MODAL_ENV",
55
+ )
56
+ @click.option(
57
+ "--full-refresh",
58
+ type=bool,
59
+ default=False,
60
+ help="Perform a full refresh of the table from source",
61
+ envvar="MODAL_FULL_REFRESH",
62
+ )
63
+ @click.option(
64
+ "--schema-contract",
65
+ type=click.Choice(["freeze", "evolve"]),
66
+ default="freeze",
67
+ help="Schema contract to use for the table",
68
+ envvar="MODAL_SCHEMA_CONTRACT",
69
+ )
70
+ @click.option(
71
+ "--load-limit",
72
+ type=int,
73
+ help="Limit the number of files to load",
74
+ envvar="MODAL_LOAD_LIMIT",
75
+ )
76
+ @click.option(
77
+ "--allow-prod",
78
+ type=bool,
79
+ default=False,
80
+ help="Allow run against prod env",
81
+ envvar="MODAL_ALLOW_PROD",
82
+ )
83
+ @click.option(
84
+ "--configs-dir",
85
+ type=str,
86
+ default="pipelines",
87
+ help="Which directory to search for pipeline yamls",
88
+ envvar="MODAL_CONFIG_DIR",
89
+ )
90
+ @click.option(
91
+ "--dlt-data-dir",
92
+ type=str,
93
+ default="data_dir/",
94
+ help="Where should dlt store data from pipeline runs?",
95
+ envvar="MODAL_DLT_DATA_DIR",
96
+ )
97
+ @click.option(
98
+ "--dlt-project-dir",
99
+ type=str,
100
+ default="project_dir/",
101
+ help="Where should dlt store project metadata?",
102
+ envvar="MODAL_DLT_PROJECT_DIR",
103
+ )
104
+ def load(
105
+ pipeline_name: str,
106
+ env: str,
107
+ full_refresh: bool,
108
+ schema_contract: str,
109
+ load_limit: int | None,
110
+ allow_prod: bool,
111
+ configs_dir: str,
112
+ dlt_data_dir: str,
113
+ dlt_project_dir: str,
114
+ ):
115
+ """Load files from S3 into an Athena table using a data pipeline.
116
+
117
+ This function configures and executes a pipeline that reads files from a
118
+ specified S3 location and loads them into an Athena table. The pipeline configuration
119
+ is read from YAML files in the pipelines directory and can be run in different
120
+ environments with options for full refresh.
121
+
122
+ Args:
123
+ pipeline_name (str): Name of the pipeline to run. Can be just [table] if unique,
124
+ [database/table] if needed for disambiguation, or [domain/database/table].
125
+ Must match a YAML file in the pipelines directory
126
+ env (str): Environment to run the pipeline in - one of "dev", "test", or
127
+ "prod". Alters the destination database and S3 paths. Defaults to "dev".
128
+ full_refresh (bool): If True, performs a full refresh of the table. If False,
129
+ only loads new files. Default is False.
130
+ schema_contract (str): Schema contract to use - "freeze" or "evolve".
131
+ Default is "freeze".
132
+ load_limit (int): Optional limit on number of files to load.
133
+ allow_prod (bool): Enables writing to production file paths
134
+
135
+ Expects YAML pipeline configurations in pipelines/[domain]/[database]/[table].yml
136
+ with the required source and reader specifications.
137
+ """
138
+
139
+ logger.info(
140
+ f"Load CLI Initialised with:"
141
+ f"\n pipeline_name: {pipeline_name}"
142
+ f"\n env: {env}"
143
+ f"\n full_refresh: {full_refresh}"
144
+ f"\n schema_contract: {schema_contract}"
145
+ f"\n load_limit: {load_limit}"
146
+ f"\n allow_prod: {allow_prod}"
147
+ )
148
+ if env == "prod" and not allow_prod:
149
+ click.echo("❌ Production run blocked for safety.")
150
+ click.echo("Use --allow-prod flag if you're absolutely certain.")
151
+ return
152
+
153
+ Pipeline.set_temp_dlt_directory_for_airflow(
154
+ dlt_data_dir=dlt_data_dir, dlt_project_dir=dlt_project_dir
155
+ )
156
+
157
+ # Redirect the OS temp dir before dlt.Pipeline is constructed, so that dlt and its
158
+ # dependencies (which resolve tempfile.gettempdir() independently of DLT_DATA_DIR)
159
+ # never touch the real system /tmp.
160
+ previous_tempdir = tempfile.tempdir
161
+ previous_tmpdir = os.environ.get("TMPDIR")
162
+ process_temp_dir = tempfile.mkdtemp(prefix="dlt_pipeline_", dir=tempfile.gettempdir())
163
+ tempfile.tempdir = process_temp_dir
164
+ os.environ["TMPDIR"] = process_temp_dir
165
+
166
+ try:
167
+ # Load and initialize the pipeline
168
+ pipeline = Pipeline.load_from_yaml(
169
+ pipeline_name,
170
+ configs_dir=configs_dir,
171
+ env=env,
172
+ full_refresh=full_refresh,
173
+ schema_contract=schema_contract,
174
+ load_limit=load_limit,
175
+ )
176
+
177
+ logger.info(str(pipeline))
178
+
179
+ # Run the pipeline
180
+ with summary_log as ctx: # noqa: F841
181
+ pipeline.run()
182
+ finally:
183
+ tempfile.tempdir = previous_tempdir
184
+ if previous_tmpdir is None:
185
+ os.environ.pop("TMPDIR", None)
186
+ else:
187
+ os.environ["TMPDIR"] = previous_tmpdir
188
+
189
+
190
+
191
+ @cli.command()
192
+ @click.argument("pipeline_name")
193
+ @click.option(
194
+ "--env",
195
+ type=click.Choice(["test", "dev", "prod"]),
196
+ help="Environment to teardown the pipeline in",
197
+ envvar="MODAL_ENV",
198
+ )
199
+ @click.option(
200
+ "--confirm",
201
+ is_flag=True,
202
+ help="Skip confirmation prompt for teardown",
203
+ )
204
+ @click.option(
205
+ "--allow-prod",
206
+ is_flag=True,
207
+ help="Allow teardown in production environment",
208
+ )
209
+ @click.option(
210
+ "--staging-only",
211
+ is_flag=True,
212
+ help="""
213
+ Whether to drop staging tables only, or all tables. Staging tables are temporary
214
+ tables used by dlt when loading to Iceberg.
215
+ """,
216
+ )
217
+ def dropdb(
218
+ pipeline_name: str,
219
+ env: str,
220
+ confirm: bool,
221
+ allow_prod: bool,
222
+ staging_only: bool,
223
+ ):
224
+ """Teardown a pipeline by deleting databases, tables, and S3 data.
225
+
226
+ This command will permanently delete:
227
+ - Athena databases and tables
228
+ - S3 staging data
229
+ - All associated pipeline resources
230
+
231
+ Args:
232
+ pipeline_name (str): Name of the pipeline. Can be just [table] if unique,
233
+ [database/table] if needed for disambiguation, or [domain/database/table]
234
+ env (str): Environment to teardown - one of "dev", "test", or "prod"
235
+ confirm (bool): Skip interactive confirmation prompt
236
+ allow_prod (bool): Allow teardown in production (dangerous!)
237
+ staging_only (bool): Only delete the staging tables dlt creates
238
+ """
239
+
240
+ logger.info(
241
+ f"Teardown CLI Initialised with:\n pipeline_name: {pipeline_name}\n env: {env}"
242
+ )
243
+ # Load the pipeline to get details
244
+ # Default values manually set for
245
+ pipeline = Pipeline.load_from_yaml(
246
+ pipeline_name,
247
+ env=env,
248
+ # Default values set - not important for teardown but needed for initialisation
249
+ full_refresh=False,
250
+ schema_contract="freeze",
251
+ load_limit=0,
252
+ )
253
+
254
+ # Safety check for production
255
+ if env == "prod" and not allow_prod:
256
+ click.echo("❌ Production teardown blocked for safety.")
257
+ click.echo("Use --allow-prod flag if you're absolutely certain.")
258
+ return
259
+
260
+ # Show what will be deleted
261
+ click.echo("\n🗑️ 🗑️ 🗑️")
262
+ click.echo(f"Pipeline Teardown: {pipeline_name}")
263
+ click.echo(f"Environment: {pipeline.env}")
264
+ click.echo(f"Database: {pipeline.destination_database}")
265
+ click.echo(f"Table: {pipeline.destination_table}")
266
+ click.echo(f"S3 Path: {pipeline.destination_path}")
267
+ click.echo(f"dlt staging tables only? {staging_only}")
268
+
269
+ # Confirmation prompt unless --confirm is used
270
+ if not confirm:
271
+ if not click.confirm(
272
+ "\n⚠️ ⚠️ ⚠️\n This will permanently delete all data. Continue?"
273
+ ):
274
+ click.echo("Teardown cancelled.")
275
+ return
276
+
277
+ # Execute teardown
278
+ try:
279
+ logger.info(f"Starting teardown for {pipeline_name} in {env} environment")
280
+ pipeline.drop_db(allow_prod=allow_prod, staging_only=staging_only)
281
+ click.echo(f"✅ Successfully torn down pipeline: {pipeline_name}")
282
+ except Exception as e:
283
+ click.echo(f"❌ Teardown failed: {str(e)}")
284
+ logger.error(f"Teardown failed for {pipeline_name}: {e}")
285
+ raise
286
+
287
+
288
+ if __name__ == "__main__":
289
+ if not load_dotenv():
290
+ logger.warning("No env file found. May be required if running locally.")
291
+ cli()
File without changes
@@ -0,0 +1,194 @@
1
+ import json
2
+ from functools import wraps
3
+ from pathlib import Path
4
+ from typing import Any, Callable, Iterator
5
+
6
+ import dlt
7
+ import pandas as pd
8
+ import pyarrow as pa
9
+ from cloudpathlib import S3Path
10
+ from loguru import logger
11
+
12
+ from moj_data_loader.readers.transforms import (
13
+ add_empty_column_types,
14
+ add_file_name_to_df,
15
+ add_magistrates_timestamp_col,
16
+ apply_transforms_to_generator,
17
+ get_file_encoding,
18
+ infer_excel_rowskip,
19
+ is_excel_header_empty,
20
+ )
21
+ from moj_data_loader.utils.exceptions import ConfigError
22
+ from moj_data_loader.utils.pgp import with_pgp_decryption
23
+
24
+ # Module-level registry - loads on import
25
+ _readers: dict[str, Callable] = {}
26
+
27
+
28
+ def register(func: Callable) -> Callable:
29
+ """Decorator that registers a processor function with its exact name."""
30
+
31
+ @wraps(func)
32
+ def wrapper(*args, **kwargs):
33
+ try:
34
+ yield from func(*args, **kwargs)
35
+ except Exception as e:
36
+ logger.error(f"Reader {func.__name__} failed: {e}")
37
+ raise
38
+
39
+ # Register using the exact function name
40
+ _readers[func.__name__] = wrapper
41
+ return wrapper
42
+
43
+
44
+ def read(
45
+ reader: str,
46
+ files: list[S3Path],
47
+ transforms: list[dict[str, Any]] | None = None,
48
+ **kwargs,
49
+ ):
50
+ """Dispatch to a registered reader and optionally apply transforms.
51
+
52
+ Parameters
53
+ ----------
54
+ reader:
55
+ Name of the registered reader function.
56
+ files:
57
+ Files to pass to the reader.
58
+ transforms:
59
+ Optional list of transform configs, each a dict with ``name``
60
+ (required) and ``kwargs`` (optional). They are applied in order
61
+ to every ``pa.Table`` yielded by the reader.
62
+ **kwargs:
63
+ Forwarded to the reader function.
64
+ """
65
+ if reader not in _readers:
66
+ raise ConfigError(
67
+ f"No reader found for name: {reader}. \n Available: {list(_readers.keys())}"
68
+ )
69
+
70
+ reader_gen = _readers[reader](files, **kwargs)
71
+
72
+ if transforms:
73
+ reader_gen = apply_transforms_to_generator(reader_gen, transforms)
74
+
75
+ reader_function = dlt.resource(reader_gen, table_format="iceberg")
76
+
77
+ return reader_function
78
+
79
+
80
+ def list_readers() -> list[str]:
81
+ """List all registered reader names."""
82
+ return list(_readers.keys())
83
+
84
+
85
+ @register
86
+ @with_pgp_decryption
87
+ def read_csv(files: list[S3Path | Path], **kwargs) -> Iterator[pa.Table]:
88
+ """Read CSV files (with optional PGP decryption via decorator)."""
89
+
90
+ # Extract file mapping if present (from PGP decorator)
91
+ file_mapping = kwargs.pop("_original_file_paths", None)
92
+
93
+ for file in files:
94
+ opts = dict(kwargs)
95
+ if "encoding" not in opts:
96
+ opts["encoding"] = get_file_encoding(file)
97
+
98
+ logger.info(f"Running load with options: {opts}")
99
+ logger.info(f"Reading (csv): {str(file)}")
100
+
101
+ with file.open("rb") as f:
102
+ df = pd.read_csv(f, **opts)
103
+
104
+ # Use original S3 path if available, otherwise use current path
105
+ original_path = (
106
+ file_mapping.get(str(file), str(file)) if file_mapping else str(file)
107
+ )
108
+ df = add_file_name_to_df(df, original_path)
109
+
110
+ # Ensure empty columns have valid types
111
+ df = add_empty_column_types(df)
112
+
113
+ yield pa.Table.from_pandas(df)
114
+
115
+
116
+ @register
117
+ @with_pgp_decryption
118
+ def read_excel(files: list[S3Path | Path], **kwargs) -> Iterator[pa.Table]:
119
+ """Read Excel files (with optional PGP decryption via decorator)."""
120
+
121
+ # Extract file mapping if present (from PGP decorator)
122
+ file_mapping = kwargs.pop("_original_file_paths", None)
123
+
124
+ for file in files:
125
+ opts = dict(kwargs)
126
+ if "dtype_backend" not in opts:
127
+ opts["dtype_backend"] = "pyarrow"
128
+ if "engine" not in opts:
129
+ opts["engine"] = "calamine"
130
+ if "skiprows" not in opts:
131
+ opts["skiprows"] = infer_excel_rowskip(file)
132
+
133
+ if is_excel_header_empty(file, opts["skiprows"]):
134
+ raise ValueError(
135
+ f"Empty header in Excel file: {file}, skiprows: {opts['skiprows']}"
136
+ )
137
+
138
+ logger.info(f"Running load with options: {opts}")
139
+ logger.info(f"Reading (excel): {str(file)}")
140
+
141
+ with file.open("rb") as f:
142
+ df = pd.read_excel(f, **opts)
143
+
144
+ # Use original S3 path if available, otherwise use current path
145
+ original_path = (
146
+ file_mapping.get(str(file), str(file)) if file_mapping else str(file)
147
+ )
148
+ df = add_file_name_to_df(df, original_path)
149
+
150
+ yield pa.Table.from_pandas(df)
151
+
152
+
153
+ @register
154
+ def read_parquet(files: list[S3Path], **kwargs) -> Iterator[pa.Table]:
155
+ """Read Parquet files"""
156
+ opts = dict(kwargs)
157
+ total = len(files)
158
+ for counter, file in enumerate(files, start=1):
159
+ logger.info(f"Reading (parquet) {counter}/{total}: {str(file)}")
160
+ logger.info(f"Running load with options: {opts}")
161
+
162
+ with file.open("rb") as f:
163
+ df = pd.read_parquet(f, **kwargs)
164
+
165
+ df = add_file_name_to_df(df, str(file))
166
+ yield pa.Table.from_pandas(df)
167
+
168
+
169
+ @register
170
+ def read_json(files: list[S3Path], **kwargs) -> Iterator[pa.Table]:
171
+ """Reader for JSON and JSONL file types"""
172
+
173
+ for file in files:
174
+ logger.info(f"Reading (JSON): {str(file)}")
175
+ logger.info(f"Running load with options: {kwargs}")
176
+ with file.open("rb") as f:
177
+ df = pd.read_json(f, **kwargs)
178
+ df = add_file_name_to_df(df, str(file))
179
+ yield pa.Table.from_pandas(df)
180
+
181
+
182
+ @register
183
+ def read_magistrates_json(files: list[S3Path], **kwargs) -> Iterator[pa.Table]:
184
+ """Reader for JSON and JSONL file types"""
185
+
186
+ for file in files:
187
+ logger.info(f"Reading (JSON): {str(file)}")
188
+ logger.info(f"Running load with options: {kwargs}")
189
+ df = pd.read_json(file, **kwargs)
190
+ df = add_file_name_to_df(df, str(file))
191
+ df = add_magistrates_timestamp_col(df, str(file))
192
+ if "ofn" in df.columns:
193
+ df["ofn"] = df["ofn"].apply(lambda x: json.dumps(x))
194
+ yield pa.Table.from_pandas(df)