moj-data-loader 2.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- moj_data_loader/Pipeline.py +659 -0
- moj_data_loader/__init__.py +0 -0
- moj_data_loader/main.py +291 -0
- moj_data_loader/readers/__init__.py +0 -0
- moj_data_loader/readers/registry.py +194 -0
- moj_data_loader/readers/transforms.py +334 -0
- moj_data_loader/utils/__init__.py +0 -0
- moj_data_loader/utils/batch.py +221 -0
- moj_data_loader/utils/exceptions.py +6 -0
- moj_data_loader/utils/files.py +46 -0
- moj_data_loader/utils/pgp.py +223 -0
- moj_data_loader/utils/summary_log.py +81 -0
- moj_data_loader/utils/validators.py +15 -0
- moj_data_loader-2.2.0.dist-info/METADATA +124 -0
- moj_data_loader-2.2.0.dist-info/RECORD +17 -0
- moj_data_loader-2.2.0.dist-info/WHEEL +4 -0
- moj_data_loader-2.2.0.dist-info/entry_points.txt +3 -0
moj_data_loader/main.py
ADDED
|
@@ -0,0 +1,291 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import tempfile
|
|
3
|
+
|
|
4
|
+
import click
|
|
5
|
+
from dotenv import load_dotenv
|
|
6
|
+
from loguru import logger
|
|
7
|
+
|
|
8
|
+
from moj_data_loader.Pipeline import Pipeline
|
|
9
|
+
from moj_data_loader.utils.summary_log import summary_log
|
|
10
|
+
|
|
11
|
+
LEGACY_ENV_VAR_MAP = {
|
|
12
|
+
"AFFILE_PIPELINE_NAME": "MODAL_PIPELINE_NAME",
|
|
13
|
+
"AFFILE_ENV": "MODAL_ENV",
|
|
14
|
+
"AFFILE_FULL_REFRESH": "MODAL_FULL_REFRESH",
|
|
15
|
+
"AFFILE_SCHEMA_CONTRACT": "MODAL_SCHEMA_CONTRACT",
|
|
16
|
+
"AFFILE_LOAD_LIMIT": "MODAL_LOAD_LIMIT",
|
|
17
|
+
"AFFILE_ALLOW_PROD": "MODAL_ALLOW_PROD",
|
|
18
|
+
"AFFILE_CONFIG_DIR": "MODAL_CONFIG_DIR",
|
|
19
|
+
"AFFILE_DLT_DATA_DIR": "MODAL_DLT_DATA_DIR",
|
|
20
|
+
"AFFILE_DLT_PROJECT_DIR": "MODAL_DLT_PROJECT_DIR",
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def fail_if_legacy_affile_env_vars_present() -> None:
|
|
25
|
+
"""Fail fast when removed AFFILE_* env vars are present in v2.0."""
|
|
26
|
+
legacy_in_use = []
|
|
27
|
+
for legacy_name, modal_name in LEGACY_ENV_VAR_MAP.items():
|
|
28
|
+
if os.getenv(legacy_name) is not None:
|
|
29
|
+
legacy_in_use.append(f"{legacy_name} -> {modal_name}")
|
|
30
|
+
|
|
31
|
+
if legacy_in_use:
|
|
32
|
+
raise click.UsageError(
|
|
33
|
+
"Detected removed AFFILE_* environment variables (v2.0 breaking change).\n"
|
|
34
|
+
"Please migrate to MODAL_* equivalents:\n"
|
|
35
|
+
+ "\n".join([f"- {entry}" for entry in legacy_in_use])
|
|
36
|
+
+ "\nSee docs/migration_v2.md for migration steps."
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@click.group()
|
|
41
|
+
def cli():
|
|
42
|
+
"""MoJ Data Loader (MoDaL) - Config-driven data pipeline system."""
|
|
43
|
+
fail_if_legacy_affile_env_vars_present()
|
|
44
|
+
pass
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@cli.command()
|
|
48
|
+
@click.argument("pipeline_name", envvar="MODAL_PIPELINE_NAME")
|
|
49
|
+
@click.option(
|
|
50
|
+
"--env",
|
|
51
|
+
type=click.Choice(["test", "dev", "prod"]),
|
|
52
|
+
default="test",
|
|
53
|
+
help="Environment to load the pipeline in",
|
|
54
|
+
envvar="MODAL_ENV",
|
|
55
|
+
)
|
|
56
|
+
@click.option(
|
|
57
|
+
"--full-refresh",
|
|
58
|
+
type=bool,
|
|
59
|
+
default=False,
|
|
60
|
+
help="Perform a full refresh of the table from source",
|
|
61
|
+
envvar="MODAL_FULL_REFRESH",
|
|
62
|
+
)
|
|
63
|
+
@click.option(
|
|
64
|
+
"--schema-contract",
|
|
65
|
+
type=click.Choice(["freeze", "evolve"]),
|
|
66
|
+
default="freeze",
|
|
67
|
+
help="Schema contract to use for the table",
|
|
68
|
+
envvar="MODAL_SCHEMA_CONTRACT",
|
|
69
|
+
)
|
|
70
|
+
@click.option(
|
|
71
|
+
"--load-limit",
|
|
72
|
+
type=int,
|
|
73
|
+
help="Limit the number of files to load",
|
|
74
|
+
envvar="MODAL_LOAD_LIMIT",
|
|
75
|
+
)
|
|
76
|
+
@click.option(
|
|
77
|
+
"--allow-prod",
|
|
78
|
+
type=bool,
|
|
79
|
+
default=False,
|
|
80
|
+
help="Allow run against prod env",
|
|
81
|
+
envvar="MODAL_ALLOW_PROD",
|
|
82
|
+
)
|
|
83
|
+
@click.option(
|
|
84
|
+
"--configs-dir",
|
|
85
|
+
type=str,
|
|
86
|
+
default="pipelines",
|
|
87
|
+
help="Which directory to search for pipeline yamls",
|
|
88
|
+
envvar="MODAL_CONFIG_DIR",
|
|
89
|
+
)
|
|
90
|
+
@click.option(
|
|
91
|
+
"--dlt-data-dir",
|
|
92
|
+
type=str,
|
|
93
|
+
default="data_dir/",
|
|
94
|
+
help="Where should dlt store data from pipeline runs?",
|
|
95
|
+
envvar="MODAL_DLT_DATA_DIR",
|
|
96
|
+
)
|
|
97
|
+
@click.option(
|
|
98
|
+
"--dlt-project-dir",
|
|
99
|
+
type=str,
|
|
100
|
+
default="project_dir/",
|
|
101
|
+
help="Where should dlt store project metadata?",
|
|
102
|
+
envvar="MODAL_DLT_PROJECT_DIR",
|
|
103
|
+
)
|
|
104
|
+
def load(
|
|
105
|
+
pipeline_name: str,
|
|
106
|
+
env: str,
|
|
107
|
+
full_refresh: bool,
|
|
108
|
+
schema_contract: str,
|
|
109
|
+
load_limit: int | None,
|
|
110
|
+
allow_prod: bool,
|
|
111
|
+
configs_dir: str,
|
|
112
|
+
dlt_data_dir: str,
|
|
113
|
+
dlt_project_dir: str,
|
|
114
|
+
):
|
|
115
|
+
"""Load files from S3 into an Athena table using a data pipeline.
|
|
116
|
+
|
|
117
|
+
This function configures and executes a pipeline that reads files from a
|
|
118
|
+
specified S3 location and loads them into an Athena table. The pipeline configuration
|
|
119
|
+
is read from YAML files in the pipelines directory and can be run in different
|
|
120
|
+
environments with options for full refresh.
|
|
121
|
+
|
|
122
|
+
Args:
|
|
123
|
+
pipeline_name (str): Name of the pipeline to run. Can be just [table] if unique,
|
|
124
|
+
[database/table] if needed for disambiguation, or [domain/database/table].
|
|
125
|
+
Must match a YAML file in the pipelines directory
|
|
126
|
+
env (str): Environment to run the pipeline in - one of "dev", "test", or
|
|
127
|
+
"prod". Alters the destination database and S3 paths. Defaults to "dev".
|
|
128
|
+
full_refresh (bool): If True, performs a full refresh of the table. If False,
|
|
129
|
+
only loads new files. Default is False.
|
|
130
|
+
schema_contract (str): Schema contract to use - "freeze" or "evolve".
|
|
131
|
+
Default is "freeze".
|
|
132
|
+
load_limit (int): Optional limit on number of files to load.
|
|
133
|
+
allow_prod (bool): Enables writing to production file paths
|
|
134
|
+
|
|
135
|
+
Expects YAML pipeline configurations in pipelines/[domain]/[database]/[table].yml
|
|
136
|
+
with the required source and reader specifications.
|
|
137
|
+
"""
|
|
138
|
+
|
|
139
|
+
logger.info(
|
|
140
|
+
f"Load CLI Initialised with:"
|
|
141
|
+
f"\n pipeline_name: {pipeline_name}"
|
|
142
|
+
f"\n env: {env}"
|
|
143
|
+
f"\n full_refresh: {full_refresh}"
|
|
144
|
+
f"\n schema_contract: {schema_contract}"
|
|
145
|
+
f"\n load_limit: {load_limit}"
|
|
146
|
+
f"\n allow_prod: {allow_prod}"
|
|
147
|
+
)
|
|
148
|
+
if env == "prod" and not allow_prod:
|
|
149
|
+
click.echo("❌ Production run blocked for safety.")
|
|
150
|
+
click.echo("Use --allow-prod flag if you're absolutely certain.")
|
|
151
|
+
return
|
|
152
|
+
|
|
153
|
+
Pipeline.set_temp_dlt_directory_for_airflow(
|
|
154
|
+
dlt_data_dir=dlt_data_dir, dlt_project_dir=dlt_project_dir
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
# Redirect the OS temp dir before dlt.Pipeline is constructed, so that dlt and its
|
|
158
|
+
# dependencies (which resolve tempfile.gettempdir() independently of DLT_DATA_DIR)
|
|
159
|
+
# never touch the real system /tmp.
|
|
160
|
+
previous_tempdir = tempfile.tempdir
|
|
161
|
+
previous_tmpdir = os.environ.get("TMPDIR")
|
|
162
|
+
process_temp_dir = tempfile.mkdtemp(prefix="dlt_pipeline_", dir=tempfile.gettempdir())
|
|
163
|
+
tempfile.tempdir = process_temp_dir
|
|
164
|
+
os.environ["TMPDIR"] = process_temp_dir
|
|
165
|
+
|
|
166
|
+
try:
|
|
167
|
+
# Load and initialize the pipeline
|
|
168
|
+
pipeline = Pipeline.load_from_yaml(
|
|
169
|
+
pipeline_name,
|
|
170
|
+
configs_dir=configs_dir,
|
|
171
|
+
env=env,
|
|
172
|
+
full_refresh=full_refresh,
|
|
173
|
+
schema_contract=schema_contract,
|
|
174
|
+
load_limit=load_limit,
|
|
175
|
+
)
|
|
176
|
+
|
|
177
|
+
logger.info(str(pipeline))
|
|
178
|
+
|
|
179
|
+
# Run the pipeline
|
|
180
|
+
with summary_log as ctx: # noqa: F841
|
|
181
|
+
pipeline.run()
|
|
182
|
+
finally:
|
|
183
|
+
tempfile.tempdir = previous_tempdir
|
|
184
|
+
if previous_tmpdir is None:
|
|
185
|
+
os.environ.pop("TMPDIR", None)
|
|
186
|
+
else:
|
|
187
|
+
os.environ["TMPDIR"] = previous_tmpdir
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
@cli.command()
|
|
192
|
+
@click.argument("pipeline_name")
|
|
193
|
+
@click.option(
|
|
194
|
+
"--env",
|
|
195
|
+
type=click.Choice(["test", "dev", "prod"]),
|
|
196
|
+
help="Environment to teardown the pipeline in",
|
|
197
|
+
envvar="MODAL_ENV",
|
|
198
|
+
)
|
|
199
|
+
@click.option(
|
|
200
|
+
"--confirm",
|
|
201
|
+
is_flag=True,
|
|
202
|
+
help="Skip confirmation prompt for teardown",
|
|
203
|
+
)
|
|
204
|
+
@click.option(
|
|
205
|
+
"--allow-prod",
|
|
206
|
+
is_flag=True,
|
|
207
|
+
help="Allow teardown in production environment",
|
|
208
|
+
)
|
|
209
|
+
@click.option(
|
|
210
|
+
"--staging-only",
|
|
211
|
+
is_flag=True,
|
|
212
|
+
help="""
|
|
213
|
+
Whether to drop staging tables only, or all tables. Staging tables are temporary
|
|
214
|
+
tables used by dlt when loading to Iceberg.
|
|
215
|
+
""",
|
|
216
|
+
)
|
|
217
|
+
def dropdb(
|
|
218
|
+
pipeline_name: str,
|
|
219
|
+
env: str,
|
|
220
|
+
confirm: bool,
|
|
221
|
+
allow_prod: bool,
|
|
222
|
+
staging_only: bool,
|
|
223
|
+
):
|
|
224
|
+
"""Teardown a pipeline by deleting databases, tables, and S3 data.
|
|
225
|
+
|
|
226
|
+
This command will permanently delete:
|
|
227
|
+
- Athena databases and tables
|
|
228
|
+
- S3 staging data
|
|
229
|
+
- All associated pipeline resources
|
|
230
|
+
|
|
231
|
+
Args:
|
|
232
|
+
pipeline_name (str): Name of the pipeline. Can be just [table] if unique,
|
|
233
|
+
[database/table] if needed for disambiguation, or [domain/database/table]
|
|
234
|
+
env (str): Environment to teardown - one of "dev", "test", or "prod"
|
|
235
|
+
confirm (bool): Skip interactive confirmation prompt
|
|
236
|
+
allow_prod (bool): Allow teardown in production (dangerous!)
|
|
237
|
+
staging_only (bool): Only delete the staging tables dlt creates
|
|
238
|
+
"""
|
|
239
|
+
|
|
240
|
+
logger.info(
|
|
241
|
+
f"Teardown CLI Initialised with:\n pipeline_name: {pipeline_name}\n env: {env}"
|
|
242
|
+
)
|
|
243
|
+
# Load the pipeline to get details
|
|
244
|
+
# Default values manually set for
|
|
245
|
+
pipeline = Pipeline.load_from_yaml(
|
|
246
|
+
pipeline_name,
|
|
247
|
+
env=env,
|
|
248
|
+
# Default values set - not important for teardown but needed for initialisation
|
|
249
|
+
full_refresh=False,
|
|
250
|
+
schema_contract="freeze",
|
|
251
|
+
load_limit=0,
|
|
252
|
+
)
|
|
253
|
+
|
|
254
|
+
# Safety check for production
|
|
255
|
+
if env == "prod" and not allow_prod:
|
|
256
|
+
click.echo("❌ Production teardown blocked for safety.")
|
|
257
|
+
click.echo("Use --allow-prod flag if you're absolutely certain.")
|
|
258
|
+
return
|
|
259
|
+
|
|
260
|
+
# Show what will be deleted
|
|
261
|
+
click.echo("\n🗑️ 🗑️ 🗑️")
|
|
262
|
+
click.echo(f"Pipeline Teardown: {pipeline_name}")
|
|
263
|
+
click.echo(f"Environment: {pipeline.env}")
|
|
264
|
+
click.echo(f"Database: {pipeline.destination_database}")
|
|
265
|
+
click.echo(f"Table: {pipeline.destination_table}")
|
|
266
|
+
click.echo(f"S3 Path: {pipeline.destination_path}")
|
|
267
|
+
click.echo(f"dlt staging tables only? {staging_only}")
|
|
268
|
+
|
|
269
|
+
# Confirmation prompt unless --confirm is used
|
|
270
|
+
if not confirm:
|
|
271
|
+
if not click.confirm(
|
|
272
|
+
"\n⚠️ ⚠️ ⚠️\n This will permanently delete all data. Continue?"
|
|
273
|
+
):
|
|
274
|
+
click.echo("Teardown cancelled.")
|
|
275
|
+
return
|
|
276
|
+
|
|
277
|
+
# Execute teardown
|
|
278
|
+
try:
|
|
279
|
+
logger.info(f"Starting teardown for {pipeline_name} in {env} environment")
|
|
280
|
+
pipeline.drop_db(allow_prod=allow_prod, staging_only=staging_only)
|
|
281
|
+
click.echo(f"✅ Successfully torn down pipeline: {pipeline_name}")
|
|
282
|
+
except Exception as e:
|
|
283
|
+
click.echo(f"❌ Teardown failed: {str(e)}")
|
|
284
|
+
logger.error(f"Teardown failed for {pipeline_name}: {e}")
|
|
285
|
+
raise
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
if __name__ == "__main__":
|
|
289
|
+
if not load_dotenv():
|
|
290
|
+
logger.warning("No env file found. May be required if running locally.")
|
|
291
|
+
cli()
|
|
File without changes
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
import json
|
|
2
|
+
from functools import wraps
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Any, Callable, Iterator
|
|
5
|
+
|
|
6
|
+
import dlt
|
|
7
|
+
import pandas as pd
|
|
8
|
+
import pyarrow as pa
|
|
9
|
+
from cloudpathlib import S3Path
|
|
10
|
+
from loguru import logger
|
|
11
|
+
|
|
12
|
+
from moj_data_loader.readers.transforms import (
|
|
13
|
+
add_empty_column_types,
|
|
14
|
+
add_file_name_to_df,
|
|
15
|
+
add_magistrates_timestamp_col,
|
|
16
|
+
apply_transforms_to_generator,
|
|
17
|
+
get_file_encoding,
|
|
18
|
+
infer_excel_rowskip,
|
|
19
|
+
is_excel_header_empty,
|
|
20
|
+
)
|
|
21
|
+
from moj_data_loader.utils.exceptions import ConfigError
|
|
22
|
+
from moj_data_loader.utils.pgp import with_pgp_decryption
|
|
23
|
+
|
|
24
|
+
# Module-level registry - loads on import
|
|
25
|
+
_readers: dict[str, Callable] = {}
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def register(func: Callable) -> Callable:
|
|
29
|
+
"""Decorator that registers a processor function with its exact name."""
|
|
30
|
+
|
|
31
|
+
@wraps(func)
|
|
32
|
+
def wrapper(*args, **kwargs):
|
|
33
|
+
try:
|
|
34
|
+
yield from func(*args, **kwargs)
|
|
35
|
+
except Exception as e:
|
|
36
|
+
logger.error(f"Reader {func.__name__} failed: {e}")
|
|
37
|
+
raise
|
|
38
|
+
|
|
39
|
+
# Register using the exact function name
|
|
40
|
+
_readers[func.__name__] = wrapper
|
|
41
|
+
return wrapper
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def read(
|
|
45
|
+
reader: str,
|
|
46
|
+
files: list[S3Path],
|
|
47
|
+
transforms: list[dict[str, Any]] | None = None,
|
|
48
|
+
**kwargs,
|
|
49
|
+
):
|
|
50
|
+
"""Dispatch to a registered reader and optionally apply transforms.
|
|
51
|
+
|
|
52
|
+
Parameters
|
|
53
|
+
----------
|
|
54
|
+
reader:
|
|
55
|
+
Name of the registered reader function.
|
|
56
|
+
files:
|
|
57
|
+
Files to pass to the reader.
|
|
58
|
+
transforms:
|
|
59
|
+
Optional list of transform configs, each a dict with ``name``
|
|
60
|
+
(required) and ``kwargs`` (optional). They are applied in order
|
|
61
|
+
to every ``pa.Table`` yielded by the reader.
|
|
62
|
+
**kwargs:
|
|
63
|
+
Forwarded to the reader function.
|
|
64
|
+
"""
|
|
65
|
+
if reader not in _readers:
|
|
66
|
+
raise ConfigError(
|
|
67
|
+
f"No reader found for name: {reader}. \n Available: {list(_readers.keys())}"
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
reader_gen = _readers[reader](files, **kwargs)
|
|
71
|
+
|
|
72
|
+
if transforms:
|
|
73
|
+
reader_gen = apply_transforms_to_generator(reader_gen, transforms)
|
|
74
|
+
|
|
75
|
+
reader_function = dlt.resource(reader_gen, table_format="iceberg")
|
|
76
|
+
|
|
77
|
+
return reader_function
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def list_readers() -> list[str]:
|
|
81
|
+
"""List all registered reader names."""
|
|
82
|
+
return list(_readers.keys())
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
@register
|
|
86
|
+
@with_pgp_decryption
|
|
87
|
+
def read_csv(files: list[S3Path | Path], **kwargs) -> Iterator[pa.Table]:
|
|
88
|
+
"""Read CSV files (with optional PGP decryption via decorator)."""
|
|
89
|
+
|
|
90
|
+
# Extract file mapping if present (from PGP decorator)
|
|
91
|
+
file_mapping = kwargs.pop("_original_file_paths", None)
|
|
92
|
+
|
|
93
|
+
for file in files:
|
|
94
|
+
opts = dict(kwargs)
|
|
95
|
+
if "encoding" not in opts:
|
|
96
|
+
opts["encoding"] = get_file_encoding(file)
|
|
97
|
+
|
|
98
|
+
logger.info(f"Running load with options: {opts}")
|
|
99
|
+
logger.info(f"Reading (csv): {str(file)}")
|
|
100
|
+
|
|
101
|
+
with file.open("rb") as f:
|
|
102
|
+
df = pd.read_csv(f, **opts)
|
|
103
|
+
|
|
104
|
+
# Use original S3 path if available, otherwise use current path
|
|
105
|
+
original_path = (
|
|
106
|
+
file_mapping.get(str(file), str(file)) if file_mapping else str(file)
|
|
107
|
+
)
|
|
108
|
+
df = add_file_name_to_df(df, original_path)
|
|
109
|
+
|
|
110
|
+
# Ensure empty columns have valid types
|
|
111
|
+
df = add_empty_column_types(df)
|
|
112
|
+
|
|
113
|
+
yield pa.Table.from_pandas(df)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
@register
|
|
117
|
+
@with_pgp_decryption
|
|
118
|
+
def read_excel(files: list[S3Path | Path], **kwargs) -> Iterator[pa.Table]:
|
|
119
|
+
"""Read Excel files (with optional PGP decryption via decorator)."""
|
|
120
|
+
|
|
121
|
+
# Extract file mapping if present (from PGP decorator)
|
|
122
|
+
file_mapping = kwargs.pop("_original_file_paths", None)
|
|
123
|
+
|
|
124
|
+
for file in files:
|
|
125
|
+
opts = dict(kwargs)
|
|
126
|
+
if "dtype_backend" not in opts:
|
|
127
|
+
opts["dtype_backend"] = "pyarrow"
|
|
128
|
+
if "engine" not in opts:
|
|
129
|
+
opts["engine"] = "calamine"
|
|
130
|
+
if "skiprows" not in opts:
|
|
131
|
+
opts["skiprows"] = infer_excel_rowskip(file)
|
|
132
|
+
|
|
133
|
+
if is_excel_header_empty(file, opts["skiprows"]):
|
|
134
|
+
raise ValueError(
|
|
135
|
+
f"Empty header in Excel file: {file}, skiprows: {opts['skiprows']}"
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
logger.info(f"Running load with options: {opts}")
|
|
139
|
+
logger.info(f"Reading (excel): {str(file)}")
|
|
140
|
+
|
|
141
|
+
with file.open("rb") as f:
|
|
142
|
+
df = pd.read_excel(f, **opts)
|
|
143
|
+
|
|
144
|
+
# Use original S3 path if available, otherwise use current path
|
|
145
|
+
original_path = (
|
|
146
|
+
file_mapping.get(str(file), str(file)) if file_mapping else str(file)
|
|
147
|
+
)
|
|
148
|
+
df = add_file_name_to_df(df, original_path)
|
|
149
|
+
|
|
150
|
+
yield pa.Table.from_pandas(df)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
@register
|
|
154
|
+
def read_parquet(files: list[S3Path], **kwargs) -> Iterator[pa.Table]:
|
|
155
|
+
"""Read Parquet files"""
|
|
156
|
+
opts = dict(kwargs)
|
|
157
|
+
total = len(files)
|
|
158
|
+
for counter, file in enumerate(files, start=1):
|
|
159
|
+
logger.info(f"Reading (parquet) {counter}/{total}: {str(file)}")
|
|
160
|
+
logger.info(f"Running load with options: {opts}")
|
|
161
|
+
|
|
162
|
+
with file.open("rb") as f:
|
|
163
|
+
df = pd.read_parquet(f, **kwargs)
|
|
164
|
+
|
|
165
|
+
df = add_file_name_to_df(df, str(file))
|
|
166
|
+
yield pa.Table.from_pandas(df)
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
@register
|
|
170
|
+
def read_json(files: list[S3Path], **kwargs) -> Iterator[pa.Table]:
|
|
171
|
+
"""Reader for JSON and JSONL file types"""
|
|
172
|
+
|
|
173
|
+
for file in files:
|
|
174
|
+
logger.info(f"Reading (JSON): {str(file)}")
|
|
175
|
+
logger.info(f"Running load with options: {kwargs}")
|
|
176
|
+
with file.open("rb") as f:
|
|
177
|
+
df = pd.read_json(f, **kwargs)
|
|
178
|
+
df = add_file_name_to_df(df, str(file))
|
|
179
|
+
yield pa.Table.from_pandas(df)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
@register
|
|
183
|
+
def read_magistrates_json(files: list[S3Path], **kwargs) -> Iterator[pa.Table]:
|
|
184
|
+
"""Reader for JSON and JSONL file types"""
|
|
185
|
+
|
|
186
|
+
for file in files:
|
|
187
|
+
logger.info(f"Reading (JSON): {str(file)}")
|
|
188
|
+
logger.info(f"Running load with options: {kwargs}")
|
|
189
|
+
df = pd.read_json(file, **kwargs)
|
|
190
|
+
df = add_file_name_to_df(df, str(file))
|
|
191
|
+
df = add_magistrates_timestamp_col(df, str(file))
|
|
192
|
+
if "ofn" in df.columns:
|
|
193
|
+
df["ofn"] = df["ofn"].apply(lambda x: json.dumps(x))
|
|
194
|
+
yield pa.Table.from_pandas(df)
|