moj-data-loader 2.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- moj_data_loader/Pipeline.py +659 -0
- moj_data_loader/__init__.py +0 -0
- moj_data_loader/main.py +291 -0
- moj_data_loader/readers/__init__.py +0 -0
- moj_data_loader/readers/registry.py +194 -0
- moj_data_loader/readers/transforms.py +334 -0
- moj_data_loader/utils/__init__.py +0 -0
- moj_data_loader/utils/batch.py +221 -0
- moj_data_loader/utils/exceptions.py +6 -0
- moj_data_loader/utils/files.py +46 -0
- moj_data_loader/utils/pgp.py +223 -0
- moj_data_loader/utils/summary_log.py +81 -0
- moj_data_loader/utils/validators.py +15 -0
- moj_data_loader-2.2.0.dist-info/METADATA +124 -0
- moj_data_loader-2.2.0.dist-info/RECORD +17 -0
- moj_data_loader-2.2.0.dist-info/WHEEL +4 -0
- moj_data_loader-2.2.0.dist-info/entry_points.txt +3 -0
|
@@ -0,0 +1,659 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import re
|
|
5
|
+
import tempfile
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Annotated, Any, Iterator, Literal
|
|
8
|
+
|
|
9
|
+
import awswrangler as wr
|
|
10
|
+
import dlt
|
|
11
|
+
import yaml
|
|
12
|
+
from botocore.exceptions import ClientError
|
|
13
|
+
from cloudpathlib import S3Path
|
|
14
|
+
from loguru import logger
|
|
15
|
+
from pydantic import AfterValidator, Field
|
|
16
|
+
from pydantic_settings import BaseSettings, SettingsConfigDict
|
|
17
|
+
|
|
18
|
+
from moj_data_loader.readers.registry import read
|
|
19
|
+
from moj_data_loader.utils.exceptions import ConfigError
|
|
20
|
+
from .utils.batch import (
|
|
21
|
+
cleanup_batch_data_dir,
|
|
22
|
+
group_files_by_batch_memory_limit,
|
|
23
|
+
)
|
|
24
|
+
from moj_data_loader.utils.files import (
|
|
25
|
+
filter_files_at_path,
|
|
26
|
+
get_files_at_path,
|
|
27
|
+
)
|
|
28
|
+
from moj_data_loader.utils.summary_log import summary_log
|
|
29
|
+
from moj_data_loader.utils.validators import (
|
|
30
|
+
validate_s3_path_exists,
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
DEFAULT_BATCH_MEMORY_LIMIT_GB = 4
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class Pipeline(BaseSettings):
|
|
37
|
+
model_config = SettingsConfigDict(populate_by_name=True, extra="ignore")
|
|
38
|
+
|
|
39
|
+
# Runtime Attributes - set in env
|
|
40
|
+
env: Literal["dev", "test", "prod"]
|
|
41
|
+
full_refresh: bool
|
|
42
|
+
schema_contract: Literal["evolve", "freeze"]
|
|
43
|
+
load_limit: int | None
|
|
44
|
+
enable_batching: bool = False
|
|
45
|
+
batch_memory_limit: float | None = None
|
|
46
|
+
|
|
47
|
+
# Pipeline attributes - created from yaml values
|
|
48
|
+
pipeline_name: str
|
|
49
|
+
source_path: Annotated[S3Path, AfterValidator(validate_s3_path_exists)]
|
|
50
|
+
source_domain: str
|
|
51
|
+
file_types: list[str] = Field(default_factory=list)
|
|
52
|
+
destination_path: S3Path
|
|
53
|
+
destination_database: str
|
|
54
|
+
destination_table: str
|
|
55
|
+
reader_name: str
|
|
56
|
+
reader_kwargs: dict[str, Any] = Field(default_factory=dict)
|
|
57
|
+
reader_transforms: list[dict[str, Any]] = Field(default_factory=list)
|
|
58
|
+
|
|
59
|
+
# Post-init attributes
|
|
60
|
+
files_at_source: list[S3Path] | None = None
|
|
61
|
+
already_loaded_files: list[str] | None = None
|
|
62
|
+
files_to_load: list[S3Path] | None = None
|
|
63
|
+
objects_at_destination: list[S3Path] | None = None
|
|
64
|
+
query_dump_path: S3Path | None = None
|
|
65
|
+
table_path: S3Path | None = None
|
|
66
|
+
|
|
67
|
+
# dlt objects
|
|
68
|
+
dlt_destination_filesystem: dlt.destinations.filesystem | None = None
|
|
69
|
+
dlt_pipeline: dlt.Pipeline | None = None
|
|
70
|
+
dlt_resource: Iterator | None = None
|
|
71
|
+
|
|
72
|
+
def model_post_init(self, __context: Any) -> None:
|
|
73
|
+
# Update destination locations with env
|
|
74
|
+
self.destination_path = self._generate_destination_path()
|
|
75
|
+
self.destination_database = self._generate_destination_database()
|
|
76
|
+
self.query_dump_path = self.destination_path / "athena_query_dump"
|
|
77
|
+
self.table_path = (
|
|
78
|
+
self.destination_path / self.destination_database / self.destination_table
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
# Compute files to load
|
|
82
|
+
self.files_at_source = self._get_files_at_source()
|
|
83
|
+
self.already_loaded_files = self._get_already_loaded_files()
|
|
84
|
+
self.files_to_load = self._get_files_to_load()
|
|
85
|
+
summary_log.files_to_load = len(self.files_to_load)
|
|
86
|
+
summary_log.already_loaded_files = len(self.already_loaded_files)
|
|
87
|
+
summary_log.files_at_source = len(self.files_at_source)
|
|
88
|
+
summary_log.files_loaded = 0
|
|
89
|
+
|
|
90
|
+
# Check files at destination path
|
|
91
|
+
self.objects_at_destination = self._get_objects_at_destination_path()
|
|
92
|
+
|
|
93
|
+
logger.info("Setting DLT destination....")
|
|
94
|
+
self.set_dlt_destination_file_system()
|
|
95
|
+
self.set_dlt_pipeline()
|
|
96
|
+
|
|
97
|
+
def set_dlt_destination_file_system(self):
|
|
98
|
+
# dlt objects
|
|
99
|
+
self.dlt_destination_filesystem = dlt.destinations.filesystem(
|
|
100
|
+
bucket_url=str(self.destination_path)
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
def set_dlt_pipeline(self):
|
|
104
|
+
self.dlt_pipeline = dlt.pipeline(
|
|
105
|
+
pipeline_name=f"{self.destination_database}__{self.destination_table}",
|
|
106
|
+
destination=dlt.destinations.athena(
|
|
107
|
+
query_result_bucket=str(self.query_dump_path)
|
|
108
|
+
),
|
|
109
|
+
staging=self.dlt_destination_filesystem,
|
|
110
|
+
dataset_name=self.destination_database,
|
|
111
|
+
progress="log",
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
self.dlt_resource = read(
|
|
115
|
+
reader=self.reader_name,
|
|
116
|
+
files=self.files_to_load,
|
|
117
|
+
transforms=self.reader_transforms or None,
|
|
118
|
+
**self.reader_kwargs,
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
# Model Initialisation Methods ----
|
|
122
|
+
|
|
123
|
+
@classmethod
|
|
124
|
+
def load_from_yaml(
|
|
125
|
+
cls, pipeline: str, configs_dir: Path | str = Path("pipelines"), **env_kwargs
|
|
126
|
+
) -> Pipeline:
|
|
127
|
+
"""
|
|
128
|
+
YAML loader for Pipeline class
|
|
129
|
+
|
|
130
|
+
Args:
|
|
131
|
+
pipeline: A string, referring to the name of a yaml file in configs_dir
|
|
132
|
+
configs_dir: A directory of configs for pipeline runs.
|
|
133
|
+
The pipeline above must exist in this directory.
|
|
134
|
+
"""
|
|
135
|
+
|
|
136
|
+
# Coerce strings into Path
|
|
137
|
+
if isinstance(configs_dir, str):
|
|
138
|
+
configs_dir = Path(configs_dir)
|
|
139
|
+
|
|
140
|
+
# Validate if pipeline yaml exists in configs_dir
|
|
141
|
+
targets = list(configs_dir.rglob(f"{pipeline}.y*ml"))
|
|
142
|
+
|
|
143
|
+
# Validate only one target yaml
|
|
144
|
+
if len(targets) == 0:
|
|
145
|
+
raise ConfigError(
|
|
146
|
+
f"No corresponding yaml file found under {str(configs_dir)} "
|
|
147
|
+
f"for {pipeline}"
|
|
148
|
+
)
|
|
149
|
+
elif len(targets) > 1:
|
|
150
|
+
raise ConfigError(
|
|
151
|
+
f"Multiple pipelines yamls found for {pipeline}: \n"
|
|
152
|
+
f"{[{str(p)} for p in targets]} \n"
|
|
153
|
+
f"Provide the database prefix as well, e.g. \n"
|
|
154
|
+
f'"affile_dummy/test_csv"'
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
# Open config, reading for instantiating object
|
|
158
|
+
config_path = targets[0]
|
|
159
|
+
|
|
160
|
+
with config_path.open() as f:
|
|
161
|
+
pipeline_config = yaml.safe_load(f)
|
|
162
|
+
|
|
163
|
+
# Filter out .yml & generate destination metadata from path
|
|
164
|
+
path_elements = re.sub(r"\.ya?ml$", "", str(config_path)).split("/")
|
|
165
|
+
|
|
166
|
+
# Take domain, database, table from last 3 path elements
|
|
167
|
+
domain, database, table = path_elements[-3:]
|
|
168
|
+
|
|
169
|
+
class_config = {
|
|
170
|
+
"pipeline_name": pipeline,
|
|
171
|
+
"source_path": S3Path(pipeline_config["source"]["path"]),
|
|
172
|
+
"file_types": pipeline_config["source"].get("file_types", []),
|
|
173
|
+
"source_domain": domain,
|
|
174
|
+
"destination_path": S3Path(
|
|
175
|
+
pipeline_config.get("destination", {}).get(
|
|
176
|
+
"path",
|
|
177
|
+
"s3://mojap-processed/", # Default to mojap-processed
|
|
178
|
+
)
|
|
179
|
+
),
|
|
180
|
+
"destination_database": database,
|
|
181
|
+
"destination_table": table,
|
|
182
|
+
"reader_name": pipeline_config["reader"]["name"],
|
|
183
|
+
"reader_kwargs": pipeline_config["reader"].get("kwargs", {}),
|
|
184
|
+
"reader_transforms": pipeline_config["reader"].get("transforms", []),
|
|
185
|
+
"enable_batching": pipeline_config.get("enable_batching", False),
|
|
186
|
+
"batch_memory_limit": pipeline_config.get("batch_memory_limit"),
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
return cls(**class_config, **env_kwargs)
|
|
190
|
+
|
|
191
|
+
@staticmethod
|
|
192
|
+
def set_temp_dlt_directory_for_airflow(dlt_data_dir: str, dlt_project_dir: str):
|
|
193
|
+
"""Set the DLT data and project directories to a temporary
|
|
194
|
+
local directory so that DLT can run in an airflow KNS pod
|
|
195
|
+
"""
|
|
196
|
+
os.environ["DLT_DATA_DIR"] = dlt_data_dir
|
|
197
|
+
os.environ["DLT_PROJECT_DIR"] = dlt_project_dir
|
|
198
|
+
|
|
199
|
+
def _generate_destination_path(self) -> S3Path:
|
|
200
|
+
"""Create a S3Path pointing at the appropriate staging destination
|
|
201
|
+
|
|
202
|
+
Defaults to s3://mojap-processed/ as the root bucket. Splits paths by domain,
|
|
203
|
+
source system, environment, and destination table. Can be overriden by setting
|
|
204
|
+
a destination path in the yaml config
|
|
205
|
+
|
|
206
|
+
Returns:
|
|
207
|
+
An S3Path object pointing to the file load location
|
|
208
|
+
"""
|
|
209
|
+
|
|
210
|
+
path = (
|
|
211
|
+
self.destination_path / f"domain__{self.source_domain}" / f"env__{self.env}/"
|
|
212
|
+
)
|
|
213
|
+
|
|
214
|
+
logger.info(f"Target S3 staging path: {path}")
|
|
215
|
+
|
|
216
|
+
return S3Path(path)
|
|
217
|
+
|
|
218
|
+
def _generate_destination_database(self) -> str:
|
|
219
|
+
"""Generate a database name based on the destination database and environment.
|
|
220
|
+
|
|
221
|
+
Returns:
|
|
222
|
+
str: The generated database name
|
|
223
|
+
"""
|
|
224
|
+
return (
|
|
225
|
+
f"{self.destination_database}_{self.env}"
|
|
226
|
+
if self.env != "prod"
|
|
227
|
+
and not self.destination_database.endswith(f"_{self.env}")
|
|
228
|
+
else self.destination_database
|
|
229
|
+
)
|
|
230
|
+
|
|
231
|
+
def _get_files_at_source(self) -> list[S3Path]:
|
|
232
|
+
logger.info("---- Checking files at source ---- ")
|
|
233
|
+
|
|
234
|
+
all_files = get_files_at_path(self.source_path)
|
|
235
|
+
|
|
236
|
+
if not all_files:
|
|
237
|
+
raise ConfigError(f"No files detected at {self.source_path}")
|
|
238
|
+
|
|
239
|
+
# Error if multiple file types found without file globs
|
|
240
|
+
file_types = set([file.suffix.lower() for file in all_files])
|
|
241
|
+
|
|
242
|
+
if len(file_types) > 1 and not self.file_types:
|
|
243
|
+
raise ConfigError(
|
|
244
|
+
f"Multiple file types found at {self.source_path}: {file_types}.\n"
|
|
245
|
+
f"Please provide 'file_types' in config to filter appropriately."
|
|
246
|
+
)
|
|
247
|
+
|
|
248
|
+
# Filter files by file globs if provided
|
|
249
|
+
if not self.file_types:
|
|
250
|
+
logger.info("No file types supplied, using all files")
|
|
251
|
+
selected_files = all_files
|
|
252
|
+
else:
|
|
253
|
+
selected_files = filter_files_at_path(
|
|
254
|
+
files=all_files, glob_patterns=self.file_types
|
|
255
|
+
)
|
|
256
|
+
|
|
257
|
+
logger.info(f"{len(selected_files)} files found at {self.source_path}")
|
|
258
|
+
|
|
259
|
+
return selected_files
|
|
260
|
+
|
|
261
|
+
def _get_already_loaded_files(self) -> list[str]:
|
|
262
|
+
"""Query Athena to get a list of files that have already been loaded.
|
|
263
|
+
|
|
264
|
+
Queries an Athena table to get the distinct meta__source_file column values,
|
|
265
|
+
which represent S3 URIs of files that have already been processed. It handles
|
|
266
|
+
cases where the database or table don't exist yet by returning an empty list.
|
|
267
|
+
|
|
268
|
+
Returns:
|
|
269
|
+
list[str]: List of S3 URIs for files that have already been processed.
|
|
270
|
+
Returns empty list if database/table doesn't exist.
|
|
271
|
+
|
|
272
|
+
Raises:
|
|
273
|
+
QueryFailed: If the Athena query fails for reasons other than missing
|
|
274
|
+
database/table
|
|
275
|
+
Exception: For any other unexpected errors during query execution
|
|
276
|
+
"""
|
|
277
|
+
|
|
278
|
+
tbl = f"{self.destination_database}.{self.destination_table}"
|
|
279
|
+
|
|
280
|
+
sql = f"SELECT DISTINCT meta__source_file FROM {tbl}"
|
|
281
|
+
|
|
282
|
+
logger.info(f"Getting loaded files with sql: \n {sql} \n")
|
|
283
|
+
|
|
284
|
+
try:
|
|
285
|
+
df = wr.athena.read_sql_query(
|
|
286
|
+
sql=sql,
|
|
287
|
+
database=self.destination_database,
|
|
288
|
+
ctas_approach=False,
|
|
289
|
+
)
|
|
290
|
+
|
|
291
|
+
return df["meta__source_file"].tolist()
|
|
292
|
+
|
|
293
|
+
# Continue through Database/Table not found errors - Create New
|
|
294
|
+
except wr.exceptions.QueryFailed as error:
|
|
295
|
+
error_message = str(error).lower()
|
|
296
|
+
keywords = ["schema_not_found", "table_not_found"]
|
|
297
|
+
|
|
298
|
+
if any(keyword in error_message for keyword in keywords):
|
|
299
|
+
logger.info(
|
|
300
|
+
f"Database or Table not found for {tbl}."
|
|
301
|
+
f"Will be created by Pipeline.run()"
|
|
302
|
+
)
|
|
303
|
+
return []
|
|
304
|
+
|
|
305
|
+
# Raise all other errors
|
|
306
|
+
else:
|
|
307
|
+
logger.error(f"Query failed for {tbl}: {repr(error)}")
|
|
308
|
+
raise
|
|
309
|
+
except Exception as error:
|
|
310
|
+
logger.error(
|
|
311
|
+
f"Unexpected error getting processed files for {tbl}: {repr(error)}"
|
|
312
|
+
)
|
|
313
|
+
raise
|
|
314
|
+
|
|
315
|
+
def _get_objects_at_destination_path(self) -> list[S3Path]:
|
|
316
|
+
"""Lists files at the target S3 Path
|
|
317
|
+
|
|
318
|
+
This has some utilities:
|
|
319
|
+
- Check files at a destination before teardown
|
|
320
|
+
- Check if files already exist at a destination we are newly loading to
|
|
321
|
+
|
|
322
|
+
Returns:
|
|
323
|
+
A list of S3Path objects to the given files
|
|
324
|
+
"""
|
|
325
|
+
logger.info("---- Checking files at destination and staging paths ----")
|
|
326
|
+
|
|
327
|
+
# Expected load destination - Staging
|
|
328
|
+
staging_destination = S3Path(
|
|
329
|
+
self.destination_path
|
|
330
|
+
/ f"{str(self.destination_database)}_staging"
|
|
331
|
+
/ str(self.destination_table)
|
|
332
|
+
)
|
|
333
|
+
|
|
334
|
+
logger.info(f"Staging destination: {str(staging_destination)}")
|
|
335
|
+
|
|
336
|
+
# Expected load destination - Iceberg
|
|
337
|
+
iceberg_destination = S3Path(
|
|
338
|
+
self.destination_path
|
|
339
|
+
/ str(self.destination_database)
|
|
340
|
+
/ str(self.destination_table)
|
|
341
|
+
)
|
|
342
|
+
|
|
343
|
+
logger.info(f"Iceberg destination: {str(iceberg_destination)}")
|
|
344
|
+
|
|
345
|
+
staging_items = get_files_at_path(staging_destination)
|
|
346
|
+
iceberg_items = get_files_at_path(iceberg_destination)
|
|
347
|
+
|
|
348
|
+
all_destination_items = staging_items + iceberg_items
|
|
349
|
+
|
|
350
|
+
return all_destination_items
|
|
351
|
+
|
|
352
|
+
def _get_files_to_load(self) -> list[S3Path]:
|
|
353
|
+
if self.files_at_source is None:
|
|
354
|
+
raise ConfigError(f"No files at source path: {self.source_path}")
|
|
355
|
+
|
|
356
|
+
# Load all files if full refresh true
|
|
357
|
+
if self.full_refresh:
|
|
358
|
+
logger.info("Loading all sources files as full_refesh is True")
|
|
359
|
+
|
|
360
|
+
files_to_load = self.files_at_source
|
|
361
|
+
|
|
362
|
+
else:
|
|
363
|
+
already_loaded_files = set(self.already_loaded_files or [])
|
|
364
|
+
|
|
365
|
+
files_to_load = [
|
|
366
|
+
file
|
|
367
|
+
for file in self.files_at_source
|
|
368
|
+
if str(file) not in already_loaded_files
|
|
369
|
+
]
|
|
370
|
+
|
|
371
|
+
logger.info(f"Detected {len(files_to_load)} files that are not loaded")
|
|
372
|
+
|
|
373
|
+
# Apply limit if set
|
|
374
|
+
if self.load_limit and self.load_limit > 0:
|
|
375
|
+
files_to_load = files_to_load[: self.load_limit]
|
|
376
|
+
|
|
377
|
+
return files_to_load
|
|
378
|
+
|
|
379
|
+
# Model Validation Methods ----
|
|
380
|
+
|
|
381
|
+
def _validate_no_orphaned_files(self) -> None:
|
|
382
|
+
"""
|
|
383
|
+
Validate that if already_loaded_files is None (no database/table exists)
|
|
384
|
+
and files_at_destination is not None, raise an error to prevent orphaned files.
|
|
385
|
+
|
|
386
|
+
This validation prevents a scenario where files exist at the destination
|
|
387
|
+
but are not tracked in the database, which would lead to orphaned files
|
|
388
|
+
being deleted during teardown.
|
|
389
|
+
"""
|
|
390
|
+
|
|
391
|
+
if self.objects_at_destination and not self.already_loaded_files:
|
|
392
|
+
raise ConfigError(
|
|
393
|
+
"Orphaned files detected at destination: "
|
|
394
|
+
f"{len(self.objects_at_destination)} objects found at {self.table_path} "
|
|
395
|
+
"but no database/table exists to track them. "
|
|
396
|
+
"These files would be deleted during teardown. "
|
|
397
|
+
"Please check and clean up the destination before proceeding."
|
|
398
|
+
)
|
|
399
|
+
|
|
400
|
+
# Pipeline Actions ----
|
|
401
|
+
def run(self) -> None:
|
|
402
|
+
"""Run the dlt pipeline"""
|
|
403
|
+
|
|
404
|
+
logger.info("Starting DLT pipeline run...")
|
|
405
|
+
summary_log.files_loaded = 0
|
|
406
|
+
|
|
407
|
+
if self.full_refresh and self.enable_batching:
|
|
408
|
+
logger.warning(
|
|
409
|
+
"Stopping pipeline run: full_refresh cannot be combined with "
|
|
410
|
+
"batch_memory_limit. Disable batching or full_refresh and retry."
|
|
411
|
+
)
|
|
412
|
+
return
|
|
413
|
+
|
|
414
|
+
if self.dlt_pipeline is None:
|
|
415
|
+
raise ConfigError("dlt_pipeline attribute in Pipeline is None")
|
|
416
|
+
|
|
417
|
+
if self.files_to_load in (None, []):
|
|
418
|
+
logger.info("No files to load. Pipeline will not run")
|
|
419
|
+
return
|
|
420
|
+
|
|
421
|
+
self._validate_no_orphaned_files()
|
|
422
|
+
|
|
423
|
+
runtime_config = {
|
|
424
|
+
"write_disposition": "replace" if self.full_refresh else "append",
|
|
425
|
+
"refresh": "drop_sources" if self.full_refresh else None,
|
|
426
|
+
"schema_contract": {
|
|
427
|
+
"tables": "evolve",
|
|
428
|
+
"columns": "evolve" if self.full_refresh else self.schema_contract,
|
|
429
|
+
},
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
if self.enable_batching:
|
|
433
|
+
files_loaded = self.run_batched_pipeline(runtime_config=runtime_config)
|
|
434
|
+
else:
|
|
435
|
+
files_loaded = self.run_unbatched_pipeline(runtime_config=runtime_config)
|
|
436
|
+
|
|
437
|
+
summary_log.files_loaded = files_loaded
|
|
438
|
+
|
|
439
|
+
run_count = self._get_run_count_from_loads()
|
|
440
|
+
logger.info(f"Pipeline run completed. Maintenance run count: {run_count}")
|
|
441
|
+
|
|
442
|
+
# Fire-and-forget optimize and vacuum after each 10 successful prod runs.
|
|
443
|
+
if self.env == "prod" and self._is_maintenance_run(run_count):
|
|
444
|
+
logger.info("Triggering optimize and vacuum")
|
|
445
|
+
self._optimize()
|
|
446
|
+
self._vacuum()
|
|
447
|
+
|
|
448
|
+
def run_batched_pipeline(
|
|
449
|
+
self, runtime_config: dict[str, Any] | None = None
|
|
450
|
+
) -> int:
|
|
451
|
+
files_loaded = 0
|
|
452
|
+
|
|
453
|
+
if self.batch_memory_limit is None or self.batch_memory_limit <= 0:
|
|
454
|
+
self.batch_memory_limit = DEFAULT_BATCH_MEMORY_LIMIT_GB
|
|
455
|
+
|
|
456
|
+
batch_limit_bytes = int(self.batch_memory_limit * (1024**3))
|
|
457
|
+
|
|
458
|
+
total_files = len(self.files_to_load)
|
|
459
|
+
|
|
460
|
+
file_batches = group_files_by_batch_memory_limit(
|
|
461
|
+
files=self.files_to_load,
|
|
462
|
+
limit_bytes=batch_limit_bytes,
|
|
463
|
+
)
|
|
464
|
+
|
|
465
|
+
for batch_index, batch_files in enumerate(file_batches, start=1):
|
|
466
|
+
logger.info(
|
|
467
|
+
"Running batch "
|
|
468
|
+
f"{batch_index}/{len(file_batches)} with {len(batch_files)} files"
|
|
469
|
+
)
|
|
470
|
+
try:
|
|
471
|
+
batch_temp_dir = tempfile.gettempdir()
|
|
472
|
+
logger.info(f"Using batch temp directory: {batch_temp_dir}")
|
|
473
|
+
dlt_resource = read(
|
|
474
|
+
reader=self.reader_name,
|
|
475
|
+
files=batch_files,
|
|
476
|
+
transforms=self.reader_transforms or None,
|
|
477
|
+
**self.reader_kwargs,
|
|
478
|
+
)
|
|
479
|
+
self.dlt_pipeline.run(
|
|
480
|
+
data=dlt_resource,
|
|
481
|
+
table_name=self.destination_table,
|
|
482
|
+
**runtime_config,
|
|
483
|
+
)
|
|
484
|
+
for _ in batch_files:
|
|
485
|
+
files_loaded += 1
|
|
486
|
+
logger.info(f"Loaded {files_loaded}/{total_files} files")
|
|
487
|
+
finally:
|
|
488
|
+
cleanup_batch_data_dir(
|
|
489
|
+
destination_database=self.destination_database,
|
|
490
|
+
destination_table=self.destination_table,
|
|
491
|
+
batch_temp_dir=batch_temp_dir,
|
|
492
|
+
)
|
|
493
|
+
|
|
494
|
+
return files_loaded
|
|
495
|
+
|
|
496
|
+
|
|
497
|
+
def run_unbatched_pipeline(
|
|
498
|
+
self, runtime_config: dict[str, Any] | None = None
|
|
499
|
+
) -> int:
|
|
500
|
+
self.dlt_pipeline.run(
|
|
501
|
+
data=self.dlt_resource,
|
|
502
|
+
table_name=self.destination_table,
|
|
503
|
+
**runtime_config,
|
|
504
|
+
)
|
|
505
|
+
|
|
506
|
+
total_files = len(self.files_to_load)
|
|
507
|
+
files_loaded = total_files
|
|
508
|
+
logger.info(f"Loaded {files_loaded}/{total_files} files")
|
|
509
|
+
return files_loaded
|
|
510
|
+
|
|
511
|
+
def _get_run_count_from_loads(self) -> int:
|
|
512
|
+
"""Count successful loads for this pipeline schema from the dlt loads table."""
|
|
513
|
+
schema_name = f"{self.destination_database}__{self.destination_table}"
|
|
514
|
+
|
|
515
|
+
sql = (
|
|
516
|
+
"SELECT COUNT(DISTINCT load_id) AS run_count "
|
|
517
|
+
f"FROM {self.destination_database}._dlt_loads "
|
|
518
|
+
f"WHERE schema_name = '{schema_name}'"
|
|
519
|
+
)
|
|
520
|
+
|
|
521
|
+
try:
|
|
522
|
+
df = wr.athena.read_sql_query(
|
|
523
|
+
sql=sql,
|
|
524
|
+
database=self.destination_database,
|
|
525
|
+
ctas_approach=False,
|
|
526
|
+
)
|
|
527
|
+
except wr.exceptions.QueryFailed as error:
|
|
528
|
+
error_message = str(error).lower()
|
|
529
|
+
keywords = ["schema_not_found", "table_not_found"]
|
|
530
|
+
|
|
531
|
+
if any(keyword in error_message for keyword in keywords):
|
|
532
|
+
logger.info(
|
|
533
|
+
"_dlt_loads table not found for "
|
|
534
|
+
f"{self.destination_database}. Returning run_count=0"
|
|
535
|
+
)
|
|
536
|
+
return 0
|
|
537
|
+
logger.error(
|
|
538
|
+
"Query failed getting run count from _dlt_loads for "
|
|
539
|
+
f"{self.destination_database}: {repr(error)}"
|
|
540
|
+
)
|
|
541
|
+
raise
|
|
542
|
+
except Exception as error:
|
|
543
|
+
logger.error(
|
|
544
|
+
"Unexpected error getting run count from _dlt_loads for "
|
|
545
|
+
f"{self.destination_database}: {repr(error)}"
|
|
546
|
+
)
|
|
547
|
+
raise
|
|
548
|
+
|
|
549
|
+
if df.empty:
|
|
550
|
+
return 0
|
|
551
|
+
|
|
552
|
+
return int(df["run_count"].iloc[0] or 0)
|
|
553
|
+
|
|
554
|
+
@staticmethod
|
|
555
|
+
def _is_maintenance_run(run_count: int) -> bool:
|
|
556
|
+
"""Check if this run should trigger optimize/vacuum."""
|
|
557
|
+
|
|
558
|
+
return run_count % 10 == 0
|
|
559
|
+
|
|
560
|
+
def drop_db(
|
|
561
|
+
self,
|
|
562
|
+
allow_prod: bool = False,
|
|
563
|
+
staging_only: bool = True,
|
|
564
|
+
) -> None:
|
|
565
|
+
"""Delete Glue Database, Tables and S3 locations for a pipeline"""
|
|
566
|
+
|
|
567
|
+
if self.env == "prod" and not allow_prod:
|
|
568
|
+
raise ValueError(
|
|
569
|
+
"WARNING. You are trying to delete production data."
|
|
570
|
+
"Pass 'allow_prod=True' as an argument if you are sure."
|
|
571
|
+
)
|
|
572
|
+
|
|
573
|
+
# Define all possible targets
|
|
574
|
+
all_databases = [
|
|
575
|
+
self.destination_database,
|
|
576
|
+
f"{self.destination_database}_staging",
|
|
577
|
+
]
|
|
578
|
+
|
|
579
|
+
all_s3_paths = [
|
|
580
|
+
str(self.destination_path / self.destination_database),
|
|
581
|
+
str(self.destination_path / f"{self.destination_database}_staging"),
|
|
582
|
+
str(self.destination_path / "athena_query_dump"),
|
|
583
|
+
]
|
|
584
|
+
|
|
585
|
+
# Filter based on staging_only parameter
|
|
586
|
+
if staging_only:
|
|
587
|
+
target_databases = [db for db in all_databases if "_staging" in db]
|
|
588
|
+
|
|
589
|
+
target_s3_paths = [
|
|
590
|
+
path
|
|
591
|
+
for path in all_s3_paths
|
|
592
|
+
if "_staging" in path or "athena_query_dump" in path
|
|
593
|
+
]
|
|
594
|
+
else:
|
|
595
|
+
target_databases = all_databases
|
|
596
|
+
target_s3_paths = all_s3_paths
|
|
597
|
+
|
|
598
|
+
# Delete databases
|
|
599
|
+
for database in target_databases:
|
|
600
|
+
try:
|
|
601
|
+
wr.catalog.delete_database(database)
|
|
602
|
+
logger.info(f"Deleted database: {database}")
|
|
603
|
+
except ClientError as e:
|
|
604
|
+
if e.response["Error"]["Code"] == "EntityNotFoundException":
|
|
605
|
+
logger.info(f"Database {database} not found - skipping deletion")
|
|
606
|
+
else:
|
|
607
|
+
logger.error(f"Failed to delete database {database}: {e}")
|
|
608
|
+
raise
|
|
609
|
+
|
|
610
|
+
# Delete S3 paths
|
|
611
|
+
for s3_path in target_s3_paths:
|
|
612
|
+
if wr.s3.list_objects(s3_path):
|
|
613
|
+
logger.info(f"Deleting objects at: {s3_path}")
|
|
614
|
+
wr.s3.delete_objects(s3_path)
|
|
615
|
+
else:
|
|
616
|
+
logger.info(f"No files found at {s3_path}")
|
|
617
|
+
|
|
618
|
+
def _vacuum(self) -> None:
|
|
619
|
+
"""VACUUM a Iceberg table"""
|
|
620
|
+
sql = f"VACUUM {self.destination_table}"
|
|
621
|
+
|
|
622
|
+
wr.athena.start_query_execution(
|
|
623
|
+
sql, database=self.destination_database, s3_output=str(self.query_dump_path)
|
|
624
|
+
)
|
|
625
|
+
|
|
626
|
+
def _optimize(self) -> None:
|
|
627
|
+
"""OPTIMIZE a Iceberg table"""
|
|
628
|
+
sql = f"OPTIMIZE {self.destination_table} REWRITE DATA USING BIN_PACK"
|
|
629
|
+
|
|
630
|
+
wr.athena.start_query_execution(
|
|
631
|
+
sql,
|
|
632
|
+
database=self.destination_database,
|
|
633
|
+
s3_output=str(self.query_dump_path),
|
|
634
|
+
wait=True,
|
|
635
|
+
)
|
|
636
|
+
|
|
637
|
+
def __str__(self):
|
|
638
|
+
return f"""Pipeline initialised with:
|
|
639
|
+
pipeline_name: {self.pipeline_name}
|
|
640
|
+
env: {self.env}
|
|
641
|
+
full_refresh: {self.full_refresh}
|
|
642
|
+
schema_contract: {self.schema_contract}
|
|
643
|
+
load_limit: {self.load_limit}
|
|
644
|
+
enable_batching: {self.enable_batching}
|
|
645
|
+
batch_memory_limit: {self.batch_memory_limit}
|
|
646
|
+
source_path: {self.source_path}
|
|
647
|
+
source_domain: {self.source_domain}
|
|
648
|
+
destination_database: {self.destination_database}
|
|
649
|
+
destination_table: {self.destination_table}
|
|
650
|
+
destination_path: f'{self.destination_path}/{self.destination_database}/{
|
|
651
|
+
self.destination_table
|
|
652
|
+
}'
|
|
653
|
+
reader_name: {self.reader_name}
|
|
654
|
+
reader_kwargs: {self.reader_kwargs}
|
|
655
|
+
files_at_source: {len(self.files_at_source) if self.files_at_source else 0} files
|
|
656
|
+
already_loaded_files: {
|
|
657
|
+
len(self.already_loaded_files) if self.already_loaded_files else 0
|
|
658
|
+
} files
|
|
659
|
+
files_to_load: {len(self.files_to_load) if self.files_to_load else 0} files"""
|
|
File without changes
|