moj-data-loader 2.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- moj_data_loader/Pipeline.py +659 -0
- moj_data_loader/__init__.py +0 -0
- moj_data_loader/main.py +291 -0
- moj_data_loader/readers/__init__.py +0 -0
- moj_data_loader/readers/registry.py +194 -0
- moj_data_loader/readers/transforms.py +334 -0
- moj_data_loader/utils/__init__.py +0 -0
- moj_data_loader/utils/batch.py +221 -0
- moj_data_loader/utils/exceptions.py +6 -0
- moj_data_loader/utils/files.py +46 -0
- moj_data_loader/utils/pgp.py +223 -0
- moj_data_loader/utils/summary_log.py +81 -0
- moj_data_loader/utils/validators.py +15 -0
- moj_data_loader-2.2.0.dist-info/METADATA +124 -0
- moj_data_loader-2.2.0.dist-info/RECORD +17 -0
- moj_data_loader-2.2.0.dist-info/WHEEL +4 -0
- moj_data_loader-2.2.0.dist-info/entry_points.txt +3 -0
|
@@ -0,0 +1,334 @@
|
|
|
1
|
+
import re
|
|
2
|
+
from datetime import datetime, timezone
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Any, Callable, Iterator
|
|
5
|
+
|
|
6
|
+
import charset_normalizer as charset
|
|
7
|
+
import pandas as pd
|
|
8
|
+
import pyarrow as pa
|
|
9
|
+
from cloudpathlib import S3Path
|
|
10
|
+
from loguru import logger
|
|
11
|
+
|
|
12
|
+
from ..utils.exceptions import ConfigError
|
|
13
|
+
|
|
14
|
+
# ---------------------------------------------------------------------------
|
|
15
|
+
# Transform registry
|
|
16
|
+
# ---------------------------------------------------------------------------
|
|
17
|
+
_transforms: dict[str, Callable[..., pa.Table]] = {}
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def register_transform(func: Callable[..., pa.Table]) -> Callable[..., pa.Table]:
|
|
21
|
+
"""Decorator that registers a transform function by its name.
|
|
22
|
+
|
|
23
|
+
Registered transforms accept a ``pa.Table`` as the first argument and
|
|
24
|
+
return a ``pa.Table``.
|
|
25
|
+
"""
|
|
26
|
+
_transforms[func.__name__] = func
|
|
27
|
+
return func
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def get_transform(name: str) -> Callable[..., pa.Table]:
|
|
31
|
+
"""Look up a registered transform by *name*."""
|
|
32
|
+
if name not in _transforms:
|
|
33
|
+
raise ConfigError(
|
|
34
|
+
f"No transform found for name: {name}. Available: {list(_transforms.keys())}"
|
|
35
|
+
)
|
|
36
|
+
return _transforms[name]
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def list_transforms() -> list[str]:
|
|
40
|
+
"""Return names of all registered transforms."""
|
|
41
|
+
return list(_transforms.keys())
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def apply_transforms(
|
|
45
|
+
table: pa.Table,
|
|
46
|
+
transforms: list[dict[str, Any]],
|
|
47
|
+
) -> pa.Table:
|
|
48
|
+
"""Apply a sequence of transforms to a PyArrow Table.
|
|
49
|
+
|
|
50
|
+
Each item in *transforms* is a dict with ``name`` (required) and optional
|
|
51
|
+
``kwargs`` that are forwarded to the transform function.
|
|
52
|
+
|
|
53
|
+
Example config (YAML)::
|
|
54
|
+
|
|
55
|
+
transforms:
|
|
56
|
+
- name: add_load_timestamp
|
|
57
|
+
kwargs:
|
|
58
|
+
column_name: ingestion_ts
|
|
59
|
+
- name: drop_columns
|
|
60
|
+
kwargs:
|
|
61
|
+
columns:
|
|
62
|
+
- tmp_col
|
|
63
|
+
"""
|
|
64
|
+
for transform_config in transforms:
|
|
65
|
+
name = transform_config["name"]
|
|
66
|
+
kwargs = transform_config.get("kwargs", {})
|
|
67
|
+
transform_fn = get_transform(name)
|
|
68
|
+
logger.info(f"Applying transform: {name} with kwargs: {kwargs}")
|
|
69
|
+
table = transform_fn(table, **kwargs)
|
|
70
|
+
return table
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def apply_transforms_to_generator(
|
|
74
|
+
reader_gen: Iterator[pa.Table],
|
|
75
|
+
transforms: list[dict[str, Any]],
|
|
76
|
+
) -> Iterator[pa.Table]:
|
|
77
|
+
"""Wrap an existing reader generator so that *transforms* are applied to
|
|
78
|
+
every yielded ``pa.Table``."""
|
|
79
|
+
for table in reader_gen:
|
|
80
|
+
yield apply_transforms(table, transforms)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
# ---------------------------------------------------------------------------
|
|
84
|
+
# Registered transforms (operate on pa.Table)
|
|
85
|
+
# ---------------------------------------------------------------------------
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
@register_transform
|
|
89
|
+
def add_load_timestamp(
|
|
90
|
+
table: pa.Table,
|
|
91
|
+
column_name: str = "load_timestamp",
|
|
92
|
+
) -> pa.Table:
|
|
93
|
+
"""Append a UTC timestamp column to the table.
|
|
94
|
+
|
|
95
|
+
The timestamp is stored as a PyArrow ``timestamp[us, tz=UTC]`` for
|
|
96
|
+
type-safe downstream processing.
|
|
97
|
+
"""
|
|
98
|
+
timestamp = datetime.now(timezone.utc)
|
|
99
|
+
ts_array = pa.array(
|
|
100
|
+
[timestamp] * table.num_rows,
|
|
101
|
+
type=pa.timestamp("us", tz="UTC"),
|
|
102
|
+
)
|
|
103
|
+
return table.append_column(column_name, ts_array)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
@register_transform
|
|
107
|
+
def drop_columns(table: pa.Table, columns: list[str]) -> pa.Table:
|
|
108
|
+
"""Drop the named *columns* from the table."""
|
|
109
|
+
return table.drop(columns)
|
|
110
|
+
|
|
111
|
+
@register_transform
|
|
112
|
+
def add_column_number_separator(
|
|
113
|
+
table: pa.Table,
|
|
114
|
+
separator: str = "_",
|
|
115
|
+
) -> pa.Table:
|
|
116
|
+
"""Add a separator to all column names in the table."""
|
|
117
|
+
columns = table.column_names
|
|
118
|
+
new_columns = []
|
|
119
|
+
for col in columns:
|
|
120
|
+
chars = col
|
|
121
|
+
new_chars = []
|
|
122
|
+
for index, char in enumerate(chars):
|
|
123
|
+
|
|
124
|
+
is_prev_digit = index != 0 and col[index - 1].isdigit()
|
|
125
|
+
is_prev_sep = index != 0 and col[index - 1] == separator
|
|
126
|
+
is_char_digit = char.isdigit()
|
|
127
|
+
is_char_sep = char == separator
|
|
128
|
+
|
|
129
|
+
if is_char_digit and not is_prev_sep and not is_prev_digit and index != 0:
|
|
130
|
+
new_chars.append(separator)
|
|
131
|
+
elif (not is_char_digit and not is_char_sep) and is_prev_digit:
|
|
132
|
+
new_chars.append(separator)
|
|
133
|
+
|
|
134
|
+
new_chars.append(char)
|
|
135
|
+
|
|
136
|
+
new_columns.append("".join(new_chars))
|
|
137
|
+
|
|
138
|
+
return table.rename_columns(new_columns)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
@register_transform
|
|
142
|
+
def rename_columns(table: pa.Table, columns: dict[str, str]) -> pa.Table:
|
|
143
|
+
"""Rename columns using a mapping of old names to new names."""
|
|
144
|
+
renamed_columns = []
|
|
145
|
+
for name in table.column_names:
|
|
146
|
+
renamed_columns.append(columns.get(name, name))
|
|
147
|
+
|
|
148
|
+
if len(renamed_columns) != len(set(renamed_columns)):
|
|
149
|
+
raise ConfigError("Column renaming would create duplicate column names")
|
|
150
|
+
|
|
151
|
+
return table.rename_columns(renamed_columns)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
@register_transform
|
|
155
|
+
def enforce_column_type(
|
|
156
|
+
table: pa.Table, columns: dict[str, str], add_missing: bool = False
|
|
157
|
+
) -> pa.Table:
|
|
158
|
+
"""Cast columns to the specified PyArrow types.
|
|
159
|
+
|
|
160
|
+
This replaces the previous DataFrame-level ``enforce_column_type`` helper.
|
|
161
|
+
It performs the same operation — iterating over the supplied column-to-type
|
|
162
|
+
mapping and casting each column — but operates directly on a PyArrow Table
|
|
163
|
+
using ``pa.Array.cast()`` rather than ``pandas.Series.astype()``.
|
|
164
|
+
|
|
165
|
+
*columns* is a mapping of ``{column_name: pyarrow_type_string}``
|
|
166
|
+
(e.g. ``{"age": "float64"}``).
|
|
167
|
+
|
|
168
|
+
Supported type strings: float64, float32, int64, int32, string,
|
|
169
|
+
large_string, bool, date32.
|
|
170
|
+
"""
|
|
171
|
+
_type_map: dict[str, pa.DataType] = {
|
|
172
|
+
"float64": pa.float64(),
|
|
173
|
+
"float32": pa.float32(),
|
|
174
|
+
"int64": pa.int64(),
|
|
175
|
+
"int32": pa.int32(),
|
|
176
|
+
"string": pa.string(),
|
|
177
|
+
"large_string": pa.large_string(),
|
|
178
|
+
"bool": pa.bool_(),
|
|
179
|
+
"date32": pa.date32(),
|
|
180
|
+
"date64": pa.date64(),
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
if add_missing:
|
|
184
|
+
missing_cols = {
|
|
185
|
+
col: dtype for col, dtype in columns.items() if col not in table.column_names
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
for col, dtype in missing_cols.items():
|
|
189
|
+
target_type = _type_map.get(dtype)
|
|
190
|
+
if target_type is None:
|
|
191
|
+
raise ConfigError(
|
|
192
|
+
f"Unknown type '{dtype}' for column '{col}'. "
|
|
193
|
+
f"Available types: {list(_type_map.keys())}"
|
|
194
|
+
)
|
|
195
|
+
logger.warning(f"Column missing: {col}, adding to table")
|
|
196
|
+
table = table.append_column(
|
|
197
|
+
col,
|
|
198
|
+
pa.nulls(table.num_rows, type=target_type),
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
for col_name, target_type_str in columns.items():
|
|
202
|
+
target_type = _type_map.get(target_type_str)
|
|
203
|
+
if target_type is None:
|
|
204
|
+
raise ConfigError(
|
|
205
|
+
f"Unknown type '{target_type_str}' for column '{col_name}'. "
|
|
206
|
+
f"Available types: {list(_type_map.keys())}"
|
|
207
|
+
)
|
|
208
|
+
idx = table.schema.get_field_index(col_name)
|
|
209
|
+
casted = table.column(col_name).cast(target_type)
|
|
210
|
+
table = table.set_column(idx, col_name, casted)
|
|
211
|
+
return table
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
# ---------------------------------------------------------------------------
|
|
215
|
+
# DataFrame-level helpers (used internally by readers, not registered)
|
|
216
|
+
# ---------------------------------------------------------------------------
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def add_file_name_to_df(df: pd.DataFrame, file_name: str) -> pd.DataFrame:
|
|
220
|
+
"""Adds the file name to the dataframe."""
|
|
221
|
+
df["meta__source_file"] = file_name
|
|
222
|
+
return df
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def get_file_encoding(file: Path | S3Path, sample_size: int = 200_000_000) -> str:
|
|
226
|
+
"""
|
|
227
|
+
Detect encoding by reading only a sample of the file.
|
|
228
|
+
|
|
229
|
+
Args:
|
|
230
|
+
file: Path or S3Path to the file
|
|
231
|
+
sample_size: Number of bytes to sample (default 200MB)
|
|
232
|
+
|
|
233
|
+
Returns:
|
|
234
|
+
Detected encoding string
|
|
235
|
+
"""
|
|
236
|
+
with file.open("rb") as f:
|
|
237
|
+
sample = f.read(sample_size)
|
|
238
|
+
result = charset.detect(sample)
|
|
239
|
+
|
|
240
|
+
confidence = result["confidence"]
|
|
241
|
+
encoding = result["encoding"]
|
|
242
|
+
|
|
243
|
+
logger.info(
|
|
244
|
+
f"File: {file} "
|
|
245
|
+
f"detected with encoding: {encoding} "
|
|
246
|
+
f"with confidence {confidence}"
|
|
247
|
+
)
|
|
248
|
+
|
|
249
|
+
if encoding is None:
|
|
250
|
+
logger.warning(f"Encoding detection failed for {file}. Defaulting to 'utf-8'.")
|
|
251
|
+
return "utf-8"
|
|
252
|
+
|
|
253
|
+
return encoding
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def is_excel_header_empty(file_path: S3Path | Path, skiprows=0):
|
|
257
|
+
"""
|
|
258
|
+
Check if row skiprows + 1 of an Excel file stored in S3 is empty.
|
|
259
|
+
This function reads only the first row (excluding skiprows) of the specified Excel
|
|
260
|
+
file without assigning column headers, and checks whether all cells in that row are
|
|
261
|
+
either empty strings or NaN values.
|
|
262
|
+
Parameters:
|
|
263
|
+
s3_filepath (str): The S3 URI of the Excel file to check, e.g., "s3://bucket-name/path/to/file.xlsx".
|
|
264
|
+
Returns:
|
|
265
|
+
bool: True if all cells in the first row are empty or NaN, False otherwise.
|
|
266
|
+
"""
|
|
267
|
+
with file_path.open("rb"):
|
|
268
|
+
df = pd.read_excel(file_path, header=None, nrows=1, skiprows=skiprows)
|
|
269
|
+
|
|
270
|
+
# If pandas returns no rows at this index, then its definitely empty at this point
|
|
271
|
+
if df.shape[0] == 0:
|
|
272
|
+
return True
|
|
273
|
+
|
|
274
|
+
first_row = df.iloc[0]
|
|
275
|
+
return all(pd.isna(cell) or cell == "" for cell in first_row)
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def infer_excel_rowskip(file_path):
|
|
279
|
+
"""
|
|
280
|
+
This function infers the number or rows to skip to enable and excel file to be
|
|
281
|
+
read appropriately
|
|
282
|
+
Parameters:
|
|
283
|
+
s3_filepath (str): The S3 URI of the Excel file to check, e.g., "s3://bucket-name/path/to/file.xlsx".
|
|
284
|
+
Returns:
|
|
285
|
+
int: Number of rows to skip in order to reach a row with data in it (zero-indexed)
|
|
286
|
+
"""
|
|
287
|
+
logger.info(f"Inferring row-skips for {file_path}")
|
|
288
|
+
for i in range(5):
|
|
289
|
+
logger.info(f"Checking if {i} row skips yields data")
|
|
290
|
+
if is_excel_header_empty(file_path, i):
|
|
291
|
+
logger.info(f"No {i} row skips doesnt yield data")
|
|
292
|
+
continue
|
|
293
|
+
else:
|
|
294
|
+
logger.info(
|
|
295
|
+
f"{i} rowskips does yield data. Returning {i} as inferred row-skip"
|
|
296
|
+
)
|
|
297
|
+
return i
|
|
298
|
+
logger.info("No valid options returning 0")
|
|
299
|
+
return 0
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def add_empty_column_types(df: pd.DataFrame) -> pd.DataFrame:
|
|
303
|
+
"""
|
|
304
|
+
Ensure all columns have valid types for PyArrow conversion.
|
|
305
|
+
|
|
306
|
+
Converts columns that are entirely empty (all NaN/None) to string type
|
|
307
|
+
to prevent PyArrow from inferring 'null' type which causes dlt to drop them.
|
|
308
|
+
|
|
309
|
+
Args:
|
|
310
|
+
df: Pandas DataFrame to process (modified in-place)
|
|
311
|
+
|
|
312
|
+
Returns:
|
|
313
|
+
DataFrame with empty columns converted to string type
|
|
314
|
+
"""
|
|
315
|
+
for col in df.columns:
|
|
316
|
+
# Check if column is entirely empty (all NaN/None)
|
|
317
|
+
if df[col].isna().all():
|
|
318
|
+
# Convert to pandas string type (nullable string)
|
|
319
|
+
df[col] = df[col].astype("string")
|
|
320
|
+
logger.debug(
|
|
321
|
+
f"Column '{col}' is entirely empty - converted to string type "
|
|
322
|
+
"to preserve in final table"
|
|
323
|
+
)
|
|
324
|
+
|
|
325
|
+
return df
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
def add_magistrates_timestamp_col(df: pd.DataFrame, file_name: str) -> pd.DataFrame:
|
|
329
|
+
"""Adds the unix timestamp to the dataframe"""
|
|
330
|
+
match = re.search(r"(?<==)(\d+)", file_name)
|
|
331
|
+
if match is None:
|
|
332
|
+
raise ConfigError(f"Could not extract timestamp from file name: {file_name}")
|
|
333
|
+
df["meta__file_land_timestamp"] = match.group(0)
|
|
334
|
+
return df
|
|
File without changes
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import shutil
|
|
3
|
+
import gc
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
import tempfile
|
|
6
|
+
|
|
7
|
+
from cloudpathlib import S3Path
|
|
8
|
+
from loguru import logger
|
|
9
|
+
import psutil
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
'''
|
|
13
|
+
Groups files into ordered batches capped by cumulative object size.
|
|
14
|
+
'''
|
|
15
|
+
def group_files_by_batch_memory_limit(files: list[S3Path], limit_bytes: int) -> list[list[S3Path]]:
|
|
16
|
+
|
|
17
|
+
batches: list[list[S3Path]] = []
|
|
18
|
+
current_batch: list[S3Path] = []
|
|
19
|
+
current_batch_bytes = 0
|
|
20
|
+
|
|
21
|
+
for file in files:
|
|
22
|
+
file_size_bytes = file.stat().st_size
|
|
23
|
+
|
|
24
|
+
if file_size_bytes > limit_bytes:
|
|
25
|
+
if current_batch:
|
|
26
|
+
batches.append(current_batch)
|
|
27
|
+
current_batch = []
|
|
28
|
+
current_batch_bytes = 0
|
|
29
|
+
|
|
30
|
+
logger.warning(
|
|
31
|
+
"File exceeds batch_memory_limit and will run in its own batch: "
|
|
32
|
+
f"{file} ({file_size_bytes} bytes > {limit_bytes} bytes)"
|
|
33
|
+
)
|
|
34
|
+
batches.append([file])
|
|
35
|
+
continue
|
|
36
|
+
|
|
37
|
+
if current_batch and (current_batch_bytes + file_size_bytes > limit_bytes):
|
|
38
|
+
batches.append(current_batch)
|
|
39
|
+
current_batch = [file]
|
|
40
|
+
current_batch_bytes = file_size_bytes
|
|
41
|
+
continue
|
|
42
|
+
|
|
43
|
+
current_batch.append(file)
|
|
44
|
+
current_batch_bytes += file_size_bytes
|
|
45
|
+
|
|
46
|
+
if current_batch:
|
|
47
|
+
batches.append(current_batch)
|
|
48
|
+
|
|
49
|
+
return batches
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
'''
|
|
53
|
+
Resolves the local dlt pipeline directory for a destination table.
|
|
54
|
+
'''
|
|
55
|
+
def _get_schema_dir(destination_database: str, destination_table: str, dlt_data_dir: str | None = None) -> Path:
|
|
56
|
+
|
|
57
|
+
data_dir_value = dlt_data_dir or os.getenv("DLT_DATA_DIR", "data_dir/")
|
|
58
|
+
data_dir = Path(data_dir_value).resolve()
|
|
59
|
+
return (data_dir / "pipelines" / f"{destination_database}__{destination_table}").resolve()
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
'''
|
|
63
|
+
Removes transient runtime directories from a local dlt schema directory.
|
|
64
|
+
'''
|
|
65
|
+
def _cleanup_runtime_dirs(schema_dir: Path) -> list[str]:
|
|
66
|
+
|
|
67
|
+
cleaned_runtime_dirs: list[str] = []
|
|
68
|
+
for runtime_dir_name in ("load", "normalize"):
|
|
69
|
+
runtime_dir = schema_dir / runtime_dir_name
|
|
70
|
+
if not runtime_dir.exists():
|
|
71
|
+
continue
|
|
72
|
+
try:
|
|
73
|
+
shutil.rmtree(runtime_dir)
|
|
74
|
+
cleaned_runtime_dirs.append(runtime_dir_name)
|
|
75
|
+
except Exception as error:
|
|
76
|
+
logger.warning(
|
|
77
|
+
"Batch data_dir cleanup failed for "
|
|
78
|
+
f"{runtime_dir}: {repr(error)}"
|
|
79
|
+
)
|
|
80
|
+
return cleaned_runtime_dirs
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
'''
|
|
84
|
+
Calculates total file size in bytes for a directory tree.
|
|
85
|
+
'''
|
|
86
|
+
def _calculate_dir_size_bytes(target_dir: Path) -> int:
|
|
87
|
+
|
|
88
|
+
return sum(
|
|
89
|
+
file_path.stat().st_size
|
|
90
|
+
for file_path in target_dir.rglob("*")
|
|
91
|
+
if file_path.is_file()
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
'''
|
|
96
|
+
Finds the largest file by size within a directory tree.
|
|
97
|
+
'''
|
|
98
|
+
def _find_largest_file(target_dir: Path) -> tuple[Path | None, int]:
|
|
99
|
+
|
|
100
|
+
largest_file: Path | None = None
|
|
101
|
+
largest_size = -1
|
|
102
|
+
for file_path in target_dir.rglob("*"):
|
|
103
|
+
if not file_path.is_file():
|
|
104
|
+
continue
|
|
105
|
+
try:
|
|
106
|
+
size = file_path.stat().st_size
|
|
107
|
+
except OSError:
|
|
108
|
+
continue
|
|
109
|
+
if size > largest_size:
|
|
110
|
+
largest_size = size
|
|
111
|
+
largest_file = file_path
|
|
112
|
+
return largest_file, max(largest_size, 0)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
'''
|
|
116
|
+
Finds the largest top-level subdirectory (by total contained file size) within a directory.
|
|
117
|
+
'''
|
|
118
|
+
def _find_largest_dir(target_dir: Path) -> tuple[Path | None, int]:
|
|
119
|
+
|
|
120
|
+
largest_dir: Path | None = None
|
|
121
|
+
largest_size = -1
|
|
122
|
+
for entry in target_dir.iterdir():
|
|
123
|
+
if not entry.is_dir():
|
|
124
|
+
continue
|
|
125
|
+
try:
|
|
126
|
+
size = _calculate_dir_size_bytes(entry)
|
|
127
|
+
except OSError:
|
|
128
|
+
continue
|
|
129
|
+
if size > largest_size:
|
|
130
|
+
largest_size = size
|
|
131
|
+
largest_dir = entry
|
|
132
|
+
return largest_dir, max(largest_size, 0)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
'''
|
|
136
|
+
Logs a summary of runtime directory cleanup and post-cleanup sizes.
|
|
137
|
+
'''
|
|
138
|
+
def _log_batch_cleanup_summary(
|
|
139
|
+
schema_dir: Path,
|
|
140
|
+
cleaned_runtime_dirs: list[str],
|
|
141
|
+
schema_dir_size_bytes: int,
|
|
142
|
+
) -> None:
|
|
143
|
+
|
|
144
|
+
logger.debug(
|
|
145
|
+
"Cleaned batch runtime directories under "
|
|
146
|
+
f"{schema_dir}: {cleaned_runtime_dirs}"
|
|
147
|
+
f" (schema dir size post cleanup: {schema_dir_size_bytes} bytes)"
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
'''
|
|
152
|
+
Cleans transient dlt runtime directories while preserving schemas metadata.
|
|
153
|
+
'''
|
|
154
|
+
def cleanup_batch_data_dir(
|
|
155
|
+
destination_database: str,
|
|
156
|
+
destination_table: str,
|
|
157
|
+
dlt_data_dir: str | None = None,
|
|
158
|
+
batch_temp_dir: str | Path | None = None,
|
|
159
|
+
) -> None:
|
|
160
|
+
|
|
161
|
+
schema_dir = _get_schema_dir(
|
|
162
|
+
destination_database=destination_database,
|
|
163
|
+
destination_table=destination_table,
|
|
164
|
+
dlt_data_dir=dlt_data_dir,
|
|
165
|
+
)
|
|
166
|
+
|
|
167
|
+
cleaned_runtime_dirs: list[str] = []
|
|
168
|
+
if schema_dir.exists():
|
|
169
|
+
cleaned_runtime_dirs = _cleanup_runtime_dirs(schema_dir)
|
|
170
|
+
|
|
171
|
+
schema_dir_size_bytes = _calculate_dir_size_bytes(schema_dir)
|
|
172
|
+
_log_batch_cleanup_summary(
|
|
173
|
+
schema_dir=schema_dir,
|
|
174
|
+
cleaned_runtime_dirs=cleaned_runtime_dirs,
|
|
175
|
+
schema_dir_size_bytes=schema_dir_size_bytes,
|
|
176
|
+
)
|
|
177
|
+
else:
|
|
178
|
+
logger.debug(f"No local runtime schema dir to clean: {schema_dir}")
|
|
179
|
+
|
|
180
|
+
gc.collect()
|
|
181
|
+
if batch_temp_dir is not None:
|
|
182
|
+
batch_temp_path = Path(batch_temp_dir)
|
|
183
|
+
for path in batch_temp_path.iterdir():
|
|
184
|
+
logger.info(f" - {path}")
|
|
185
|
+
# Delete each item independently so one failure doesn't leave the rest behind.
|
|
186
|
+
for path in batch_temp_path.iterdir():
|
|
187
|
+
try:
|
|
188
|
+
if path.is_dir():
|
|
189
|
+
shutil.rmtree(path)
|
|
190
|
+
else:
|
|
191
|
+
path.unlink()
|
|
192
|
+
except Exception as error:
|
|
193
|
+
logger.warning(
|
|
194
|
+
"Cleanup of batch temp item {} failed: {}",
|
|
195
|
+
path,
|
|
196
|
+
repr(error),
|
|
197
|
+
)
|
|
198
|
+
logger.debug(f"Emptied batch temp directory: {batch_temp_path}")
|
|
199
|
+
else :
|
|
200
|
+
logger.debug("No batch temp directory specified for cleanup.")
|
|
201
|
+
|
|
202
|
+
log_pod_disk_usage()
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
'''
|
|
206
|
+
Log pod disk usage for trend tracking
|
|
207
|
+
'''
|
|
208
|
+
def log_pod_disk_usage():
|
|
209
|
+
try:
|
|
210
|
+
disk_usage = psutil.disk_usage("/")
|
|
211
|
+
used_gb = disk_usage.used / (1024**3)
|
|
212
|
+
total_gb = disk_usage.total / (1024**3)
|
|
213
|
+
percent_used = disk_usage.percent
|
|
214
|
+
logger.debug(
|
|
215
|
+
"Pod disk usage: {:.2f} GB / {:.2f} GB ({:.1f}%)",
|
|
216
|
+
used_gb,
|
|
217
|
+
total_gb,
|
|
218
|
+
percent_used,
|
|
219
|
+
)
|
|
220
|
+
except Exception as error:
|
|
221
|
+
logger.warning("Failed to log pod disk usage: {}", repr(error))
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
from cloudpathlib import S3Path
|
|
2
|
+
from loguru import logger
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def get_endpoints_at_path(path: S3Path) -> list[S3Path]:
|
|
6
|
+
items = [item for item in path.rglob("*") if item != path]
|
|
7
|
+
|
|
8
|
+
return items
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def get_files_at_path(path: S3Path) -> list[S3Path]:
|
|
12
|
+
items = get_endpoints_at_path(path)
|
|
13
|
+
|
|
14
|
+
files = [item for item in items if item.is_file()]
|
|
15
|
+
logger.info(f"Total files found at {str(path)}: {len(files)}")
|
|
16
|
+
|
|
17
|
+
return files
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def get_dirs_at_path(path: S3Path) -> list[S3Path]:
|
|
21
|
+
items = get_endpoints_at_path(path)
|
|
22
|
+
|
|
23
|
+
dirs = [item for item in items if item.is_dir()]
|
|
24
|
+
logger.info(f"Total directories found at {str(path)}: {len(dirs)}")
|
|
25
|
+
|
|
26
|
+
return dirs
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def filter_files_at_path(
|
|
30
|
+
files: list[S3Path], glob_patterns: str | list[str]
|
|
31
|
+
) -> list[S3Path]:
|
|
32
|
+
if not glob_patterns:
|
|
33
|
+
ValueError("glob_patterns is empty")
|
|
34
|
+
|
|
35
|
+
if isinstance(glob_patterns, str):
|
|
36
|
+
glob_patterns = [glob_patterns]
|
|
37
|
+
|
|
38
|
+
selected_files = [
|
|
39
|
+
file for file in files if any([file.match(pattern) for pattern in glob_patterns])
|
|
40
|
+
]
|
|
41
|
+
logger.info(
|
|
42
|
+
f"Filtering against patterns: {glob_patterns} -- "
|
|
43
|
+
f"{len(selected_files)} match patterns"
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
return selected_files
|