moj-data-loader 2.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,659 @@
1
+ from __future__ import annotations
2
+
3
+ import os
4
+ import re
5
+ import tempfile
6
+ from pathlib import Path
7
+ from typing import Annotated, Any, Iterator, Literal
8
+
9
+ import awswrangler as wr
10
+ import dlt
11
+ import yaml
12
+ from botocore.exceptions import ClientError
13
+ from cloudpathlib import S3Path
14
+ from loguru import logger
15
+ from pydantic import AfterValidator, Field
16
+ from pydantic_settings import BaseSettings, SettingsConfigDict
17
+
18
+ from moj_data_loader.readers.registry import read
19
+ from moj_data_loader.utils.exceptions import ConfigError
20
+ from .utils.batch import (
21
+ cleanup_batch_data_dir,
22
+ group_files_by_batch_memory_limit,
23
+ )
24
+ from moj_data_loader.utils.files import (
25
+ filter_files_at_path,
26
+ get_files_at_path,
27
+ )
28
+ from moj_data_loader.utils.summary_log import summary_log
29
+ from moj_data_loader.utils.validators import (
30
+ validate_s3_path_exists,
31
+ )
32
+
33
+ DEFAULT_BATCH_MEMORY_LIMIT_GB = 4
34
+
35
+
36
+ class Pipeline(BaseSettings):
37
+ model_config = SettingsConfigDict(populate_by_name=True, extra="ignore")
38
+
39
+ # Runtime Attributes - set in env
40
+ env: Literal["dev", "test", "prod"]
41
+ full_refresh: bool
42
+ schema_contract: Literal["evolve", "freeze"]
43
+ load_limit: int | None
44
+ enable_batching: bool = False
45
+ batch_memory_limit: float | None = None
46
+
47
+ # Pipeline attributes - created from yaml values
48
+ pipeline_name: str
49
+ source_path: Annotated[S3Path, AfterValidator(validate_s3_path_exists)]
50
+ source_domain: str
51
+ file_types: list[str] = Field(default_factory=list)
52
+ destination_path: S3Path
53
+ destination_database: str
54
+ destination_table: str
55
+ reader_name: str
56
+ reader_kwargs: dict[str, Any] = Field(default_factory=dict)
57
+ reader_transforms: list[dict[str, Any]] = Field(default_factory=list)
58
+
59
+ # Post-init attributes
60
+ files_at_source: list[S3Path] | None = None
61
+ already_loaded_files: list[str] | None = None
62
+ files_to_load: list[S3Path] | None = None
63
+ objects_at_destination: list[S3Path] | None = None
64
+ query_dump_path: S3Path | None = None
65
+ table_path: S3Path | None = None
66
+
67
+ # dlt objects
68
+ dlt_destination_filesystem: dlt.destinations.filesystem | None = None
69
+ dlt_pipeline: dlt.Pipeline | None = None
70
+ dlt_resource: Iterator | None = None
71
+
72
+ def model_post_init(self, __context: Any) -> None:
73
+ # Update destination locations with env
74
+ self.destination_path = self._generate_destination_path()
75
+ self.destination_database = self._generate_destination_database()
76
+ self.query_dump_path = self.destination_path / "athena_query_dump"
77
+ self.table_path = (
78
+ self.destination_path / self.destination_database / self.destination_table
79
+ )
80
+
81
+ # Compute files to load
82
+ self.files_at_source = self._get_files_at_source()
83
+ self.already_loaded_files = self._get_already_loaded_files()
84
+ self.files_to_load = self._get_files_to_load()
85
+ summary_log.files_to_load = len(self.files_to_load)
86
+ summary_log.already_loaded_files = len(self.already_loaded_files)
87
+ summary_log.files_at_source = len(self.files_at_source)
88
+ summary_log.files_loaded = 0
89
+
90
+ # Check files at destination path
91
+ self.objects_at_destination = self._get_objects_at_destination_path()
92
+
93
+ logger.info("Setting DLT destination....")
94
+ self.set_dlt_destination_file_system()
95
+ self.set_dlt_pipeline()
96
+
97
+ def set_dlt_destination_file_system(self):
98
+ # dlt objects
99
+ self.dlt_destination_filesystem = dlt.destinations.filesystem(
100
+ bucket_url=str(self.destination_path)
101
+ )
102
+
103
+ def set_dlt_pipeline(self):
104
+ self.dlt_pipeline = dlt.pipeline(
105
+ pipeline_name=f"{self.destination_database}__{self.destination_table}",
106
+ destination=dlt.destinations.athena(
107
+ query_result_bucket=str(self.query_dump_path)
108
+ ),
109
+ staging=self.dlt_destination_filesystem,
110
+ dataset_name=self.destination_database,
111
+ progress="log",
112
+ )
113
+
114
+ self.dlt_resource = read(
115
+ reader=self.reader_name,
116
+ files=self.files_to_load,
117
+ transforms=self.reader_transforms or None,
118
+ **self.reader_kwargs,
119
+ )
120
+
121
+ # Model Initialisation Methods ----
122
+
123
+ @classmethod
124
+ def load_from_yaml(
125
+ cls, pipeline: str, configs_dir: Path | str = Path("pipelines"), **env_kwargs
126
+ ) -> Pipeline:
127
+ """
128
+ YAML loader for Pipeline class
129
+
130
+ Args:
131
+ pipeline: A string, referring to the name of a yaml file in configs_dir
132
+ configs_dir: A directory of configs for pipeline runs.
133
+ The pipeline above must exist in this directory.
134
+ """
135
+
136
+ # Coerce strings into Path
137
+ if isinstance(configs_dir, str):
138
+ configs_dir = Path(configs_dir)
139
+
140
+ # Validate if pipeline yaml exists in configs_dir
141
+ targets = list(configs_dir.rglob(f"{pipeline}.y*ml"))
142
+
143
+ # Validate only one target yaml
144
+ if len(targets) == 0:
145
+ raise ConfigError(
146
+ f"No corresponding yaml file found under {str(configs_dir)} "
147
+ f"for {pipeline}"
148
+ )
149
+ elif len(targets) > 1:
150
+ raise ConfigError(
151
+ f"Multiple pipelines yamls found for {pipeline}: \n"
152
+ f"{[{str(p)} for p in targets]} \n"
153
+ f"Provide the database prefix as well, e.g. \n"
154
+ f'"affile_dummy/test_csv"'
155
+ )
156
+
157
+ # Open config, reading for instantiating object
158
+ config_path = targets[0]
159
+
160
+ with config_path.open() as f:
161
+ pipeline_config = yaml.safe_load(f)
162
+
163
+ # Filter out .yml & generate destination metadata from path
164
+ path_elements = re.sub(r"\.ya?ml$", "", str(config_path)).split("/")
165
+
166
+ # Take domain, database, table from last 3 path elements
167
+ domain, database, table = path_elements[-3:]
168
+
169
+ class_config = {
170
+ "pipeline_name": pipeline,
171
+ "source_path": S3Path(pipeline_config["source"]["path"]),
172
+ "file_types": pipeline_config["source"].get("file_types", []),
173
+ "source_domain": domain,
174
+ "destination_path": S3Path(
175
+ pipeline_config.get("destination", {}).get(
176
+ "path",
177
+ "s3://mojap-processed/", # Default to mojap-processed
178
+ )
179
+ ),
180
+ "destination_database": database,
181
+ "destination_table": table,
182
+ "reader_name": pipeline_config["reader"]["name"],
183
+ "reader_kwargs": pipeline_config["reader"].get("kwargs", {}),
184
+ "reader_transforms": pipeline_config["reader"].get("transforms", []),
185
+ "enable_batching": pipeline_config.get("enable_batching", False),
186
+ "batch_memory_limit": pipeline_config.get("batch_memory_limit"),
187
+ }
188
+
189
+ return cls(**class_config, **env_kwargs)
190
+
191
+ @staticmethod
192
+ def set_temp_dlt_directory_for_airflow(dlt_data_dir: str, dlt_project_dir: str):
193
+ """Set the DLT data and project directories to a temporary
194
+ local directory so that DLT can run in an airflow KNS pod
195
+ """
196
+ os.environ["DLT_DATA_DIR"] = dlt_data_dir
197
+ os.environ["DLT_PROJECT_DIR"] = dlt_project_dir
198
+
199
+ def _generate_destination_path(self) -> S3Path:
200
+ """Create a S3Path pointing at the appropriate staging destination
201
+
202
+ Defaults to s3://mojap-processed/ as the root bucket. Splits paths by domain,
203
+ source system, environment, and destination table. Can be overriden by setting
204
+ a destination path in the yaml config
205
+
206
+ Returns:
207
+ An S3Path object pointing to the file load location
208
+ """
209
+
210
+ path = (
211
+ self.destination_path / f"domain__{self.source_domain}" / f"env__{self.env}/"
212
+ )
213
+
214
+ logger.info(f"Target S3 staging path: {path}")
215
+
216
+ return S3Path(path)
217
+
218
+ def _generate_destination_database(self) -> str:
219
+ """Generate a database name based on the destination database and environment.
220
+
221
+ Returns:
222
+ str: The generated database name
223
+ """
224
+ return (
225
+ f"{self.destination_database}_{self.env}"
226
+ if self.env != "prod"
227
+ and not self.destination_database.endswith(f"_{self.env}")
228
+ else self.destination_database
229
+ )
230
+
231
+ def _get_files_at_source(self) -> list[S3Path]:
232
+ logger.info("---- Checking files at source ---- ")
233
+
234
+ all_files = get_files_at_path(self.source_path)
235
+
236
+ if not all_files:
237
+ raise ConfigError(f"No files detected at {self.source_path}")
238
+
239
+ # Error if multiple file types found without file globs
240
+ file_types = set([file.suffix.lower() for file in all_files])
241
+
242
+ if len(file_types) > 1 and not self.file_types:
243
+ raise ConfigError(
244
+ f"Multiple file types found at {self.source_path}: {file_types}.\n"
245
+ f"Please provide 'file_types' in config to filter appropriately."
246
+ )
247
+
248
+ # Filter files by file globs if provided
249
+ if not self.file_types:
250
+ logger.info("No file types supplied, using all files")
251
+ selected_files = all_files
252
+ else:
253
+ selected_files = filter_files_at_path(
254
+ files=all_files, glob_patterns=self.file_types
255
+ )
256
+
257
+ logger.info(f"{len(selected_files)} files found at {self.source_path}")
258
+
259
+ return selected_files
260
+
261
+ def _get_already_loaded_files(self) -> list[str]:
262
+ """Query Athena to get a list of files that have already been loaded.
263
+
264
+ Queries an Athena table to get the distinct meta__source_file column values,
265
+ which represent S3 URIs of files that have already been processed. It handles
266
+ cases where the database or table don't exist yet by returning an empty list.
267
+
268
+ Returns:
269
+ list[str]: List of S3 URIs for files that have already been processed.
270
+ Returns empty list if database/table doesn't exist.
271
+
272
+ Raises:
273
+ QueryFailed: If the Athena query fails for reasons other than missing
274
+ database/table
275
+ Exception: For any other unexpected errors during query execution
276
+ """
277
+
278
+ tbl = f"{self.destination_database}.{self.destination_table}"
279
+
280
+ sql = f"SELECT DISTINCT meta__source_file FROM {tbl}"
281
+
282
+ logger.info(f"Getting loaded files with sql: \n {sql} \n")
283
+
284
+ try:
285
+ df = wr.athena.read_sql_query(
286
+ sql=sql,
287
+ database=self.destination_database,
288
+ ctas_approach=False,
289
+ )
290
+
291
+ return df["meta__source_file"].tolist()
292
+
293
+ # Continue through Database/Table not found errors - Create New
294
+ except wr.exceptions.QueryFailed as error:
295
+ error_message = str(error).lower()
296
+ keywords = ["schema_not_found", "table_not_found"]
297
+
298
+ if any(keyword in error_message for keyword in keywords):
299
+ logger.info(
300
+ f"Database or Table not found for {tbl}."
301
+ f"Will be created by Pipeline.run()"
302
+ )
303
+ return []
304
+
305
+ # Raise all other errors
306
+ else:
307
+ logger.error(f"Query failed for {tbl}: {repr(error)}")
308
+ raise
309
+ except Exception as error:
310
+ logger.error(
311
+ f"Unexpected error getting processed files for {tbl}: {repr(error)}"
312
+ )
313
+ raise
314
+
315
+ def _get_objects_at_destination_path(self) -> list[S3Path]:
316
+ """Lists files at the target S3 Path
317
+
318
+ This has some utilities:
319
+ - Check files at a destination before teardown
320
+ - Check if files already exist at a destination we are newly loading to
321
+
322
+ Returns:
323
+ A list of S3Path objects to the given files
324
+ """
325
+ logger.info("---- Checking files at destination and staging paths ----")
326
+
327
+ # Expected load destination - Staging
328
+ staging_destination = S3Path(
329
+ self.destination_path
330
+ / f"{str(self.destination_database)}_staging"
331
+ / str(self.destination_table)
332
+ )
333
+
334
+ logger.info(f"Staging destination: {str(staging_destination)}")
335
+
336
+ # Expected load destination - Iceberg
337
+ iceberg_destination = S3Path(
338
+ self.destination_path
339
+ / str(self.destination_database)
340
+ / str(self.destination_table)
341
+ )
342
+
343
+ logger.info(f"Iceberg destination: {str(iceberg_destination)}")
344
+
345
+ staging_items = get_files_at_path(staging_destination)
346
+ iceberg_items = get_files_at_path(iceberg_destination)
347
+
348
+ all_destination_items = staging_items + iceberg_items
349
+
350
+ return all_destination_items
351
+
352
+ def _get_files_to_load(self) -> list[S3Path]:
353
+ if self.files_at_source is None:
354
+ raise ConfigError(f"No files at source path: {self.source_path}")
355
+
356
+ # Load all files if full refresh true
357
+ if self.full_refresh:
358
+ logger.info("Loading all sources files as full_refesh is True")
359
+
360
+ files_to_load = self.files_at_source
361
+
362
+ else:
363
+ already_loaded_files = set(self.already_loaded_files or [])
364
+
365
+ files_to_load = [
366
+ file
367
+ for file in self.files_at_source
368
+ if str(file) not in already_loaded_files
369
+ ]
370
+
371
+ logger.info(f"Detected {len(files_to_load)} files that are not loaded")
372
+
373
+ # Apply limit if set
374
+ if self.load_limit and self.load_limit > 0:
375
+ files_to_load = files_to_load[: self.load_limit]
376
+
377
+ return files_to_load
378
+
379
+ # Model Validation Methods ----
380
+
381
+ def _validate_no_orphaned_files(self) -> None:
382
+ """
383
+ Validate that if already_loaded_files is None (no database/table exists)
384
+ and files_at_destination is not None, raise an error to prevent orphaned files.
385
+
386
+ This validation prevents a scenario where files exist at the destination
387
+ but are not tracked in the database, which would lead to orphaned files
388
+ being deleted during teardown.
389
+ """
390
+
391
+ if self.objects_at_destination and not self.already_loaded_files:
392
+ raise ConfigError(
393
+ "Orphaned files detected at destination: "
394
+ f"{len(self.objects_at_destination)} objects found at {self.table_path} "
395
+ "but no database/table exists to track them. "
396
+ "These files would be deleted during teardown. "
397
+ "Please check and clean up the destination before proceeding."
398
+ )
399
+
400
+ # Pipeline Actions ----
401
+ def run(self) -> None:
402
+ """Run the dlt pipeline"""
403
+
404
+ logger.info("Starting DLT pipeline run...")
405
+ summary_log.files_loaded = 0
406
+
407
+ if self.full_refresh and self.enable_batching:
408
+ logger.warning(
409
+ "Stopping pipeline run: full_refresh cannot be combined with "
410
+ "batch_memory_limit. Disable batching or full_refresh and retry."
411
+ )
412
+ return
413
+
414
+ if self.dlt_pipeline is None:
415
+ raise ConfigError("dlt_pipeline attribute in Pipeline is None")
416
+
417
+ if self.files_to_load in (None, []):
418
+ logger.info("No files to load. Pipeline will not run")
419
+ return
420
+
421
+ self._validate_no_orphaned_files()
422
+
423
+ runtime_config = {
424
+ "write_disposition": "replace" if self.full_refresh else "append",
425
+ "refresh": "drop_sources" if self.full_refresh else None,
426
+ "schema_contract": {
427
+ "tables": "evolve",
428
+ "columns": "evolve" if self.full_refresh else self.schema_contract,
429
+ },
430
+ }
431
+
432
+ if self.enable_batching:
433
+ files_loaded = self.run_batched_pipeline(runtime_config=runtime_config)
434
+ else:
435
+ files_loaded = self.run_unbatched_pipeline(runtime_config=runtime_config)
436
+
437
+ summary_log.files_loaded = files_loaded
438
+
439
+ run_count = self._get_run_count_from_loads()
440
+ logger.info(f"Pipeline run completed. Maintenance run count: {run_count}")
441
+
442
+ # Fire-and-forget optimize and vacuum after each 10 successful prod runs.
443
+ if self.env == "prod" and self._is_maintenance_run(run_count):
444
+ logger.info("Triggering optimize and vacuum")
445
+ self._optimize()
446
+ self._vacuum()
447
+
448
+ def run_batched_pipeline(
449
+ self, runtime_config: dict[str, Any] | None = None
450
+ ) -> int:
451
+ files_loaded = 0
452
+
453
+ if self.batch_memory_limit is None or self.batch_memory_limit <= 0:
454
+ self.batch_memory_limit = DEFAULT_BATCH_MEMORY_LIMIT_GB
455
+
456
+ batch_limit_bytes = int(self.batch_memory_limit * (1024**3))
457
+
458
+ total_files = len(self.files_to_load)
459
+
460
+ file_batches = group_files_by_batch_memory_limit(
461
+ files=self.files_to_load,
462
+ limit_bytes=batch_limit_bytes,
463
+ )
464
+
465
+ for batch_index, batch_files in enumerate(file_batches, start=1):
466
+ logger.info(
467
+ "Running batch "
468
+ f"{batch_index}/{len(file_batches)} with {len(batch_files)} files"
469
+ )
470
+ try:
471
+ batch_temp_dir = tempfile.gettempdir()
472
+ logger.info(f"Using batch temp directory: {batch_temp_dir}")
473
+ dlt_resource = read(
474
+ reader=self.reader_name,
475
+ files=batch_files,
476
+ transforms=self.reader_transforms or None,
477
+ **self.reader_kwargs,
478
+ )
479
+ self.dlt_pipeline.run(
480
+ data=dlt_resource,
481
+ table_name=self.destination_table,
482
+ **runtime_config,
483
+ )
484
+ for _ in batch_files:
485
+ files_loaded += 1
486
+ logger.info(f"Loaded {files_loaded}/{total_files} files")
487
+ finally:
488
+ cleanup_batch_data_dir(
489
+ destination_database=self.destination_database,
490
+ destination_table=self.destination_table,
491
+ batch_temp_dir=batch_temp_dir,
492
+ )
493
+
494
+ return files_loaded
495
+
496
+
497
+ def run_unbatched_pipeline(
498
+ self, runtime_config: dict[str, Any] | None = None
499
+ ) -> int:
500
+ self.dlt_pipeline.run(
501
+ data=self.dlt_resource,
502
+ table_name=self.destination_table,
503
+ **runtime_config,
504
+ )
505
+
506
+ total_files = len(self.files_to_load)
507
+ files_loaded = total_files
508
+ logger.info(f"Loaded {files_loaded}/{total_files} files")
509
+ return files_loaded
510
+
511
+ def _get_run_count_from_loads(self) -> int:
512
+ """Count successful loads for this pipeline schema from the dlt loads table."""
513
+ schema_name = f"{self.destination_database}__{self.destination_table}"
514
+
515
+ sql = (
516
+ "SELECT COUNT(DISTINCT load_id) AS run_count "
517
+ f"FROM {self.destination_database}._dlt_loads "
518
+ f"WHERE schema_name = '{schema_name}'"
519
+ )
520
+
521
+ try:
522
+ df = wr.athena.read_sql_query(
523
+ sql=sql,
524
+ database=self.destination_database,
525
+ ctas_approach=False,
526
+ )
527
+ except wr.exceptions.QueryFailed as error:
528
+ error_message = str(error).lower()
529
+ keywords = ["schema_not_found", "table_not_found"]
530
+
531
+ if any(keyword in error_message for keyword in keywords):
532
+ logger.info(
533
+ "_dlt_loads table not found for "
534
+ f"{self.destination_database}. Returning run_count=0"
535
+ )
536
+ return 0
537
+ logger.error(
538
+ "Query failed getting run count from _dlt_loads for "
539
+ f"{self.destination_database}: {repr(error)}"
540
+ )
541
+ raise
542
+ except Exception as error:
543
+ logger.error(
544
+ "Unexpected error getting run count from _dlt_loads for "
545
+ f"{self.destination_database}: {repr(error)}"
546
+ )
547
+ raise
548
+
549
+ if df.empty:
550
+ return 0
551
+
552
+ return int(df["run_count"].iloc[0] or 0)
553
+
554
+ @staticmethod
555
+ def _is_maintenance_run(run_count: int) -> bool:
556
+ """Check if this run should trigger optimize/vacuum."""
557
+
558
+ return run_count % 10 == 0
559
+
560
+ def drop_db(
561
+ self,
562
+ allow_prod: bool = False,
563
+ staging_only: bool = True,
564
+ ) -> None:
565
+ """Delete Glue Database, Tables and S3 locations for a pipeline"""
566
+
567
+ if self.env == "prod" and not allow_prod:
568
+ raise ValueError(
569
+ "WARNING. You are trying to delete production data."
570
+ "Pass 'allow_prod=True' as an argument if you are sure."
571
+ )
572
+
573
+ # Define all possible targets
574
+ all_databases = [
575
+ self.destination_database,
576
+ f"{self.destination_database}_staging",
577
+ ]
578
+
579
+ all_s3_paths = [
580
+ str(self.destination_path / self.destination_database),
581
+ str(self.destination_path / f"{self.destination_database}_staging"),
582
+ str(self.destination_path / "athena_query_dump"),
583
+ ]
584
+
585
+ # Filter based on staging_only parameter
586
+ if staging_only:
587
+ target_databases = [db for db in all_databases if "_staging" in db]
588
+
589
+ target_s3_paths = [
590
+ path
591
+ for path in all_s3_paths
592
+ if "_staging" in path or "athena_query_dump" in path
593
+ ]
594
+ else:
595
+ target_databases = all_databases
596
+ target_s3_paths = all_s3_paths
597
+
598
+ # Delete databases
599
+ for database in target_databases:
600
+ try:
601
+ wr.catalog.delete_database(database)
602
+ logger.info(f"Deleted database: {database}")
603
+ except ClientError as e:
604
+ if e.response["Error"]["Code"] == "EntityNotFoundException":
605
+ logger.info(f"Database {database} not found - skipping deletion")
606
+ else:
607
+ logger.error(f"Failed to delete database {database}: {e}")
608
+ raise
609
+
610
+ # Delete S3 paths
611
+ for s3_path in target_s3_paths:
612
+ if wr.s3.list_objects(s3_path):
613
+ logger.info(f"Deleting objects at: {s3_path}")
614
+ wr.s3.delete_objects(s3_path)
615
+ else:
616
+ logger.info(f"No files found at {s3_path}")
617
+
618
+ def _vacuum(self) -> None:
619
+ """VACUUM a Iceberg table"""
620
+ sql = f"VACUUM {self.destination_table}"
621
+
622
+ wr.athena.start_query_execution(
623
+ sql, database=self.destination_database, s3_output=str(self.query_dump_path)
624
+ )
625
+
626
+ def _optimize(self) -> None:
627
+ """OPTIMIZE a Iceberg table"""
628
+ sql = f"OPTIMIZE {self.destination_table} REWRITE DATA USING BIN_PACK"
629
+
630
+ wr.athena.start_query_execution(
631
+ sql,
632
+ database=self.destination_database,
633
+ s3_output=str(self.query_dump_path),
634
+ wait=True,
635
+ )
636
+
637
+ def __str__(self):
638
+ return f"""Pipeline initialised with:
639
+ pipeline_name: {self.pipeline_name}
640
+ env: {self.env}
641
+ full_refresh: {self.full_refresh}
642
+ schema_contract: {self.schema_contract}
643
+ load_limit: {self.load_limit}
644
+ enable_batching: {self.enable_batching}
645
+ batch_memory_limit: {self.batch_memory_limit}
646
+ source_path: {self.source_path}
647
+ source_domain: {self.source_domain}
648
+ destination_database: {self.destination_database}
649
+ destination_table: {self.destination_table}
650
+ destination_path: f'{self.destination_path}/{self.destination_database}/{
651
+ self.destination_table
652
+ }'
653
+ reader_name: {self.reader_name}
654
+ reader_kwargs: {self.reader_kwargs}
655
+ files_at_source: {len(self.files_at_source) if self.files_at_source else 0} files
656
+ already_loaded_files: {
657
+ len(self.already_loaded_files) if self.already_loaded_files else 0
658
+ } files
659
+ files_to_load: {len(self.files_to_load) if self.files_to_load else 0} files"""
File without changes