lightweight-table-diff 0.1.2__tar.gz → 0.1.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {lightweight_table_diff-0.1.2 → lightweight_table_diff-0.1.4}/PKG-INFO +1 -1
- {lightweight_table_diff-0.1.2 → lightweight_table_diff-0.1.4}/pyproject.toml +1 -1
- {lightweight_table_diff-0.1.2 → lightweight_table_diff-0.1.4}/src/lightweight_table_diff/core.py +4 -4
- {lightweight_table_diff-0.1.2 → lightweight_table_diff-0.1.4}/src/lightweight_table_diff/runner.py +6 -6
- {lightweight_table_diff-0.1.2 → lightweight_table_diff-0.1.4}/src/lightweight_table_diff.egg-info/PKG-INFO +1 -1
- {lightweight_table_diff-0.1.2 → lightweight_table_diff-0.1.4}/README.md +0 -0
- {lightweight_table_diff-0.1.2 → lightweight_table_diff-0.1.4}/setup.cfg +0 -0
- {lightweight_table_diff-0.1.2 → lightweight_table_diff-0.1.4}/src/lightweight_table_diff/__init__.py +0 -0
- {lightweight_table_diff-0.1.2 → lightweight_table_diff-0.1.4}/src/lightweight_table_diff/__main__.py +0 -0
- {lightweight_table_diff-0.1.2 → lightweight_table_diff-0.1.4}/src/lightweight_table_diff/adapters/__init__.py +0 -0
- {lightweight_table_diff-0.1.2 → lightweight_table_diff-0.1.4}/src/lightweight_table_diff/adapters/csv.py +0 -0
- {lightweight_table_diff-0.1.2 → lightweight_table_diff-0.1.4}/src/lightweight_table_diff/adapters/hive_s3.py +0 -0
- {lightweight_table_diff-0.1.2 → lightweight_table_diff-0.1.4}/src/lightweight_table_diff/adapters/parquet.py +0 -0
- {lightweight_table_diff-0.1.2 → lightweight_table_diff-0.1.4}/src/lightweight_table_diff/adapters/sav.py +0 -0
- {lightweight_table_diff-0.1.2 → lightweight_table_diff-0.1.4}/src/lightweight_table_diff/config.py +0 -0
- {lightweight_table_diff-0.1.2 → lightweight_table_diff-0.1.4}/src/lightweight_table_diff/dimensions.py +0 -0
- {lightweight_table_diff-0.1.2 → lightweight_table_diff-0.1.4}/src/lightweight_table_diff/normalisers.py +0 -0
- {lightweight_table_diff-0.1.2 → lightweight_table_diff-0.1.4}/src/lightweight_table_diff.egg-info/SOURCES.txt +0 -0
- {lightweight_table_diff-0.1.2 → lightweight_table_diff-0.1.4}/src/lightweight_table_diff.egg-info/dependency_links.txt +0 -0
- {lightweight_table_diff-0.1.2 → lightweight_table_diff-0.1.4}/src/lightweight_table_diff.egg-info/requires.txt +0 -0
- {lightweight_table_diff-0.1.2 → lightweight_table_diff-0.1.4}/src/lightweight_table_diff.egg-info/top_level.txt +0 -0
{lightweight_table_diff-0.1.2 → lightweight_table_diff-0.1.4}/src/lightweight_table_diff/core.py
RENAMED
|
@@ -20,7 +20,7 @@ def diff_tbls(
|
|
|
20
20
|
after: pl.LazyFrame,
|
|
21
21
|
id_cols: list[str],
|
|
22
22
|
compare_cols: list[str] | None = None,
|
|
23
|
-
join_type: str = "
|
|
23
|
+
join_type: str = "inner",
|
|
24
24
|
) -> pl.LazyFrame:
|
|
25
25
|
if compare_cols is None:
|
|
26
26
|
compare_cols = list(before.drop(*id_cols).collect_schema().keys())
|
|
@@ -36,8 +36,8 @@ def diff_tbls(
|
|
|
36
36
|
pl.when(~pl.col(f"b__{c}").eq_missing(pl.col(f"a__{c}")))
|
|
37
37
|
.then(
|
|
38
38
|
pl.struct(
|
|
39
|
-
pl.col(f"b__{c}").alias("before_val"),
|
|
40
|
-
pl.col(f"a__{c}").alias("after_val"),
|
|
39
|
+
pl.col(f"b__{c}").cast(pl.String).alias("before_val"),
|
|
40
|
+
pl.col(f"a__{c}").cast(pl.String).alias("after_val"),
|
|
41
41
|
)
|
|
42
42
|
)
|
|
43
43
|
.otherwise(None)
|
|
@@ -69,7 +69,7 @@ def batch_diff_tbls(
|
|
|
69
69
|
id_cols: list[str],
|
|
70
70
|
compare_cols: list[str] | None = None,
|
|
71
71
|
batch_size: int = 50,
|
|
72
|
-
join_type: str = "
|
|
72
|
+
join_type: str = "inner",
|
|
73
73
|
) -> pl.LazyFrame:
|
|
74
74
|
if compare_cols is None:
|
|
75
75
|
compare_cols = list(before.drop(*id_cols).collect_schema().keys())
|
{lightweight_table_diff-0.1.2 → lightweight_table_diff-0.1.4}/src/lightweight_table_diff/runner.py
RENAMED
|
@@ -32,14 +32,14 @@ class ComparisonResult:
|
|
|
32
32
|
added_cols: list[str]
|
|
33
33
|
|
|
34
34
|
|
|
35
|
-
def run_comparison(job: dict, **
|
|
35
|
+
def run_comparison(job: dict, **kwargs) -> ComparisonResult:
|
|
36
36
|
job = copy.deepcopy(job)
|
|
37
37
|
name = job.get("name", "unnamed")
|
|
38
38
|
keys = job["join_keys"]
|
|
39
39
|
logger.info("Running comparison: %s", name)
|
|
40
40
|
|
|
41
|
-
before = adapters.load(job["before"], **
|
|
42
|
-
after = adapters.load(job["after"], **
|
|
41
|
+
before = adapters.load(job["before"], **kwargs)
|
|
42
|
+
after = adapters.load(job["after"], **kwargs)
|
|
43
43
|
|
|
44
44
|
removed_cols, added_cols = column_indels(before, after, keys)
|
|
45
45
|
if removed_cols:
|
|
@@ -72,7 +72,7 @@ def run_comparison(job: dict, **context) -> ComparisonResult:
|
|
|
72
72
|
keys,
|
|
73
73
|
cols,
|
|
74
74
|
batch_size=job.get("batch_size", 50),
|
|
75
|
-
join_type=
|
|
75
|
+
join_type="inner",
|
|
76
76
|
)
|
|
77
77
|
|
|
78
78
|
n = diff.select(pl.len()).collect().item()
|
|
@@ -137,11 +137,11 @@ def write_results(result: ComparisonResult, output_dir: str | Path) -> None:
|
|
|
137
137
|
logger.info(" Wrote %s", summary_path)
|
|
138
138
|
|
|
139
139
|
|
|
140
|
-
def run_config(config_path: str | Path, **
|
|
140
|
+
def run_config(config_path: str | Path, **kwargs) -> list[ComparisonResult]:
|
|
141
141
|
jobs = load_config(config_path)
|
|
142
142
|
results: list[ComparisonResult] = []
|
|
143
143
|
for job in jobs:
|
|
144
|
-
result = run_comparison(job, **
|
|
144
|
+
result = run_comparison(job, **kwargs)
|
|
145
145
|
write_results(result, job.get("output_dir", "./diff_output"))
|
|
146
146
|
results.append(result)
|
|
147
147
|
return results
|
|
File without changes
|
|
File without changes
|
{lightweight_table_diff-0.1.2 → lightweight_table_diff-0.1.4}/src/lightweight_table_diff/__init__.py
RENAMED
|
File without changes
|
{lightweight_table_diff-0.1.2 → lightweight_table_diff-0.1.4}/src/lightweight_table_diff/__main__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{lightweight_table_diff-0.1.2 → lightweight_table_diff-0.1.4}/src/lightweight_table_diff/config.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|