floe-python 0.8.8__py3-none-any.whl → 0.8.9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- floe/checks/unique.py +4 -44
- floe/iceberg/catalog.py +53 -45
- floe/iceberg/sink.py +11 -7
- floe/iceberg/write.py +46 -9
- floe/run/__init__.py +10 -6
- floe/run/entity.py +2 -1
- {floe_python-0.8.8.dist-info → floe_python-0.8.9.dist-info}/METADATA +1 -1
- {floe_python-0.8.8.dist-info → floe_python-0.8.9.dist-info}/RECORD +11 -11
- {floe_python-0.8.8.dist-info → floe_python-0.8.9.dist-info}/WHEEL +0 -0
- {floe_python-0.8.8.dist-info → floe_python-0.8.9.dist-info}/entry_points.txt +0 -0
- {floe_python-0.8.8.dist-info → floe_python-0.8.9.dist-info}/licenses/LICENSE +0 -0
floe/checks/unique.py
CHANGED
|
@@ -17,7 +17,6 @@ from floe.checks.rows import RowCheck
|
|
|
17
17
|
|
|
18
18
|
SAMPLE_LIMIT = 5
|
|
19
19
|
_SEEN = "__floe_seen"
|
|
20
|
-
_DEBUG = "__floe_debug"
|
|
21
20
|
_COUNT = "__floe_count"
|
|
22
21
|
_BITS = "_bits" # suffix of the bit-pattern column that keys a float column
|
|
23
22
|
|
|
@@ -56,45 +55,11 @@ def resolve_unique_keys(
|
|
|
56
55
|
return resolved
|
|
57
56
|
|
|
58
57
|
|
|
59
|
-
# `str` Debug; ponytail: non-ASCII non-printable characters (Rust `\\u{..}`) are left raw
|
|
60
|
-
_STR_ESCAPES = {
|
|
61
|
-
"\\": "\\\\",
|
|
62
|
-
'"': '\\"',
|
|
63
|
-
"\n": "\\n",
|
|
64
|
-
"\r": "\\r",
|
|
65
|
-
"\t": "\\t",
|
|
66
|
-
"\0": "\\0",
|
|
67
|
-
**{chr(i): f"\\u{{{i:x}}}" for i in [*range(1, 32), 127] if chr(i) not in "\n\r\t"},
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
|
|
71
58
|
def _float_bits(s: pl.Series) -> pl.Series:
|
|
72
59
|
"""Rust's `to_bits()`: Polars cannot reinterpret a float, so view the Arrow buffer."""
|
|
73
60
|
return pl.Series(s.cast(pl.Float64).to_arrow().view(pa.uint64()))
|
|
74
61
|
|
|
75
62
|
|
|
76
|
-
def _debug(name: str, dtype: pl.DataType) -> pl.Expr:
|
|
77
|
-
"""Rust's `{:?}` of the `UniqueKey` variant: the sample tie-break sorts on these strings."""
|
|
78
|
-
c = pl.col(name)
|
|
79
|
-
if dtype == pl.String:
|
|
80
|
-
escaped = c
|
|
81
|
-
for raw, esc in _STR_ESCAPES.items():
|
|
82
|
-
escaped = escaped.str.replace_all(raw, esc, literal=True)
|
|
83
|
-
return pl.format('String("{}")', escaped)
|
|
84
|
-
if dtype == pl.Boolean:
|
|
85
|
-
return pl.format("Bool({})", c.cast(pl.String))
|
|
86
|
-
if dtype.is_signed_integer():
|
|
87
|
-
return pl.format("I64({})", c.cast(pl.String))
|
|
88
|
-
if dtype.is_unsigned_integer():
|
|
89
|
-
return pl.format("U64({})", c.cast(pl.String))
|
|
90
|
-
if dtype.is_float():
|
|
91
|
-
return pl.format("F64({})", pl.col(name + _BITS).cast(pl.String))
|
|
92
|
-
if isinstance(dtype, pl.Datetime): # chrono prints the fraction only when not zero
|
|
93
|
-
return pl.format('Other("{}")', c.dt.to_string("%Y-%m-%d %H:%M:%S%.f"))
|
|
94
|
-
# ponytail: date keys use `Other(display)`, unverified against Rust; ties may order differently
|
|
95
|
-
return pl.format('Other("{}")', c.cast(pl.String))
|
|
96
|
-
|
|
97
|
-
|
|
98
63
|
def _text(value: object) -> str:
|
|
99
64
|
"""Rust's `to_string` of a key value."""
|
|
100
65
|
if isinstance(value, bool):
|
|
@@ -221,7 +186,7 @@ class UniqueTracker:
|
|
|
221
186
|
return RowCheck("unique", name, message, dup)
|
|
222
187
|
|
|
223
188
|
def results(self) -> list[UniqueResult]:
|
|
224
|
-
"""Per constraint, the top `SAMPLE_LIMIT` keys by count, ties by
|
|
189
|
+
"""Per constraint, the top `SAMPLE_LIMIT` keys by count, ties by the key values (numbers numerically, strings by code point)."""
|
|
225
190
|
return [self._result(c) for c in self._constraints]
|
|
226
191
|
|
|
227
192
|
def _result(self, c: _Constraint) -> UniqueResult:
|
|
@@ -229,14 +194,9 @@ class UniqueTracker:
|
|
|
229
194
|
return UniqueResult(c.columns, 0, [])
|
|
230
195
|
parts = pl.concat(c.dup_counts)
|
|
231
196
|
counts = parts.group_by(parts.columns[:-1]).agg(pl.col(_COUNT).sum())
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
)
|
|
236
|
-
top = (
|
|
237
|
-
counts.with_columns(debug.alias(_DEBUG))
|
|
238
|
-
.sort([_COUNT, _DEBUG], descending=[True, False])
|
|
239
|
-
.head(SAMPLE_LIMIT)
|
|
197
|
+
by = [_COUNT, *counts.columns[:-1]]
|
|
198
|
+
top = counts.sort(by, descending=[True] + [False] * (len(by) - 1)).head(
|
|
199
|
+
SAMPLE_LIMIT
|
|
240
200
|
)
|
|
241
201
|
samples = [ # at most SAMPLE_LIMIT rows
|
|
242
202
|
UniqueSample(
|
floe/iceberg/catalog.py
CHANGED
|
@@ -12,7 +12,11 @@ from urllib.parse import urlsplit
|
|
|
12
12
|
|
|
13
13
|
import duckdb
|
|
14
14
|
from pyiceberg.catalog import Catalog, load_catalog
|
|
15
|
-
from pyiceberg.exceptions import
|
|
15
|
+
from pyiceberg.exceptions import (
|
|
16
|
+
NamespaceAlreadyExistsError,
|
|
17
|
+
NoSuchNamespaceError,
|
|
18
|
+
NoSuchTableError,
|
|
19
|
+
)
|
|
16
20
|
from pyiceberg.partitioning import PartitionField, PartitionSpec
|
|
17
21
|
from pyiceberg.schema import Schema
|
|
18
22
|
from pyiceberg.table import Table
|
|
@@ -345,55 +349,59 @@ def ensure_table(
|
|
|
345
349
|
|
|
346
350
|
With `add_columns`, columns new in `schema` are added as optional (D13) and returned by name; partition changes stay rejected.
|
|
347
351
|
"""
|
|
348
|
-
ident = (namespace,)
|
|
349
|
-
with run_error("rest catalog namespace_exists check failed"):
|
|
350
|
-
namespace_exists = catalog.namespace_exists(ident)
|
|
351
|
-
if not namespace_exists:
|
|
352
|
-
if not create_namespace:
|
|
353
|
-
raise FloeRunError(f"entity.name={entity} iceberg namespace {namespace} does not exist and create_database_if_missing is false") # fmt: skip
|
|
354
|
-
with (
|
|
355
|
-
run_error("rest catalog namespace create failed"),
|
|
356
|
-
suppress(NamespaceAlreadyExistsError),
|
|
357
|
-
): # another entity or process created it first
|
|
358
|
-
catalog.create_namespace(ident)
|
|
359
352
|
identifier = (namespace, name)
|
|
360
|
-
with run_error("rest catalog
|
|
361
|
-
|
|
362
|
-
if exists:
|
|
363
|
-
with run_error("rest catalog load_table failed"):
|
|
353
|
+
with run_error("rest catalog load_table failed"):
|
|
354
|
+
try:
|
|
364
355
|
table = catalog.load_table(identifier)
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
356
|
+
except (
|
|
357
|
+
NoSuchTableError,
|
|
358
|
+
NoSuchNamespaceError,
|
|
359
|
+
): # fmt: skip -- a missing namespace raises either, per catalog
|
|
360
|
+
table = None
|
|
361
|
+
if table is None:
|
|
362
|
+
ident = (namespace,)
|
|
363
|
+
with run_error("rest catalog namespace_exists check failed"):
|
|
364
|
+
namespace_exists = catalog.namespace_exists(ident)
|
|
365
|
+
if not namespace_exists:
|
|
366
|
+
if not create_namespace:
|
|
367
|
+
raise FloeRunError(f"entity.name={entity} iceberg namespace {namespace} does not exist and create_database_if_missing is false") # fmt: skip
|
|
368
|
+
with (
|
|
369
|
+
run_error("rest catalog namespace create failed"),
|
|
370
|
+
suppress(NamespaceAlreadyExistsError),
|
|
371
|
+
): # another entity or process created it first
|
|
372
|
+
catalog.create_namespace(ident)
|
|
373
|
+
spec = (
|
|
375
374
|
None
|
|
376
375
|
if partition_fields is None
|
|
377
|
-
else build_partition_spec(
|
|
378
|
-
entity,
|
|
376
|
+
else build_partition_spec(schema, partition_fields, entity)
|
|
379
377
|
)
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
378
|
+
with run_error("rest catalog create_table failed"):
|
|
379
|
+
created = catalog.create_table( # pyright: ignore[reportUnknownMemberType] -- PyIceberg's `schema` is `Schema | Unknown`
|
|
380
|
+
identifier,
|
|
381
|
+
schema,
|
|
382
|
+
location=location,
|
|
383
|
+
partition_spec=spec if spec is not None else PartitionSpec(),
|
|
384
|
+
)
|
|
385
|
+
return created, []
|
|
386
|
+
added: list[NestedField] = []
|
|
387
|
+
if schema_evolution == "add_columns":
|
|
388
|
+
added = additive_columns(table.schema(), schema, entity)
|
|
389
|
+
# field ids of `schema` may differ from the table's once columns are added
|
|
390
|
+
check_schema = table.schema()
|
|
391
|
+
else:
|
|
392
|
+
check_schema_matches(table.schema(), schema, entity)
|
|
393
|
+
check_schema = schema
|
|
394
|
+
check_partition_spec_matches(
|
|
395
|
+
table.spec(),
|
|
388
396
|
None
|
|
389
397
|
if partition_fields is None
|
|
390
|
-
else build_partition_spec(
|
|
398
|
+
else build_partition_spec(check_schema, partition_fields, entity),
|
|
399
|
+
entity,
|
|
391
400
|
)
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
return created, []
|
|
401
|
+
if added:
|
|
402
|
+
with run_error("rest catalog update_schema failed"):
|
|
403
|
+
with table.update_schema() as update:
|
|
404
|
+
for field in added:
|
|
405
|
+
update.add_column(field.name, field.field_type, required=False)
|
|
406
|
+
table.refresh()
|
|
407
|
+
return table, [f.name for f in added]
|
floe/iceberg/sink.py
CHANGED
|
@@ -15,6 +15,7 @@ from typing import Any
|
|
|
15
15
|
import polars as pl
|
|
16
16
|
import pyarrow as pa
|
|
17
17
|
from pyiceberg.catalog import Catalog
|
|
18
|
+
from pyiceberg.exceptions import NoSuchNamespaceError, NoSuchTableError
|
|
18
19
|
from pyiceberg.schema import Schema, assign_fresh_schema_ids
|
|
19
20
|
from pyiceberg.table import Table
|
|
20
21
|
|
|
@@ -39,6 +40,7 @@ from floe.iceberg.merge import merge_scd1, merge_scd2
|
|
|
39
40
|
from floe.iceberg.write import (
|
|
40
41
|
SMALL_FILE_THRESHOLD_BYTES,
|
|
41
42
|
IcebergWriteResult,
|
|
43
|
+
parquet_properties,
|
|
42
44
|
write_table,
|
|
43
45
|
)
|
|
44
46
|
from floe.storage.inputs import InputFile
|
|
@@ -176,13 +178,15 @@ def write_iceberg(
|
|
|
176
178
|
primary_key = entity.schema_.primary_key or [] # validated non-empty for merges
|
|
177
179
|
key = [output.get(k.strip(), k.strip()) for k in primary_key]
|
|
178
180
|
identifier = (target.namespace, target.table)
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
exists = entity.sink.write_mode in ("merge_scd1", "merge_scd2") and catalog.table_exists(identifier) # fmt: skip
|
|
182
|
-
if exists:
|
|
181
|
+
# the configured mode: a refreshed merge writes as overwrite and keeps the key guard (#1091)
|
|
182
|
+
if entity.sink.write_mode in ("merge_scd1", "merge_scd2"):
|
|
183
183
|
with run_error("rest catalog load_table failed"):
|
|
184
|
-
|
|
185
|
-
|
|
184
|
+
try:
|
|
185
|
+
existing = set(catalog.load_table(identifier).schema().column_names)
|
|
186
|
+
except (NoSuchTableError, NoSuchNamespaceError):
|
|
187
|
+
existing = None
|
|
188
|
+
# Rust `merge/shared.rs`
|
|
189
|
+
if existing is not None and (added := [k for k in key if k not in existing]):
|
|
186
190
|
raise FloeRunError(f"entity.name={entity.name} iceberg schema evolution failed: merge key columns cannot be added: {', '.join(added)}") # fmt: skip
|
|
187
191
|
table, added = ensure_table(
|
|
188
192
|
catalog, target.namespace, target.table, schema, entity.name, target.location.removeprefix("local://"),
|
|
@@ -215,7 +219,7 @@ def write_iceberg(
|
|
|
215
219
|
threshold = SMALL_FILE_THRESHOLD_BYTES if size is None else max(1, size // 2)
|
|
216
220
|
append = "overwrite" if mode == "overwrite" else "append"
|
|
217
221
|
unique = [[output.get(c, c) for c in k] for k in resolve_unique_keys(entity.schema_.unique_keys, [c.name for c in entity.schema_.columns if c.unique], entity.schema_.primary_key)] # fmt: skip
|
|
218
|
-
result = write_table(table, reader, append, run_id, digest, threshold, unique_keys=unique) # fmt: skip
|
|
222
|
+
result = write_table(table, reader, append, run_id, digest, threshold, unique_keys=unique, parquet=parquet_properties(options and options.compression, options and options.row_group_size)) # fmt: skip
|
|
219
223
|
ids = {"table_version": result.metadata_version, "snapshot_id": result.snapshot_id}
|
|
220
224
|
fields = {k: v for k, v in ids.items() if v is not None} | {
|
|
221
225
|
"iceberg_catalog_name": target.catalog.name,
|
floe/iceberg/write.py
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
|
|
6
6
|
import itertools
|
|
7
7
|
import uuid
|
|
8
|
-
from collections.abc import Callable, Iterator, Sequence
|
|
8
|
+
from collections.abc import Callable, Iterator, Mapping, Sequence
|
|
9
9
|
from dataclasses import dataclass, replace
|
|
10
10
|
from datetime import datetime
|
|
11
11
|
from pathlib import PurePosixPath
|
|
@@ -18,9 +18,17 @@ from pyiceberg.io.pyarrow import (
|
|
|
18
18
|
_check_pyarrow_schema_compatible, # pyright: ignore[reportPrivateUsage] -- same check `Transaction.append` makes
|
|
19
19
|
_dataframe_to_data_files, # pyright: ignore[reportPrivateUsage] -- feeds one snapshot producer from many chunks
|
|
20
20
|
)
|
|
21
|
-
from pyiceberg.manifest import
|
|
21
|
+
from pyiceberg.manifest import (
|
|
22
|
+
DataFile,
|
|
23
|
+
DataFileContent,
|
|
24
|
+
ManifestEntryStatus,
|
|
25
|
+
ManifestFile,
|
|
26
|
+
)
|
|
22
27
|
from pyiceberg.table import Table, TableProperties
|
|
23
|
-
from pyiceberg.table.snapshots import ancestors_of
|
|
28
|
+
from pyiceberg.table.snapshots import Operation, ancestors_of
|
|
29
|
+
from pyiceberg.table.update.snapshot import (
|
|
30
|
+
_OverwriteFiles, # pyright: ignore[reportPrivateUsage] -- PyIceberg 0.12 has no public hook on the manifest list
|
|
31
|
+
)
|
|
24
32
|
|
|
25
33
|
from floe.errors import FloeRunError
|
|
26
34
|
from floe.iceberg.catalog import run_error
|
|
@@ -55,6 +63,14 @@ class IcebergWriteResult:
|
|
|
55
63
|
)
|
|
56
64
|
|
|
57
65
|
|
|
66
|
+
class _LiveOverwrite(_OverwriteFiles):
|
|
67
|
+
"""PyIceberg 0.12 keeps the all-DELETED manifest of every earlier overwrite in the manifest list, one more per
|
|
68
|
+
overwrite, so the scan and the commit read more manifests each time (#1132); Java drops them, so do we."""
|
|
69
|
+
|
|
70
|
+
def _existing_manifests(self) -> list[ManifestFile]:
|
|
71
|
+
return [m for m in super()._existing_manifests() if m.has_added_files() or m.has_existing_files()] # fmt: skip
|
|
72
|
+
|
|
73
|
+
|
|
58
74
|
def _chunks(reader: pa.RecordBatchReader, chunk_rows: int) -> Iterator[pa.Table]:
|
|
59
75
|
pending: list[pa.RecordBatch] = []
|
|
60
76
|
rows = 0
|
|
@@ -72,6 +88,16 @@ def _chunks(reader: pa.RecordBatchReader, chunk_rows: int) -> Iterator[pa.Table]
|
|
|
72
88
|
yield pa.Table.from_batches(pending, reader.schema)
|
|
73
89
|
|
|
74
90
|
|
|
91
|
+
def parquet_properties(
|
|
92
|
+
compression: str | None, row_group_size: int | None
|
|
93
|
+
) -> dict[str, str]:
|
|
94
|
+
"""The sink's `compression` and `row_group_size` as PyIceberg's write properties; unset keeps the table's."""
|
|
95
|
+
props = {TableProperties.PARQUET_COMPRESSION: compression} if compression else {}
|
|
96
|
+
if row_group_size:
|
|
97
|
+
props[TableProperties.PARQUET_ROW_GROUP_LIMIT] = str(row_group_size)
|
|
98
|
+
return props
|
|
99
|
+
|
|
100
|
+
|
|
75
101
|
def _metadata_version(location: str | None) -> int | None:
|
|
76
102
|
prefix = PurePosixPath(location or "").name.split("-", 1)[0]
|
|
77
103
|
return int(prefix) if prefix.isdigit() else None
|
|
@@ -86,22 +112,25 @@ def write_table(
|
|
|
86
112
|
small_file_threshold_bytes: int = SMALL_FILE_THRESHOLD_BYTES,
|
|
87
113
|
chunk_rows: int = CHUNK_ROWS,
|
|
88
114
|
unique_keys: Sequence[Sequence[str]] = (),
|
|
115
|
+
parquet: Mapping[str, str] | None = None,
|
|
89
116
|
) -> IcebergWriteResult:
|
|
90
117
|
"""Data files are written chunk by chunk (partitioned tables split each chunk), then committed once.
|
|
91
118
|
|
|
92
119
|
Overwrite replaces every existing file in the same snapshot. An append of no rows commits nothing.
|
|
93
120
|
On a conflict an append checks `unique_keys` (table column names) against the files added since the base
|
|
94
121
|
snapshot and fails closed, committing nothing, when a rival already holds a key (#549).
|
|
122
|
+
`parquet` holds `write.parquet.*` properties for these data files only; the table's own properties are not changed.
|
|
95
123
|
"""
|
|
96
124
|
with run_error("iceberg data write failed"):
|
|
97
125
|
_check_pyarrow_schema_compatible(table.schema(), reader.schema)
|
|
126
|
+
metadata = table.metadata.model_copy(update={"properties": {**table.properties, **(parquet or {})}}) # fmt: skip
|
|
98
127
|
counter = itertools.count()
|
|
99
128
|
write_uuid = uuid.uuid4()
|
|
100
129
|
data_files: list[DataFile] = [
|
|
101
130
|
data_file
|
|
102
131
|
for chunk in _chunks(reader, chunk_rows)
|
|
103
132
|
for data_file in _dataframe_to_data_files(
|
|
104
|
-
|
|
133
|
+
metadata, chunk, table.io, write_uuid, counter
|
|
105
134
|
)
|
|
106
135
|
]
|
|
107
136
|
digest = (
|
|
@@ -131,13 +160,21 @@ def write_table(
|
|
|
131
160
|
# PyIceberg retries a 409 itself (refresh, rebuild) and would skip the guard; in memory only, never sent
|
|
132
161
|
table.metadata = table.metadata.model_copy(update={"properties": {**table.properties, TableProperties.COMMIT_NUM_RETRIES: "0"}}) # fmt: skip
|
|
133
162
|
tx = table.transaction()
|
|
134
|
-
|
|
135
|
-
snapshot_properties={RUN_ID: run_id, INPUT_HASH: digest}
|
|
136
|
-
)
|
|
163
|
+
props = {RUN_ID: run_id, INPUT_HASH: digest}
|
|
137
164
|
with (
|
|
138
|
-
|
|
165
|
+
_LiveOverwrite(
|
|
166
|
+
operation=Operation.OVERWRITE
|
|
167
|
+
if table.current_snapshot()
|
|
168
|
+
else Operation.APPEND,
|
|
169
|
+
transaction=tx,
|
|
170
|
+
io=table.io,
|
|
171
|
+
branch="main",
|
|
172
|
+
snapshot_properties=props,
|
|
173
|
+
)
|
|
139
174
|
if mode == "overwrite"
|
|
140
|
-
else
|
|
175
|
+
else tx.update_snapshot(
|
|
176
|
+
snapshot_properties=props
|
|
177
|
+
).fast_append() as snap
|
|
141
178
|
):
|
|
142
179
|
for old in replaced:
|
|
143
180
|
snap.delete_data_file(old)
|
floe/run/__init__.py
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
import time
|
|
4
4
|
from collections.abc import Sequence
|
|
5
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
5
6
|
from dataclasses import dataclass
|
|
6
7
|
from datetime import UTC, datetime
|
|
7
8
|
from pathlib import Path
|
|
@@ -158,12 +159,15 @@ def run(
|
|
|
158
159
|
|
|
159
160
|
emit("run_started", config=config, report_base=report_path)
|
|
160
161
|
# Rust resolves every entity's inputs before the first entity_started (#864); run_entity reuses them
|
|
161
|
-
|
|
162
|
-
for e in selected:
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
162
|
+
# D24: a full refresh may rebuild from the archive alone, so run_entity decides
|
|
163
|
+
to_list = [e for e in selected if not (full_refresh and e.sink.archive and e.incremental_mode in ("none", "archive"))] # fmt: skip
|
|
164
|
+
# `map` yields in config order, so the first failure in config order is raised
|
|
165
|
+
if workers > 1 and len(to_list) > 1:
|
|
166
|
+
with ThreadPoolExecutor(workers, thread_name_prefix="floe-list") as pool:
|
|
167
|
+
found = list(pool.map(inputs, to_list))
|
|
168
|
+
else:
|
|
169
|
+
found = [inputs(e) for e in to_list]
|
|
170
|
+
listed = {e.id: f for e, f in zip(to_list, found, strict=True)}
|
|
167
171
|
client = storage_client(report_storage)
|
|
168
172
|
|
|
169
173
|
def finish(e: EntityConfig) -> dict[str, Any]:
|
floe/run/entity.py
CHANGED
|
@@ -402,7 +402,8 @@ def _run_file(plan: _Plan, file: InputFile, path: Path, spill: Path) -> tuple[di
|
|
|
402
402
|
def failed(err: FloeIoError, mismatch: MismatchOutcome | None) -> tuple[dict[str, Any], Iterable[pl.DataFrame]]: # fmt: skip
|
|
403
403
|
# Rust skips the raw copy under PII: the file cannot be masked
|
|
404
404
|
plan.tracker.restore(keys) # the unreadable file leaves no keys behind
|
|
405
|
-
|
|
405
|
+
# a file removed after listing has nothing to copy: the read error alone is reported
|
|
406
|
+
keep = entity.pii is None and entity.sink.rejected is not None and path.exists()
|
|
406
407
|
raw = _copy_raw(plan, path, severity, file.source_name) if keep else None
|
|
407
408
|
return _failed_file(entity, uri, len(declared) + len(derived), mismatch, err, raw), [] # fmt: skip
|
|
408
409
|
|
|
@@ -15,7 +15,7 @@ floe/checks/cast.py,sha256=lM40xtKEggWI8FW3jijM3T2JFQtaXs8FhsFXo3vy5oM,6788
|
|
|
15
15
|
floe/checks/mismatch.py,sha256=RNqRL1nihH6Y83RI-dxPXf0WTWZccXq6mnR4m6V0IIo,5827
|
|
16
16
|
floe/checks/normalize.py,sha256=9fdRYFjWGXBAEw1Po71N5tsUWG8eekmSWIQktPduOig,4431
|
|
17
17
|
floe/checks/rows.py,sha256=-yPieSus0-7ZewLv2vR8waHqQ9DGMbVvmJh1_hK4eDs,4433
|
|
18
|
-
floe/checks/unique.py,sha256=
|
|
18
|
+
floe/checks/unique.py,sha256=ZVckMOoGuKIzq3M84TGeZ1ks0nab9YTaI54w5Q_rjZc,10864
|
|
19
19
|
floe/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
20
20
|
floe/config/entity.py,sha256=CLXLXzkUbQhxenD_LwjBt0eoPreaYaxiBVEsdmnFIco,15883
|
|
21
21
|
floe/config/include.py,sha256=XKYhp2QK_VUmAODAXJ35Wl4FCpQEMQm24m-WKlpI_og,5033
|
|
@@ -27,12 +27,12 @@ floe/config/types.py,sha256=4eveoOlyGmNxyZpI1PSjli9lnXLPc1-jYj4fL8qHN_8,5038
|
|
|
27
27
|
floe/config/validate.py,sha256=7tyofwoTBL-BBCNxnmkXeSOybwmMNngeEcj8zviRyLs,36176
|
|
28
28
|
floe/config/vars.py,sha256=7Bq0IU0KrFeRlyA8W9-F0gDSDKfXCfFpBrX08H8Flvw,4577
|
|
29
29
|
floe/config/yaml.py,sha256=HeFGhKFxYkiqQOwljYOgUD5naJyxL_ZIX6hUIS36tzU,10071
|
|
30
|
-
floe/iceberg/catalog.py,sha256=
|
|
30
|
+
floe/iceberg/catalog.py,sha256=qEhMvH4NFBd4GxqBEg4jqgTNxtqWkL_RRzIlgXr2GmA,16722
|
|
31
31
|
floe/iceberg/compact.py,sha256=5O372vAIsvgRc2PGihryAsrQ0otn0F30AQu-cetRCP0,7123
|
|
32
32
|
floe/iceberg/merge.py,sha256=VE1Odf2B34Gxj7ZcuhbrfbgTGM9AdiDz6_2BbcQ-kSQ,12448
|
|
33
|
-
floe/iceberg/sink.py,sha256=
|
|
33
|
+
floe/iceberg/sink.py,sha256=vdkgLjqvWH7opOrfc-147xCMQhLnxRjUzLjHLchHlAM,12043
|
|
34
34
|
floe/iceberg/unique.py,sha256=OmiPQthhB5jGTaXsCu9ZztNFEFUf6eT0MLU2Gnh-C2A,1095
|
|
35
|
-
floe/iceberg/write.py,sha256=
|
|
35
|
+
floe/iceberg/write.py,sha256=YNE8IwLV9jK3vksO0XCj36HKHqW6ZEHCu4173Xpi6hg,12854
|
|
36
36
|
floe/lineage/events.py,sha256=4txd8bGUwKBZd-8Aw_ebTurhlxAYk8SVMgH8vSjC3bs,6682
|
|
37
37
|
floe/lineage/observer.py,sha256=IFHjzsT65L-NkcQ8Noa_8CDVpLu4iGadRB2-K0Lj8t8,10216
|
|
38
38
|
floe/read/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -45,8 +45,8 @@ floe/read/xlsx.py,sha256=dengSY8HuNh9ahyR_LR8beqXTZQubqQ1Jx8qEak5jrY,3009
|
|
|
45
45
|
floe/read/xml.py,sha256=7m6DtoU8iExuLROavyojysqOUE_Bd_N0uwFaLAzjI_Y,8004
|
|
46
46
|
floe/report/entity.py,sha256=FRwLtXrlJXDNXrOKscitm-NCNJWQXd0vr_JIf_cUp20,9867
|
|
47
47
|
floe/report/summary.py,sha256=tb547VRkM5WGWHQ-4vZB2DUW4VTWgv9FUoym-NpuWE4,4452
|
|
48
|
-
floe/run/__init__.py,sha256=
|
|
49
|
-
floe/run/entity.py,sha256=
|
|
48
|
+
floe/run/__init__.py,sha256=yCLCoLbtA_Iuy7wabkFSImeuU8xrLp8Q0ZgYmw8WDRg,9229
|
|
49
|
+
floe/run/entity.py,sha256=2NxCKNVUn0v2Oat4FslZfNjWAu3YhGKnSks3NRwkexU,49764
|
|
50
50
|
floe/run/events.py,sha256=OE7zYkt0Dz2xrvgNavT1nHfZJ-97IIJ8LpHcameUJZU,2859
|
|
51
51
|
floe/run/parallel.py,sha256=lKBW9NOpOu6KFOcEo-RQ56ukEOKHXPPoLI3o6gmlY_g,3435
|
|
52
52
|
floe/storage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -61,8 +61,8 @@ floe/storage/s3.py,sha256=XcG1qJTEY3KLSMLT950mUdrAxuCA8-Gx3fpNSzyV_Xo,7508
|
|
|
61
61
|
floe/storage/types.py,sha256=FvAEXWMdVdopRK276JyETRYGOva2-uR8pZvsWj6Xvy0,993
|
|
62
62
|
floe/write/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
63
63
|
floe/write/outputs.py,sha256=cXxGUIJnZ38XPaqEI6a83GFUsapBRdY52XQm3GlUPcI,6937
|
|
64
|
-
floe_python-0.8.
|
|
65
|
-
floe_python-0.8.
|
|
66
|
-
floe_python-0.8.
|
|
67
|
-
floe_python-0.8.
|
|
68
|
-
floe_python-0.8.
|
|
64
|
+
floe_python-0.8.9.dist-info/METADATA,sha256=jwfiIw93AklPynvV_-ABBNgnNCZjBArFMK4APsKu9Bk,442
|
|
65
|
+
floe_python-0.8.9.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
66
|
+
floe_python-0.8.9.dist-info/entry_points.txt,sha256=DU75r-E3zcK0quvwg5iT5DeWbUJck_Tz80bEj1IIzHA,40
|
|
67
|
+
floe_python-0.8.9.dist-info/licenses/LICENSE,sha256=dUhuoK-TCRQMpuLEAdfme-qPSJI0TlcH9jlNxeg9_EQ,1056
|
|
68
|
+
floe_python-0.8.9.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|