floe-python 0.8.0__py3-none-any.whl

floe/write/outputs.py

@@ -0,0 +1,121 @@
+"""Accepted Parquet and rejected CSV writers (Rust: `io/write/{parquet,csv,parts}.rs`); `base` is a `local://` or `s3://` URI."""
+
+import io
+import math
+import re
+import uuid
+from collections.abc import Generator, Iterator
+from contextlib import contextmanager
+from dataclasses import dataclass
+from itertools import count
+
+import polars as pl
+
+from floe.config.types import SinkOptions, WriteMode
+from floe.errors import FloeError, FloeStorageError
+from floe.storage.paths import parse_s3_uri
+from floe.storage.types import StorageClient
+
+DEFAULT_MAX_SIZE_PER_FILE = 256 * 1024 * 1024
+MAX_REPORTED_PARTS = 50
+_MIB = 1024 * 1024
+_PART = re.compile(r"part-.+")
+
+
+@contextmanager
+def _os_errors() -> Generator[None]:
+    """An `OSError` while writing becomes a bare `FloeError`: Rust surfaces a raw `io::Error` here, which logs with code `error`."""
+    try:
+        yield
+    except (
+        OSError,
+        FloeStorageError,
+    ) as e:  # the storage client wraps the OSError; Rust prints the bare io::Error
+        os = e if isinstance(e, OSError) else e.__cause__
+        if not isinstance(os, OSError):
+            raise
+        raise FloeError(f"{os.strerror} (os error {os.errno})" if os.errno is not None else str(os)) from os  # fmt: skip
+
+
+@dataclass(frozen=True)
+class AcceptedWrite:
+    """Rust's accepted write output; the byte metrics are recorded, not compared (G5)."""
+
+    files_written: int
+    part_files: list[str]
+    total_bytes_written: int
+    avg_file_size_mb: float | None
+    small_files_count: int
+
+
+def part_uris(client: StorageClient, base: str, ext: str) -> list[str]:
+    """The `part-*.<ext>` files directly under `base`, sorted."""
+    key = parse_s3_uri(base)[1] if base.startswith("s3://") else None
+    prefix = (
+        base.removeprefix("local://") if key is None else key and f"{key}/"
+    )  # a bucket root lists every key
+    uris = (r.uri for r in client.list(prefix))
+    return sorted(u for u in uris if u.rpartition("/")[0] == base and u.endswith(f".{ext}") and _PART.fullmatch(u.rpartition("/")[2].removesuffix(f".{ext}")))  # fmt: skip
+
+
+def _part_names(client: StorageClient, base: str, mode: WriteMode, ext: str) -> Iterator[str]:  # fmt: skip
+    """Overwrite clears the old parts and numbers from 0 (`part-00000`); other modes use `part-<uuid>`."""
+    if mode != "overwrite":
+        while True:
+            yield f"part-{uuid.uuid4()}.{ext}"
+    for uri in part_uris(client, base, ext):
+        client.delete(uri)
+    for i in count():
+        yield f"part-{i:05}.{ext}"
+
+
+def write_accepted(
+    df: pl.DataFrame,
+    client: StorageClient,
+    base: str,
+    mode: WriteMode = "overwrite",
+    options: SinkOptions | None = None,
+) -> AcceptedWrite:
+    """Write `df` as `part-*.parquet` files under `base`, split so each holds about `max_size_per_file`."""
+    base = base.rstrip("/")
+    names = _part_names(client, base, mode, "parquet")
+    max_size = (options and options.max_size_per_file) or DEFAULT_MAX_SIZE_PER_FILE
+    avg_row = max(1, math.ceil(df.estimated_size() / df.height)) if df.height else 1
+    rows = max(1, max_size // avg_row)
+    kwargs: dict[str, str | int] = {}
+    if options and options.compression:
+        kwargs["compression"] = options.compression
+    if options and options.row_group_size:
+        kwargs["row_group_size"] = options.row_group_size
+    files: list[str] = []
+    sizes: list[int] = []
+    with _os_errors():
+        for chunk in [df.slice(i, rows) for i in range(0, df.height, rows)] or [df]:
+            name = next(names)
+            # ponytail: a part is held in memory (≤ max_size_per_file) before the put; stream multipart uploads if that is too much
+            body = io.BytesIO()
+            chunk.write_parquet(body, **kwargs)  # pyright: ignore[reportArgumentType]  # ponytail: kwargs loses the compression Literal; validated by the config
+            client.write(f"{base}/{name}", body.getvalue())
+            files.append(name)
+            sizes.append(body.tell())
+    total = sum(sizes)
+    return AcceptedWrite(
+        len(files),
+        files[:MAX_REPORTED_PARTS],
+        total,
+        total / len(files) / _MIB,
+        sum(s < max(1, max_size // 2) for s in sizes),
+    )
+
+
+def write_rejected(df: pl.DataFrame, client: StorageClient, base: str, mode: WriteMode = "overwrite", key: str | None = None) -> str:  # fmt: skip
+    """Write the rejected frame from `split_rows` as one CSV part under `base`; returns its URI.
+
+    With a `key` (the run's input hash and the file) a non-overwrite part is named from it, so a retry rewrites the same part (S6, #919); Rust has no retry guard.
+    """
+    with _os_errors():
+        base = base.rstrip("/")
+        name = f"part-{uuid.uuid5(uuid.NAMESPACE_URL, key)}.csv" if key and mode != "overwrite" else next(_part_names(client, base, mode, "csv"))  # fmt: skip
+        uri = f"{base}/{name}"
+        client.write(uri, df.write_csv().encode())
+    return uri

floe_python-0.8.0.dist-info/METADATA

@@ -0,0 +1,15 @@
+Metadata-Version: 2.5
+Name: floe-python
+Version: 0.8.0
+Summary: Floe: a data-contract runtime for file ingestion
+License-File: LICENSE
+Requires-Python: >=3.11
+Requires-Dist: boto3>=1.43.118
+Requires-Dist: duckdb==1.5.6
+Requires-Dist: fastexcel>=0.21.0
+Requires-Dist: polars>=2.0.0
+Requires-Dist: pyarrow>=26.0.0
+Requires-Dist: pydantic>=2.14.1
+Requires-Dist: pyiceberg-core>=0.10.1
+Requires-Dist: pyiceberg>=0.12.0
+Requires-Dist: pyyaml>=6.0.3

floe_python-0.8.0.dist-info/RECORD

@@ -0,0 +1,66 @@
+floe/__init__.py,sha256=VkSVWF7vzIICTkxaNI2SVXfD2JQmrHtV_NiTBCUwzaM,13221
+floe/__main__.py,sha256=Nzz6wrGXRpJiO59qlX7WEJRTlb-XcRFswmP0TA2KjyM,57
+floe/_cli.py,sha256=2ecJIMELxfXkm3ISzvQsS-CnnjUBj1Kqh9HANlUlMCM,13549
+floe/_cli_output.py,sha256=fwUqMkBvUEojri8WnXGzTIHMZLcoOHQfByeOz5L6ugA,8792
+floe/add_entity.py,sha256=42b7l8o55dzgaR_DDgpW7YWdnU2weYMf4U_rf7qCY6M,21986
+floe/errors.py,sha256=hLSkgPQATBnjY1CAQqX4aB0oI0-WKoHW09bDXAgzC7k,186
+floe/glob.py,sha256=A0HdePLXLGl1T6heJbgdrNHr8pZfdidG7pQ1QsXWOl8,10579
+floe/maintain.py,sha256=CGSoZhO7p0u14o10FCKCO4FprzpVO548A71eq5FowF8,7343
+floe/manifest.py,sha256=HV3y-WzZeorVpTAGrNU9LuZCbvX8ALfGC9J3Q2jtnYY,19425
+floe/manifest_run.py,sha256=8m4LXaymkw3ZX706sTjZK5L1hOYDT231CWl9tjoBe1c,12700
+floe/pii.py,sha256=uJjU-C4c7h45E7rEL9K3Zp1LTTLh-FNKfUV5av2QMO4,4351
+floe/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+floe/state.py,sha256=XFHF6zVxbdGg1bkONZDLryTD4noDF9IgaimaEI-3Nlg,11104
+floe/checks/cast.py,sha256=lM40xtKEggWI8FW3jijM3T2JFQtaXs8FhsFXo3vy5oM,6788
+floe/checks/mismatch.py,sha256=2Gz-tiWKxy-iehSjAlsGzPLtYeNIfBWvi4yQPlASVYQ,5309
+floe/checks/normalize.py,sha256=INYh3NO3JkdgDjTXDi7pylr79nzrJmDuBlrIrm_8LRc,4326
+floe/checks/rows.py,sha256=E-YUZUvBUyOB7zat2dtbPWFM-0T1-01lYKuQU1A6wto,4156
+floe/checks/unique.py,sha256=BUu6ezLtQKj8_9_SybGND6rxbUbzlmqh_WJomTHRfGw,9782
+floe/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+floe/config/entity.py,sha256=CLXLXzkUbQhxenD_LwjBt0eoPreaYaxiBVEsdmnFIco,15883
+floe/config/include.py,sha256=XKYhp2QK_VUmAODAXJ35Wl4FCpQEMQm24m-WKlpI_og,5033
+floe/config/parse.py,sha256=B3gOi80-M0y9Y4OyNjM4MYmM2mFHKGVpqipX1JqaLjM,5290
+floe/config/profile.py,sha256=xlLbrMbD_prJ0UteoSUNwbmjOq5NMm6XmQvxQNg4WxI,14035
+floe/config/project.py,sha256=zgTOhTDlbMr1chViMEZyv0drozZdumTaIoKDL0vv9sw,14141
+floe/config/storage.py,sha256=Yjia1mfxWvIq7DpS24AFSCQDeJ4aqxY9s76WDDrqMv4,5452
+floe/config/types.py,sha256=4eveoOlyGmNxyZpI1PSjli9lnXLPc1-jYj4fL8qHN_8,5038
+floe/config/validate.py,sha256=4m71oZgvKFH61Ts6TASlLZ_JDnPEjWo4cfGcoJOLGQY,34963
+floe/config/vars.py,sha256=7Bq0IU0KrFeRlyA8W9-F0gDSDKfXCfFpBrX08H8Flvw,4577
+floe/config/yaml.py,sha256=HeFGhKFxYkiqQOwljYOgUD5naJyxL_ZIX6hUIS36tzU,10071
+floe/iceberg/catalog.py,sha256=nRRC4VyjKEXATlifmlE2394e8b5vAO_HiIp0LugS5tI,13819
+floe/iceberg/compact.py,sha256=5O372vAIsvgRc2PGihryAsrQ0otn0F30AQu-cetRCP0,7123
+floe/iceberg/merge.py,sha256=VE1Odf2B34Gxj7ZcuhbrfbgTGM9AdiDz6_2BbcQ-kSQ,12448
+floe/iceberg/sink.py,sha256=2pWrixvqC9_ZNKvGAJju4-TfKOqFoYR28agllUt9AXA,10804
+floe/iceberg/unique.py,sha256=PKUFEcCkU71k66__KfUE7NTos73ZYlZbb8GJPn0yFoo,2958
+floe/iceberg/write.py,sha256=zrFeeJu-YT4Pxo9u-fRujUtFb1Epx1KFoq2qFyP1iGs,11036
+floe/lineage/events.py,sha256=4txd8bGUwKBZd-8Aw_ebTurhlxAYk8SVMgH8vSjC3bs,6682
+floe/lineage/observer.py,sha256=dDYfWvPpFGjdqvFqQDZ2pTjj7uDfqji2-BTID9hzg90,10075
+floe/read/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+floe/read/avro.py,sha256=rhGYlrWGFnqt_tyLIph57qV2TH91ULjoJ5qUlpW_6jI,1419
+floe/read/csv.py,sha256=5nXUGBa2s2Tt790UvOcrvfDym906JYjyhgunV6xPzr4,5153
+floe/read/json.py,sha256=dPUXNlQUmMeYId3lef8ssEQeHi0tIOoEblvMQDmNVt4,8284
+floe/read/orc.py,sha256=TWgdlLBR2NF_oVZ5zEmRKNXoTkq5QfF2BOFzKyzSjhY,1553
+floe/read/parquet.py,sha256=JwfuWaHj2TxBKJeaKignq8p0Op_pUfsOdVh7dDC3wcE,1491
+floe/read/xlsx.py,sha256=dengSY8HuNh9ahyR_LR8beqXTZQubqQ1Jx8qEak5jrY,3009
+floe/read/xml.py,sha256=7m6DtoU8iExuLROavyojysqOUE_Bd_N0uwFaLAzjI_Y,8004
+floe/report/entity.py,sha256=ekuB-1JDATvxIFnBTCQpku8uXtrSQNwvryjdTidXk6I,9707
+floe/report/summary.py,sha256=tb547VRkM5WGWHQ-4vZB2DUW4VTWgv9FUoym-NpuWE4,4452
+floe/run/__init__.py,sha256=PON2NYpvsRej6cXBPP8GDRZMcinVTtnKpwLT93Vzg0c,7687
+floe/run/entity.py,sha256=4UGhUtjYR84MOe3jjwFmLGWAaDln1z5KOHC0MOvDLpQ,45204
+floe/run/events.py,sha256=OE7zYkt0Dz2xrvgNavT1nHfZJ-97IIJ8LpHcameUJZU,2859
+floe/storage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+floe/storage/archive.py,sha256=1gPImm9PDSJFCpWjg37ax4ubb0wwc6FZm38quQXhORg,1101
+floe/storage/inputs.py,sha256=nmgEo2ub3hPUghLUXHU1wnKzpL4aYZbHb6S0XPYvMew,10410
+floe/storage/local.py,sha256=NKjBf_-PkDnbZ3FzXWE7KZ1xY0G_o3tRy0_7nxW7FLs,6307
+floe/storage/partitioning.py,sha256=MG8RJDrwgZhucTswlXC2RwCmzrYIoOE6SH_aBM11KW8,6178
+floe/storage/paths.py,sha256=Z0fry3Gp0Tu2DpUeKqNXdqYP6n_KcKzK2llGx0TGHB8,2343
+floe/storage/resolve.py,sha256=pFAuVytTfnSEcTAVlJjFTj7S_Vipjry1ormWZP3Egmc,2593
+floe/storage/s3.py,sha256=iWLDr3-qFH-Of3HKUiaKFD-1HQRip4FSQ-q9xxv7VNI,7212
+floe/storage/types.py,sha256=FvAEXWMdVdopRK276JyETRYGOva2-uR8pZvsWj6Xvy0,993
+floe/write/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+floe/write/outputs.py,sha256=5LgN_8PMPPOgkRVZw_KvxYXmc02-meL7iZLeMKVtufQ,4932
+floe_python-0.8.0.dist-info/METADATA,sha256=jQyy8j3vSQ6Qs047OQASPHOEnq5UV02wpXavZ3MHf7U,444
+floe_python-0.8.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+floe_python-0.8.0.dist-info/entry_points.txt,sha256=DU75r-E3zcK0quvwg5iT5DeWbUJck_Tz80bEj1IIzHA,40
+floe_python-0.8.0.dist-info/licenses/LICENSE,sha256=dUhuoK-TCRQMpuLEAdfme-qPSJI0TlcH9jlNxeg9_EQ,1056
+floe_python-0.8.0.dist-info/RECORD,,

floe_python-0.8.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.33.0
+Root-Is-Purelib: true
+Tag: py3-none-any

floe_python-0.8.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+floe = floe._cli:main

floe_python-0.8.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.