floe-python 0.8.1__py3-none-any.whl → 0.8.3__py3-none-any.whl

floe/_cli.py

@@ -40,7 +40,7 @@ def _realpath(path: str) -> Path:
 def _profile(location: str | None) -> ProfileConfig | None:
     if not location:
         return None
-    return parse_profile(location if location.startswith("s3://") else _realpath(location))  # fmt: skip
+    return parse_profile(location if location.startswith(("s3://", "env:")) else _realpath(location))  # fmt: skip
 
 
 def _load(path: Path, profile: ProfileConfig | None) -> RootConfig:
@@ -83,7 +83,7 @@ def _validate(args: argparse.Namespace) -> int:
         print(f"  Severity: {e.policy.severity}")
     profile_arg = ""
     if args.profile:
-        profile_arg = f" -p {args.profile if args.profile.startswith('s3://') else os.path.realpath(args.profile)}"  # fmt: skip
+        profile_arg = f" -p {args.profile if args.profile.startswith(('s3://', 'env:')) else os.path.realpath(args.profile)}"  # fmt: skip
     print(f"Next: floe run -c {path}{profile_arg}")
     return 0
 

floe/config/profile.py

@@ -1,5 +1,6 @@
 """Environment profiles, ported from `profile/parse.rs` and `profile/validate.rs`."""
 
+import os
 from collections.abc import Callable
 from dataclasses import dataclass
 from pathlib import Path
@@ -404,7 +405,20 @@ def apply_profile(config: RootConfig, profile: ProfileConfig) -> RootConfig:
     return config.model_copy(update={k: v for k, v in blocks.items() if v is not None})
 
 
+ENV_PREFIX = "env:"
+
+
+def read_env_profile(location: str) -> str:
+    """`env:NAME`: the profile YAML held in environment variable `NAME`."""
+    name = location.removeprefix(ENV_PREFIX)
+    if not (text := os.environ.get(name, "")).strip():
+        raise FloeConfigError(f"profile environment variable {name} is unset or empty")
+    return text
+
+
 def _docs(location: str | Path) -> list[object]:
+    if isinstance(location, str) and location.startswith(ENV_PREFIX):
+        return parse_yaml(read_env_profile(location))
     if isinstance(location, Path) or not location.startswith("s3://"):
         return load_yaml(Path(location))
     try:

floe/config/validate.py

@@ -330,6 +330,8 @@ def _check_iceberg(
         raise _fail(
             entity, f"sink.accepted.iceberg.catalog references unknown catalog {name}"
         )
+    if definition.type == "glue" and "{domain}" in (definition.database or "") and not (iceberg.namespace or entity.domain):  # fmt: skip
+        raise _fail(entity, f"sink.accepted.iceberg.namespace or domain is required for catalog {name} database containing {{domain}}")  # fmt: skip
     # Unity is rejected at parse (D1); the registry checked `warehouse_storage`.
     if definition.type == "glue" and entity.sink.write_mode.startswith("merge"):
         # ponytail: merges run through DuckDB, attached to REST catalogs only; attach Glue when a merge needs it

floe/iceberg/catalog.py

@@ -2,6 +2,9 @@
 
 import os
 import re
+import shutil
+import tempfile
+import weakref
 from collections.abc import Generator, Sequence
 from contextlib import contextmanager
 from urllib.parse import urlsplit
@@ -26,6 +29,11 @@ _TRANSFORMS = frozenset({"identity", "year", "month", "day", "hour"})
 # first id of a partition field, as in the Iceberg spec
 _PARTITION_FIELD_ID_START = 1000
 DUCKDB_CATALOG = "floe_catalog"
+# the ADR's limits (python-core-for-olf.md, "Supported limits"); DuckDB's own default is 80% of the
+# host's RAM, not the cgroup limit, so a Kubernetes pod is OOM-killed instead of spilling.
+# Size pods at the memory limit + 2 GiB. The spill bound fails a runaway merge before it fills the disk.
+DUCKDB_MEMORY_LIMIT = "4GB"
+DUCKDB_MAX_TEMP_DIRECTORY_SIZE = "50GB"
 
 
 def transform_name(field: PartitionField) -> str:
@@ -174,8 +182,10 @@ def open_duckdb(
         raise FloeConfigError(f"iceberg catalog {definition.name}: merges and compaction require a rest catalog (got {definition.type})")  # fmt: skip
     statements = _duckdb_setup(definition, storage)
     with run_error("duckdb catalog attach failed"):
-        # ponytail: DuckDB's default memory_limit and spill dir; set the ADR's limits when the run wires merges
-        con = duckdb.connect()
+        spill = tempfile.mkdtemp(prefix="floe-duckdb-")
+        con = duckdb.connect(config={"memory_limit": DUCKDB_MEMORY_LIMIT, "temp_directory": spill, "max_temp_directory_size": DUCKDB_MAX_TEMP_DIRECTORY_SIZE})  # fmt: skip
+        # the spill directory lives as long as the connection (also on `with` exit and at interpreter exit)
+        weakref.finalize(con, shutil.rmtree, spill, True)
         for statement in statements:
             con.execute(statement)
     return con

floe/iceberg/sink.py

@@ -70,9 +70,12 @@ def resolve_target(storages: StoragesConfig | None, catalogs: CatalogsConfig | N
     catalog = next((c for c in (catalogs.definitions if catalogs else []) if c.name == name), None)  # fmt: skip
     if catalog is None:  # validated; kept for callers that skip validation
         raise FloeConfigError(f"entity.name={entity.name} sink.accepted.iceberg.catalog references unknown catalog {name}")  # fmt: skip
-    namespace = catalog_ident(
-        sink.iceberg.namespace or entity.domain or catalog.database or "default"
-    )
+    # what `{domain}` in the database stands for (validated: set when the database has the token)
+    scope = sink.iceberg.namespace or entity.domain
+    database = catalog.database
+    if database is not None:
+        database = database.replace("{domain}", catalog_ident(scope or ""))
+    namespace = catalog_ident(scope or database or "default")
     table = catalog_ident(sink.iceberg.table or entity.name)
     field = f"entity.name={entity.name} sink.accepted.storage"
     accepted_uri, storage = resolve_path(storages, config_dir, sink.storage, sink.path, field)  # fmt: skip
@@ -85,8 +88,8 @@ def resolve_target(storages: StoragesConfig | None, catalogs: CatalogsConfig | N
     else:
         location = accepted_uri
     # Rust: the Glue database holds the table; the namespace only names its location
-    if catalog.database is not None:
-        namespace = catalog.database
+    if database is not None:
+        namespace = database
     return IcebergTarget(catalog, storage, namespace, table, location, accepted_uri)  # fmt: skip
 
 

floe/manifest.py

@@ -17,7 +17,7 @@ from typing import Any, NamedTuple, cast
 from pydantic import BaseModel
 
 from floe import __version__
-from floe.config.profile import ProfileConfig
+from floe.config.profile import ENV_PREFIX, ProfileConfig, read_env_profile
 from floe.config.project import ReportConfig, RootConfig
 from floe.config.storage import StoragesConfig
 from floe.config.types import ArchiveTarget, EntityConfig, SinkTarget, SourceConfig
@@ -464,7 +464,10 @@ def build_manifest(
     profile_uri = profile_checksum = None
     if options.profile_arg is not None:  # recorded like `config_uri` and its checksum
         # an s3 profile is read from there by the run, so it is recorded as typed
-        if options.profile_arg.startswith("s3://"):
+        if options.profile_arg.startswith(ENV_PREFIX):  # the Job reads the variable too
+            profile_uri = options.profile_arg
+            profile_checksum = _sha256(read_env_profile(profile_uri).encode())
+        elif options.profile_arg.startswith("s3://"):
             profile_uri = options.profile_arg
             profile_checksum = _sha256(read_s3(profile_uri, "profile"))
         else:

floe/storage/local.py

@@ -101,6 +101,7 @@ class LocalClient:
         try:
             dst.parent.mkdir(parents=True, exist_ok=True)
             mode = src.stat().st_mode & 0o777
+            dst.unlink(missing_ok=True)  # not through a symlink or shared inode
             fd = os.open(dst, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, mode)
             os.fchmod(fd, mode)  # #779: the open mode applies on create only
             with src.open("rb") as fin, os.fdopen(fd, "wb") as fout:

floe/write/outputs.py

@@ -4,10 +4,9 @@ import io
 import math
 import re
 import uuid
-from collections.abc import Generator, Iterator
-from contextlib import contextmanager
+from collections.abc import Callable, Generator
+from contextlib import contextmanager, suppress
 from dataclasses import dataclass
-from itertools import count
 
 import polars as pl
 
@@ -58,15 +57,46 @@ def part_uris(client: StorageClient, base: str, ext: str) -> list[str]:
     return sorted(u for u in uris if u.rpartition("/")[0] == base and u.endswith(f".{ext}") and _PART.fullmatch(u.rpartition("/")[2].removesuffix(f".{ext}")))  # fmt: skip
 
 
-def _part_names(client: StorageClient, base: str, mode: WriteMode, ext: str) -> Iterator[str]:  # fmt: skip
-    """Overwrite clears the old parts and numbers from 0 (`part-00000`); other modes use `part-<uuid>`."""
+@contextmanager
+def _parts(client: StorageClient, base: str, mode: WriteMode, ext: str) -> Generator[Callable[[bytes], str]]:  # fmt: skip
+    """Yield `put(body) -> name`. Non-overwrite modes write `part-<uuid>` directly.
+
+    Overwrite stages every part as `_staging-<id>-part-NNNNN` (invisible to `part_uris`) and
+    only when all were written copies them over `part-NNNNN`, deletes the old parts that no new
+    part replaced, then the staged copies (#1054). A failed write leaves the old parts untouched.
+    """
     if mode != "overwrite":
-        while True:
-            yield f"part-{uuid.uuid4()}.{ext}"
-    for uri in part_uris(client, base, ext):
+
+        def put(body: bytes) -> str:
+            name = f"part-{uuid.uuid4()}.{ext}"
+            client.write(f"{base}/{name}", body)
+            return name
+
+        yield put
+        return
+    stage = f"_staging-{uuid.uuid4()}-"
+    names: list[str] = []
+
+    def put_staged(body: bytes) -> str:
+        name = f"part-{len(names):05}.{ext}"
+        names.append(name)  # before the write: a partial file needs cleanup too
+        client.write(f"{base}/{stage}{name}", body)
+        return name
+
+    try:
+        yield put_staged
+    except BaseException:
+        for name in names:
+            with suppress(FloeError, OSError):  # best effort: the original error wins
+                client.delete(f"{base}/{stage}{name}")
+        raise
+    old = part_uris(client, base, ext)
+    for name in names:
+        client.copy(f"{base}/{stage}{name}", f"{base}/{name}")
+    for uri in set(old) - {f"{base}/{n}" for n in names}:
         client.delete(uri)
-    for i in count():
-        yield f"part-{i:05}.{ext}"
+    for name in names:
+        client.delete(f"{base}/{stage}{name}")
 
 
 def write_accepted(
@@ -78,7 +108,6 @@ def write_accepted(
 ) -> AcceptedWrite:
     """Write `df` as `part-*.parquet` files under `base`, split so each holds about `max_size_per_file`."""
     base = base.rstrip("/")
-    names = _part_names(client, base, mode, "parquet")
     max_size = (options and options.max_size_per_file) or DEFAULT_MAX_SIZE_PER_FILE
     avg_row = max(1, math.ceil(df.estimated_size() / df.height)) if df.height else 1
     rows = max(1, max_size // avg_row)
@@ -89,14 +118,12 @@ def write_accepted(
         kwargs["row_group_size"] = options.row_group_size
     files: list[str] = []
     sizes: list[int] = []
-    with _os_errors():
+    with _os_errors(), _parts(client, base, mode, "parquet") as put:
         for chunk in [df.slice(i, rows) for i in range(0, df.height, rows)] or [df]:
-            name = next(names)
             # ponytail: a part is held in memory (≤ max_size_per_file) before the put; stream multipart uploads if that is too much
             body = io.BytesIO()
             chunk.write_parquet(body, **kwargs)  # pyright: ignore[reportArgumentType]  # ponytail: kwargs loses the compression Literal; validated by the config
-            client.write(f"{base}/{name}", body.getvalue())
-            files.append(name)
+            files.append(put(body.getvalue()))
             sizes.append(body.tell())
     total = sum(sizes)
     return AcceptedWrite(
@@ -115,7 +142,10 @@ def write_rejected(df: pl.DataFrame, client: StorageClient, base: str, mode: Wri
     """
     with _os_errors():
         base = base.rstrip("/")
-        name = f"part-{uuid.uuid5(uuid.NAMESPACE_URL, key)}.csv" if key and mode != "overwrite" else next(_part_names(client, base, mode, "csv"))  # fmt: skip
-        uri = f"{base}/{name}"
-        client.write(uri, df.write_csv().encode())
-    return uri
+        if key and mode != "overwrite":
+            name = f"part-{uuid.uuid5(uuid.NAMESPACE_URL, key)}.csv"
+            client.write(f"{base}/{name}", df.write_csv().encode())
+        else:
+            with _parts(client, base, mode, "csv") as put:
+                name = put(df.write_csv().encode())
+    return f"{base}/{name}"

{floe_python-0.8.1.dist-info → floe_python-0.8.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: floe-python
-Version: 0.8.1
+Version: 0.8.3
 Summary: Floe: a data-contract runtime for file ingestion
 License-File: LICENSE
 Requires-Python: >=3.11

{floe_python-0.8.1.dist-info → floe_python-0.8.3.dist-info}/RECORD

@@ -1,12 +1,12 @@
 floe/__init__.py,sha256=0X9pm8uWiwoDslKZyyjsEAYd6g1mlvIWhkWgE5uhGng,13279
 floe/__main__.py,sha256=Nzz6wrGXRpJiO59qlX7WEJRTlb-XcRFswmP0TA2KjyM,57
-floe/_cli.py,sha256=6zQRRczAgARgc7PcLrWS7lEzJa6GKsbpf8Zz8GsDjFo,14081
+floe/_cli.py,sha256=hwast3uMX0iUtkIpcChSIVHYkoYi4fR4jwCwQT53lXE,14101
 floe/_cli_output.py,sha256=VLxWQr-EPtIwklo9xr2CnjMD17U-7JrMUWdTtAP3jrY,8809
 floe/add_entity.py,sha256=42b7l8o55dzgaR_DDgpW7YWdnU2weYMf4U_rf7qCY6M,21986
 floe/errors.py,sha256=hLSkgPQATBnjY1CAQqX4aB0oI0-WKoHW09bDXAgzC7k,186
 floe/glob.py,sha256=A0HdePLXLGl1T6heJbgdrNHr8pZfdidG7pQ1QsXWOl8,10579
 floe/maintain.py,sha256=CGSoZhO7p0u14o10FCKCO4FprzpVO548A71eq5FowF8,7343
-floe/manifest.py,sha256=GDaa01K9n5vxI0kRQSkM_xjjIwoeF6JN-N1Fktas8Xc,20216
+floe/manifest.py,sha256=sAWjv6yS4GokQDTnZwutyplLz1ICPBYl2qJmm9IORdM,20462
 floe/manifest_run.py,sha256=1XLInkpaCofMezMBs_5uIFoR5W7lbwCBFZeqRcnwUAw,12398
 floe/pii.py,sha256=uJjU-C4c7h45E7rEL9K3Zp1LTTLh-FNKfUV5av2QMO4,4351
 floe/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -20,17 +20,17 @@ floe/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 floe/config/entity.py,sha256=CLXLXzkUbQhxenD_LwjBt0eoPreaYaxiBVEsdmnFIco,15883
 floe/config/include.py,sha256=XKYhp2QK_VUmAODAXJ35Wl4FCpQEMQm24m-WKlpI_og,5033
 floe/config/parse.py,sha256=B3gOi80-M0y9Y4OyNjM4MYmM2mFHKGVpqipX1JqaLjM,5290
-floe/config/profile.py,sha256=mYbyFse2pBUFuV5TMtl0RFrfyPUnJhdklbAX5eSAM88,15564
+floe/config/profile.py,sha256=jWackztS-0H3dTMXHALTZ7LsIOmwjriwoeY4vFgARr0,16046
 floe/config/project.py,sha256=zgTOhTDlbMr1chViMEZyv0drozZdumTaIoKDL0vv9sw,14141
 floe/config/storage.py,sha256=BK3P65ukFqNgmbjCcuYmXR04OScf1yD8f19vGJq-aDI,6234
 floe/config/types.py,sha256=4eveoOlyGmNxyZpI1PSjli9lnXLPc1-jYj4fL8qHN_8,5038
-floe/config/validate.py,sha256=MRwGztx0xbcqGjpIKjfCMDMeyTz-pd4Mr9XrfyiW5CQ,35571
+floe/config/validate.py,sha256=-cDn9CKfR1BmAATgYy1PWlTWSkpu4mjt4PMKNL7k2FE,35858
 floe/config/vars.py,sha256=7Bq0IU0KrFeRlyA8W9-F0gDSDKfXCfFpBrX08H8Flvw,4577
 floe/config/yaml.py,sha256=HeFGhKFxYkiqQOwljYOgUD5naJyxL_ZIX6hUIS36tzU,10071
-floe/iceberg/catalog.py,sha256=TeINK-jVZJAurROEeovVNK3s4PY-czeWih1y_KWohP8,14455
+floe/iceberg/catalog.py,sha256=QW8L6DKVr4Ug0CbuEP4wgy7Lh5-TsoOXRZWQxfM5zSQ,15115
 floe/iceberg/compact.py,sha256=5O372vAIsvgRc2PGihryAsrQ0otn0F30AQu-cetRCP0,7123
 floe/iceberg/merge.py,sha256=VE1Odf2B34Gxj7ZcuhbrfbgTGM9AdiDz6_2BbcQ-kSQ,12448
-floe/iceberg/sink.py,sha256=854HAAz4UmsgBGR6TrBy3tcqWk-ss1RcuLK-ynQe0gA,11040
+floe/iceberg/sink.py,sha256=uE0jLhhxRFOeKi7ChAzstKexy1ZrovE4EKATGTYn-Gk,11255
 floe/iceberg/unique.py,sha256=PKUFEcCkU71k66__KfUE7NTos73ZYlZbb8GJPn0yFoo,2958
 floe/iceberg/write.py,sha256=zrFeeJu-YT4Pxo9u-fRujUtFb1Epx1KFoq2qFyP1iGs,11036
 floe/lineage/events.py,sha256=4txd8bGUwKBZd-8Aw_ebTurhlxAYk8SVMgH8vSjC3bs,6682
@@ -51,16 +51,16 @@ floe/run/events.py,sha256=OE7zYkt0Dz2xrvgNavT1nHfZJ-97IIJ8LpHcameUJZU,2859
 floe/storage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 floe/storage/archive.py,sha256=1gPImm9PDSJFCpWjg37ax4ubb0wwc6FZm38quQXhORg,1101
 floe/storage/inputs.py,sha256=nmgEo2ub3hPUghLUXHU1wnKzpL4aYZbHb6S0XPYvMew,10410
-floe/storage/local.py,sha256=NKjBf_-PkDnbZ3FzXWE7KZ1xY0G_o3tRy0_7nxW7FLs,6307
+floe/storage/local.py,sha256=UiGFYEY33j45CQspWzRyJBW0HC2Nk9_zG--wL-aUuJI,6388
 floe/storage/partitioning.py,sha256=MG8RJDrwgZhucTswlXC2RwCmzrYIoOE6SH_aBM11KW8,6178
 floe/storage/paths.py,sha256=Z0fry3Gp0Tu2DpUeKqNXdqYP6n_KcKzK2llGx0TGHB8,2343
 floe/storage/resolve.py,sha256=pFAuVytTfnSEcTAVlJjFTj7S_Vipjry1ormWZP3Egmc,2593
 floe/storage/s3.py,sha256=XcG1qJTEY3KLSMLT950mUdrAxuCA8-Gx3fpNSzyV_Xo,7508
 floe/storage/types.py,sha256=FvAEXWMdVdopRK276JyETRYGOva2-uR8pZvsWj6Xvy0,993
 floe/write/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-floe/write/outputs.py,sha256=5LgN_8PMPPOgkRVZw_KvxYXmc02-meL7iZLeMKVtufQ,4932
-floe_python-0.8.1.dist-info/METADATA,sha256=t7zQL0mXQAhLksYIkXMkuuYolrQL3046WRDPjZurnQI,442
-floe_python-0.8.1.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
-floe_python-0.8.1.dist-info/entry_points.txt,sha256=DU75r-E3zcK0quvwg5iT5DeWbUJck_Tz80bEj1IIzHA,40
-floe_python-0.8.1.dist-info/licenses/LICENSE,sha256=dUhuoK-TCRQMpuLEAdfme-qPSJI0TlcH9jlNxeg9_EQ,1056
-floe_python-0.8.1.dist-info/RECORD,,
+floe/write/outputs.py,sha256=mPS3W2bHgeNR_FJ3Ghb13gMgiPpk5-mpSad1bwq97Kg,6034
+floe_python-0.8.3.dist-info/METADATA,sha256=DApXCIMAHZD6G1V_b9v5T2APwgUeKUEfWYmUk10EsK4,442
+floe_python-0.8.3.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+floe_python-0.8.3.dist-info/entry_points.txt,sha256=DU75r-E3zcK0quvwg5iT5DeWbUJck_Tz80bEj1IIzHA,40
+floe_python-0.8.3.dist-info/licenses/LICENSE,sha256=dUhuoK-TCRQMpuLEAdfme-qPSJI0TlcH9jlNxeg9_EQ,1056
+floe_python-0.8.3.dist-info/RECORD,,