s4fs 1.1.0__tar.gz

s4fs-1.1.0/PKG-INFO

@@ -0,0 +1,107 @@
+Metadata-Version: 2.4
+Name: s4fs
+Version: 1.1.0
+Summary: fsspec adapter for reading S4 gateway-written objects (S4F2 framed + compressed) directly from the backend — no gateway required. Read-only by design.
+Author: abyo software 合同会社
+Author-email: masumi-ryugo <abyo.software@gmail.com>
+License: Apache-2.0
+Project-URL: Homepage, https://github.com/abyo-software/s4
+Project-URL: Repository, https://github.com/abyo-software/s4
+Project-URL: Issues, https://github.com/abyo-software/s4/issues
+Keywords: s3,fsspec,compression,zstd,pandas,pyarrow
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: System :: Archiving :: Compression
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: fsspec>=2023.1.0
+Requires-Dist: s4-codec<2,>=1.1.0
+Provides-Extra: s3
+Requires-Dist: s3fs>=2023.1.0; extra == "s3"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+
+# s4fs — fsspec filesystem for S4 objects (no gateway required)
+
+`s4fs` lets pandas / pyarrow / DuckDB / Polars (anything fsspec-aware) read
+[S4](https://github.com/abyo-software/s4) gateway-written objects **directly
+from the S3 backend**. Objects are transparently decompressed on read,
+`ls`/`info` report the original (decompressed) sizes, and range reads use the
+`<key>.s4index` sidecar to fetch + decode only the frames that overlap the
+requested range. Objects that never went through the gateway pass through
+byte-for-byte. This is the lock-in escape hatch: if you stop running the
+gateway, your data stays readable.
+
+## Install
+
+```bash
+pip install -e python/s4fs[s3]    # from a source checkout
+# requires the s4-codec wheel: cd crates/s4-codec-py && maturin build --release
+```
+
+## Use
+
+```python
+import pandas as pd
+opts = {"target_options": {"endpoint_url": "http://backend:9000"}}
+df = pd.read_parquet("s4://bucket/data.parquet", storage_options=opts)
+```
+
+```python
+import fsspec, pyarrow.parquet as pq
+fs = fsspec.filesystem("s4", target_options={"endpoint_url": "http://backend:9000"})
+table = pq.read_table("bucket/data.parquet", filesystem=fs)
+```
+
+```python
+import duckdb
+con = duckdb.connect(); con.register_filesystem(fs)
+con.sql("SELECT count(*) FROM read_parquet('s4://bucket/data.parquet')")
+```
+
+Any underlying fsspec filesystem can be injected instead of s3fs:
+`S4FileSystem(fs=my_fs)` (used by the unit tests with an in-memory stub).
+
+## Decoded formats
+
+- S4F2-framed objects (single-PUT and multipart), S4P1 padding skipped
+- codecs: `passthrough`, `cpu-zstd`, `cpu-gzip`, `cpu-zstd-dict`
+  (dictionaries are fetched from `.s4dict/<id>` and fingerprint-verified)
+- unframed gateway objects carrying a metadata manifest (`cpu-gzip`,
+  legacy raw zstd, `passthrough`)
+- `.s4index` sidecars v1/v2/v3 with ETag staleness checks (a stale sidecar
+  falls back to a full-object read)
+
+## Limitations
+
+- **Read-only.** All write APIs raise `NotImplementedError` — write through
+  the S4 gateway, which owns the framing / sidecar / metadata contract.
+- **GPU frames are refused loudly.** `nvcomp-*` / `dietgpu-ans` frames raise
+  `NotImplementedError` (decode them through the gateway); s4fs never
+  returns silently-wrong bytes.
+- **SSE-encrypted objects are refused loudly.** Reads raise
+  `NotImplementedError` (the keyring / KMS / SSE-C key lives in the
+  gateway — read encrypted objects through the gateway). Detection is
+  threefold: the `s4-encrypted` object metadata stamp, the sidecar's v3
+  SSE binding, and the `S4E1`–`S4E6` envelope magic in the body; s4fs
+  never returns ciphertext as if it were data.
+- Exact-size resolution in `ls`/`info` may cost one extra backend request
+  per object (sidecar GET or metadata HEAD); results are cached per
+  filesystem instance.
+- Range reads on framed objects without a usable sidecar fall back to a
+  full-object read (with a warning when the object is multi-frame).
+  Legacy v1 sidecars (no source ETag/size binding) are treated as
+  unusable — they cannot be tied to the live object.
+- `open()` refuses framed objects whose original size is inexact (no
+  usable sidecar, no `s4-original-size` metadata) instead of silently
+  truncating buffered reads at the compressed size; opt back in with
+  `S4FileSystem(allow_inexact_open=True)`. `cat_file()` is unaffected.
+
+## Tests
+
+```bash
+pytest python/s4fs/tests                      # unit (gateway-captured fixtures)
+pytest python/s4fs/tests/test_e2e_minio.py -m e2e   # docker + MinIO + real gateway
+```

s4fs-1.1.0/README.md

@@ -0,0 +1,82 @@
+# s4fs — fsspec filesystem for S4 objects (no gateway required)
+
+`s4fs` lets pandas / pyarrow / DuckDB / Polars (anything fsspec-aware) read
+[S4](https://github.com/abyo-software/s4) gateway-written objects **directly
+from the S3 backend**. Objects are transparently decompressed on read,
+`ls`/`info` report the original (decompressed) sizes, and range reads use the
+`<key>.s4index` sidecar to fetch + decode only the frames that overlap the
+requested range. Objects that never went through the gateway pass through
+byte-for-byte. This is the lock-in escape hatch: if you stop running the
+gateway, your data stays readable.
+
+## Install
+
+```bash
+pip install -e python/s4fs[s3]    # from a source checkout
+# requires the s4-codec wheel: cd crates/s4-codec-py && maturin build --release
+```
+
+## Use
+
+```python
+import pandas as pd
+opts = {"target_options": {"endpoint_url": "http://backend:9000"}}
+df = pd.read_parquet("s4://bucket/data.parquet", storage_options=opts)
+```
+
+```python
+import fsspec, pyarrow.parquet as pq
+fs = fsspec.filesystem("s4", target_options={"endpoint_url": "http://backend:9000"})
+table = pq.read_table("bucket/data.parquet", filesystem=fs)
+```
+
+```python
+import duckdb
+con = duckdb.connect(); con.register_filesystem(fs)
+con.sql("SELECT count(*) FROM read_parquet('s4://bucket/data.parquet')")
+```
+
+Any underlying fsspec filesystem can be injected instead of s3fs:
+`S4FileSystem(fs=my_fs)` (used by the unit tests with an in-memory stub).
+
+## Decoded formats
+
+- S4F2-framed objects (single-PUT and multipart), S4P1 padding skipped
+- codecs: `passthrough`, `cpu-zstd`, `cpu-gzip`, `cpu-zstd-dict`
+  (dictionaries are fetched from `.s4dict/<id>` and fingerprint-verified)
+- unframed gateway objects carrying a metadata manifest (`cpu-gzip`,
+  legacy raw zstd, `passthrough`)
+- `.s4index` sidecars v1/v2/v3 with ETag staleness checks (a stale sidecar
+  falls back to a full-object read)
+
+## Limitations
+
+- **Read-only.** All write APIs raise `NotImplementedError` — write through
+  the S4 gateway, which owns the framing / sidecar / metadata contract.
+- **GPU frames are refused loudly.** `nvcomp-*` / `dietgpu-ans` frames raise
+  `NotImplementedError` (decode them through the gateway); s4fs never
+  returns silently-wrong bytes.
+- **SSE-encrypted objects are refused loudly.** Reads raise
+  `NotImplementedError` (the keyring / KMS / SSE-C key lives in the
+  gateway — read encrypted objects through the gateway). Detection is
+  threefold: the `s4-encrypted` object metadata stamp, the sidecar's v3
+  SSE binding, and the `S4E1`–`S4E6` envelope magic in the body; s4fs
+  never returns ciphertext as if it were data.
+- Exact-size resolution in `ls`/`info` may cost one extra backend request
+  per object (sidecar GET or metadata HEAD); results are cached per
+  filesystem instance.
+- Range reads on framed objects without a usable sidecar fall back to a
+  full-object read (with a warning when the object is multi-frame).
+  Legacy v1 sidecars (no source ETag/size binding) are treated as
+  unusable — they cannot be tied to the live object.
+- `open()` refuses framed objects whose original size is inexact (no
+  usable sidecar, no `s4-original-size` metadata) instead of silently
+  truncating buffered reads at the compressed size; opt back in with
+  `S4FileSystem(allow_inexact_open=True)`. `cat_file()` is unaffected.
+
+## Tests
+
+```bash
+pytest python/s4fs/tests                      # unit (gateway-captured fixtures)
+pytest python/s4fs/tests/test_e2e_minio.py -m e2e   # docker + MinIO + real gateway
+```

s4fs-1.1.0/pyproject.toml

@@ -0,0 +1,58 @@
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "s4fs"
+version = "1.1.0"
+description = "fsspec adapter for reading S4 gateway-written objects (S4F2 framed + compressed) directly from the backend — no gateway required. Read-only by design."
+readme = "README.md"
+requires-python = ">=3.9"
+license = { text = "Apache-2.0" }
+authors = [
+    { name = "abyo software 合同会社" },
+    { name = "masumi-ryugo", email = "abyo.software@gmail.com" },
+]
+keywords = ["s3", "fsspec", "compression", "zstd", "pandas", "pyarrow"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3",
+    "Topic :: System :: Archiving :: Compression",
+]
+dependencies = [
+    "fsspec>=2023.1.0",
+    # PyO3 binding crate (crates/s4-codec-py) — provides the S4F2 frame /
+    # .s4index sidecar decoders and the CPU codecs. Floor is 1.1.0: the
+    # binding APIs s4fs uses (SIDECAR_SUFFIX / frame_iter / decode_index /
+    # crc32c / CpuZstdDict) do not exist in the 1.0.0 wheel; they ship
+    # with the next s4-codec release (1.1.0).
+    "s4-codec>=1.1.0,<2",
+]
+
+[project.optional-dependencies]
+# Default underlying filesystem for s3:// backends.
+s3 = ["s3fs>=2023.1.0"]
+dev = ["pytest>=7.0"]
+
+[project.urls]
+Homepage = "https://github.com/abyo-software/s4"
+Repository = "https://github.com/abyo-software/s4"
+Issues = "https://github.com/abyo-software/s4/issues"
+
+# fsspec discovers the implementation lazily via the `fsspec.specs` entry
+# point group — `fsspec.filesystem("s4")` works without importing s4fs first.
+[project.entry-points."fsspec.specs"]
+s4 = "s4fs.core:S4FileSystem"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.pytest.ini_options]
+markers = [
+    "e2e: end-to-end tests that require docker (MinIO) and the s4 binary",
+]
+# e2e is opt-in: `pytest -m e2e` (an explicit -m on the command line
+# overrides this default).
+addopts = "-m 'not e2e'"

s4fs-1.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

s4fs-1.1.0/src/s4fs/__init__.py

@@ -0,0 +1,27 @@
+"""s4fs — fsspec adapter for S4 gateway-written objects (read-only).
+
+Usage::
+
+    import s4fs  # registers the "s4" protocol
+    import pandas as pd
+
+    df = pd.read_parquet(
+        "s4://bucket/data.parquet",
+        storage_options={"target_options": {"endpoint_url": "http://backend:9000"}},
+    )
+"""
+
+from fsspec import register_implementation
+
+from s4fs.core import S4File, S4FileSystem
+
+__all__ = ["S4File", "S4FileSystem"]
+__version__ = "1.0.0"
+
+# Idempotent: the `fsspec.specs` entry point in pyproject.toml already
+# advertises the implementation; registering here covers source checkouts
+# and older fsspec versions. clobber=False keeps an operator override.
+try:
+    register_implementation("s4", S4FileSystem, clobber=False)
+except ValueError:
+    pass