putfs 0.0.0__tar.gz

putfs-0.0.0/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

putfs-0.0.0/PKG-INFO

@@ -0,0 +1,55 @@
+Metadata-Version: 2.4
+Name: putfs
+Version: 0.0.0
+Summary: Efficient HTTP blob storage over local filesystem
+License-Expression: Apache-2.0
+License-File: LICENSE
+Author: Simon Wörpel
+Author-email: simon.woerpel@pm.me
+Requires-Python: >=3.13,<4
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Provides-Extra: client
+Provides-Extra: s3
+Requires-Dist: anyio (>=4.13.0,<5.0.0)
+Requires-Dist: anystore (>=1.1.7,<2.0.0) ; extra == "client"
+Requires-Dist: awssig (>=0.5.0) ; extra == "s3"
+Requires-Dist: granian (>=2.7.2,<3.0.0)
+Requires-Dist: starlette (>=1.0.0,<2.0.0)
+Requires-Dist: uvloop (>=0.22.1,<0.23.0)
+Project-URL: Homepage, https://putf.sh
+Project-URL: Issues, https://github.com/dataresearchcenter/putfs/issues
+Project-URL: Repository, https://github.com/dataresearchcenter/putfs
+Description-Content-Type: text/markdown
+
+# PutFS
+
+Just a `PUT` and `DELETE` API over a plain directory hierarchy with zero opinions.
+
+Wired together with optimized Nginx and a well-tuned filesystem, these few lines of async Python [outperform MinIO](https://putf.sh/benchmarks/) and related systems at scale. And we didn't even port to Rust.
+
+## Why?
+
+MinIO became [AIStor](https://min.io). $96,000/year. [Ceph](https://ceph.io), [SeaweedFS](https://github.com/seaweedfs/seaweedfs), [Garage](https://garagehq.deuxfleurs.fr) – each solve a different problem than *"I have a server, I need to store files over HTTP."*
+
+**PutFS** delegates everything to battle-tested tools: [nginx](https://nginx.org) for file serving and auth, [ZFS](https://openzfs.org) for erasure coding and caching, [zrepl](https://zrepl.github.io)/[rsync](https://rsync.samba.org) for replication. S3 compat layer included.
+
+If **PutFS** disappears tomorrow, we can still `ls -la` your data.
+
+## Docs
+
+[**putf.sh**](https://putf.sh)
+
+- [Quickstart](https://putf.sh/quickstart/)
+- [MinIO migration](https://putf.sh/minio-migration/)
+- [Benchmarks](https://putf.sh/benchmarks/)
+- [Reference](https://putf.sh/reference/model/) – auth, replication, encryption, versioning, scaling, ...
+- [Tuning](https://putf.sh/tuning/nginx/) – nginx, granian, ZFS
+
+## License
+
+Apache 2.0. (well, reasons.)
+

putfs-0.0.0/README.md

@@ -0,0 +1,27 @@
+# PutFS
+
+Just a `PUT` and `DELETE` API over a plain directory hierarchy with zero opinions.
+
+Wired together with optimized Nginx and a well-tuned filesystem, these few lines of async Python [outperform MinIO](https://putf.sh/benchmarks/) and related systems at scale. And we didn't even port to Rust.
+
+## Why?
+
+MinIO became [AIStor](https://min.io). $96,000/year. [Ceph](https://ceph.io), [SeaweedFS](https://github.com/seaweedfs/seaweedfs), [Garage](https://garagehq.deuxfleurs.fr) – each solve a different problem than *"I have a server, I need to store files over HTTP."*
+
+**PutFS** delegates everything to battle-tested tools: [nginx](https://nginx.org) for file serving and auth, [ZFS](https://openzfs.org) for erasure coding and caching, [zrepl](https://zrepl.github.io)/[rsync](https://rsync.samba.org) for replication. S3 compat layer included.
+
+If **PutFS** disappears tomorrow, we can still `ls -la` your data.
+
+## Docs
+
+[**putf.sh**](https://putf.sh)
+
+- [Quickstart](https://putf.sh/quickstart/)
+- [MinIO migration](https://putf.sh/minio-migration/)
+- [Benchmarks](https://putf.sh/benchmarks/)
+- [Reference](https://putf.sh/reference/model/) – auth, replication, encryption, versioning, scaling, ...
+- [Tuning](https://putf.sh/tuning/nginx/) – nginx, granian, ZFS
+
+## License
+
+Apache 2.0. (well, reasons.)

putfs-0.0.0/putfs/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.0.0"

putfs-0.0.0/putfs/api.py

@@ -0,0 +1,125 @@
+import errno
+import fnmatch
+from pathlib import Path
+
+import anyio
+from starlette.applications import Starlette
+from starlette.requests import Request
+from starlette.responses import Response, StreamingResponse
+from starlette.routing import Route
+
+from putfs.fs import format_mtime, iter_keys, safe_path
+from putfs.settings import Settings
+
+
+def create_app() -> Starlette:
+    settings = Settings()
+    root = Path(settings.root).resolve()
+    chunk_size = settings.chunk_size
+
+    async def get(request: Request) -> Response:
+        key = request.path_params["key"]
+        exclude_prefix = request.query_params.get("exclude_prefix")
+        glob = request.query_params.get("glob")
+        depth_str = request.query_params.get("depth")
+        depth = int(depth_str) if depth_str else None
+
+        if key == "" or key.endswith("/"):
+            prefix = key.rstrip("/")
+            base = safe_path(root, prefix)
+            # qualify user-supplied filters to root-relative paths
+            abs_exclude = f"{prefix}/{exclude_prefix}" if exclude_prefix else None
+            abs_glob = f"{prefix}/{glob}" if glob else None
+
+            async def stream_keys():
+                async for k in iter_keys(root, base, abs_exclude, abs_glob, depth):
+                    # return paths relative to the listed prefix
+                    rel = k[len(prefix) + 1 :] if k.startswith(prefix + "/") else k
+                    yield f"{rel}\n"
+
+            return StreamingResponse(
+                stream_keys(), media_type="application/octet-stream"
+            )
+
+        path = safe_path(root, key)
+        if not path.is_file():
+            return Response(status_code=404, content=key)
+
+        stat = path.stat()
+        headers = {
+            "content-length": str(stat.st_size),
+            "last-modified": format_mtime(stat.st_mtime),
+        }
+
+        async def stream():
+            async with await anyio.open_file(path, "rb") as f:
+                while chunk := await f.read(chunk_size):
+                    yield chunk
+
+        return StreamingResponse(stream(), headers=headers)
+
+    async def head(request: Request) -> Response:
+        key = request.path_params["key"]
+        path = safe_path(root, key)
+        if not path.is_file():
+            return Response(status_code=404)
+        stat = path.stat()
+        return Response(
+            status_code=200,
+            headers={
+                "content-length": str(stat.st_size),
+                "last-modified": format_mtime(stat.st_mtime),
+            },
+        )
+
+    async def put(request: Request) -> Response:
+        key = request.path_params["key"]
+        path = safe_path(root, key)
+        # return fast if worm
+        for glob in settings.worm_globs:
+            if fnmatch.fnmatch(key, glob):
+                if path.exists():
+                    if settings.worm_strict:
+                        return Response(status_code=403, content="Forbidden (WORM)")
+                    return Response(status_code=204)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        try:
+            async with await anyio.open_file(path, "wb") as f:
+                async for chunk in request.stream():
+                    await f.write(chunk)
+        except OSError as e:
+            if e.errno == errno.ENOSPC:
+                return Response(status_code=507, content="Insufficient Storage")
+            if e.errno in (errno.EPERM, errno.EACCES, errno.EROFS):
+                return Response(status_code=403, content="Forbidden")
+            raise
+        return Response(status_code=204)
+
+    async def delete(request: Request) -> Response:
+        key = request.path_params["key"]
+        path = safe_path(root, key)
+        if not path.exists():
+            return Response(status_code=404)
+        if not settings.worm_allow_delete:
+            for glob in settings.worm_globs:
+                if fnmatch.fnmatch(key, glob):
+                    return Response(status_code=403, content="Forbidden (WORM)")
+        try:
+            path.unlink()
+        except OSError as e:
+            if e.errno in (errno.EPERM, errno.EACCES, errno.EROFS):
+                return Response(status_code=403, content="Forbidden")
+            raise
+        return Response(status_code=204)
+
+    return Starlette(
+        routes=[
+            Route("/{key:path}", get, methods=["GET"]),
+            Route("/{key:path}", head, methods=["HEAD"]),
+            Route("/{key:path}", put, methods=["PUT"]),
+            Route("/{key:path}", delete, methods=["DELETE"]),
+        ],
+    )
+
+
+app = create_app()

putfs-0.0.0/putfs/cli.py

@@ -0,0 +1,7 @@
+def cli():
+    try:
+        from putfs.client.cli import cli as app
+    except ImportError:
+        print("CLI requires: pip install putfs[client]")
+        raise SystemExit(1)
+    app()

putfs-0.0.0/putfs/client/__init__.py

@@ -0,0 +1,3 @@
+from putfs.client.http import PutFS
+
+__all__ = ["PutFS"]

putfs-0.0.0/putfs/client/cli.py

@@ -0,0 +1,128 @@
+from typing import Annotated, Optional
+
+from anystore.logic.io import stream
+import typer
+from rich import print
+
+from putfs import __version__
+from putfs.client.http import get_resource, get_store
+from putfs.client.sync import sync
+from putfs.settings import Settings
+
+settings = Settings()
+cli = typer.Typer(
+    no_args_is_help=True,
+    help=(
+        "PutFS CLI -- copy, sync, list, and delete objects on a PutFS server.\n\n"
+        "Authenticate via --key/--secret flags or PUTFS_API_KEY/PUTFS_API_SECRET env vars. "
+        "URIs use putfs://host:port/path for PutFS stores; "
+        "local paths and all anystore URIs (http(s)://, s3://, gs://, etc.) for others."
+    ),
+)
+
+state: dict = {"key": settings.api_key, "secret": settings.api_secret}
+
+
+@cli.callback(invoke_without_command=True)
+def cli_main(
+    version: Annotated[
+        Optional[bool], typer.Option("--version", help="Show version")
+    ] = False,
+    settings: Annotated[
+        Optional[bool], typer.Option("--settings", help="Show current settings")
+    ] = False,
+    key: Annotated[
+        Optional[str], typer.Option("--key", help="API key")
+    ] = settings.api_key,
+    secret: Annotated[
+        Optional[str], typer.Option("--secret", help="API secret")
+    ] = settings.api_secret,
+):
+    if version:
+        typer.echo(__version__)
+        raise typer.Exit()
+    if settings:
+        print(Settings().__dict__)
+    if key:
+        state["key"] = key
+    if secret:
+        state["secret"] = secret
+
+
+@cli.command("cp")
+def cli_cp(src: str, dst: str):
+    """Stream-Copy a single file between stores."""
+    src_res = get_resource(src, **state)
+    dst_res = get_resource(dst, **state)
+    with src_res.open(mode="rb") as i:
+        with dst_res.open(mode="wb") as o:
+            stream(i, o, settings.chunk_size)
+
+
+@cli.command("ls")
+def cli_ls(
+    uri: str,
+    glob: Annotated[Optional[str], typer.Option("--glob", help="Glob pattern")] = None,
+    exclude_prefix: Annotated[
+        Optional[str], typer.Option("--exclude-prefix", help="Exclude prefix")
+    ] = None,
+):
+    """List keys in a store."""
+    store = get_store(uri, **state)
+    for key in store.iterate_keys(
+        exclude_prefix=exclude_prefix,
+        glob=glob,
+    ):
+        typer.echo(key)
+
+
+@cli.command("sync")
+def cli_sync(
+    src: str,
+    dst: str,
+    delete: Annotated[
+        bool, typer.Option("--delete", help="Delete extra keys in target")
+    ] = False,
+    overwrite: Annotated[
+        bool, typer.Option("--overwrite", help="Skip diff checks, upload everything")
+    ] = False,
+    worm: Annotated[
+        bool, typer.Option("--worm", help="WORM mode: Ignore existing files entirely")
+    ] = False,
+    workers: Annotated[
+        Optional[int],
+        typer.Option("--workers", "-w", help="Number of threads (default: CPU count)"),
+    ] = None,
+):
+    """Sync keys between two stores."""
+    src_store = get_store(src, **state)
+    dst_store = get_store(dst, **state)
+    count = sync(
+        src_store,
+        dst_store,
+        delete=delete,
+        overwrite=overwrite,
+        worm=worm,
+        workers=workers,
+    )
+    typer.echo(f"{count} keys synced")
+
+
+@cli.command("rm")
+def cli_rm(
+    uri: str,
+    recursive: Annotated[
+        bool, typer.Option("--recursive", "-r", help="Delete by prefix")
+    ] = False,
+):
+    """Delete a key from a store."""
+    if recursive:
+        store = get_store(uri, **state)
+        count = 0
+        for k in store.iterate_keys():
+            store.delete(k)
+            count += 1
+        typer.echo(f"{count} keys deleted")
+    else:
+        res = get_resource(uri, **state)
+        res.delete()

putfs-0.0.0/putfs/client/http.py

@@ -0,0 +1,90 @@
+from typing import Any
+from anystore.logic.uri import UriHandler
+from anystore.store import Store, get_store as get_anystore
+from anystore.store.resource import UriResource
+from anystore.types import Uri
+from putfs.settings import Settings
+
+
+settings = Settings()
+
+
+def _is_putfs(uri: Uri | UriHandler) -> bool:
+    """
+    Check if given uri is a PutFS endpoint. Valid PutFS schemes are:
+        - putfs://
+        - anystore+http[s]:// (already anstore compat)
+    """
+    if not isinstance(uri, UriHandler):
+        uri = UriHandler(uri)
+    return uri.scheme.startswith(("putfs", "anystore+"))
+
+
+def _make_anystore_http_uri(uri: Uri | UriHandler) -> str:
+    """Transform a http-like uri into anystore format to tell fsspec which
+    storage to use"""
+    if not isinstance(uri, UriHandler):
+        uri = UriHandler(uri)
+    rest = uri.parsed.netloc + uri.parsed.path
+    if _is_putfs(uri):
+        # putfs[s]:// -> anystore+http[s]://
+        proto = "https" if settings.https else "http"
+        scheme = uri.scheme.replace("putfs", f"anystore+{proto}")
+        return f"{scheme}://{rest}"
+    if uri.is_http:
+        return f"anystore+{uri.scheme}://{rest}"
+    raise ValueError(f"Invalid URI: `{uri}`")
+
+
+class PutFS(Store):
+    def __init__(
+        self,
+        uri: Uri | None = None,
+        key: str | None = None,
+        secret: str | None = None,
+        **headers: Any,
+    ) -> None:
+        settings = Settings()
+        uri = _make_anystore_http_uri(uri or settings.endpoint_url)
+        backend_config = {
+            "client_kwargs": {
+                "headers": {
+                    "X-Api-Key": key or settings.api_key,
+                    "X-Api-Secret": secret or settings.api_secret,
+                    **headers,
+                },
+            },
+        }
+        super().__init__(uri=uri, backend_config=backend_config)
+
+
+def get_store(uri: Uri, **kwargs) -> Store:
+    if _is_putfs(uri):
+        return PutFS(uri, **kwargs)
+    return get_anystore(uri)
+
+
+class PutFSResource:
+    """Minimal resource wrapper for PutFS stores."""
+
+    def __init__(self, store: Store, key: str):
+        self.store = store
+        self.key = key
+
+    def open(self, **kwargs):
+        return self.store.open(self.key, **kwargs)
+
+    def delete(self, **kwargs):
+        self.store.delete(self.key, **kwargs)
+
+    def exists(self) -> bool:
+        return self.store.exists(self.key)
+
+
+def get_resource(uri: Uri, **kwargs) -> UriResource | PutFSResource:
+    if _is_putfs(uri):
+        # Split: http://host/dataset/path/file.txt → store=http://host/dataset/path, key=file.txt
+        base, key = str(uri).rsplit("/", 1)
+        store = PutFS(uri=base, **kwargs)
+        return PutFSResource(store, key)
+    return UriResource(uri)

putfs-0.0.0/putfs/client/sync.py

@@ -0,0 +1,80 @@
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from multiprocessing import cpu_count
+
+from anystore.io import logged_items, stream_bytes
+from anystore.store import Store
+
+
+def _needs_transfer(source: Store, target: Store, key: str, worm: bool) -> bool:
+    """Check if key needs transfer by comparing mtime (size fallback)."""
+    # short circuit WORM mode:
+    if worm and target.exists(key):
+        return False
+    source_info = source.info(key)
+    try:
+        target_info = target.info(key)
+    except (FileNotFoundError, Exception):
+        return True
+    if source_info.updated_at and target_info.updated_at:
+        return source_info.updated_at > target_info.updated_at
+    return source_info.size != target_info.size
+
+
+def _sync_key(
+    source: Store, target: Store, key: str, overwrite: bool, worm: bool
+) -> bool:
+    """Transfer a single key if needed."""
+    if not overwrite and not _needs_transfer(source, target, key, worm):
+        return False
+    stream_bytes(key, source, target)
+    return True
+
+
+def sync(
+    source: Store,
+    target: Store,
+    delete: bool = False,
+    overwrite: bool = False,
+    worm: bool = False,
+    workers: int | None = None,
+) -> int:
+    """Sync keys from source store to target store using threads.
+
+    When delete=False (default), only the source is listed – the target
+    is not scanned upfront. Each key is checked individually via HEAD
+    (unless overwrite=True, which skips all checks).
+
+    When delete=True, both stores are listed to find extra keys to remove.
+
+    Returns the number of keys transferred + deleted.
+    """
+    workers = workers or cpu_count()
+    source_keys = set(source.iterate_keys())
+
+    # Only list target when we need to find extras to delete
+    target_keys = set(target.iterate_keys()) if delete else set()
+
+    total = len(source_keys)
+    count = 0
+
+    with ThreadPoolExecutor(max_workers=workers) as pool:
+        futures = {}
+        for key in source_keys:
+            futures[pool.submit(_sync_key, source, target, key, overwrite, worm)] = key
+
+        for future in logged_items(
+            as_completed(futures),
+            "Syncing",
+            total=total,
+            item_name="File",
+        ):
+            if future.result():
+                count += 1
+
+    if delete:
+        extra = target_keys - source_keys
+        for key in extra:
+            target.delete(key)
+            count += 1
+
+    return count

putfs-0.0.0/putfs/fs.py

@@ -0,0 +1,74 @@
+import fnmatch
+import os
+from email.utils import formatdate
+from pathlib import Path
+
+import anyio
+
+
+def safe_path(root: Path, key: str) -> Path:
+    """Resolve key to an absolute path, raising ValueError on traversal."""
+    resolved = (root / key).resolve()
+    if not resolved.is_relative_to(root.resolve()):
+        raise ValueError("path traversal")
+    return resolved
+
+
+def format_mtime(mtime: float) -> str:
+    """Format a stat mtime as an RFC 2822 date string."""
+    return formatdate(mtime, usegmt=True)
+
+
+def format_mtime_iso(mtime: float) -> str:
+    """Format a stat mtime as an ISO 8601 date string for S3."""
+    from datetime import datetime, timezone
+
+    return datetime.fromtimestamp(mtime, tz=timezone.utc).strftime(
+        "%Y-%m-%dT%H:%M:%S.000Z"
+    )
+
+
+def etag(stat: os.stat_result) -> str:
+    """Cheap deterministic ETag from file size and mtime."""
+    return f'"{hash(f"{stat.st_size}-{stat.st_mtime}"):016x}"'
+
+
+async def iter_keys(
+    root: Path,
+    base: Path,
+    exclude_prefix: str | None = None,
+    glob: str | None = None,
+    depth: int | None = None,
+):
+    """Async streaming listing via os.scandir, relative to root.
+
+    depth=None: recursive (all files). depth=1: immediate children only.
+    Yields file paths for depth>1 or None, directory names for depth=1.
+    """
+    root_str = str(root)
+    base_str = str(base)
+    queue: list[tuple[str, int]] = [(base_str, 0)]
+    while queue:
+        current, level = queue.pop()
+        try:
+            entries = await anyio.to_thread.run_sync(
+                lambda d=current: list(os.scandir(d))
+            )
+        except FileNotFoundError:
+            continue
+        for entry in entries:
+            if entry.is_dir(follow_symlinks=False):
+                if depth is not None and depth == 1 and level == 0:
+                    rel = os.path.relpath(entry.path, root_str)
+                    yield rel
+                elif depth is None or level + 1 < depth:
+                    queue.append((entry.path, level + 1))
+            elif entry.is_file(follow_symlinks=False):
+                if depth is not None and depth == 1 and level > 0:
+                    continue
+                rel = os.path.relpath(entry.path, root_str)
+                if exclude_prefix and rel.startswith(exclude_prefix):
+                    continue
+                if glob and not fnmatch.fnmatch(rel, glob):
+                    continue
+                yield rel

putfs-0.0.0/putfs/s3/__init__.py

@@ -0,0 +1,3 @@
+from putfs.s3.api import app, create_app
+
+__all__ = ["app", "create_app"]

putfs-0.0.0/putfs/s3/api.py

@@ -0,0 +1,280 @@
+import errno
+from base64 import b64decode, b64encode
+from pathlib import Path
+from xml.sax.saxutils import escape
+
+import anyio
+from starlette.applications import Starlette
+from starlette.middleware import Middleware
+from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.requests import Request
+from starlette.responses import FileResponse, Response
+from starlette.routing import Route
+
+from putfs.fs import etag, format_mtime_iso, iter_keys, safe_path
+from putfs.s3.auth import load_key_mapping, verify_sigv4
+from putfs.settings import Settings
+
+
+def make_xml_response(body: str, status_code: int = 200) -> Response:
+    xml = f'<?xml version="1.0" encoding="UTF-8"?>\n{body}'
+    return Response(content=xml, media_type="application/xml", status_code=status_code)
+
+
+def create_app() -> Starlette:
+    settings = Settings()
+    root = Path(settings.root).resolve()
+    key_mapping = load_key_mapping(settings.s3_keys_dir)
+
+    # ── Auth middleware ──────────────────────────────────────────────
+
+    class S3AuthMiddleware(BaseHTTPMiddleware):
+        async def dispatch(self, request: Request, call_next):
+            from starlette.exceptions import HTTPException
+
+            try:
+                await verify_sigv4(
+                    request, key_mapping, settings.s3_region, settings.s3_allow_unsigned
+                )
+            except HTTPException as exc:
+                return make_xml_response(
+                    f"<Error><Code>{exc.status_code}</Code>"
+                    f"<Message>{escape(exc.detail)}</Message>"
+                    f"</Error>",
+                    status_code=exc.status_code,
+                )
+            return await call_next(request)
+
+    # ── Bucket ops ───────────────────────────────────────────────────
+
+    async def list_buckets(request: Request) -> Response:
+        buckets = []
+        async for name in iter_keys(root, root, depth=1):
+            path = root / name
+            stat = path.stat()
+            buckets.append(
+                f"<Bucket>"
+                f"<Name>{escape(name)}</Name>"
+                f"<CreationDate>{format_mtime_iso(stat.st_mtime)}</CreationDate>"
+                f"</Bucket>"
+            )
+        return make_xml_response(
+            f"<ListAllMyBucketsResult>"
+            f"<Buckets>{''.join(buckets)}</Buckets>"
+            f"</ListAllMyBucketsResult>"
+        )
+
+    async def create_bucket(request: Request) -> Response:
+        bucket = request.path_params["bucket"]
+        path = safe_path(root, bucket)
+        path.mkdir(parents=True, exist_ok=True)
+        return Response(status_code=200)
+
+    async def head_bucket(request: Request) -> Response:
+        bucket = request.path_params["bucket"]
+        path = safe_path(root, bucket)
+        if path.is_dir():
+            return Response(status_code=200)
+        return Response(status_code=404)
+
+    async def delete_bucket(request: Request) -> Response:
+        bucket = request.path_params["bucket"]
+        path = safe_path(root, bucket)
+        if not path.is_dir():
+            return Response(status_code=404)
+        try:
+            path.rmdir()
+        except OSError:
+            return make_xml_response(
+                f"<Error><Code>BucketNotEmpty</Code>"
+                f"<BucketName>{escape(bucket)}</BucketName>"
+                f"<Message>The bucket is not empty.</Message>"
+                f"</Error>",
+                status_code=409,
+            )
+        return Response(status_code=204)
+
+    # ── List objects ─────────────────────────────────────────────────
+
+    async def list_objects(request: Request) -> Response:
+        bucket = request.path_params["bucket"]
+        prefix = request.query_params.get("prefix", "")
+        delimiter = request.query_params.get("delimiter", "")
+        max_keys = int(request.query_params.get("max-keys", "10000"))
+        continuation_token = request.query_params.get("continuation-token", "")
+        start_after = request.query_params.get("start-after", "")
+
+        base = safe_path(root, bucket)
+        if prefix:
+            base = safe_path(root, f"{bucket}/{prefix}")
+
+        start_key = ""
+        if continuation_token:
+            start_key = b64decode(continuation_token).decode()
+        elif start_after:
+            start_key = start_after
+
+        contents: list[str] = []
+        common_prefixes: set[str] = set()
+        count = 0
+        truncated = False
+        last_key = ""
+
+        async for rel_key in iter_keys(root, base):
+            obj_key = (
+                rel_key[len(bucket) + 1:]
+                if rel_key.startswith(bucket + "/")
+                else rel_key
+            )
+
+            if start_key and obj_key <= start_key:
+                continue
+
+            if delimiter and prefix:
+                after_prefix = obj_key[len(prefix):]
+                delim_pos = after_prefix.find(delimiter)
+                if delim_pos >= 0:
+                    cp = prefix + after_prefix[: delim_pos + 1]
+                    common_prefixes.add(cp)
+                    continue
+            elif delimiter:
+                delim_pos = obj_key.find(delimiter)
+                if delim_pos >= 0:
+                    cp = obj_key[: delim_pos + 1]
+                    common_prefixes.add(cp)
+                    continue
+
+            if count >= max_keys:
+                truncated = True
+                break
+
+            file_path = root / rel_key
+            try:
+                stat = file_path.stat()
+            except FileNotFoundError:
+                continue
+
+            contents.append(
+                f"<Contents>"
+                f"<Key>{escape(obj_key)}</Key>"
+                f"<LastModified>{format_mtime_iso(stat.st_mtime)}</LastModified>"
+                f"<ETag>{escape(etag(stat))}</ETag>"
+                f"<Size>{stat.st_size}</Size>"
+                f"<StorageClass>STANDARD</StorageClass>"
+                f"</Contents>"
+            )
+            last_key = obj_key
+            count += 1
+
+        next_token = ""
+        if truncated and last_key:
+            next_token = b64encode(last_key.encode()).decode()
+
+        cp_xml = "".join(
+            f"<CommonPrefixes><Prefix>{escape(p)}</Prefix></CommonPrefixes>"
+            for p in sorted(common_prefixes)
+        )
+
+        body = (
+            f"<ListBucketResult>"
+            f"<Name>{escape(bucket)}</Name>"
+            f"<Prefix>{escape(prefix)}</Prefix>"
+            f"<MaxKeys>{max_keys}</MaxKeys>"
+            f"<KeyCount>{count}</KeyCount>"
+            f"<IsTruncated>{'true' if truncated else 'false'}</IsTruncated>"
+            f"{'<Delimiter>' + escape(delimiter) + '</Delimiter>' if delimiter else ''}"
+            f"{'<NextContinuationToken>' + escape(next_token) + '</NextContinuationToken>' if next_token else ''}"
+            f"{''.join(contents)}"
+            f"{cp_xml}"
+            f"</ListBucketResult>"
+        )
+        return make_xml_response(body)
+
+    # ── Object ops ───────────────────────────────────────────────────
+
+    async def get_object(request: Request) -> Response:
+        bucket = request.path_params["bucket"]
+        key = request.path_params["key"]
+        path = safe_path(root, f"{bucket}/{key}")
+        if not path.is_file():
+            return make_xml_response(
+                f"<Error><Code>NoSuchKey</Code>"
+                f"<Key>{escape(key)}</Key>"
+                f"<Message>The specified key does not exist.</Message>"
+                f"</Error>",
+                status_code=404,
+            )
+
+        stat = path.stat()
+        headers = {
+            "last-modified": format_mtime_iso(stat.st_mtime),
+            "etag": etag(stat),
+        }
+        if settings.s3_accel_redirect:
+            headers["x-accel-redirect"] = f"/_internal/{bucket}/{key}"
+            headers["content-length"] = str(stat.st_size)
+            return Response(status_code=200, headers=headers)
+        return FileResponse(path, headers=headers)
+
+    async def head_object(request: Request) -> Response:
+        bucket = request.path_params["bucket"]
+        key = request.path_params["key"]
+        path = safe_path(root, f"{bucket}/{key}")
+        if not path.is_file():
+            return Response(status_code=404)
+        stat = path.stat()
+        return Response(
+            status_code=200,
+            headers={
+                "content-length": str(stat.st_size),
+                "last-modified": format_mtime_iso(stat.st_mtime),
+                "etag": etag(stat),
+                "accept-ranges": "bytes",
+            },
+        )
+
+    async def put_object(request: Request) -> Response:
+        bucket = request.path_params["bucket"]
+        key = request.path_params["key"]
+        path = safe_path(root, f"{bucket}/{key}")
+        path.parent.mkdir(parents=True, exist_ok=True)
+        try:
+            async with await anyio.open_file(path, "wb") as f:
+                async for chunk in request.stream():
+                    await f.write(chunk)
+        except OSError as e:
+            if e.errno == errno.ENOSPC:
+                return Response(status_code=507, content="Insufficient Storage")
+            raise
+        stat = path.stat()
+        return Response(status_code=200, headers={"etag": etag(stat)})
+
+    async def delete_object(request: Request) -> Response:
+        bucket = request.path_params["bucket"]
+        key = request.path_params["key"]
+        path = safe_path(root, f"{bucket}/{key}")
+        try:
+            path.unlink()
+        except FileNotFoundError:
+            pass
+        return Response(status_code=204)
+
+    # ── App ──────────────────────────────────────────────────────────
+
+    return Starlette(
+        routes=[
+            Route("/", list_buckets, methods=["GET"]),
+            Route("/{bucket}", create_bucket, methods=["PUT"]),
+            Route("/{bucket}", head_bucket, methods=["HEAD"]),
+            Route("/{bucket}", delete_bucket, methods=["DELETE"]),
+            Route("/{bucket}", list_objects, methods=["GET"]),
+            Route("/{bucket}/{key:path}", get_object, methods=["GET"]),
+            Route("/{bucket}/{key:path}", head_object, methods=["HEAD"]),
+            Route("/{bucket}/{key:path}", put_object, methods=["PUT"]),
+            Route("/{bucket}/{key:path}", delete_object, methods=["DELETE"]),
+        ],
+        middleware=[Middleware(S3AuthMiddleware)],
+    )
+
+
+app = create_app()

putfs-0.0.0/putfs/s3/auth.py

@@ -0,0 +1,109 @@
+import fnmatch
+import os
+import re
+import posixpath
+from pathlib import Path
+
+from starlette.exceptions import HTTPException
+from starlette.requests import Request
+
+# key_id → [(secret, glob), ...]
+KeyMapping = dict[str, list[tuple[str, str]]]
+
+
+def load_key_mapping(keys_dir: str) -> KeyMapping:
+    """Load key_id → [(secret, glob), ...] mapping from key files.
+
+    Each file in keys_dir is named after the access key ID.
+    Each line is `<secret>:<glob>` where glob controls path access.
+
+    Example file `keys/s3/AKID_acme`:
+        mysecret123:invoices/*
+        mysecret123:contracts/*
+        othersecret:*
+    """
+    mapping: KeyMapping = {}
+    keys_path = Path(keys_dir)
+    if not keys_path.is_dir():
+        return mapping
+    for entry in os.scandir(keys_path):
+        if entry.is_file() and not entry.name.startswith("."):
+            rules = []
+            for line in Path(entry.path).read_text().splitlines():
+                line = line.strip()
+                if not line or line.startswith("#"):
+                    continue
+                if ":" in line:
+                    secret, glob = line.split(":", 1)
+                    rules.append((secret.strip(), glob.strip()))
+                else:
+                    # bare secret, no glob restriction (allow all)
+                    rules.append((line, "*"))
+            if rules:
+                mapping[entry.name] = rules
+    return mapping
+
+
+def _extract_key_id(auth_header: str) -> str | None:
+    """Extract access key ID from SigV4 Authorization header."""
+    m = re.search(r"Credential=([^/]+)/", auth_header)
+    return m.group(1) if m else None
+
+
+async def verify_sigv4(
+    request: Request,
+    key_mapping: KeyMapping,
+    region: str,
+    allow_unsigned: bool = False,
+) -> None:
+    """Verify AWS SigV4 signature and path permissions."""
+    auth_header = request.headers.get("authorization", "")
+    if not auth_header.startswith("AWS4-HMAC-SHA256"):
+        if allow_unsigned:
+            return
+        raise HTTPException(status_code=403, detail="Missing SigV4 authorization")
+
+    try:
+        import awssig
+    except ImportError:
+        raise HTTPException(
+            status_code=500,
+            detail="S3 auth requires awssig: pip install putfs[s3]",
+        )
+
+    key_id = _extract_key_id(auth_header)
+    if not key_id or key_id not in key_mapping:
+        raise HTTPException(status_code=403, detail="Access Denied")
+
+    rules = key_mapping[key_id]
+    # Normalize path: resolve .. and decode, reject traversal patterns
+    raw_path = request.url.path.lstrip("/")
+    request_path = posixpath.normpath(raw_path).lstrip("/")
+    if ".." in request_path.split("/"):
+        raise HTTPException(status_code=403, detail="Access Denied")
+
+    # Try each secret for this key ID
+    verified = False
+    for secret, glob_pattern in rules:
+        try:
+            verifier = awssig.AWSSigV4S3Verifier(
+                request_method=request.method,
+                uri_path=request.url.path,
+                query_string=str(request.url.query) if request.url.query else "",
+                headers=dict(request.headers),
+                body=b"",
+                region=region,
+                service="s3",
+                key_mapping={key_id: secret},
+                timestamp_mismatch=15 * 60,
+            )
+            verifier.verify()
+            # Signature matches – check path permission against normalized path
+            if fnmatch.fnmatch(request_path, glob_pattern):
+                verified = True
+                break
+        except awssig.InvalidSignatureError:
+            continue
+
+    if not verified:
+        raise HTTPException(status_code=403, detail="Access Denied")

putfs-0.0.0/putfs/settings.py

@@ -0,0 +1,35 @@
+import os
+
+
+BOOL = ("true", "1", "yes")
+
+
+class Settings:
+    """Settings from environment variables with PUTFS_ prefix."""
+
+    def __init__(self):
+        self.root = os.environ.get("PUTFS_ROOT", "data")
+        self.chunk_size = int(os.environ.get("PUTFS_CHUNK_SIZE", str(10 * 1024 * 1024)))
+        self.worm_globs = os.environ.get("PUTFS_WORM_GLOBS", "").split(",")
+        self.worm_strict = os.environ.get("PUTFS_WORM_STRICT", "false").lower() in BOOL
+        self.worm_allow_delete = (
+            os.environ.get("PUTFS_WORM_ALLOW_DELETE", "false").lower() in BOOL
+        )
+
+        # client
+        self.endpoint_url = os.environ.get(
+            "PUTFS_ENDPOINT_URL", "http://localhost:8000"
+        )
+        self.api_key = os.environ.get("PUTFS_API_KEY", "test-key")
+        self.api_secret = os.environ.get("PUTFS_API_SECRET", "test-secret")
+        self.https = os.environ.get("PUTFS_HTTPS", "true").lower() in BOOL
+
+        # s3
+        self.s3_keys_dir = os.environ.get("PUTFS_S3_KEYS_DIR", "keys")
+        self.s3_region = os.environ.get("PUTFS_S3_REGION", "eu-central-1")
+        self.s3_allow_unsigned = (
+            os.environ.get("PUTFS_S3_ALLOW_UNSIGNED", "false").lower() in BOOL
+        )
+        self.s3_accel_redirect = (
+            os.environ.get("PUTFS_S3_ACCEL_REDIRECT", "true").lower() in BOOL
+        )

putfs-0.0.0/pyproject.toml

@@ -0,0 +1,78 @@
+[project]
+name = "putfs"
+version = "0.0.0"
+description = "Efficient HTTP blob storage over local filesystem"
+authors = [
+    { name = "Simon Wörpel", email = "simon.woerpel@pm.me" },
+    { name = "Data and Research Center – DARC", email = "hi@dataresearchcenter.org" },
+]
+license = "Apache-2.0"
+readme = "README.md"
+classifiers = [
+    "Intended Audience :: Developers",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+]
+requires-python = ">=3.13,<4"
+dependencies = [
+    "starlette (>=1.0.0,<2.0.0)",
+    "anyio (>=4.13.0,<5.0.0)",
+    "granian (>=2.7.2,<3.0.0)",
+    "uvloop (>=0.22.1,<0.23.0)",
+]
+
+[project.optional-dependencies]
+client = ["anystore (>=1.1.7,<2.0.0)"]
+s3 = ["awssig (>=0.5.0)"]
+
+[project.scripts]
+putfs = "putfs.cli:cli"
+
+[project.urls]
+Homepage = "https://putf.sh"
+Repository = "https://github.com/dataresearchcenter/putfs"
+Issues = "https://github.com/dataresearchcenter/putfs/issues"
+
+[tool.poetry.group.dev.dependencies]
+pytest = "^9.0.2"
+pytest-cov = ">=4.1,<8.0"
+pytest-env = "^1.6.0"
+black = ">=23.11,<27.0"
+isort = "^8.0.1"
+mypy = "^1.17.1"
+pre-commit = "^4.0.1"
+flake8 = ">=6.1,<8.0"
+ipdb = "^0.13.13"
+bump2version = "^1.0.1"
+pytest-asyncio = ">=0.23.8,<1.4.0"
+httpx = "^0.28.1"
+boto3 = "^1.38.0"
+uvicorn = "^0.34.0"
+zensical = "^0.0.36"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.pytest.ini_options]
+markers = [
+    "docker: requires docker compose stack running",
+]
+
+[tool.pytest_env]
+DEBUG = 1
+PUTFS_ROOT = "./test-data"
+PUTFS_ENDPOINT_URL = "http://localhost:8000"
+PUTFS_HTTPS = 0
+PUTFS_API_KEY = "test-key"
+PUTFS_API_SECRET = "test-secret"
+PUTFS_S3_KEYS_DIR = "./contrib/keys/s3"
+PUTFS_S3_REGION = "eu-central-1"
+PUTFS_S3_ALLOW_UNSIGNED = "false"
+PUTFS_S3_ACCEL_REDIRECT = "false"
+AWS_ACCESS_KEY_ID = "test-key"
+AWS_SECRET_ACCESS_KEY = "test-secret"
+AWS_DEFAULT_REGION = "eu-central-1"
+AWS_ENDPOINT_URL = "http://localhost:8000"