x2s3 0.8.0__tar.gz

x2s3-0.8.0/LICENSE

@@ -0,0 +1,28 @@
+BSD 3-Clause License
+
+Copyright (c) 2024, Howard Hughes Medical Institute
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

x2s3-0.8.0/PKG-INFO

@@ -0,0 +1,120 @@
+Metadata-Version: 2.4
+Name: x2s3
+Version: 0.8.0
+Summary: RESTful web service which makes any storage system X available as an S3-compatible REST API
+Author-email: Konrad Rokicki <rokicki@janelia.hhmi.org>
+License: BSD-3-Clause
+Project-URL: Homepage, https://github.com/JaneliaSciComp/x2s3
+Project-URL: Repository, https://github.com/JaneliaSciComp/x2s3
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: aiobotocore
+Requires-Dist: boto3
+Requires-Dist: botocore
+Requires-Dist: fastapi
+Requires-Dist: loguru
+Requires-Dist: pydantic
+Requires-Dist: pydantic-settings
+Requires-Dist: python-dateutil
+Requires-Dist: pytest
+Requires-Dist: starlette
+Requires-Dist: typing_extensions
+Requires-Dist: uvicorn
+Dynamic: license-file
+
+# x2s3
+
+![Python CI](https://github.com/JaneliaSciComp/x2s3/actions/workflows/python-ci.yml/badge.svg)
+
+RESTful web service which makes any storage system *X* available as an S3-compatible REST API, hence the name "X to S3". It was initially built to support cloud-compatible microscopy image viewers such as [N5 Viewer](https://github.com/saalfeldlab/n5-viewer) (BigDataViewer) and [Neuroglancer](https://github.com/google/neuroglancer).
+
+At Janelia, we use **x2s3** to make private buckets on Seagate Lyve appear public, and to proxy internal resources (e.g. VAST S3). It can also be used as a pop-up file service for quickly viewing local images in BigDataViewer or Neuroglancer.
+
+<p align="center">
+    <img src="https://raw.githubusercontent.com/JaneliaSciComp/x2s3/main/docs/use_cases.png">
+</p>
+
+# Features
+
+* Extensible support for backend storage systems
+* Optional web-based bucket explorer
+* Hidden buckets
+* Partial buckets (chroot-like prefixes)
+* Non-blocking object streaming
+
+Inspired by S3 proxies such as [oxyno-zeta/s3-proxy](https://github.com/oxyno-zeta/s3-proxy) and [pottava/aws-s3-proxy](https://github.com/pottava/aws-s3-proxy), this service goes a step further to implement enough of the [AWS S3 API](https://docs.aws.amazon.com/AmazonS3/latest/API/Type_API_Reference.html) to be useable by AWS clients, such as  BigDataViewer. The S3 proxy implements which do this well (e.g. [gaul/s3proxy](https://github.com/gaul/s3proxy)) only proxy a single bucket at a time.
+
+S3 endpoints implemented:
+* [GetBucketAcl](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketAcl.html)
+* [GetObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html)
+* [HeadObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html)
+* [ListBuckets](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListBuckets.html)
+* [ListObjectsV2](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html)
+
+S3 features omitted:
+* Permissions
+* Encryption
+* Versioning
+* RequestPayer
+* etc.
+
+# Running
+
+Create a `config.yaml` file that contains all of the buckets you want to serve. You can get started quickly by using the provided example template:
+
+```bash
+cp config.template.yaml config.yaml
+```
+
+See the [documentation](docs/Config.md) for more information about the configuration file.
+
+The simplest way to run the service is to use Docker:
+
+```bash
+docker run -it -p 8000:8000 -v ./config.yaml:/app/x2s3/config.yaml ghcr.io/janeliascicomp/x2s3:latest
+```
+
+You can also run the service with Python/Uvicorn directly. See the [development documentation](docs/Development.md) for more information on setting that up.
+
+
+## Production Deployment
+
+For production deployments, we recommend using an orchestrator (like Docker Compose) to run the prebuilt Docker container along with an Nginx reverse proxy which provides caching and TLS termination.
+
+Create a `./docker/.env` file that looks like this:
+
+```bash
+CONFIG_FILE=/path/to/config.yaml
+VAR_DIR=/path/to/var/dir
+CERT_DIR=/path/to/certs
+NGINX_CACHE_DIR=/path/to/cache
+```
+
+These properties configure the service as follows:
+* `CONFIG_FILE`: path to the `config.yaml` settings file
+* `VAR_DIR`: optional path to the var directory containing access keys referenced by `config.yaml`
+* `CERT_FILE`: optional path to the SSL cert file
+* `KEY_FILE`: optional path to the SSL key file
+* `NGINX_CACHE_DIR`: path for Nginx response caching (you can disable caching by editing `nginx.conf`)
+
+Now you can bring up the container:
+
+```bash
+cd docker/
+docker compose up -d
+```
+
+# Additional Documentation
+
+* [Configuation](docs/Config.md) - how to use `config.yaml` to configure the service
+* [Development](docs/Development.md) - notes on developing the service codebase
+
+
+# Attributions
+
+* Proxy icons created by [Uniconlabs - Flaticon](https://www.flaticon.com/free-icons/proxy)
+* [AWS S3 API](https://docs.aws.amazon.com/AmazonS3/latest/API/Type_API_Reference.html) Reference

x2s3-0.8.0/README.md

@@ -0,0 +1,92 @@
+# x2s3
+
+![Python CI](https://github.com/JaneliaSciComp/x2s3/actions/workflows/python-ci.yml/badge.svg)
+
+RESTful web service which makes any storage system *X* available as an S3-compatible REST API, hence the name "X to S3". It was initially built to support cloud-compatible microscopy image viewers such as [N5 Viewer](https://github.com/saalfeldlab/n5-viewer) (BigDataViewer) and [Neuroglancer](https://github.com/google/neuroglancer).
+
+At Janelia, we use **x2s3** to make private buckets on Seagate Lyve appear public, and to proxy internal resources (e.g. VAST S3). It can also be used as a pop-up file service for quickly viewing local images in BigDataViewer or Neuroglancer.
+
+<p align="center">
+    <img src="https://raw.githubusercontent.com/JaneliaSciComp/x2s3/main/docs/use_cases.png">
+</p>
+
+# Features
+
+* Extensible support for backend storage systems
+* Optional web-based bucket explorer
+* Hidden buckets
+* Partial buckets (chroot-like prefixes)
+* Non-blocking object streaming
+
+Inspired by S3 proxies such as [oxyno-zeta/s3-proxy](https://github.com/oxyno-zeta/s3-proxy) and [pottava/aws-s3-proxy](https://github.com/pottava/aws-s3-proxy), this service goes a step further to implement enough of the [AWS S3 API](https://docs.aws.amazon.com/AmazonS3/latest/API/Type_API_Reference.html) to be useable by AWS clients, such as  BigDataViewer. The S3 proxy implements which do this well (e.g. [gaul/s3proxy](https://github.com/gaul/s3proxy)) only proxy a single bucket at a time.
+
+S3 endpoints implemented:
+* [GetBucketAcl](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketAcl.html)
+* [GetObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html)
+* [HeadObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html)
+* [ListBuckets](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListBuckets.html)
+* [ListObjectsV2](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html)
+
+S3 features omitted:
+* Permissions
+* Encryption
+* Versioning
+* RequestPayer
+* etc.
+
+# Running
+
+Create a `config.yaml` file that contains all of the buckets you want to serve. You can get started quickly by using the provided example template:
+
+```bash
+cp config.template.yaml config.yaml
+```
+
+See the [documentation](docs/Config.md) for more information about the configuration file.
+
+The simplest way to run the service is to use Docker:
+
+```bash
+docker run -it -p 8000:8000 -v ./config.yaml:/app/x2s3/config.yaml ghcr.io/janeliascicomp/x2s3:latest
+```
+
+You can also run the service with Python/Uvicorn directly. See the [development documentation](docs/Development.md) for more information on setting that up.
+
+
+## Production Deployment
+
+For production deployments, we recommend using an orchestrator (like Docker Compose) to run the prebuilt Docker container along with an Nginx reverse proxy which provides caching and TLS termination.
+
+Create a `./docker/.env` file that looks like this:
+
+```bash
+CONFIG_FILE=/path/to/config.yaml
+VAR_DIR=/path/to/var/dir
+CERT_DIR=/path/to/certs
+NGINX_CACHE_DIR=/path/to/cache
+```
+
+These properties configure the service as follows:
+* `CONFIG_FILE`: path to the `config.yaml` settings file
+* `VAR_DIR`: optional path to the var directory containing access keys referenced by `config.yaml`
+* `CERT_FILE`: optional path to the SSL cert file
+* `KEY_FILE`: optional path to the SSL key file
+* `NGINX_CACHE_DIR`: path for Nginx response caching (you can disable caching by editing `nginx.conf`)
+
+Now you can bring up the container:
+
+```bash
+cd docker/
+docker compose up -d
+```
+
+# Additional Documentation
+
+* [Configuation](docs/Config.md) - how to use `config.yaml` to configure the service
+* [Development](docs/Development.md) - notes on developing the service codebase
+
+
+# Attributions
+
+* Proxy icons created by [Uniconlabs - Flaticon](https://www.flaticon.com/free-icons/proxy)
+* [AWS S3 API](https://docs.aws.amazon.com/AmazonS3/latest/API/Type_API_Reference.html) Reference

x2s3-0.8.0/pyproject.toml

@@ -0,0 +1,40 @@
+[build-system]
+requires = ["setuptools>=42", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "x2s3"
+version = "0.8.0"
+description = "RESTful web service which makes any storage system X available as an S3-compatible REST API"
+readme = "README.md"
+authors = [
+    { name = "Konrad Rokicki", email = "rokicki@janelia.hhmi.org" }
+]
+license = { text = "BSD-3-Clause" }
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: BSD License",
+    "Operating System :: OS Independent",
+]
+requires-python = ">=3.6"
+dependencies = [
+    "aiobotocore",
+    "boto3",
+    "botocore",
+    "fastapi",
+    "loguru",
+    "pydantic",
+    "pydantic-settings",
+    "python-dateutil",
+    "pytest",
+    "starlette",
+    "typing_extensions",
+    "uvicorn"
+]
+
+[project.urls]
+Homepage = "https://github.com/JaneliaSciComp/x2s3"
+Repository = "https://github.com/JaneliaSciComp/x2s3"
+
+[tool.setuptools]
+packages = ["x2s3"] 

x2s3-0.8.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

x2s3-0.8.0/tests/test_awss3.py

@@ -0,0 +1,297 @@
+import urllib.parse
+
+import pytest
+from fastapi.testclient import TestClient
+from pydantic import HttpUrl
+
+from xml.etree.ElementTree import Element
+from x2s3.app import create_app
+from x2s3.settings import Target, Settings
+from x2s3.utils import parse_xml
+
+obj_path = '/janelia-data-examples/jrc_mus_lung_covid.n5/render/v1_acquire_align___20210609_224836/s0/0/0/0'
+
+@pytest.fixture
+def get_settings():
+    settings = Settings()
+    settings.base_url = HttpUrl('http://testserver')
+    settings.virtual_buckets = True
+    settings.targets = [
+        Target(
+            name='janelia-data-examples',
+            options={'bucket':'janelia-data-examples'}
+        ),
+        Target(
+            name='with-prefix',
+            options={
+                'bucket':'janelia-data-examples',
+                'prefix':'jrc_mus_lung_covid.n5/'
+            }
+        ),
+        Target(
+            name='hidden-with-endpoint',
+            browseable=False,
+            options={
+                'bucket':'janelia-data-examples',
+                'endpoint':'https://s3.amazonaws.com',
+            }
+        )
+    ]
+    return settings
+
+
+@pytest.fixture
+def app(get_settings):
+    return create_app(get_settings)
+
+
+def test_acl(app):
+    with TestClient(app) as client:
+        response = client.get(f"/janelia-data-examples?acl")
+        assert response.status_code == 200
+        assert response.headers['content-type'] == "application/xml"
+        assert response.text.count("<Grant>") == 1
+
+
+def test_get_html_root(app):
+    with TestClient(app) as client:
+        response = client.get("/")
+        assert response.status_code == 200
+        assert response.headers['content-type'].startswith("text/html")
+        for target in app.settings.targets:
+            if target.browseable:
+                assert target.name in response.text
+            else:
+                assert target.name not in response.text
+
+
+def test_get_html_listing(app):
+    with TestClient(app) as client:
+        response = client.get("/janelia-data-examples/jrc_mus_lung_covid.n5/")
+        assert response.status_code == 200
+        assert response.headers['content-type'].startswith("text/html")
+        assert '<html>' in response.text
+        assert '<head>' in response.text
+        assert 'attributes.json' in response.text
+
+
+def test_list_objects(app):
+    with TestClient(app) as client:
+        bucket_name = 'janelia-data-examples'
+        max_keys = 7
+        response = client.get(f"/{bucket_name}?list-type=2&max-keys={max_keys}")
+        assert response.status_code == 200
+        root = parse_xml(response.text)
+        assert root.tag == "ListBucketResult"
+        assert root.find('Name').text == bucket_name
+
+        # CommonPrefixes are only returned when there is a delimiter
+        assert len(root.findall('CommonPrefixes')) == 0
+
+        contents = root.findall('Contents')
+        assert len(contents) <= max_keys
+    
+        if root.find('NextContinuationToken') is not None:
+            assert root.find('IsTruncated').text == "true"
+
+        assert isinstance(contents[0].find('Key'), Element)
+        assert isinstance(contents[0].find('Size'), Element)
+        assert isinstance(contents[0].find('LastModified'), Element)
+
+
+def test_list_objects_delimiter(app):
+    with TestClient(app) as client:
+        bucket_name = 'janelia-data-examples'
+        max_keys = 9
+        response = client.get(f"/{bucket_name}?list-type=2&delimiter=/&max-keys={max_keys}")
+        assert response.status_code == 200
+        root = parse_xml(response.text)
+        assert root.tag == "ListBucketResult"
+        assert root.find('Name').text == bucket_name
+        assert root.find('Delimiter').text == '/'
+        assert len(root.findall('CommonPrefixes')) >= 1
+        assert root.find('IsTruncated').text == "false"
+
+
+def test_list_objects_continuation(app):
+    with TestClient(app) as client:
+        bucket_name = 'janelia-data-examples'
+        max_keys = 4
+        uri = f"/{bucket_name}?list-type=2&max-keys={max_keys}&prefix=jrc_mus_lung_covid.n5/render/v1_acquire_align___20210609_224836/s0/0/0"
+        
+        token_param = ''
+        total = 0
+        c = 0
+        while True:
+            url = f"{uri}{token_param}"
+            print(f"Fetching {url}")
+            response = client.get(url)
+            assert response.status_code == 200
+            root = parse_xml(response.text)
+
+            assert root.tag == "ListBucketResult"
+            assert root.find('Name').text == bucket_name
+            assert root.find('MaxKeys').text == str(max_keys)
+            assert len(root.findall('CommonPrefixes')) == 0
+
+            contents = root.findall('Contents')
+            print(f"Got {len(contents)} results")
+            total += len(contents)
+            assert len(contents) <= max_keys
+
+            next_token_elem = root.find('NextContinuationToken')
+            if next_token_elem is not None:
+                assert root.find('IsTruncated').text == "true"
+            else:
+                break
+
+            safe_token = urllib.parse.quote_plus(next_token_elem.text)
+            token_param = f"&continuation-token={safe_token}"
+            
+            c += 1
+            if c>10:
+                assert False
+
+        assert total == 6
+
+
+def test_head_object(app):
+    with TestClient(app) as client:
+        response = client.head("/janelia-data-examples/jrc_mus_lung_covid.n5/attributes.json")
+        assert response.status_code == 200
+        response = client.head("/janelia-data-examples/jrc_mus_lung_covid.n5/")
+        assert response.status_code == 404
+
+
+def test_prefixed_head_object(app):
+    with TestClient(app) as client:
+        response = client.head("/with-prefix/attributes.json")
+        assert response.status_code == 200
+        response = client.head("/with-prefix/render/attributes.json")
+        assert response.status_code == 200
+        response = client.head("/with-prefix/render/")
+        assert response.status_code == 404
+
+
+def test_get_object(app):
+    with TestClient(app) as client:
+        response = client.get("/janelia-data-examples/jrc_mus_lung_covid.n5/attributes.json")
+        assert response.status_code == 200
+        json_obj = response.json()
+        assert 'n5' in json_obj
+
+
+def test_prefixed_get_object(app):
+    with TestClient(app) as client:
+        response = client.get("/with-prefix/attributes.json")
+        assert response.status_code == 200
+        json_obj = response.json()
+        assert 'n5' in json_obj
+
+
+def test_virtual_host_get_object(app):
+    with TestClient(app) as client:
+        response = client.get("/jrc_mus_lung_covid.n5/attributes.json",
+            headers={'Host':'janelia-data-examples.testserver'})
+        assert response.status_code == 200
+        json_obj = response.json()
+        assert 'n5' in json_obj
+
+
+def test_get_object_hidden(app):
+    with TestClient(app) as client:
+        response = client.get("/hidden-with-endpoint/jrc_mus_lung_covid.n5/attributes.json")
+        assert response.status_code == 200
+        json_obj = response.json()
+        assert 'n5' in json_obj
+
+
+def test_get_object_range_first(app):
+    with TestClient(app) as client:
+        # Test a valid range request (first 100 bytes)
+        response = client.get(
+            obj_path,
+            headers={"Range": "bytes=0-99"}
+        )
+        assert response.status_code == 206  # Partial Content
+        assert 'Content-Range' in response.headers
+        assert response.headers['Content-Range'] == 'bytes 0-99/987143'
+        assert len(response.content) == 100
+
+def test_get_object_range_mid(app):
+    with TestClient(app) as client:
+        # Test a valid range request (bytes 100-199)
+        response = client.get(
+            obj_path,
+            headers={"Range": "bytes=100-199"}
+        )
+        assert response.status_code == 206  # Partial Content
+        assert 'Content-Range' in response.headers
+        assert response.headers['Content-Range'] == 'bytes 100-199/987143'
+        assert len(response.content) == 100
+
+def test_get_object_range_last(app):
+    with TestClient(app) as client:
+        # Test a valid range request (last 100 bytes)
+        response = client.get(
+            obj_path,
+            headers={"Range": "bytes=-100"}
+        )
+        assert response.status_code == 206  # Partial Content
+        assert 'Content-Range' in response.headers
+        assert len(response.content) == 100
+
+def test_get_object_range_out_of_bounds(app):
+    with TestClient(app) as client:
+        # Test invalid range request (out of bounds)
+        response = client.get(
+            obj_path,
+            headers={"Range": "bytes=1000000-2000000"}
+        )
+        assert response.status_code == 416  # Range Not Satisfiable
+        root = parse_xml(response.text)
+        assert root.find('Code').text == 'InvalidRange'
+
+
+def test_prefixed_list_objects(app):
+    with TestClient(app) as client:
+        bucket_name = 'with-prefix'
+        response = client.get(f"/{bucket_name}?list-type=2&delimiter=/")
+        assert response.status_code == 200
+        root = parse_xml(response.text)
+        assert root.tag == "ListBucketResult"
+        assert root.find('Name').text == bucket_name
+        assert root.find('Delimiter').text == '/'
+        assert len(root.findall('CommonPrefixes')) == 2
+        assert root.find('IsTruncated').text == "false"
+
+
+def test_get_object_missing(app):
+    with TestClient(app) as client:
+        response = client.get("/janelia-data-examples/missing")
+        assert response.status_code == 404
+        assert response.headers['content-type'] == "application/xml"
+
+
+def test_bucket_missing(app):
+    with TestClient(app) as client:
+        response = client.get("/missing/attributes.json")
+        assert response.status_code == 404
+        assert response.headers['content-type'] == "application/xml"
+        root = parse_xml(response.text)
+        assert root.find('Code').text == 'NoSuchBucket'
+        
+
+def test_list_objects_error(app):
+    with TestClient(app) as client:
+        response = client.get(f"/janelia-data-examples?list-type=2&max-keys=aaa")
+        assert response.status_code == 400
+        assert response.headers['content-type'] == "application/json"
+
+
+def test_get_object_precedence(app):
+    with TestClient(app) as client:
+        response = client.get(f"/janelia-data-examples/jrc_mus_lung_covid.n5/attributes.json?list-type=2")
+        assert response.status_code == 200
+        json_obj = response.json()
+        assert 'n5' in json_obj

x2s3-0.8.0/tests/test_base.py

@@ -0,0 +1,41 @@
+import urllib.parse
+
+import pytest
+from fastapi.testclient import TestClient
+from pydantic import HttpUrl
+
+from xml.etree.ElementTree import Element
+from x2s3.app import create_app
+from x2s3.settings import Target, Settings
+from x2s3.utils import parse_xml
+
+
+@pytest.fixture
+def get_settings():
+    settings = Settings()
+    settings.targets = [
+    ]
+    return settings
+
+
+@pytest.fixture
+def app(get_settings):
+    return create_app(get_settings)
+
+    
+@pytest.fixture
+def client(app):
+    return TestClient(app)
+
+
+def test_get_favicon(client):
+    response = client.get("/favicon.ico")
+    assert response.status_code == 200
+    assert response.headers['content-type'].startswith("image")
+
+
+def test_get_robotstxt(client):
+    response = client.get("/robots.txt")
+    assert response.status_code == 200
+    assert response.headers['content-type'].startswith("text/plain")
+    assert response.text == "User-agent: *\nDisallow: /"