x2s3 0.8.0__py3-none-any.whl

x2s3/__init__.py

@@ -0,0 +1,4 @@
+from .client import ProxyClient
+from .settings import Settings
+
+__all__ = ['Settings', 'ProxyClient']

x2s3/app.py

@@ -0,0 +1,309 @@
+import os
+import sys
+from typing import Optional
+
+from loguru import logger
+from fastapi import FastAPI, HTTPException, Request, Query
+from fastapi.responses import JSONResponse, FileResponse, PlainTextResponse
+from fastapi.staticfiles import StaticFiles
+from fastapi.templating import Jinja2Templates
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.exceptions import RequestValidationError
+from starlette.exceptions import HTTPException as StarletteHTTPException
+
+from x2s3.utils import *
+from x2s3 import client_registry
+from x2s3.settings import get_settings, Target
+
+def create_app(settings):
+
+    app = FastAPI()
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=["*"],
+        allow_credentials=True,
+        allow_methods=["GET","HEAD"],
+        allow_headers=["*"],
+        expose_headers=["Range", "Content-Range"],
+    )
+    app.mount("/static", StaticFiles(directory="static"), name="static")
+    templates = Jinja2Templates(directory="templates")
+
+
+    @app.exception_handler(StarletteHTTPException)
+    async def http_exception_handler(request, exc):
+        return JSONResponse({"error":str(exc.detail)}, status_code=exc.status_code)
+
+
+    @app.exception_handler(RequestValidationError)
+    async def validation_exception_handler(request, exc):
+        return JSONResponse({"error":str(exc)}, status_code=400)
+
+
+    @app.on_event("startup")
+    async def startup_event():
+        """ Runs once when the service is first starting.
+            Reads the configuration and sets up the proxy clients. 
+        """
+        if callable(settings):
+            app.settings = settings()
+        else:
+            app.settings = settings
+
+        # Configure logging
+        logger.remove()
+        logger.add(sys.stderr, level=app.settings.log_level)
+
+        logger.trace("Available protocols:")
+        for proto in client_registry.available_protocols():
+            logger.trace(f"- {proto}")
+
+        app.clients = {}
+
+        # Add local path client if configured
+        if app.settings.local_path:
+            local_target = Target(
+                name=app.settings.local_name,
+                client='file',
+                options={
+                    'path': str(app.settings.local_path),
+                }
+            )
+            app.settings.targets += [local_target]
+
+        # Configure targets
+        for target_name in app.settings.get_target_map():
+            target_key = target_name.lower()
+            target_config = app.settings.get_target_config(target_key)
+            proxy_kwargs = {
+                'target_name': target_name,
+            }
+
+            client = client_registry.client(target_config.client,
+                proxy_kwargs, **target_config.options)
+
+            if target_key in app.clients:
+                logger.warning(f"Overriding target key: {target_key}")
+
+            app.clients[target_key] = client
+            logger.debug(f"Configured target {target_name}")
+
+        logger.info(f"Server ready with {len(app.clients)} targets")
+
+
+    def get_client(target_name):
+        target_key = target_name.lower()
+        if target_key in app.clients:
+            return app.clients[target_key]
+        return None
+
+
+    def get_target(request, path):
+        target_path = path
+        base_url = app.settings.base_url
+        
+        logger.trace(f"base_url: {base_url}")
+        logger.trace(f"request.url.hostname: {request.url.hostname}")
+
+        subdomain = None
+        if app.settings.virtual_buckets:
+            if base_url:
+                subdomain = request.url.hostname.removesuffix(base_url.host).removesuffix('.')
+            else:
+                logger.warning("virtual_buckets enabled but no base URL is configured")
+
+        if subdomain:
+            # Target is given in the subdomain
+            is_virtual = True
+            target_name = subdomain.split('.')[0]
+        else:
+            # Target is encoded as the first element in the path
+            is_virtual = False
+            # Extract target from path
+            ts = target_path.removeprefix('/').split('/', maxsplit=1)
+            logger.trace(f"target path components: {ts}")
+            if len(ts)==2:
+                target_name, target_path = ts
+            elif len(ts)==1:
+                # This happens if we are at the root of the proxy
+                target_name, target_path = ts[0], ''
+            else:
+                # This shouldn't happen
+                target_name, target_path = None, ''
+
+        logger.trace(f"target_name={target_name}, target_path={target_path}, is_virtual={is_virtual}")
+        return target_name, target_path, is_virtual
+
+
+    async def browse_bucket(request: Request,
+                            target_name: str,
+                            prefix: str,
+                            continuation_token: str = None,
+                            max_keys: int = 10,
+                            is_virtual: bool = False):
+        
+        target_config = app.settings.get_target_config(target_name)
+        if not target_config:
+            raise HTTPException(status_code=404, detail="Target bucket not found")
+
+        client = get_client(target_name)
+        if client is None:
+            raise HTTPException(status_code=500, detail="Client for target bucket not found")
+
+        response = await client.list_objects_v2(continuation_token, '/', None,
+                                                False, max_keys, prefix, None)
+
+        if response.status_code != 200:
+            # Return error respone
+            return response
+
+        xml = response.body.decode("utf-8")
+        root = parse_xml(xml)
+
+        common_prefixes = []
+        cps = [c for c in root.findall('CommonPrefixes')]
+        if cps:
+            for cp in cps:
+                common_prefixes += [dir_path(e.text) for e in cp.iter('Prefix')] if cps else []
+
+        contents = []
+        cs = [c for c in root.findall('Contents')]
+        if cs:
+            for c in cs:
+                key_elem = c.find('Key')
+                if key_elem is not None and key_elem.text != prefix:
+
+                    content = {'key': key_elem.text}
+
+                    size_elem = c.find('Size')
+                    if size_elem is not None and size_elem.text:
+                        num_bytes = int(size_elem.text)
+                        content['size'] = humanize_bytes(num_bytes)
+
+                    lm_elem = c.find('LastModified')
+                    if lm_elem is not None and lm_elem.text:
+                        content['lastmod'] = format_isoformat_as_local(lm_elem.text)
+
+                    contents.append(content)
+
+        next_token = None
+        truncated_elem = root.find('IsTruncated')
+        if truncated_elem is not None and truncated_elem.text=='true':
+            next_ct_elem = root.find('NextContinuationToken')
+            next_token = next_ct_elem.text
+
+        target_prefix = '' if is_virtual else '/'+target_name
+        parent_prefix = dir_path(os.path.dirname(prefix.rstrip('/')))
+
+        return templates.TemplateResponse("browse.html", {
+            "request": request,
+            "prefix": prefix,
+            "index_url": app.settings.base_url or '/',
+            "target_prefix": target_prefix,
+            "common_prefixes": common_prefixes,
+            "contents": contents,
+            "parent_prefix": parent_prefix,
+            "remove_prefix": remove_prefix,
+            "continuation_token": next_token
+        })
+
+
+    @app.get('/favicon.ico', include_in_schema=False)
+    async def favicon():
+        return FileResponse('static/favicon.ico')
+
+
+    @app.get('/robots.txt', response_class=PlainTextResponse)
+    def robots():
+        return """User-agent: *\nDisallow: /"""
+
+
+    @app.get("/{path:path}")
+    async def target_dispatcher(request: Request,
+                                path: str,
+                                list_type: int = Query(None, alias="list-type"),
+                                continuation_token: Optional[str] = Query(None, alias="continuation-token"),
+                                delimiter: Optional[str] = Query(None, alias="delimiter"),
+                                encoding_type: Optional[str] = Query(None, alias="encoding-type"),
+                                fetch_owner: Optional[bool] = Query(None, alias="fetch-owner"),
+                                max_keys: Optional[int] = Query(1000, alias="max-keys"),
+                                prefix: Optional[str] = Query(None, alias="prefix"),
+                                start_after: Optional[str] = Query(None, alias="start-after")):
+
+        target_name, target_path, is_virtual = get_target(request, path)
+        logger.debug(f"target_name={target_name}, target_path={target_path}, is_virtual={is_virtual}")
+
+        if not target_name or (is_virtual and target_name=='www'):
+            # Return target index
+            bucket_list = { target: f"/{target}/" for target in app.settings.get_browseable_targets()}
+            if app.settings.ui:
+                return templates.TemplateResponse("index.html", {"request": request, "links": bucket_list})
+            else:
+                xml = get_bucket_list_xml(bucket_list)
+                return Response(content=xml, status_code=200, media_type="application/xml")
+        
+        target_config = app.settings.get_target_config(target_name)
+        if not target_config:
+            return get_nosuchbucket_response(target_name)
+
+        client = get_client(target_name)
+        if client is None:
+            raise HTTPException(status_code=500, detail="Client for target bucket not found")
+
+        if 'acl' in request.query_params:
+            return get_read_access_acl()
+
+        if list_type:
+            if not target_path:
+                if list_type == 2:
+                    return await client.list_objects_v2(continuation_token, delimiter, \
+                        encoding_type, fetch_owner, max_keys, prefix, start_after)
+                else:
+                    raise HTTPException(status_code=400, detail="Invalid list type")
+            else:
+                range_header = request.headers.get("range")
+                return await client.get_object(target_path, range_header)
+
+        if not target_path or target_path.endswith("/"):
+            if app.settings.ui:
+                return await browse_bucket(request, target_name, target_path,
+                    continuation_token=continuation_token,
+                    max_keys=100,
+                    is_virtual=is_virtual)
+            else:
+                return get_nosuchbucket_response(target_name)
+        else:
+            range_header = request.headers.get("range")
+            return await client.get_object(target_path, range_header)
+
+
+
+    @app.head("{path:path}")
+    async def head_object(request: Request, path: str):
+
+        target_name, target_path, _ = get_target(request, path)
+        if not target_name:
+            return get_nosuchbucket_response('')
+
+        try:
+            target_config = app.settings.get_target_config(target_name)
+            if not target_config:
+                return get_nosuchbucket_response(target_name)
+
+            client = get_client(target_name)
+            if client is None:
+                raise HTTPException(status_code=500, detail="Client for target bucket not found")
+
+            return await client.head_object(target_path)
+        except:
+            logger.opt(exception=sys.exc_info()).info("Error requesting head")
+            return JSONResponse({"error":"Error requesting HEAD"}, status_code=500)
+
+    return app
+
+
+app = create_app(get_settings)
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=8000)

x2s3/client.py

@@ -0,0 +1,34 @@
+class ProxyClient:
+    """ Interface for a client that implements an S3-like interface 
+        to key-value access against some backend service. 
+
+        Note that this interface does not try to encode the entire S3 API.
+        We only care about the bare-bones functionality that is required 
+        for viewers like Neuroglancer, N5 Viewer, Vizarr, etc.
+    """
+
+    async def head_object(self, key: str):
+        """
+        Basic interface for AWS S3's HeadObject API.
+        https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html
+        """
+
+    async def get_object(self, key: str, range_header: str = None):
+        """
+        Basic interface for AWS S3's GetObject API.
+        https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html
+        """
+
+    async def list_objects_v2(self,
+                            continuation_token: str,
+                            delimiter: str,
+                            encoding_type: str,
+                            fetch_owner: str,
+                            max_keys: str,
+                            prefix: str,
+                            start_after: str):
+        """
+        Basic interface for AWS S3's ListObjectsV2 API.
+        https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html
+        """
+    

x2s3/client_aioboto.py

@@ -0,0 +1,292 @@
+import os
+import sys
+import typing
+from typing_extensions import override
+
+from loguru import logger
+from starlette.background import BackgroundTask
+import botocore
+from botocore.exceptions import NoCredentialsError, PartialCredentialsError
+from aiobotocore.session import get_session
+from aiobotocore.config import AioConfig
+from fastapi.responses import Response, StreamingResponse, JSONResponse
+
+from x2s3.utils import *
+from x2s3.client import ProxyClient
+
+def handle_s3_exception(e, key=None):
+    """ Handle various cases of generic errors from the boto AWS API.
+    """
+    if isinstance(e, (NoCredentialsError, PartialCredentialsError)):
+        logger.opt(exception=sys.exc_info()).error("AWS credentials not configured properly")
+        return JSONResponse({"error":"AWS credentials not configured properly"}, status_code=408)
+    elif isinstance(e, botocore.exceptions.ReadTimeoutError):
+        return JSONResponse({"error":"Upstream endpoint timed out"}, status_code=408)
+    elif isinstance(e, botocore.exceptions.ClientError):
+        status_code = e.response['ResponseMetadata']['HTTPStatusCode']
+        error = e.response['Error']
+        error_code = error['Code'] if 'Code' in error else 'Unknown'
+        if error_code == "NoSuchKey":
+            return get_nosuchkey_response(key)
+        else:
+            message = error['Message'] if 'Message' in error else 'Unknown'
+            resource = error['Resource'] if 'Resource' in error else 'Unknown'
+            return get_error_response(status_code, error_code, message, resource)
+    else:
+        logger.opt(exception=sys.exc_info()).error("Error communicating with AWS S3")
+        return JSONResponse({"error":"Error communicating with AWS S3"}, status_code=500)
+
+
+class AiobotoProxyClient(ProxyClient):
+
+    def __init__(self, proxy_kwargs, **kwargs):
+
+        self.proxy_kwargs = proxy_kwargs or {}
+        self.target_name = self.proxy_kwargs['target_name']
+        self.bucket_name = kwargs['bucket']
+        self.bucket_prefix = kwargs.get('prefix')
+
+        self.anonymous = True
+        access_key,secret_key = '',''
+
+        if 'access_key_path' in kwargs:
+            self.anonymous = False
+            access_key_path = kwargs['access_key_path']
+            secret_key_path = kwargs['secret_key_path']
+
+            with open(access_key_path, 'r') as ak_file:
+                access_key = ak_file.read().strip()
+
+            with open(secret_key_path, 'r') as sk_file:
+                secret_key = sk_file.read().strip()
+
+        self.client_kwargs = {
+            'aws_access_key_id': access_key,
+            'aws_secret_access_key': secret_key,
+        }
+
+        if 'endpoint' in kwargs:
+            self.client_kwargs['endpoint_url'] = kwargs.get('endpoint')
+
+
+    def get_client_creator(self):
+        session = get_session()
+        conf = AioConfig(signature_version=botocore.UNSIGNED) if self.anonymous else AioConfig()
+        return session.create_client('s3', config=conf, **self.client_kwargs)
+
+
+    @override
+    async def head_object(self, key: str):
+        real_key = key
+        if self.bucket_prefix:
+            real_key = os.path.join(self.bucket_prefix, key) if key else self.bucket_prefix
+
+        async with self.get_client_creator() as client:
+            try:
+                s3_res = await client.head_object(Bucket=self.bucket_name, Key=real_key)
+                headers = {
+                    "ETag": s3_res.get("ETag"),
+                    "Accept-Ranges": "bytes",
+                    "Content-Length": str(s3_res.get("ContentLength")),
+                    "Last-Modified": s3_res.get("LastModified").strftime("%a, %d %b %Y %H:%M:%S GMT"),
+                }
+
+                content_type = guess_content_type(real_key)
+                headers['Content-Type'] = content_type
+
+                return Response(headers=headers)
+            except Exception as e:
+                return handle_s3_exception(e, key)
+
+
+    @override
+    async def get_object(self, key: str, range_header: str = None):
+        real_key = key
+        if self.bucket_prefix:
+            real_key = os.path.join(self.bucket_prefix, key) if key else self.bucket_prefix
+
+        filename = os.path.basename(real_key)
+        content_type = guess_content_type(filename)
+
+        headers = {
+            'Accept-Ranges': "bytes",
+            'Content-Type': content_type,
+        }
+
+        if content_type=='application/octet-stream':
+            headers['Content-Disposition'] = f'attachment; filename="{filename}"'
+
+        try:
+            return S3Stream(
+                self.get_client_creator,
+                headers=headers,
+                media_type=content_type,
+                bucket=self.bucket_name,
+                key=key,
+                real_key=real_key,
+                range_header=range_header,
+                )
+        except Exception as e:
+            return handle_s3_exception(e, key)
+
+
+    @override
+    async def list_objects_v2(self,
+                            continuation_token: str,
+                            delimiter: str,
+                            encoding_type: str,
+                            fetch_owner: str,
+                            max_keys: str,
+                            prefix: str,
+                            start_after: str):
+
+        # prefix user-supplied prefix with configured prefix
+        real_prefix = prefix
+        if self.bucket_prefix:
+            real_prefix = os.path.join(self.bucket_prefix, prefix) if prefix else self.bucket_prefix
+
+        # ensure the prefix ends with a slash
+        if real_prefix and not real_prefix.endswith('/'):
+            real_prefix += '/'
+
+        async with self.get_client_creator() as client:
+            try:
+                params = {
+                    "Bucket": self.bucket_name,
+                    "ContinuationToken": continuation_token,
+                    "Delimiter": delimiter,
+                    "EncodingType": encoding_type,
+                    "FetchOwner": fetch_owner,
+                    "MaxKeys": max_keys,
+                    "Prefix": real_prefix,
+                    "StartAfter": start_after
+                }
+                # Remove any None values because boto3 doesn't like those
+                params = {k: v for k, v in params.items() if v is not None}
+
+                response = await client.list_objects_v2(**params)
+                next_token = remove_prefix(self.bucket_prefix, response.get("NextContinuationToken", ""))
+                is_truncated = "true" if response.get("IsTruncated", False) else "false"
+
+                contents = []
+                for obj in response.get("Contents", []):
+                    contents.append({
+                        'Key': remove_prefix(self.bucket_prefix, obj["Key"]),
+                        'LastModified': obj["LastModified"].isoformat(),
+                        'ETag': obj.get("ETag"),
+                        'Size': obj.get("Size"),
+                        'StorageClass': obj.get("StorageClass")
+                    })
+
+                common_prefixes = []
+                for cp in response.get("CommonPrefixes", []):
+                    common_prefix = remove_prefix(self.bucket_prefix, cp["Prefix"])
+                    common_prefixes.append(common_prefix)
+
+                kwargs = {
+                    'Name': self.target_name,
+                    'Prefix': prefix,
+                    'Delimiter': delimiter,
+                    'MaxKeys': max_keys,
+                    'EncodingType': encoding_type,
+                    'KeyCount': response.get("KeyCount", 0),
+                    'IsTruncated': is_truncated,
+                    'ContinuationToken': continuation_token,
+                    'NextContinuationToken': next_token,
+                    'StartAfter': start_after
+                }
+
+                xml = get_list_xml(contents, common_prefixes, **kwargs)
+                return Response(content=xml, media_type="application/xml")
+
+            except Exception as e:
+                return handle_s3_exception(e, key=prefix)
+
+
+# Adapted from https://stackoverflow.com/questions/69617252/response-file-stream-from-s3-fastapi
+class S3Stream(StreamingResponse):
+    """ Stream the result of GetObject.
+    """
+    def __init__(
+            self,
+            client_creator: typing.Callable,
+            content: typing.Any = None,
+            status_code: int = 200,
+            headers: dict = None,
+            media_type: str = None,
+            background: BackgroundTask = None,
+            bucket: str = None,
+            key: str = None,
+            real_key: str = None,
+            range_header: str = None
+    ):
+        super(S3Stream, self).__init__(content, status_code, headers, media_type, background)
+        self.client_creator = client_creator
+        self.bucket = bucket
+        self.key = key
+        self.real_key = real_key
+        self.range_header = range_header
+
+    async def stream_response(self, send) -> None:
+
+        async def send_response(r):
+            await send({
+                "type": "http.response.start",
+                "status": r.status_code,
+                "headers": r.raw_headers,
+            })
+            await send({
+                "type": "http.response.body",
+                "body": r.body,
+                "more_body": False,
+            })
+
+        async with self.client_creator() as client:
+            result = None
+            try:
+                # Get the object with the range specified in headers
+                get_object_params = {
+                    "Bucket": self.bucket,
+                    "Key": self.real_key,
+                }
+                if self.range_header:
+                    get_object_params["Range"] = self.range_header
+
+                result = await client.get_object(**get_object_params)
+                res_headers = result["ResponseMetadata"]["HTTPHeaders"]
+
+                # Determine if this is a Range result
+                if "content-range" in res_headers:
+                    self.status_code = 206 # Partial Content
+                    self.raw_headers.append((b"content-range",
+                        res_headers["content-range"].encode('utf-8')))
+                    
+                if "content-length" in res_headers:
+                    self.raw_headers.append((b"content-length",
+                        res_headers["content-length"].encode('utf-8')))
+
+                await send({
+                    "type": "http.response.start",
+                    "status": self.status_code,
+                    "headers": self.raw_headers,
+                })
+
+                async for chunk in result["Body"]:
+
+                    if not isinstance(chunk, bytes):
+                        chunk = chunk.encode(self.charset)
+
+                    await send({
+                        "type": "http.response.body",
+                        "body": chunk,
+                        "more_body": True
+                    })
+
+                await send({
+                    "type": "http.response.body",
+                    "body": b"",
+                    "more_body": False})
+
+            except Exception as e:
+                r = handle_s3_exception(e, self.key)
+                await send_response(r)