mx8fs 1.0.0.1__tar.gz

mx8fs-1.0.0.1/PKG-INFO

@@ -0,0 +1,78 @@
+Metadata-Version: 2.3
+Name: mx8fs
+Version: 1.0.0.1
+Summary: MX8 Filing system
+License: MIT
+Author: Tom Weiss
+Author-email: tom@mx8labs.com
+Requires-Python: >=3.10
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: boto3 (>=1.26.137)
+Description-Content-Type: text/markdown
+
+# MX8 File system
+
+This library provides environment agnostic file system access across local and AWS, including:
+- File / IO
+- List / Glob
+- Locking
+- Caching
+- Comparing Dictionaries
+
+# Pre-commit hooks
+
+We use precommit to run formatting checks, so whenever you clone a project run:
+
+```bash
+pre-commit install
+```
+
+Before you do anything else.
+
+You can run this at any time using:
+
+```bash
+pre-commit run --all-files
+```
+
+## Setting up the development environment
+
+You can install the full dev requirements by running [setup.sh](setup.sh) to
+1. Install the current repo and the python lib
+1. Run the pre-commit hooks on all files
+
+The project should open reasonable well in vs.code and includes three [launch configurations](.vscode/launch.json) for running unit tests and the debug server.
+
+## Code conventions and structure
+
+We use python type hinting with pylance and flake8 for linting. Unit tests are created to 100% branch coverage.
+
+The code is structured as follows:
+
+- The [mx8fs](mx8fs) folder contains the full library.
+- Tests are stored in the [tests](test) folder and run using pytest.
+
+* The github actions are in [main.yam](.github/workflows/main.yml)
+
+## License
+
+Copyright © 2025 MX8 Labs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+documentation files (the “Software”), to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
+and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions
+of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
+OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

mx8fs-1.0.0.1/README.md

@@ -0,0 +1,60 @@
+# MX8 File system
+
+This library provides environment agnostic file system access across local and AWS, including:
+- File / IO
+- List / Glob
+- Locking
+- Caching
+- Comparing Dictionaries
+
+# Pre-commit hooks
+
+We use precommit to run formatting checks, so whenever you clone a project run:
+
+```bash
+pre-commit install
+```
+
+Before you do anything else.
+
+You can run this at any time using:
+
+```bash
+pre-commit run --all-files
+```
+
+## Setting up the development environment
+
+You can install the full dev requirements by running [setup.sh](setup.sh) to
+1. Install the current repo and the python lib
+1. Run the pre-commit hooks on all files
+
+The project should open reasonable well in vs.code and includes three [launch configurations](.vscode/launch.json) for running unit tests and the debug server.
+
+## Code conventions and structure
+
+We use python type hinting with pylance and flake8 for linting. Unit tests are created to 100% branch coverage.
+
+The code is structured as follows:
+
+- The [mx8fs](mx8fs) folder contains the full library.
+- Tests are stored in the [tests](test) folder and run using pytest.
+
+* The github actions are in [main.yam](.github/workflows/main.yml)
+
+## License
+
+Copyright © 2025 MX8 Labs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+documentation files (the “Software”), to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
+and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions
+of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
+OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

mx8fs-1.0.0.1/mx8fs/__init__.py

@@ -0,0 +1,47 @@
+"""
+MX8 - Common utilities for MX8 projects
+
+Copyright (c) 2023 MX8 Inc, all rights reserved.
+
+This software is confidential and proprietary information of MX8.
+You shall not disclose such Confidential Information and shall use it only
+in accordance with the terms of the agreement you entered into with MX8.
+"""
+
+from .file_io import (
+    BinaryFileHandler,
+    read_file,
+    write_file,
+    list_files,
+    copy_file,
+    move_file,
+    delete_file,
+    file_exists,
+    get_public_url,
+    most_recent_timestamp,
+)
+from .cache import cache_to_disk, cache_to_disk_binary, get_cache_filename
+from .lock import FileLock, Waiter
+from .storage import JsonFileStorage, json_file_storage_factory
+from .comparer import ResultsComparer
+
+__all__ = [
+    "BinaryFileHandler",
+    "cache_to_disk_binary",
+    "cache_to_disk",
+    "copy_file",
+    "delete_file",
+    "file_exists",
+    "get_public_url",
+    "FileLock",
+    "get_cache_filename",
+    "json_file_storage_factory",
+    "JsonFileStorage",
+    "move_file",
+    "list_files",
+    "read_file",
+    "ResultsComparer",
+    "Waiter",
+    "write_file",
+    "most_recent_timestamp",
+]

mx8fs-1.0.0.1/mx8fs/cache.py

@@ -0,0 +1,204 @@
+"""
+Cache decorator for the MX8 AI API.
+
+Copyright (c) 2023-2025 MX8 Inc, all rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+documentation files (the “Software”), to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
+and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions
+of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
+OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+"""
+
+import hashlib
+import json
+import logging
+import pickle
+from json.decoder import JSONDecodeError
+from time import time
+from typing import Any, Callable, Dict, List, Optional, Tuple
+
+from .file_io import BinaryFileHandler, read_file, write_file
+
+
+def get_cache_filename(path: str, name: str, extension: str, expiration_seconds: int = 0, **kwargs: Dict) -> str:
+    """Create an optionally time expiring cache filename using hashed parameters."""
+
+    # Create hashes based on the name and parameter hashes
+    args = kwargs.pop("extra_args", ())
+    param_hash = hashlib.sha256(pickle.dumps((args, kwargs))).hexdigest()
+
+    # If the cache is set to expire, add the current epoch rounded down
+    # to the nearest expiration_seconds to the hash
+    if expiration_seconds > 0:
+        epoch = int(int(time()))
+        time_hash = "_" + str(epoch - epoch % expiration_seconds)
+    else:
+        time_hash = ""
+
+    # Create the filename for the cache
+    return f"{path}/{name}_{param_hash}{time_hash}.cache.{extension}"
+
+
+def _get_clean_kwargs(kwargs: Dict, ignore_kwargs: Optional[List[str]]) -> Dict:
+    """Remove ignored kwargs from the kwargs dict"""
+
+    clean_kwargs = kwargs.copy()
+    for ignore in ignore_kwargs or []:
+        clean_kwargs.pop(ignore, None)
+
+    return clean_kwargs
+
+
+def _do_logging(
+    log_group: str,
+    result: Any,
+    args: Tuple,
+    kwargs: Dict,
+    filename: str,
+    func: Callable,
+    expiration_seconds: int,
+) -> None:
+    """Log cache hit"""
+
+    try:
+        result = json.loads(result)
+    except (JSONDecodeError, TypeError):
+        pass
+
+    if log_group:
+        try:
+            logging.getLogger(log_group).debug(
+                "Cache hit",
+                extra={
+                    "cache_result": result,
+                    "cache_args": args,
+                    "cache_kwargs": kwargs,
+                    "cache_filename": filename,
+                    "cache_function": func.__name__,
+                    "cache_expiration_seconds": expiration_seconds,
+                },
+            )
+        except TypeError:  # pragma: no cover
+            # If we get a type error, case the dangerous types to strings
+            logging.getLogger(log_group).debug(
+                "Cache hit",
+                extra={
+                    "cache_result": str(result),
+                    "cache_args": args,
+                    "cache_kwargs": str(kwargs),
+                    "cache_filename": filename,
+                    "cache_function": func.__name__,
+                    "cache_expiration_seconds": expiration_seconds,
+                },
+            )
+
+
+def cache_to_disk_binary(
+    path: str,
+    expiration_seconds: int = 0,
+    log_group: str = "",
+    ignore_kwargs: Optional[List[str]] = None,
+) -> Callable[..., Callable[..., Any]]:
+    """Cache decorator for any MX8 functions.
+
+    This decorator will cache the result of the function to disk, and return
+    the cached result on subsequent calls. This is useful for caching the
+    results of expensive operations, such as calling an AI API
+
+    Parameters:
+        path: The path to the cache directory
+        expiration_seconds: The number of seconds before the cache expires
+        log_group: The log group to log cache hits to
+        ignore_kwargs: A list of kwargs to ignore when creating the cache key
+    """
+
+    def decorator(func: Callable) -> Callable[..., Any]:
+        def wrapper(*args: Tuple, **kwargs: Dict) -> Any:
+            clean_kwargs = _get_clean_kwargs(kwargs, ignore_kwargs)
+
+            filename = get_cache_filename(
+                path,
+                func.__name__,
+                "pickle",
+                expiration_seconds,
+                extra_args=args,  # type: ignore
+                **clean_kwargs,
+            )
+
+            try:
+                # Try to read the cached result from disk
+                with BinaryFileHandler(filename) as file_handler:
+                    result = pickle.load(file_handler)
+
+                _do_logging(log_group, result, args, clean_kwargs, filename, func, expiration_seconds)
+
+            except FileNotFoundError:
+                # Cache miss, execute the function and save the result to disk
+                result = func(*args, **kwargs)
+                with BinaryFileHandler(filename, "wb") as file_handler:
+                    pickle.dump(result, file_handler)
+
+            return result
+
+        return wrapper
+
+    return decorator
+
+
+def cache_to_disk(
+    path: str,
+    expiration_seconds: int = 0,
+    log_group: str = "",
+    ignore_kwargs: Optional[List[str]] = None,
+) -> Callable[..., Callable[..., str | Any]]:
+    """Cache decorator for any MX8 functions.
+
+    This decorator will cache the result of the function to disk, and return
+    the cached result on subsequent calls. This is useful for caching the
+    results of expensive operations, such as calling an AI API
+
+    Parameters:
+        path: The path to the cache directory
+        expiration_seconds: The number of seconds before the cache expires
+        log_group: The log group to log cache hits to
+        ignore_kwargs: A list of kwargs to ignore when creating the cache key
+    """
+
+    def decorator(func: Callable) -> Callable[..., str | Any]:
+        def wrapper(*args: Tuple, **kwargs: Dict) -> str | Any:
+
+            clean_kwargs = _get_clean_kwargs(kwargs, ignore_kwargs)
+
+            filename = get_cache_filename(
+                path,
+                func.__name__,
+                "txt",
+                expiration_seconds,
+                extra_args=args,  # type: ignore
+                **clean_kwargs,
+            )
+
+            try:
+                # Try to read the cached result from disk
+                result = read_file(filename)
+
+                _do_logging(log_group, result, args, clean_kwargs, filename, func, expiration_seconds)
+
+            except FileNotFoundError:
+                # Cache miss, execute the function and save the result to disk
+                result = func(*args, **kwargs)
+                write_file(filename, result)
+
+            return result
+
+        return wrapper
+
+    return decorator

mx8fs-1.0.0.1/mx8fs/comparer.py

@@ -0,0 +1,195 @@
+"""
+Support functions for testing
+
+Copyright (c) 2023-2025 MX8 Inc, all rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+documentation files (the “Software”), to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
+and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions
+of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
+OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+"""
+
+import json
+import os
+import shutil
+from difflib import ndiff
+from logging import getLogger
+from tempfile import NamedTemporaryFile
+from typing import Any, Dict, List, Optional
+
+
+from mx8fs import read_file, write_file
+
+logger = getLogger("mx8.comparer")
+
+
+def get_diff(a: str, b: str) -> str:
+    return "\n".join(d for d in ndiff(a.splitlines(), b.splitlines()) if not d.startswith(" "))
+
+
+class Differences:
+    def __init__(self) -> None:
+        self._differences: List[Dict[str, str]] = []
+
+    def __repr__(self) -> str:
+        return json.dumps(self._differences, indent=4)
+
+    def __eq__(self, value: object) -> bool:
+        return self._differences == value
+
+    def __bool__(self) -> bool:
+        return bool(self._differences)
+
+    def __len__(self) -> int:
+        return len(self._differences)
+
+    def append(self, differences: Dict[str, str]) -> None:
+        self._differences.append(differences)
+
+    def clear(self) -> None:
+        self._differences.clear()
+
+    @property
+    def keys(self) -> List[str]:
+        return [list(d.keys())[0] for d in self._differences]
+
+
+class ResultsComparer:
+    def __init__(
+        self,
+        ignore_keys: Optional[List[str]],
+        create_test_data: bool = False,
+    ) -> None:
+        self._ignore_keys = ignore_keys if ignore_keys else []
+        self._create_test_data = create_test_data
+        self._differences = Differences()
+
+    def _log_differences(self, key: str, correct: str, test: str) -> None:
+        """Log the differences between two strings"""
+        if correct != test:
+            self._differences.append({key: get_diff(correct, test)})
+
+    def _compare_dicts(self, correct: Any, test: Any, recursive: bool = False, root_key: str = "root") -> None:
+        """
+        Compare two dictionaries recursively, ignoring elements with the given key
+        """
+
+        if not recursive:
+            logger.debug(
+                {
+                    "message": "Comparing dictionaries",
+                    "dict1": correct,
+                    "dict2": test,
+                    "ignore_keys": self._ignore_keys,
+                }
+            )
+
+        if isinstance(correct, list) and isinstance(test, list):
+            if len(correct) != len(test):
+                self._log_differences(root_key, json.dumps(correct), json.dumps(test))
+                return
+
+            for i, the_dict in enumerate(correct):
+                self._compare_dicts(the_dict, test[i], recursive=True, root_key=f"{root_key}[{i}]")
+
+            return
+
+        # Check if both inputs are dictionaries
+        if not isinstance(correct, dict) or not isinstance(test, dict):
+            self._log_differences(root_key, json.dumps(correct), json.dumps(test))
+            return
+
+        # Get the set of keys for each dictionary
+        correct_keys = set(correct.keys())
+        test_keys = set(test.keys())
+
+        # Check if the keys are the same
+        if correct_keys != test_keys:
+            self._log_differences(root_key, json.dumps(correct), json.dumps(test))
+        else:
+            # Recursively compare the values for each key
+            for key in correct_keys:
+                if key not in self._ignore_keys:
+                    self._compare_dicts(correct[key], test[key], recursive=True, root_key=f"{root_key}/{key}")
+
+    def compare_dicts(self, correct: Any, test: Any) -> Differences:
+        """Compare two dictionaries"""
+        self._compare_dicts(correct, test)
+        return self._differences
+
+    def get_text_differences(self, test: str, correct: str) -> Differences:
+        """Compare a test file with a correct file"""
+        if self._create_test_data:
+            # make the directory if it doesn't exist
+            os.makedirs(os.path.dirname(correct), exist_ok=True)
+
+            # copy the test file to the correct file
+            shutil.copyfile(test, correct)
+
+        differences = Differences()
+
+        if diff := get_diff(read_file(correct), read_file(test)):
+            differences.append({"file": diff})
+
+        return differences
+
+    def get_dict_differences(self, test: str, correct: str) -> Differences:
+        """Compare a test file with a correct file"""
+
+        # Load the test file
+        test_dict = json.loads(read_file(test))
+
+        self._differences.clear()
+        if self._create_test_data:
+            try:
+                correct_dict = json.loads(read_file(correct))
+                self._compare_dicts(correct_dict, test_dict)
+                assert self._differences == [], "The files should be identical"
+            except (FileNotFoundError, AssertionError):
+                # Save the test file as the correct file
+                os.makedirs(os.path.dirname(correct), exist_ok=True)
+                write_file(correct, json.dumps(test_dict, indent=4, ensure_ascii=False).strip())
+                self._differences.clear()
+        else:
+            correct_dict = json.loads(read_file(correct))
+            self._compare_dicts(correct_dict, test_dict)
+
+        return self._differences
+
+    def get_api_response_differences(
+        self,
+        response: Any,
+        correct_file: str,
+    ) -> Differences:
+        """Check the response from the reporting API and return the differences"""
+
+        file_name = os.path.basename(correct_file)
+
+        # Write the response to a temporary file
+        try:
+            result = json.dumps(response.json(), indent=4, ensure_ascii=False)
+            compare_func = self.get_dict_differences
+        except json.JSONDecodeError:
+            compare_func = self.get_text_differences
+            result = response.text
+
+        with NamedTemporaryFile(mode="wt", delete=False, prefix=file_name) as temp_file:
+            temp_file.write(result.strip())
+            temp_file.flush()
+            temp_file_name = temp_file.name
+
+        # Compare the response to the correct file
+        mismatches = compare_func(temp_file_name, correct_file)
+
+        # Clean up the temporary file
+        os.remove(temp_file_name)
+
+        return mismatches

mx8fs-1.0.0.1/mx8fs/file_io.py

@@ -0,0 +1,243 @@
+"""
+AWS file IO functions
+
+Copyright (c) 2023-2025 MX8 Inc, all rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+documentation files (the “Software”), to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
+and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions
+of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
+OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+"""
+
+import os
+from datetime import datetime
+from glob import glob
+from io import BytesIO
+from shutil import copyfile
+from typing import IO, Any, Dict, List, Literal, Tuple, cast
+
+import boto3
+from botocore.config import Config
+
+boto_config = Config(
+    max_pool_connections=int(os.getenv("BOTO_MAX_CONNECTIONS", 50)),
+    connect_timeout=float(os.getenv("BOTO_CONNECT_TIMEOUT", 5.0)),
+    read_timeout=float(os.getenv("BOTO_READ_TIMEOUT", 840.0)),  # 1 minute less than the lambda timeout
+    retries={
+        "total_max_attempts": int(os.getenv("BOTO_MAX_RETRIES", 10)),
+        "mode": cast(Literal["legacy", "standard", "adaptive"], os.getenv("BOTO_RETRY_MODE", "adaptive")),
+    },
+)
+
+s3_client = boto3.client(
+    service_name="s3",
+    config=boto_config,
+)
+
+
+def get_bucket_key(path: str) -> Tuple[str, str]:
+    """Get the bucket and key from a S3 path"""
+    path = path.replace("s3://", "")
+    bucket, key = path.split("/", 1)
+    return bucket, key
+
+
+def file_exists(file: str) -> bool:
+    """Check if a file exists on S3 or local storage"""
+    if file.startswith("s3://"):
+        bucket, key = get_bucket_key(file)
+        try:
+            s3_client.head_object(Bucket=bucket, Key=key)
+            return True
+        except s3_client.exceptions.ClientError:
+            return False
+
+    return os.path.exists(file)
+
+
+def read_file(file: str) -> str:
+    """Read a file from S3 or local storage with UTF-8 encoding"""
+    if file.startswith("s3://"):
+        bucket, key = get_bucket_key(file)
+        try:
+            return str(s3_client.get_object(Bucket=bucket, Key=key)["Body"].read().decode("utf-8"))
+        except s3_client.exceptions.NoSuchKey as exc:
+            raise FileNotFoundError(f"File {file} not found") from exc
+    else:
+        with open(file, mode="r", encoding="UTF-8") as file_io:
+            return file_io.read()
+
+
+def write_file(file: str, data: str) -> None:
+    """Write a file to S3 or local storage with UTF-8 encoding"""
+    if file.startswith("s3://"):
+        bucket, key = get_bucket_key(file)
+        s3_client.put_object(Bucket=bucket, Key=key, Body=data.encode("UTF-8"))
+    else:
+        os.makedirs(os.path.dirname(file), exist_ok=True)
+        with open(file, mode="w", encoding="UTF-8") as file_io:
+            file_io.write(data)
+
+
+def delete_file(file: str) -> None:
+    """Delete a file from S3 or local storage"""
+    if file.startswith("s3://"):
+        bucket, key = get_bucket_key(file)
+        s3_client.delete_object(Bucket=bucket, Key=key)
+    else:
+        try:
+            os.remove(file)
+        except FileNotFoundError:
+            # Ignore if the file does not exist for S3 consistency
+            pass
+
+
+def copy_file(src: str, dst: str) -> None:
+    """Copy a file from S3 or local storage"""
+    if src.startswith("s3://"):
+        src_bucket, src_key = get_bucket_key(src)
+        dst_bucket, dst_key = get_bucket_key(dst)
+
+        try:
+            s3_client.copy_object(
+                Bucket=dst_bucket,
+                Key=dst_key,
+                CopySource={"Bucket": src_bucket, "Key": src_key},
+            )
+        except s3_client.exceptions.NoSuchKey as exc:
+            raise FileNotFoundError(f"File {src} not found") from exc
+    else:
+        os.makedirs(os.path.dirname(dst), exist_ok=True)
+        copyfile(src, dst)
+
+
+def move_file(src: str, dst: str) -> None:
+    """Move a file from S3 or local storage"""
+    copy_file(src, dst)
+    delete_file(src)
+
+
+def list_files(root_path: str, file_type: str, prefix: str = "") -> List[str]:
+    """Returns a list of files from S3 or local storage with the relevant suffix and optional prefix.
+
+    The prefix signficantly improves performance for S3 by reducing the number of objects listed.
+    """
+    if root_path.startswith("s3://"):
+        bucket, key = get_bucket_key(root_path)
+        key = key + "/" if not key.endswith("/") else key
+
+        paginator = s3_client.get_paginator("list_objects_v2")
+        files = []
+
+        for page in paginator.paginate(Bucket=bucket, Prefix=key + prefix, PaginationConfig={"PageSize": 10_000}):
+            if "Contents" in page:
+                files.extend(
+                    [
+                        obj["Key"].removeprefix(key).removesuffix(f".{file_type}")
+                        for obj in page["Contents"]
+                        if obj["Key"].endswith(file_type)
+                    ]
+                )
+
+        return files
+    return [os.path.split(f)[1][: -len(file_type) - 1] for f in glob(os.path.join(root_path, f"{prefix}*.{file_type}"))]
+
+
+def most_recent_timestamp(root_path: str, file_type: str) -> float:
+    """Returns the most recent timestamp from S3 or local storage with the suffix"""
+    if root_path.startswith("s3://"):
+        bucket, key = get_bucket_key(root_path)
+        boto_response = s3_client.list_objects_v2(Bucket=bucket, Prefix=key, Delimiter="/")
+        if "Contents" not in boto_response:
+            return 0
+
+        return max(
+            [obj["LastModified"] for obj in boto_response["Contents"] if obj["Key"].endswith(file_type)],
+            default=datetime(1970, 1, 1),
+        ).timestamp()
+
+    return max(
+        [os.path.getmtime(f) for f in glob(os.path.join(root_path, f"*.{file_type}"))],
+        default=0,
+    )
+
+
+def get_public_url(file: str, expires_in: int = 3600) -> str:
+    """Get a signed URL for a file on S3"""
+
+    if file.startswith("s3://"):
+        bucket, key = get_bucket_key(file)
+        presigned_url = s3_client.generate_presigned_url(
+            ClientMethod="get_object",
+            Params={"Bucket": bucket, "Key": key},
+            ExpiresIn=expires_in,
+        )
+
+        return str(presigned_url)
+
+    return file
+
+
+class BinaryFileHandler:
+    """File handler for S3 or local storage"""
+
+    _buffer: IO[Any]
+
+    def __init__(self, path: str, mode: str = "rb", content_type: str | None = None):
+        """
+        Creates the class, emulating the file object.
+
+        For S3, returns a BytesIO object for writing, and downloads the file
+        For local storage, returns a file object
+        """
+
+        if mode not in ["rb", "wb"]:
+            raise NotImplementedError(f"mode {mode} is not supported")
+
+        self.path = path
+        self.mode = mode
+        self.content_type = content_type
+        self.is_s3 = path.startswith("s3://")
+
+        if self.is_s3:
+            self._buffer = BytesIO()
+        else:
+            os.makedirs(os.path.dirname(self.path), exist_ok=True)
+            self._buffer = open(  # pylint: disable=consider-using-with
+                self.path, self.mode, encoding="UTF-8" if self.mode == "w" else None
+            )
+
+    def __enter__(self) -> BytesIO | IO:
+        """Read from S3 and open the stream"""
+        if self.is_s3:
+            bucket, key = get_bucket_key(self.path)
+            if self.mode == "rb":
+                # Download the file from S3 to the stream
+                try:
+                    s3_client.download_fileobj(Bucket=bucket, Key=key, Fileobj=self._buffer)
+                except s3_client.exceptions.ClientError as exc:
+                    raise FileNotFoundError(f"File {self.path} not found") from exc
+                self._buffer.seek(0)
+
+        return self._buffer
+
+    def __exit__(self, *_: List[Any], **__: Dict[str, Any]) -> None:
+        """Write to S3 or local storage and close the stream"""
+        if self.is_s3 and self.mode == "wb":
+            self._buffer.seek(0)
+            bucket, key = get_bucket_key(self.path)
+            s3_client.upload_fileobj(
+                Fileobj=self._buffer,
+                Bucket=bucket,
+                Key=key,
+                ExtraArgs=({"ContentType": self.content_type} if self.content_type else None),
+            )
+        self._buffer.close()

mx8fs-1.0.0.1/mx8fs/lock.py

@@ -0,0 +1,235 @@
+"""
+Locks a file across multiple process and clients.
+
+Based on DoggoLock (https://bitbucket.org/deductive/newtools/src/master/newtools/doggo/lock.py)
+
+Copyright (c) 2012-2025, Deductive Limited
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice,
+    this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright notice,
+    this list of conditions and the following disclaimer in the documentation
+    and/or other materials provided with the distribution.
+    * Neither the name of the Deductive Limited nor the names of
+    its contributors may be used to endorse or promote products derived from
+    this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+This version
+
+Copyright (c) 2024-2025 MX8 Inc, all rights reserved.
+
+"""
+
+import logging
+import string
+from datetime import datetime, timedelta
+from functools import cached_property
+from random import choice
+from time import sleep, time
+from typing import Any, Dict, List
+
+from mx8fs import delete_file, list_files, write_file
+
+logger = logging.getLogger("mx8.lock")
+
+TIME_FORMAT = "%Y%m%d%H%M%S"
+
+
+class Waiter:
+    """
+
+    A generic wait and timeout class.
+
+    Raises TimeoutError() if the task doesn't occur in the period.
+
+    ```python
+
+        waiter = Waiter()
+        waiter.start_timeout()
+        while not check_my_condition():
+            waiter.check_timeout()
+    ```
+    """
+
+    def __init__(self, wait_period: float, time_out_seconds: float):
+        """
+        :param wait_period: the period to wait for between iterations
+        :param time_out_seconds: the time after which a TimeoutError is raised
+        """
+        self.time_out_seconds = time_out_seconds
+        self.wait_period = wait_period
+        self._timeout: float | None = None
+
+    def __enter__(self) -> "Waiter":
+        """
+        Starts the timer
+        """
+        self.start_timeout()
+        return self
+
+    def __exit__(self, *_: Any) -> None:
+        """
+        Stops the timer
+        """
+        self._timeout = None
+
+    def wait(self, number_times: int = 1) -> None:
+        """
+        Waits for the defined period
+        """
+        sleep(self.wait_period * number_times)
+
+    def start_timeout(self) -> None:
+        """
+        Starts a time out
+        """
+        self._timeout = time() + self.time_out_seconds
+
+    def timed_out(self) -> bool:
+        """
+        Checks for a time out
+
+        :return: true if the timer has timed out, otherwise false
+        """
+        if self._timeout is not None:
+            return time() > self._timeout
+        else:
+            raise ValueError("Someone has tried to call timed_out() before calling start_timeout()")
+
+    def check_timeout(self) -> None:
+        """
+        Waits, and raises an exception if the timer has timed out
+        """
+        self.wait()
+        if self.timed_out():
+            raise TimeoutError("Timed out waiting for completion")
+
+
+class FileLock:
+    """
+    Implements locking using an additional file on the file system.
+
+    For file systems that are only eventually consistent, use a longer
+    wait_period to wait for consistency when multiple clients are reading at the same time.
+
+    Locks a file across multiple process and clients using an additional file on the file system.
+
+    The lock file has the following format:
+    "{file}.{timestamp}.{random}.lock"
+
+    where:
+
+    * file - is the file being locked
+    * timestamp - is the timestamp the lock was requested in the format %Y%m%d%H%M%S
+    * random - is a random set of letters
+
+    On creating the lock we
+
+    1. Check to see if anyone already has a lock
+    2. If they don't, attempt to create a lock and wait for wait_period.
+    3. If two processes have attempted to get the lock then the one with the earlier lock gets it.
+    4. When the lock is released the lock class deletes the lock and other processes can proceed
+
+    Locks only last for maximum_age seconds, and any request will time out after time_out_seconds
+    """
+
+    def __init__(
+        self,
+        file: str,
+        wait_period: float = 0.1,
+        time_out_seconds: int = 840,  # 1 minute less than the lambda timeout
+        maximum_age: int = 900,  # 15 minutes, the maximum time a lambda can run
+    ):
+        """
+        Initializes the lock
+
+        :param file: the file to lock
+        :param wait_period: the period to wait before confirming file lock
+        :param time_out_seconds: the time out to stop waiting after
+        :param maximum_age: the maximum age of lock files to respect
+        """
+        self.file = file
+        self.waiter = Waiter(wait_period, time_out_seconds)
+        self.maximum_age = timedelta(seconds=maximum_age)
+
+    @cached_property
+    def _lock_file(self) -> str:
+        """Get the lock file name."""
+        timestamp = datetime.now().strftime(TIME_FORMAT)
+        random_key = "".join(choice(string.ascii_lowercase) for _ in range(12))  # NOSONAR
+
+        return f"{self.file}.{timestamp}.{random_key}.lock"
+
+    def __enter__(self) -> "FileLock":
+        """Acquire the lock on the file. This will wait until the lock is available."""
+
+        logger.debug("Getting lock on %s", self.file)
+
+        # If the file is locked then wait for it to be unlocked
+        self.waiter.start_timeout()
+        while len(self._get_lock_files()) > 0:
+            self.waiter.check_timeout()
+
+        # create a lock file
+        write_file(self._lock_file, "locked")
+
+        # Check an wait in case another process is trying to get the lock
+        self.waiter.wait()
+        while (
+            len(lock_files := self._get_lock_files()) > 1 and lock_files[0] != self._lock_file
+        ):  # pragma: no cover - coverage is not collected for multi-process tests
+            try:
+                self.waiter.check_timeout()
+            except TimeoutError as ex:
+                delete_file(self._lock_file)
+                raise ex
+
+        logger.debug("Acquired lock on %s", self.file)
+        return self
+
+    def __exit__(self, *_: List[Any], **__: Dict[str, Any]) -> None:
+        """Release the lock on the file."""
+        delete_file(self._lock_file)
+        logger.debug("Released lock on %s", self.file)
+
+    def _get_lock_files(self) -> List[str]:
+        """Get all the lock files for the current file."""
+        path = "/".join(self.file.split("/")[:-1])
+        prefix = self.file.split("/")[-1]
+
+        # Get all the lock files in the same directory
+        files = [f"{path}/{file}.lock" for file in list_files(path, "lock", prefix)]
+
+        # Return the sorted current lock files for the current file
+        return sorted(file for file in files if self._lock_is_current(file))
+
+    def _lock_is_current(self, lock_file: str) -> bool:
+        """Check if the lock file is current."""
+
+        # If the lock file is not for the current file then it is not current
+        if self.file not in lock_file:
+            return False
+
+        # If we cannot parse the timestamp then it is not current
+        try:
+            timestamp = datetime.strptime(lock_file.split(".")[-3], TIME_FORMAT)
+        except (IndexError, ValueError):
+            return False
+
+        # If we are less than the maximum age then it is current
+        return datetime.now() < timestamp + self.maximum_age

mx8fs-1.0.0.1/mx8fs/storage.py

@@ -0,0 +1,144 @@
+"""
+Generic file storage class.
+
+Copyright (c) 2023-2025 MX8 Inc, all rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+documentation files (the “Software”), to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
+and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions
+of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
+OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+"""
+
+import os
+import random
+import string
+from typing import Any, Callable, List, Optional, Union
+
+from mx8fs import delete_file, file_exists, list_files, read_file, write_file
+
+
+class JsonFileStorage:
+    """A storage class for JSON serializable pydantic models."""
+
+    _extension: str
+    _key_field: str
+    _randomizer: Callable[[], None] = random.seed
+
+    def __init__(self, base_path: str, randomizer: Optional[Callable[[], None]] = None) -> None:
+        self.base_path = base_path
+        self._randomizer = randomizer or self._randomizer
+
+        if "AWS_LAMBDA_FUNCTION_NAME" in os.environ and self._randomizer == random.seed:
+            raise ValueError("Cannot use random.seed as a randomizer in AWS Lambda environment")
+
+        self.randomizer = randomizer or random.seed
+
+    @staticmethod
+    def _json_to_model(json: str) -> Any:  # pragma: no cover
+        raise NotImplementedError()
+
+    @staticmethod
+    def _dict_to_model(json: dict) -> Any:  # pragma: no cover
+        raise NotImplementedError()
+
+    @staticmethod
+    def _model_to_json(content: Any) -> str:  # pragma: no cover
+        raise NotImplementedError()
+
+    def _get_unique_key(self, key_length: int = 8) -> str:
+        """Create a eight letter unique key. This gives us nearly 3 trillion possibities"""
+
+        self.randomizer()
+
+        # Generate a random key
+        key: str = "".join(random.choices(string.ascii_uppercase + string.digits, k=key_length))  # NOSONAR
+
+        # If the key already exists, try again
+        if file_exists(self._get_path(key)):
+            return self._get_unique_key(key_length)
+
+        return key
+
+    def list(self) -> List[str]:
+        """List files in storage."""
+
+        return list_files(self.base_path, self._extension)
+
+    def read(self, key: str) -> Any:
+        """Read a file from storage."""
+
+        return self._json_to_model(read_file(self._get_path(key)))
+
+    def write(self, content: Any, key: Union[str, None] = None) -> Any:
+        """Write a file to storage."""
+        return self.write_dict(content.model_dump(), key)
+
+    def write_dict(self, content: dict, key: Union[str, None] = None) -> Any:
+        """Write a file to storage."""
+
+        # If no key is provided, generate a unique key
+        key = key or content.get(self._key_field, None)
+        if not key:
+            key = self._get_unique_key()
+
+        # Add the key to the content
+        content[self._key_field] = key
+        content_out = self._dict_to_model(content)
+
+        # Now write the file
+        return self.update(content_out)
+
+    def update(self, content: Any) -> Any:
+        """Update a file in storage."""
+
+        write_file(
+            self._get_path(getattr(content, self._key_field)),
+            self._model_to_json(content),
+        )
+        return content
+
+    def delete(self, key: str) -> None:
+        """Delete a file from storage."""
+
+        delete_file(self._get_path(key))
+
+    def _get_path(self, key: str) -> str:
+        """Get the path for a file."""
+        return os.path.join(self.base_path, f"{key}.{self._extension}")
+
+
+def json_file_storage_factory(extension: str, model: Any, key_field: str = "key") -> type[JsonFileStorage]:
+    """Create a file storage class."""
+
+    cls: type[JsonFileStorage] = type(f"{model.__class__}Storage", (JsonFileStorage,), {})
+
+    def _json_to_model(json: str) -> Any:
+        """Convert a JSON object to a model."""
+        return model.model_validate_json(json)
+
+    def _dict_to_model(json: dict) -> Any:
+        """Convert a dictionary to a model."""
+        return model(**json)
+
+    def _model_to_json(content: Any) -> str:
+        """Convert a model to a JSON object."""
+        if not isinstance(content, model):  # pragma: no cover
+            raise ValueError(f"Expected {model}, got {type(content)}")
+
+        return str(content.model_dump_json())
+
+    setattr(cls, "_json_to_model", staticmethod(_json_to_model))
+    setattr(cls, "_dict_to_model", staticmethod(_dict_to_model))
+    setattr(cls, "_model_to_json", staticmethod(_model_to_json))
+    setattr(cls, "_extension", extension)
+    setattr(cls, "_key_field", key_field)
+
+    return cls

mx8fs-1.0.0.1/pyproject.toml

@@ -0,0 +1,81 @@
+[tool.poetry]
+name = "mx8fs"
+version = "1.0.0.1"
+description = "MX8 Filing system"
+authors = ["Tom Weiss <tom@mx8labs.com>"]
+readme = "README.md"
+license = "MIT"
+packages =  [{ include = "mx8fs" }]
+
+[tool.poetry.dependencies]
+python = ">=3.10"
+boto3 = ">=1.26.137"
+
+[tool.poetry.group.dev.dependencies]
+coverage = ">=7.4.4"
+pydantic = ">=2.5.2"
+
+pytest-cov = ">=5.0.0"
+pytest = ">=8.1.1"
+pre-commit = ">=3.7.0"
+black = ">=24.4.0"
+mypy = ">=1.10.1"
+isort = ">=5.13.2"
+boto3-stubs = {version = ">=1.34.134", extras = ["s3", "s3control"]}
+pytest-mypy = ">=0.10.3"
+flake8-pyproject = ">=1.2.3"
+flake8 = ">=7.1.1"
+autoflake = ">=2.3.1"
+httpx = ">=0.20.0"
+
+[tool.mypy]
+python_version = "3.10"
+check_untyped_defs = true
+ignore_missing_imports = true
+disallow_untyped_defs = true
+warn_return_any = true
+exclude = "(test_data|.venv|.cdk.out)"
+
+[tool.pytest.ini_options]
+norecursedirs = "tests/test_data"
+addopts = [
+    "--cov=mx8fs",
+    "--cov-fail-under=100",
+    "--cov-branch",
+    "--cov-config=.coveragerc",
+    "--color=yes",
+    "--cov-report=lcov:coverage/lcov.info",
+    "--cov-report=term-missing:skip-covered",
+    "--disable-pytest-warnings",
+    "--durations=50",
+    "--verbose",
+    "--capture=no",
+    "--showlocals",
+    "--tb=short",
+]
+asyncio_default_fixture_loop_scope = "function"
+
+[tool.black]
+line-length = 120
+target-version = ['py310']
+include = '\.pyi?$'
+force-exclude = "test_data|.venv"
+
+[tool.flake8]
+exclude = "test_data|.venv"
+max-line-length = 120
+max-complexity = 18
+ignore = ["E203", "W503"]
+
+[tool.pylint]
+extension-pkg-whitelist = "pydantic"
+ignore = "test_data|.venv"
+max-line-length = 120
+
+[tool.isort]
+profile = "black"
+line_length = 120
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"