mx8fs 1.0.0.6__tar.gz

mx8fs-1.0.0.6/PKG-INFO

@@ -0,0 +1,74 @@
+Metadata-Version: 2.3
+Name: mx8fs
+Version: 1.0.0.6
+Summary: MX8 Filing system
+License: MIT
+Author: Tom Weiss
+Author-email: tom@mx8labs.com
+Requires-Python: >=3.10
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: boto3 (>=1.26.137)
+Requires-Dist: pydantic (>=2.5.2)
+Description-Content-Type: text/markdown
+
+# MX8 File system
+
+This library provides environment agnostic file system access across local and AWS, including:
+- File / IO
+- List / Glob
+- Locking
+- Caching
+- Comparing Dictionaries
+
+# Pre-commit hooks
+
+We use precommit to run formatting checks, so whenever you clone a project run:
+
+```bash
+pre-commit install
+```
+
+Before you do anything else.
+
+You can run this at any time using:
+
+```bash
+pre-commit run --all-files
+```
+
+## Setting up the development environment
+
+You can install the full dev requirements by running [setup.sh](setup.sh) to
+1. Install the current repo and the python lib
+1. Run the pre-commit hooks on all files
+
+The project should open reasonable well in vs.code and includes three [launch configurations](.vscode/launch.json) for running unit tests and the debug server.
+
+## Code conventions and structure
+
+We use python type hinting with pylance and flake8 for linting. Unit tests are created to 100% branch coverage.
+
+Key libraries are Boto3 and FastAPI. Full details are in [requirements.txt](requirements.txt).
+
+The code is structured as follows:
+
+- The [mx8fs](mx8fs) folder contains the full library.
+- Tests are stored in the [tests](test) folder and run using pytest.
+
+* The github actions are in [main.yam](.github/workflows/main.yml)
+
+## License
+
+Copyright © 2025 MX8 Labs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+

mx8fs-1.0.0.6/README.md

@@ -0,0 +1,55 @@
+# MX8 File system
+
+This library provides environment agnostic file system access across local and AWS, including:
+- File / IO
+- List / Glob
+- Locking
+- Caching
+- Comparing Dictionaries
+
+# Pre-commit hooks
+
+We use precommit to run formatting checks, so whenever you clone a project run:
+
+```bash
+pre-commit install
+```
+
+Before you do anything else.
+
+You can run this at any time using:
+
+```bash
+pre-commit run --all-files
+```
+
+## Setting up the development environment
+
+You can install the full dev requirements by running [setup.sh](setup.sh) to
+1. Install the current repo and the python lib
+1. Run the pre-commit hooks on all files
+
+The project should open reasonable well in vs.code and includes three [launch configurations](.vscode/launch.json) for running unit tests and the debug server.
+
+## Code conventions and structure
+
+We use python type hinting with pylance and flake8 for linting. Unit tests are created to 100% branch coverage.
+
+Key libraries are Boto3 and FastAPI. Full details are in [requirements.txt](requirements.txt).
+
+The code is structured as follows:
+
+- The [mx8fs](mx8fs) folder contains the full library.
+- Tests are stored in the [tests](test) folder and run using pytest.
+
+* The github actions are in [main.yam](.github/workflows/main.yml)
+
+## License
+
+Copyright © 2025 MX8 Labs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING

mx8fs-1.0.0.6/mx8fs/__init__.py

@@ -0,0 +1,47 @@
+"""
+MX8 - Common utilities for MX8 projects
+
+Copyright (c) 2023 MX8 Inc, all rights reserved.
+
+This software is confidential and proprietary information of MX8.
+You shall not disclose such Confidential Information and shall use it only
+in accordance with the terms of the agreement you entered into with MX8.
+"""
+
+from .file_io import (
+    BinaryFileHandler,
+    read_file,
+    write_file,
+    list_files,
+    copy_file,
+    move_file,
+    delete_file,
+    file_exists,
+    get_public_url,
+    most_recent_timestamp,
+)
+from .cache import cache_to_disk, cache_to_disk_binary, get_cache_filename
+from .lock import FileLock, Waiter
+from .storage import JsonFileStorage, json_file_storage_factory
+from .comparer import ResultsComparer
+
+__all__ = [
+    "BinaryFileHandler",
+    "cache_to_disk_binary",
+    "cache_to_disk",
+    "copy_file",
+    "delete_file",
+    "file_exists",
+    "get_public_url",
+    "FileLock",
+    "get_cache_filename",
+    "json_file_storage_factory",
+    "JsonFileStorage",
+    "move_file",
+    "list_files",
+    "read_file",
+    "ResultsComparer",
+    "Waiter",
+    "write_file",
+    "most_recent_timestamp",
+]

mx8fs-1.0.0.6/mx8fs/cache.py

@@ -0,0 +1,195 @@
+"""
+Cache decorator for the MX8 AI API.
+
+Copyright (c) 2023-2025 MX8 Inc, all rights reserved.
+
+This software is confidential and proprietary information of MX8.
+You shall not disclose such Confidential Information and shall use it only
+in accordance with the terms of the agreement you entered into with MX8.
+"""
+
+import hashlib
+import json
+import logging
+import pickle
+from json.decoder import JSONDecodeError
+from time import time
+from typing import Any, Callable, Dict, List, Optional, Tuple
+
+from .file_io import BinaryFileHandler, read_file, write_file
+
+
+def get_cache_filename(path: str, name: str, extension: str, expiration_seconds: int = 0, **kwargs: Dict) -> str:
+    """Create an optionally time expiring cache filename using hashed parameters."""
+
+    # Create hashes based on the name and parameter hashes
+    args = kwargs.pop("extra_args", ())
+    param_hash = hashlib.sha256(pickle.dumps((args, kwargs))).hexdigest()
+
+    # If the cache is set to expire, add the current epoch rounded down
+    # to the nearest expiration_seconds to the hash
+    if expiration_seconds > 0:
+        epoch = int(int(time()))
+        time_hash = "_" + str(epoch - epoch % expiration_seconds)
+    else:
+        time_hash = ""
+
+    # Create the filename for the cache
+    return f"{path}/{name}_{param_hash}{time_hash}.cache.{extension}"
+
+
+def _get_clean_kwargs(kwargs: Dict, ignore_kwargs: Optional[List[str]]) -> Dict:
+    """Remove ignored kwargs from the kwargs dict"""
+
+    clean_kwargs = kwargs.copy()
+    for ignore in ignore_kwargs or []:
+        clean_kwargs.pop(ignore, None)
+
+    return clean_kwargs
+
+
+def _do_logging(
+    log_group: str,
+    result: Any,
+    args: Tuple,
+    kwargs: Dict,
+    filename: str,
+    func: Callable,
+    expiration_seconds: int,
+) -> None:
+    """Log cache hit"""
+
+    try:
+        result = json.loads(result)
+    except (JSONDecodeError, TypeError):
+        pass
+
+    if log_group:
+        try:
+            logging.getLogger(log_group).info(
+                "Cache hit",
+                extra={
+                    "cache_result": result,
+                    "cache_args": args,
+                    "cache_kwargs": kwargs,
+                    "cache_filename": filename,
+                    "cache_function": func.__name__,
+                    "cache_expiration_seconds": expiration_seconds,
+                },
+            )
+        except TypeError:  # pragma: no cover
+            # If we get a type error, case the dangerous types to strings
+            logging.getLogger(log_group).info(
+                "Cache hit",
+                extra={
+                    "cache_result": str(result),
+                    "cache_args": args,
+                    "cache_kwargs": str(kwargs),
+                    "cache_filename": filename,
+                    "cache_function": func.__name__,
+                    "cache_expiration_seconds": expiration_seconds,
+                },
+            )
+
+
+def cache_to_disk_binary(
+    path: str,
+    expiration_seconds: int = 0,
+    log_group: str = "",
+    ignore_kwargs: Optional[List[str]] = None,
+) -> Callable[..., Callable[..., Any]]:
+    """Cache decorator for any MX8 functions.
+
+    This decorator will cache the result of the function to disk, and return
+    the cached result on subsequent calls. This is useful for caching the
+    results of expensive operations, such as calling the AI API
+
+    Parameters:
+        path: The path to the cache directory
+        expiration_seconds: The number of seconds before the cache expires
+        log_group: The log group to log cache hits to
+        ignore_kwargs: A list of kwargs to ignore when creating the cache key
+    """
+
+    def decorator(func: Callable) -> Callable[..., Any]:
+        def wrapper(*args: Tuple, **kwargs: Dict) -> Any:
+            clean_kwargs = _get_clean_kwargs(kwargs, ignore_kwargs)
+
+            filename = get_cache_filename(
+                path,
+                func.__name__,
+                "pickle",
+                expiration_seconds,
+                extra_args=args,  # type: ignore
+                **clean_kwargs,
+            )
+
+            try:
+                # Try to read the cached result from disk
+                with BinaryFileHandler(filename) as file_handler:
+                    result = pickle.load(file_handler)
+
+                _do_logging(log_group, result, args, clean_kwargs, filename, func, expiration_seconds)
+
+            except FileNotFoundError:
+                # Cache miss, execute the function and save the result to disk
+                result = func(*args, **kwargs)
+                with BinaryFileHandler(filename, "wb") as file_handler:
+                    pickle.dump(result, file_handler)
+
+            return result
+
+        return wrapper
+
+    return decorator
+
+
+def cache_to_disk(
+    path: str,
+    expiration_seconds: int = 0,
+    log_group: str = "",
+    ignore_kwargs: Optional[List[str]] = None,
+) -> Callable[..., Callable[..., str | Any]]:
+    """Cache decorator for any MX8 functions.
+
+    This decorator will cache the result of the function to disk, and return
+    the cached result on subsequent calls. This is useful for caching the
+    results of expensive operations, such as calling the AI API
+
+    Parameters:
+        path: The path to the cache directory
+        expiration_seconds: The number of seconds before the cache expires
+        log_group: The log group to log cache hits to
+        ignore_kwargs: A list of kwargs to ignore when creating the cache key
+    """
+
+    def decorator(func: Callable) -> Callable[..., str | Any]:
+        def wrapper(*args: Tuple, **kwargs: Dict) -> str | Any:
+
+            clean_kwargs = _get_clean_kwargs(kwargs, ignore_kwargs)
+
+            filename = get_cache_filename(
+                path,
+                func.__name__,
+                "txt",
+                expiration_seconds,
+                extra_args=args,  # type: ignore
+                **clean_kwargs,
+            )
+
+            try:
+                # Try to read the cached result from disk
+                result = read_file(filename)
+
+                _do_logging(log_group, result, args, clean_kwargs, filename, func, expiration_seconds)
+
+            except FileNotFoundError:
+                # Cache miss, execute the function and save the result to disk
+                result = func(*args, **kwargs)
+                write_file(filename, result)
+
+            return result
+
+        return wrapper
+
+    return decorator

mx8fs-1.0.0.6/mx8fs/comparer.py

@@ -0,0 +1,186 @@
+"""
+Support functions for testing
+
+Copyright (c) 2023-2025 MX8 Inc, all rights reserved.
+
+This software is confidential and proprietary information of MX8.
+You shall not disclose such Confidential Information and shall use it only
+in accordance with the terms of the agreement you entered into with MX8.
+"""
+
+import json
+import os
+import shutil
+from difflib import ndiff
+from logging import getLogger
+from tempfile import NamedTemporaryFile
+from typing import Any, Dict, List, Optional
+
+
+from mx8fs import read_file, write_file
+
+logger = getLogger("mx8.comparer")
+
+
+def get_diff(a: str, b: str) -> str:
+    return "\n".join(d for d in ndiff(a.splitlines(), b.splitlines()) if not d.startswith(" "))
+
+
+class Differences:
+    def __init__(self) -> None:
+        self._differences: List[Dict[str, str]] = []
+
+    def __repr__(self) -> str:
+        return json.dumps(self._differences, indent=4)
+
+    def __eq__(self, value: object) -> bool:
+        return self._differences == value
+
+    def __bool__(self) -> bool:
+        return bool(self._differences)
+
+    def __len__(self) -> int:
+        return len(self._differences)
+
+    def append(self, differences: Dict[str, str]) -> None:
+        self._differences.append(differences)
+
+    def clear(self) -> None:
+        self._differences.clear()
+
+    @property
+    def keys(self) -> List[str]:
+        return [list(d.keys())[0] for d in self._differences]
+
+
+class ResultsComparer:
+    def __init__(
+        self,
+        ignore_keys: Optional[List[str]],
+        create_test_data: bool = False,
+    ) -> None:
+        self._ignore_keys = ignore_keys if ignore_keys else []
+        self._create_test_data = create_test_data
+        self._differences = Differences()
+
+    def _log_differences(self, key: str, correct: str, test: str) -> None:
+        """Log the differences between two strings"""
+        if correct != test:
+            self._differences.append({key: get_diff(correct, test)})
+
+    def _compare_dicts(self, correct: Any, test: Any, recursive: bool = False, root_key: str = "root") -> None:
+        """
+        Compare two dictionaries recursively, ignoring elements with the given key
+        """
+
+        if not recursive:
+            logger.debug(
+                {
+                    "message": "Comparing dictionaries",
+                    "dict1": correct,
+                    "dict2": test,
+                    "ignore_keys": self._ignore_keys,
+                }
+            )
+
+        if isinstance(correct, list) and isinstance(test, list):
+            if len(correct) != len(test):
+                self._log_differences(root_key, json.dumps(correct), json.dumps(test))
+                return
+
+            for i, the_dict in enumerate(correct):
+                self._compare_dicts(the_dict, test[i], recursive=True, root_key=f"{root_key}[{i}]")
+
+            return
+
+        # Check if both inputs are dictionaries
+        if not isinstance(correct, dict) or not isinstance(test, dict):
+            self._log_differences(root_key, json.dumps(correct), json.dumps(test))
+            return
+
+        # Get the set of keys for each dictionary
+        correct_keys = set(correct.keys())
+        test_keys = set(test.keys())
+
+        # Check if the keys are the same
+        if correct_keys != test_keys:
+            self._log_differences(root_key, json.dumps(correct), json.dumps(test))
+        else:
+            # Recursively compare the values for each key
+            for key in correct_keys:
+                if key not in self._ignore_keys:
+                    self._compare_dicts(correct[key], test[key], recursive=True, root_key=f"{root_key}/{key}")
+
+    def compare_dicts(self, correct: Any, test: Any) -> Differences:
+        """Compare two dictionaries"""
+        self._compare_dicts(correct, test)
+        return self._differences
+
+    def get_text_differences(self, test: str, correct: str) -> Differences:
+        """Compare a test file with a correct file"""
+        if self._create_test_data:
+            # make the directory if it doesn't exist
+            os.makedirs(os.path.dirname(correct), exist_ok=True)
+
+            # copy the test file to the correct file
+            shutil.copyfile(test, correct)
+
+        differences = Differences()
+
+        if diff := get_diff(read_file(correct), read_file(test)):
+            differences.append({"file": diff})
+
+        return differences
+
+    def get_dict_differences(self, test: str, correct: str) -> Differences:
+        """Compare a test file with a correct file"""
+
+        # Load the test file
+        test_dict = json.loads(read_file(test))
+
+        self._differences.clear()
+        if self._create_test_data:
+            try:
+                correct_dict = json.loads(read_file(correct))
+                self._compare_dicts(correct_dict, test_dict)
+                assert self._differences == [], "The files should be identical"
+            except (FileNotFoundError, AssertionError):
+                # Save the test file as the correct file
+                os.makedirs(os.path.dirname(correct), exist_ok=True)
+                write_file(correct, json.dumps(test_dict, indent=4, ensure_ascii=False).strip())
+                self._differences.clear()
+        else:
+            correct_dict = json.loads(read_file(correct))
+            self._compare_dicts(correct_dict, test_dict)
+
+        return self._differences
+
+    def get_api_response_differences(
+        self,
+        response: Any,
+        correct_file: str,
+    ) -> Differences:
+        """Check the response from the reporting API and return the differences"""
+
+        file_name = os.path.basename(correct_file)
+
+        # Write the response to a temporary file
+        try:
+            result = json.dumps(response.json(), indent=4, ensure_ascii=False)
+            compare_func = self.get_dict_differences
+        except json.JSONDecodeError:
+            compare_func = self.get_text_differences
+            result = response.text
+
+        with NamedTemporaryFile(mode="wt", delete=False, prefix=file_name) as temp_file:
+            temp_file.write(result.strip())
+            temp_file.flush()
+            temp_file_name = temp_file.name
+
+        # Compare the response to the correct file
+        mismatches = compare_func(temp_file_name, correct_file)
+
+        # Clean up the temporary file
+        os.remove(temp_file_name)
+
+        return mismatches

mx8fs-1.0.0.6/mx8fs/file_io.py

@@ -0,0 +1,247 @@
+"""
+AWS file IO functions
+
+Copyright (c) 2023-2025 MX8 Inc, all rights reserved.
+
+This software is confidential and proprietary information of MX8.
+You shall not disclose such Confidential Information and shall use it only
+in accordance with the terms of the agreement you entered into with MX8.
+"""
+
+import os
+from datetime import datetime
+from glob import glob
+from io import BytesIO
+from shutil import copyfile
+from typing import IO, Any, Dict, List, Literal, Tuple, cast
+
+import boto3
+from botocore.config import Config
+
+boto_config = Config(
+    max_pool_connections=int(os.getenv("BOTO_MAX_CONNECTIONS", 10)),
+    connect_timeout=float(os.getenv("BOTO_CONNECT_TIMEOUT", 10.0)),
+    read_timeout=float(os.getenv("BOTO_READ_TIMEOUT", 10.0)),
+    retries={
+        "total_max_attempts": int(os.getenv("BOTO_MAX_RETRIES", 5)),
+        "mode": cast(Literal["legacy", "standard", "adaptive"], os.getenv("BOTO_RETRY_MODE", "standard")),
+    },
+)
+
+s3_client = boto3.client(
+    service_name="s3",
+    config=boto_config,
+)
+
+
+def get_bucket_key(path: str) -> Tuple[str, str]:
+    """Get the bucket and key from a S3 path"""
+    path = path.replace("s3://", "")
+    bucket, key = path.split("/", 1)
+    return bucket, key
+
+
+def file_exists(file: str) -> bool:
+    """Check if a file exists on S3 or local storage"""
+    if file.startswith("s3://"):
+        bucket, key = get_bucket_key(file)
+        try:
+            s3_client.head_object(Bucket=bucket, Key=key)
+            return True
+        except s3_client.exceptions.ClientError:
+            return False
+
+    return os.path.exists(file)
+
+
+def read_file(file: str) -> str:
+    """Read a file from S3 or local storage with UTF-8 encoding"""
+    if file.startswith("s3://"):
+        bucket, key = get_bucket_key(file)
+        try:
+            return str(s3_client.get_object(Bucket=bucket, Key=key)["Body"].read().decode("utf-8"))
+        except s3_client.exceptions.NoSuchKey as exc:
+            raise FileNotFoundError(f"File {file} not found") from exc
+    else:
+        with open(file, mode="r", encoding="UTF-8") as file_io:
+            return file_io.read()
+
+
+def write_file(file: str, data: str) -> None:
+    """Write a file to S3 or local storage with UTF-8 encoding"""
+    if file.startswith("s3://"):
+        bucket, key = get_bucket_key(file)
+        s3_client.put_object(Bucket=bucket, Key=key, Body=data.encode("UTF-8"))
+    else:
+        os.makedirs(os.path.dirname(file), exist_ok=True)
+        with open(file, mode="w", encoding="UTF-8") as file_io:
+            file_io.write(data)
+
+
+def delete_file(file: str) -> None:
+    """Delete a file from S3 or local storage"""
+    if file.startswith("s3://"):
+        bucket, key = get_bucket_key(file)
+        s3_client.delete_object(Bucket=bucket, Key=key)
+    else:
+        try:
+            os.remove(file)
+        except FileNotFoundError:
+            # Ignore if the file does not exist for S3 consistency
+            pass
+
+
+def copy_file(src: str, dst: str) -> None:
+    """Copy a file from S3 or local storage"""
+    if src.startswith("s3://"):
+        src_bucket, src_key = get_bucket_key(src)
+        dst_bucket, dst_key = get_bucket_key(dst)
+
+        s3_client.copy_object(
+            Bucket=dst_bucket,
+            Key=dst_key,
+            CopySource={"Bucket": src_bucket, "Key": src_key},
+        )
+    else:
+        os.makedirs(os.path.dirname(dst), exist_ok=True)
+        copyfile(src, dst)
+
+
+def move_file(src: str, dst: str) -> None:
+    """Move a file from S3 or local storage"""
+    copy_file(src, dst)
+    delete_file(src)
+
+
+def list_files(root_path: str, file_type: str, prefix: str = "") -> List[str]:
+    """Returns a list of files from S3 or local storage with the relevant suffix and optional prefix.
+
+    The prefix signficantly improves performance for S3 by reducing the number of objects listed.
+    """
+    if root_path.startswith("s3://"):
+        bucket, key = get_bucket_key(root_path)
+        key = key + "/" if not key.endswith("/") else key
+        continuation_token = None
+        files = []
+
+        while True:
+            boto_kwargs = {
+                "Bucket": bucket,
+                "Prefix": key + prefix,
+                "Delimiter": "/",
+            }
+
+            if continuation_token:
+                boto_kwargs["ContinuationToken"] = continuation_token
+
+            boto_response = s3_client.list_objects_v2(**boto_kwargs)
+
+            if "Contents" in boto_response:
+                files.extend(
+                    [
+                        obj["Key"].replace(key, "").replace(f".{file_type}", "")
+                        for obj in boto_response["Contents"]
+                        if obj["Key"].endswith(file_type)
+                    ]
+                )
+
+            if boto_response.get("IsTruncated"):  # There are more objects to list
+                continuation_token = boto_response.get("NextContinuationToken")
+            else:
+                break
+
+        return files
+
+    return [os.path.split(f)[1][: -len(file_type) - 1] for f in glob(os.path.join(root_path, f"{prefix}*.{file_type}"))]
+
+
+def most_recent_timestamp(root_path: str, file_type: str) -> float:
+    """Returns the most recent timestamp from S3 or local storage with the suffix"""
+    if root_path.startswith("s3://"):
+        bucket, key = get_bucket_key(root_path)
+        boto_response = s3_client.list_objects_v2(Bucket=bucket, Prefix=key, Delimiter="/")
+        if "Contents" not in boto_response:
+            return 0
+
+        return max(
+            [obj["LastModified"] for obj in boto_response["Contents"] if obj["Key"].endswith(file_type)],
+            default=datetime(1970, 1, 1),
+        ).timestamp()
+
+    return max(
+        [os.path.getmtime(f) for f in glob(os.path.join(root_path, f"*.{file_type}"))],
+        default=0,
+    )
+
+
+def get_public_url(file: str, expires_in: int = 3600) -> str:
+    """Get a signed URL for a file on S3"""
+
+    if file.startswith("s3://"):
+        bucket, key = get_bucket_key(file)
+        presigned_url = s3_client.generate_presigned_url(
+            ClientMethod="get_object",
+            Params={"Bucket": bucket, "Key": key},
+            ExpiresIn=expires_in,
+        )
+
+        return str(presigned_url)
+
+    return file
+
+
+class BinaryFileHandler:
+    """File handler for S3 or local storage"""
+
+    _buffer: IO[Any]
+
+    def __init__(self, path: str, mode: str = "rb", content_type: str | None = None):
+        """
+        Creates the class, emulating the file object.
+
+        For S3, returns a BytesIO object for writing, and downloads the file
+        For local storage, returns a file object
+        """
+
+        if mode not in ["rb", "wb"]:
+            raise NotImplementedError(f"mode {mode} is not supported")
+
+        self.path = path
+        self.mode = mode
+        self.content_type = content_type
+        self.is_s3 = path.startswith("s3://")
+
+        if self.is_s3:
+            self._buffer = BytesIO()
+        else:
+            os.makedirs(os.path.dirname(self.path), exist_ok=True)
+            self._buffer = open(  # pylint: disable=consider-using-with
+                self.path, self.mode, encoding="UTF-8" if self.mode == "w" else None
+            )
+
+    def __enter__(self) -> BytesIO | IO:
+        """Read from S3 and open the stream"""
+        if self.is_s3:
+            bucket, key = get_bucket_key(self.path)
+            if self.mode == "rb":
+                # Download the file from S3 to the stream
+                try:
+                    s3_client.download_fileobj(Bucket=bucket, Key=key, Fileobj=self._buffer)
+                except s3_client.exceptions.ClientError as exc:
+                    raise FileNotFoundError(f"File {self.path} not found") from exc
+                self._buffer.seek(0)
+
+        return self._buffer
+
+    def __exit__(self, *_: List[Any], **__: Dict[str, Any]) -> None:
+        """Write to S3 or local storage and close the stream"""
+        if self.is_s3 and self.mode == "wb":
+            self._buffer.seek(0)
+            bucket, key = get_bucket_key(self.path)
+            s3_client.upload_fileobj(
+                Fileobj=self._buffer,
+                Bucket=bucket,
+                Key=key,
+                ExtraArgs=({"ContentType": self.content_type} if self.content_type else None),
+            )
+        self._buffer.close()

mx8fs-1.0.0.6/mx8fs/lock.py

@@ -0,0 +1,235 @@
+"""
+Locks a file across multiple process and clients.
+
+Based on DoggoLock (https://bitbucket.org/deductive/newtools/src/master/newtools/doggo/lock.py)
+
+Copyright (c) 2012-2025, Deductive Limited
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice,
+    this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright notice,
+    this list of conditions and the following disclaimer in the documentation
+    and/or other materials provided with the distribution.
+    * Neither the name of the Deductive Limited nor the names of
+    its contributors may be used to endorse or promote products derived from
+    this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+This version
+
+Copyright (c) 2024 MX8 Inc, all rights reserved.
+
+"""
+
+import logging
+import string
+from datetime import datetime, timedelta
+from functools import cached_property
+from random import choice
+from time import sleep, time
+from typing import Any, Dict, List
+
+from mx8fs import delete_file, list_files, write_file
+
+logger = logging.getLogger("mx8.lock")
+
+TIME_FORMAT = "%Y%m%d%H%M%S"
+
+
+class Waiter:
+    """
+
+    A generic wait and timeout class.
+
+    Raises TimeoutError() if the task doesn't occur in the period.
+
+    ```python
+
+        waiter = Waiter()
+        waiter.start_timeout()
+        while not check_my_condition():
+            waiter.check_timeout()
+    ```
+    """
+
+    def __init__(self, wait_period: float, time_out_seconds: float):
+        """
+        :param wait_period: the period to wait for between iterations
+        :param time_out_seconds: the time after which a TimeoutError is raised
+        """
+        self.time_out_seconds = time_out_seconds
+        self.wait_period = wait_period
+        self._timeout: float | None = None
+
+    def __enter__(self) -> "Waiter":
+        """
+        Starts the timer
+        """
+        self.start_timeout()
+        return self
+
+    def __exit__(self, *_: Any) -> None:
+        """
+        Stops the timer
+        """
+        self._timeout = None
+
+    def wait(self, number_times: int = 1) -> None:
+        """
+        Waits for the defined period
+        """
+        sleep(self.wait_period * number_times)
+
+    def start_timeout(self) -> None:
+        """
+        Starts a time out
+        """
+        self._timeout = time() + self.time_out_seconds
+
+    def timed_out(self) -> bool:
+        """
+        Checks for a time out
+
+        :return: true if the timer has timed out, otherwise false
+        """
+        if self._timeout is not None:
+            return time() > self._timeout
+        else:
+            raise ValueError("Someone has tried to call timed_out() before calling start_timeout()")
+
+    def check_timeout(self) -> None:
+        """
+        Waits, and raises an exception if the timer has timed out
+        """
+        self.wait()
+        if self.timed_out():
+            raise TimeoutError("Timed out waiting for completion")
+
+
+class FileLock:
+    """
+    Implements locking using an additional file on the file system.
+
+    For file systems that are only eventually consistent, use a longer
+    wait_period to wait for consistency when multiple clients are reading at the same time.
+
+    Locks a file across multiple process and clients using an additional file on the file system.
+
+    The lock file has the following format:
+    "{file}.{timestamp}.{random}.lock"
+
+    where:
+
+    * file - is the file being locked
+    * timestamp - is the timestamp the lock was requested in the format %Y%m%d%H%M%S
+    * random - is a random set of letters
+
+    On creating the lock we
+
+    1. Check to see if anyone already has a lock
+    2. If they don't, attempt to create a lock and wait for wait_period.
+    3. If two processes have attempted to get the lock then the one with the earlier lock gets it.
+    4. When the lock is released the lock class deletes the lock and other processes can proceed
+
+    Locks only last for maximum_age seconds, and any request will time out after time_out_seconds
+    """
+
+    def __init__(
+        self,
+        file: str,
+        wait_period: float = 0.1,
+        time_out_seconds: int = 840,  # 1 minute less than the lambda timeout
+        maximum_age: int = 900,  # 15 minutes, the maximum time a lambda can run
+    ):
+        """
+        Initializes the lock
+
+        :param file: the file to lock
+        :param wait_period: the period to wait before confirming file lock
+        :param time_out_seconds: the time out to stop waiting after
+        :param maximum_age: the maximum age of lock files to respect
+        """
+        self.file = file
+        self.waiter = Waiter(wait_period, time_out_seconds)
+        self.maximum_age = timedelta(seconds=maximum_age)
+
+    @cached_property
+    def _lock_file(self) -> str:
+        """Get the lock file name."""
+        timestamp = datetime.now().strftime(TIME_FORMAT)
+        random_key = "".join(choice(string.ascii_lowercase) for _ in range(12))  # NOSONAR
+
+        return f"{self.file}.{timestamp}.{random_key}.lock"
+
+    def __enter__(self) -> "FileLock":
+        """Acquire the lock on the file. This will wait until the lock is available."""
+
+        logger.info("Getting lock on %s", self.file)
+
+        # If the file is locked then wait for it to be unlocked
+        self.waiter.start_timeout()
+        while len(self._get_lock_files()) > 0:
+            self.waiter.check_timeout()
+
+        # create a lock file
+        write_file(self._lock_file, "locked")
+
+        # Check an wait in case another process is trying to get the lock
+        self.waiter.wait()
+        while (
+            len(lock_files := self._get_lock_files()) > 1 and lock_files[0] != self._lock_file
+        ):  # pragma: no cover - coverage is not collected for multi-process tests
+            try:
+                self.waiter.check_timeout()
+            except TimeoutError as ex:
+                delete_file(self._lock_file)
+                raise ex
+
+        logger.info("Acquired lock on %s", self.file)
+        return self
+
+    def __exit__(self, *_: List[Any], **__: Dict[str, Any]) -> None:
+        """Release the lock on the file."""
+        delete_file(self._lock_file)
+        logger.info("Released lock on %s", self.file)
+
+    def _get_lock_files(self) -> List[str]:
+        """Get all the lock files for the current file."""
+        path = "/".join(self.file.split("/")[:-1])
+        prefix = self.file.split("/")[-1]
+
+        # Get all the lock files in the same directory
+        files = [f"{path}/{file}.lock" for file in list_files(path, "lock", prefix)]
+
+        # Return the sorted current lock files for the current file
+        return sorted(file for file in files if self._lock_is_current(file))
+
+    def _lock_is_current(self, lock_file: str) -> bool:
+        """Check if the lock file is current."""
+
+        # If the lock file is not for the current file then it is not current
+        if self.file not in lock_file:
+            return False
+
+        # If we cannot parse the timestamp then it is not current
+        try:
+            timestamp = datetime.strptime(lock_file.split(".")[-3], TIME_FORMAT)
+        except (IndexError, ValueError):
+            return False
+
+        # If we are less than the maximum age then it is current
+        return datetime.now() < timestamp + self.maximum_age

mx8fs-1.0.0.6/mx8fs/storage.py

@@ -0,0 +1,133 @@
+"""
+Generic file storage class.
+
+Copyright (c) 2023-2025 MX8 Inc, all rights reserved.
+
+This software is confidential and proprietary information of MX8.
+You shall not disclose such Confidential Information and shall use it only
+in accordance with the terms of the agreement you entered into with MX8.
+"""
+
+import os
+import random
+import string
+from typing import Any, Callable, List, Optional, Union
+
+from mx8fs import delete_file, file_exists, list_files, read_file, write_file
+
+
+class JsonFileStorage:
+    """A storage class for JSON serializable pydantic models."""
+
+    _extension: str
+    _key_field: str
+
+    def __init__(self, base_path: str, randomizer: Optional[Callable[[], None]] = None) -> None:
+        self.base_path = base_path
+
+        if "AWS_LAMBDA_FUNCTION_NAME" in os.environ and randomizer is None:
+            raise ValueError("Randomizer must be provided in AWS Lambda environment")
+
+        self.randomizer = randomizer or random.seed
+
+    @staticmethod
+    def _json_to_model(json: str) -> Any:  # pragma: no cover
+        raise NotImplementedError()
+
+    @staticmethod
+    def _dict_to_model(json: dict) -> Any:  # pragma: no cover
+        raise NotImplementedError()
+
+    @staticmethod
+    def _model_to_json(content: Any) -> str:  # pragma: no cover
+        raise NotImplementedError()
+
+    def _get_unique_key(self, key_length: int = 8) -> str:
+        """Create a eight letter unique key. This gives us nearly 3 trillion possibities"""
+
+        self.randomizer()
+
+        # Generate a random key
+        key: str = "".join(random.choices(string.ascii_uppercase + string.digits, k=key_length))  # NOSONAR
+
+        # If the key already exists, try again
+        if file_exists(self._get_path(key)):
+            return self._get_unique_key(key_length)
+
+        return key
+
+    def list(self) -> List[str]:
+        """List files in storage."""
+
+        return list_files(self.base_path, self._extension)
+
+    def read(self, key: str) -> Any:
+        """Read a file from storage."""
+
+        return self._json_to_model(read_file(self._get_path(key)))
+
+    def write(self, content: Any, key: Union[str, None] = None) -> Any:
+        """Write a file to storage."""
+        return self.write_dict(content.model_dump(), key)
+
+    def write_dict(self, content: dict, key: Union[str, None] = None) -> Any:
+        """Write a file to storage."""
+
+        # If no key is provided, generate a unique key
+        key = key or content.get(self._key_field, None)
+        if not key:
+            key = self._get_unique_key()
+
+        # Add the key to the content
+        content[self._key_field] = key
+        content_out = self._dict_to_model(content)
+
+        # Now write the file
+        return self.update(content_out)
+
+    def update(self, content: Any) -> Any:
+        """Update a file in storage."""
+
+        write_file(
+            self._get_path(getattr(content, self._key_field)),
+            self._model_to_json(content),
+        )
+        return content
+
+    def delete(self, key: str) -> None:
+        """Delete a file from storage."""
+
+        delete_file(self._get_path(key))
+
+    def _get_path(self, key: str) -> str:
+        """Get the path for a file."""
+        return os.path.join(self.base_path, f"{key}.{self._extension}")
+
+
+def json_file_storage_factory(extension: str, model: Any, key_field: str = "key") -> type[JsonFileStorage]:
+    """Create a file storage class."""
+
+    cls: type[JsonFileStorage] = type(f"{model.__class__}Storage", (JsonFileStorage,), {})
+
+    def _json_to_model(json: str) -> Any:
+        """Convert a JSON object to a model."""
+        return model.model_validate_json(json)
+
+    def _dict_to_model(json: dict) -> Any:
+        """Convert a dictionary to a model."""
+        return model(**json)
+
+    def _model_to_json(content: Any) -> str:
+        """Convert a model to a JSON object."""
+        if not isinstance(content, model):  # pragma: no cover
+            raise ValueError(f"Expected {model}, got {type(content)}")
+
+        return str(content.model_dump_json())
+
+    setattr(cls, "_json_to_model", staticmethod(_json_to_model))
+    setattr(cls, "_dict_to_model", staticmethod(_dict_to_model))
+    setattr(cls, "_model_to_json", staticmethod(_model_to_json))
+    setattr(cls, "_extension", extension)
+    setattr(cls, "_key_field", key_field)
+
+    return cls

mx8fs-1.0.0.6/pyproject.toml

@@ -0,0 +1,87 @@
+[tool.poetry]
+name = "mx8fs"
+version = "1.0.0.6"
+description = "MX8 Filing system"
+authors = ["Tom Weiss <tom@mx8labs.com>"]
+readme = "README.md"
+license = "MIT"
+packages =  [{ include = "mx8fs" }]
+
+[tool.poetry.dependencies]
+python = ">=3.10"
+boto3 = ">=1.26.137"
+pydantic = ">=2.5.2"
+
+[tool.poetry.group.dev.dependencies]
+autopep8 = ">=2.1.0"
+coverage = ">=7.4.4"
+mkdocs-material = ">=9.5.18"
+pytest-cov = ">=5.0.0"
+pytest = ">=8.1.1"
+pre-commit = ">=3.7.0"
+black = ">=24.4.0"
+mypy = ">=1.10.1"
+isort = ">=5.13.2"
+boto3-stubs = {version = ">=1.34.134", extras = ["s3", "s3control"]}
+pygls = ">=1.3.1"
+lsprotocol = ">=1.0.0"
+pytest-mypy = ">=0.10.3"
+flake8-pyproject = ">=1.2.3"
+flake8 = ">=7.1.1"
+autoflake = ">=2.3.1"
+types-retry = "0.9.9.4"
+pytest-xdist = "3.6.1"
+httpx = ">=0.20.0"
+
+[tool.mypy]
+python_version = "3.10"
+check_untyped_defs = true
+ignore_missing_imports = true
+disallow_untyped_defs = true
+warn_return_any = true
+exclude = "(test_data|.venv|.cdk.out)"
+
+[tool.pytest.ini_options]
+norecursedirs = "tests/test_data"
+addopts = [
+    "--cov=mx8fs",
+    "--cov-fail-under=100",
+    "--cov-branch",
+    "--cov-config=.coveragerc",
+    "--color=yes",
+    "--cov-report=lcov:coverage/lcov.info",
+    "--cov-report=term-missing:skip-covered",
+    "--disable-pytest-warnings",
+    "--durations=50",
+    "--verbose",
+    "--capture=no",
+    "--showlocals",
+    "--tb=short",
+    "-n=4",
+]
+asyncio_default_fixture_loop_scope = "function"
+
+[tool.black]
+line-length = 120
+target-version = ['py310']
+include = '\.pyi?$'
+force-exclude = "test_data|.venv"
+
+[tool.flake8]
+exclude = "test_data|.venv"
+max-line-length = 120
+max-complexity = 18
+ignore = ["E203", "W503"]
+
+[tool.pylint]
+extension-pkg-whitelist = "pydantic"
+ignore = "test_data|.venv"
+max-line-length = 120
+
+[tool.isort]
+profile = "black"
+line_length = 120
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"