file-keeper 0.0.1__tar.gz

file_keeper-0.0.1/PKG-INFO

@@ -0,0 +1,46 @@
+Metadata-Version: 2.2
+Name: file-keeper
+Version: 0.0.1
+Author-email: DataShades <datashades@linkdigital.com.au>, Sergey Motornyuk <sergey.motornyuk@linkdigital.com.au>
+Maintainer-email: DataShades <datashades@linkdigital.com.au>
+License: AGPL
+Project-URL: Homepage, https://github.com/DataShades/file-keeper
+Keywords: file-management,cloud,filesystem,DAL
+Classifier: License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: typing_extensions
+Requires-Dist: pluggy
+Requires-Dist: pytz
+Requires-Dist: python-magic
+Provides-Extra: test
+Requires-Dist: pytest-cov; extra == "test"
+Requires-Dist: pytest-faker; extra == "test"
+Requires-Dist: responses; extra == "test"
+Provides-Extra: docs
+Requires-Dist: mkdocs; extra == "docs"
+Requires-Dist: mkdocs-material; extra == "docs"
+Requires-Dist: pymdown-extensions; extra == "docs"
+Requires-Dist: mkdocstrings[python]; extra == "docs"
+Provides-Extra: dev
+Requires-Dist: pytest-cov; extra == "dev"
+Requires-Dist: pytest-faker; extra == "dev"
+Requires-Dist: responses; extra == "dev"
+Requires-Dist: mkdocs; extra == "dev"
+Requires-Dist: mkdocs-material; extra == "dev"
+Requires-Dist: pymdown-extensions; extra == "dev"
+Requires-Dist: mkdocstrings[python]; extra == "dev"
+Provides-Extra: s3
+Requires-Dist: boto3; extra == "s3"
+Requires-Dist: boto3-stubs[essential]; extra == "s3"
+Provides-Extra: gcs
+Requires-Dist: google-cloud-storage; extra == "gcs"
+Provides-Extra: opendal
+Requires-Dist: opendal; extra == "opendal"
+Provides-Extra: libcloud
+Requires-Dist: apache-libcloud; extra == "libcloud"
+Requires-Dist: cryptography; extra == "libcloud"
+
+# file-keeper
+
+Abstraction level for object storages.

file_keeper-0.0.1/README.md

@@ -0,0 +1,3 @@
+# file-keeper
+
+Abstraction level for object storages.

file_keeper-0.0.1/pyproject.toml

@@ -0,0 +1,208 @@
+[build-system]
+requires = [ "setuptools",]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.dynamic]
+version = {attr = "file_keeper.__version__"}
+
+[project]
+name = "file-keeper"
+dynamic = ["version"]
+description = ""
+classifiers = [
+            "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)",
+]
+keywords = [ "file-management", "cloud", "filesystem", "DAL" ]
+requires-python = ">= 3.8"
+dependencies = [ "typing_extensions", "pluggy", "pytz", "python-magic" ]
+authors = [
+    {name = "DataShades", email = "datashades@linkdigital.com.au"},
+    {name = "Sergey Motornyuk", email = "sergey.motornyuk@linkdigital.com.au"},
+]
+maintainers = [
+    {name = "DataShades", email = "datashades@linkdigital.com.au"},
+]
+
+[project.readme]
+file = "README.md"
+content-type = "text/markdown"
+
+[project.license]
+text = "AGPL"
+
+[project.urls]
+Homepage = "https://github.com/DataShades/file-keeper"
+
+[project.optional-dependencies]
+test = [ "pytest-cov", "pytest-faker", "responses" ]
+docs = [ "mkdocs", "mkdocs-material", "pymdown-extensions", "mkdocstrings[python]",]
+dev = [ "pytest-cov", "pytest-faker", "responses", "mkdocs", "mkdocs-material", "pymdown-extensions", "mkdocstrings[python]",]
+
+s3 = [ "boto3", "boto3-stubs[essential]"]
+gcs = [ "google-cloud-storage",]
+opendal = [ "opendal",]
+libcloud = [ "apache-libcloud", "cryptography",]
+
+[project.entry-points."file_keeper_ext"]
+default = "file_keeper.default"
+
+[tool.ruff]
+target-version = "py38"
+
+[tool.ruff.lint]
+select = [
+       "ANN0", # type annotations for function arguments
+       "B",  # likely bugs and design problems
+       "BLE",  # do not catch blind exception
+       "C4",  # better list/set/dict comprehensions
+       "C90",  # check McCabe complexity
+       "DTZ", # enforce timezone in date objects
+       "E",  # pycodestyle error
+       "W",  # pycodestyle warning
+       "F",  # pyflakes
+       "FA", # verify annotations from future
+       "G",  # format strings for logging statements
+       "N",  # naming conventions
+       "I",  # isort
+       "ICN", # import conventions
+       # "D1",  # require doc
+       "D2",  # doc formatting
+       "D4",  # doc convention
+       "PL",  # pylint
+       "PERF", # performance anti-patterns
+       "PT",  # pytest style
+       "PIE",  # misc lints
+       "RET",  # improvements for return statements
+       "RSE",  # improvements for rise statements
+       "S",  # security testing
+       "SIM",  # simplify code
+       "T10",  # debugging statements
+       "T20",  # print statements
+       "TID",  # tidier imports
+       "TRY",  # better exceptions
+       "UP",  # upgrade syntax for newer versions of the language
+]
+
+ignore = [
+       "RET503", # don't enforce return-None
+]
+
+[tool.ruff.lint.per-file-ignores]
+"tests/*" = ["S", "PL", "ANN", "D"]
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
+[tool.ruff.lint.isort]
+section-order = [
+              "future",
+              "standard-library",
+              "first-party",
+              "third-party",
+              "self",
+              "local-folder",
+]
+
+[tool.ruff.lint.isort.sections]
+self = ["file_keeper"]
+
+[tool.pytest.ini_options]
+addopts = "-p no:ckan"
+markers = [
+        "fk_storage_option: modify storage configuration",
+]
+
+
+[tool.git-changelog]
+output = "CHANGELOG.md"
+convention = "conventional"
+parse-trailers = true
+
+[tool.pyright]
+pythonVersion = "3.8"
+include = ["."]
+exclude = [
+    # "**/test*",
+]
+strict = []
+
+strictParameterNoneValue = true
+# Check the meaning of rules here
+# https://github.com/microsoft/pyright/blob/main/docs/configuration.md
+reportFunctionMemberAccess = true # non-standard member accesses for functions
+reportMissingImports = true
+reportMissingModuleSource = true
+reportMissingTypeStubs = false
+reportImportCycles = false
+reportUnusedImport = true
+reportUnusedClass = true
+reportUnusedFunction = true
+reportUnusedVariable = true
+reportDuplicateImport = true
+reportOptionalSubscript = true
+reportOptionalMemberAccess = true
+reportOptionalCall = true
+reportOptionalIterable = true
+reportOptionalContextManager = true
+reportOptionalOperand = true
+reportTypedDictNotRequiredAccess = true
+reportConstantRedefinition = true
+reportIncompatibleMethodOverride = true
+reportIncompatibleVariableOverride = true
+reportOverlappingOverload = true
+reportUntypedFunctionDecorator = false
+reportUnknownParameterType = true
+reportUnknownArgumentType = false
+reportUnknownLambdaType = false
+reportUnknownMemberType = false
+reportMissingTypeArgument = true
+reportInvalidTypeVarUse = true
+reportCallInDefaultInitializer = true
+reportUnknownVariableType = true
+reportUntypedBaseClass = true
+reportUnnecessaryIsInstance = true
+reportUnnecessaryCast = true
+reportUnnecessaryComparison = true
+reportAssertAlwaysTrue = true
+reportSelfClsParameterName = true
+reportUnusedCallResult = false
+useLibraryCodeForTypes = true
+reportGeneralTypeIssues = true
+reportPropertyTypeMismatch = true
+reportWildcardImportFromLibrary = true
+reportUntypedClassDecorator = false
+reportUntypedNamedTuple = true
+reportPrivateUsage = true
+reportPrivateImportUsage = true
+reportInconsistentConstructor = true
+reportMissingSuperCall = false
+reportUninitializedInstanceVariable = true
+reportInvalidStringEscapeSequence = true
+reportMissingParameterType = true
+reportImplicitStringConcatenation = false
+reportUndefinedVariable = true
+reportUnboundVariable = true
+reportInvalidStubStatement = true
+reportIncompleteStub = true
+reportUnsupportedDunderAll = true
+reportUnusedCoroutine = true
+reportUnnecessaryTypeIgnoreComment = true
+reportMatchNotExhaustive = true
+
+[tool.coverage.run]
+branch = true
+omit = ["tests/*"]
+
+[tool.tox]
+requires = ["tox>=4"]
+env_list = ["3.13", "3.12", "3.11", "3.10", "3.9", "3.8"]
+
+[tool.tox.env_run_base]
+
+description = "run unit tests"
+deps = [
+    "pytest>=8",
+    "pytest-sugar",
+]
+commands = [["pytest", { replace = "posargs", default = ["tests"], extend = true }]]
+extras = ["test"]

file_keeper-0.0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

file_keeper-0.0.1/setup.py

@@ -0,0 +1,3 @@
+from setuptools import setup
+
+setup()

file_keeper-0.0.1/src/file_keeper/__init__.py

@@ -0,0 +1,48 @@
+__version__ = "0.0.1"
+
+from .core import exceptions as exc
+from .core.data import BaseData, FileData, MultipartData
+from .core.registry import Registry
+from .core.storage import (
+    Manager,
+    Reader,
+    Settings,
+    Storage,
+    Uploader,
+    adapters,
+    make_storage,
+)
+from .core.upload import Upload, make_upload
+from .core.utils import (
+    Capability,
+    HashingReader,
+    IterableBytesReader,
+    humanize_filesize,
+    is_supported_type,
+    parse_filesize,
+)
+from .ext import hookimpl  # must be the last line to avoid circular imports
+
+__all__ = [
+    "adapters",
+    "FileData",
+    "BaseData",
+    "MultipartData",
+    "is_supported_type",
+    "Registry",
+    "Capability",
+    "parse_filesize",
+    "humanize_filesize",
+    "IterableBytesReader",
+    "HashingReader",
+    "make_upload",
+    "Upload",
+    "exc",
+    "make_storage",
+    "Storage",
+    "Reader",
+    "Uploader",
+    "Manager",
+    "Settings",
+    "hookimpl",
+]

file_keeper-0.0.1/src/file_keeper/core/data.py

@@ -0,0 +1,107 @@
+"""Base abstract functionality of the extentsion.
+
+All classes required for specific storage implementations are defined
+here. Some utilities, like `make_storage` are also added to this module instead
+of `utils` to avoid import cycles.
+
+This module relies only on types, exceptions and utils to prevent import
+cycles.
+
+"""
+
+from __future__ import annotations
+
+import copy
+import dataclasses
+from typing import Any, Generic
+
+from typing_extensions import TypeVar
+
+from . import types
+
+TData = TypeVar("TData", bound=types.PData, default=Any)
+
+
+@dataclasses.dataclass
+class BaseData(Generic[TData]):
+    location: str
+    size: int = 0
+    content_type: str = ""
+    hash: str = ""
+    storage_data: dict[str, Any] = dataclasses.field(default_factory=dict)
+
+    _plain_keys = ["location", "size", "content_type", "hash"]
+    _complex_keys = ["storage_data"]
+
+    @classmethod
+    def from_dict(cls, record: dict[str, Any]):
+        return cls(
+            *[record[key] for key in cls._plain_keys],
+            *[copy.deepcopy(record[key]) for key in cls._complex_keys],
+        )
+
+    @classmethod
+    def from_object(cls, obj: TData):
+        return cls(
+            *[getattr(obj, key) for key in cls._plain_keys],
+            *[copy.deepcopy(getattr(obj, key)) for key in cls._complex_keys],
+        )
+
+    def into_object(self, obj: TData):
+        for key in self._plain_keys:
+            setattr(obj, key, getattr(self, key))
+
+        for key in self._complex_keys:
+            setattr(obj, key, copy.deepcopy(getattr(self, key)))
+
+        return obj
+
+
+@dataclasses.dataclass
+class FileData(BaseData[TData]):
+    """Information required by storage to operate the file.
+
+    Args:
+        location: filepath, filename or any other type of unique identifier
+        size: size of the file in bytes
+        content_type: MIMEtype of the file
+        hash: checksum of the file
+        storage_data: additional details set by storage adapter
+
+    Example:
+        ```
+        FileData(
+            "local/path.txt",
+            123,
+            "text/plain",
+            md5_of_content,
+        )
+        ```
+    """
+
+    content_type: str = "application/octet-stream"
+
+
+@dataclasses.dataclass
+class MultipartData(BaseData[TData]):
+    """Information required by storage to operate the incomplete upload.
+
+    Args:
+        location: filepath, filename or any other type of unique identifier
+        size: expected size of the file in bytes
+        content_type: expected MIMEtype of the file
+        hash: expected checksum of the file
+        storage_data: additional details set by storage adapter
+
+    Example:
+        ```
+        MultipartData(
+            "local/path.txt",
+            expected_size,
+            expected_content_type,
+            expected_hash,
+        )
+        ```
+    """
+
+    location: str = ""

file_keeper-0.0.1/src/file_keeper/core/exceptions.py

@@ -0,0 +1,275 @@
+"""Exception definitions for the extension.
+
+Hierarchy:
+
+* Exception
+    * FilesError
+        * QueueError
+            * OutOfQueueError
+        * StorageError
+            * UnknownAdapterError
+            * UnknownStorageError
+            * UnsupportedOperationError
+            * PermissionError
+            * LocationError
+                * MissingFileError
+                * ExistingFileError
+            * ExtrasError
+                * MissingExtrasError
+            * InvalidStorageConfigurationError
+                * MissingStorageConfigurationError
+            * UploadError
+                * WrongUploadTypeError
+                * LocationStrategyError
+                * ContentError
+                * LargeUploadError
+                    * UploadOutOfBoundError
+                * UploadMismatchError
+                    * UploadTypeMismatchError
+                    * UploadHashMismatchError
+                    * UploadSizeMismatchError
+
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+Storage = Any
+
+
+class FilesError(Exception):
+    """Base error for catch-all scenario."""
+
+
+class QueueError(FilesError):
+    """Error related to task queue."""
+
+
+class OutOfQueueError(QueueError):
+    """Attempt to add task without initializing task queue context."""
+
+    def __str__(self):
+        return "Task queue accessed outside of queue context"
+
+
+class StorageError(FilesError):
+    """Error related to storage."""
+
+
+class UnknownStorageError(StorageError):
+    """Storage with the given name is not configured."""
+
+    def __init__(self, storage: str):
+        self.storage = storage
+
+    def __str__(self):
+        return f"Storage {self.storage} is not configured"
+
+
+class UnknownAdapterError(StorageError):
+    """Specified storage adapter is not registered."""
+
+    def __init__(self, adapter: str):
+        self.adapter = adapter
+
+    def __str__(self):
+        return f"Storage adapter {self.adapter} is not registered"
+
+
+class UnsupportedOperationError(StorageError):
+    """Requested operation is not supported by storage."""
+
+    def __init__(self, operation: str, storage: Storage):
+        self.operation = operation
+        self.storage = storage
+
+    def __str__(self):
+        return f"Operation {self.operation} is not supported by {self.storage} storage"
+
+
+class InvalidStorageConfigurationError(StorageError):
+    """Storage cannot be initialized with given configuration."""
+
+    def __init__(self, adapter_or_storage: type | str, problem: str):
+        self.adapter_or_storage = adapter_or_storage
+        self.problem = problem
+
+    def __str__(self):
+        if isinstance(self.adapter_or_storage, str):
+            what = f"storage adapter {self.adapter_or_storage}"
+        else:
+            what = f"storage adapter {self.adapter_or_storage.__name__}"
+
+        return f"Cannot initialize {what}" + f" due to following error: {self.problem}"
+
+
+class PermissionError(StorageError):
+    """Storage client does not have required permissions."""
+
+    def __init__(self, storage: Storage, operation: str, problem: str):
+        self.storage = storage
+        self.operation = operation
+        self.problem = problem
+
+    def __str__(self):
+        msg = "Storage {} is not allowed to perform {} operation: {}"
+        return msg.format(
+            self.storage,
+            self.operation,
+            self.problem,
+        )
+
+
+class MissingStorageConfigurationError(InvalidStorageConfigurationError):
+    """Storage cannot be initialized due to missing option."""
+
+    def __init__(self, adapter_or_storage: type | str, option: str):
+        super().__init__(
+            adapter_or_storage,
+            f"{option} option is required",
+        )
+
+
+class LocationError(StorageError):
+    """Storage cannot use given location."""
+
+    def __init__(self, storage: Storage, location: str):
+        self.storage = storage
+        self.location = location
+
+    def __str__(self):
+        return f"{self.storage} cannot use location {self.location}"
+
+
+class MissingFileError(LocationError):
+    """File does not exist."""
+
+    def __str__(self):
+        return f"File {self.location} does not exist inside storage {self.storage}"
+
+
+class ExistingFileError(LocationError):
+    """File already exists."""
+
+    def __str__(self):
+        return f"File {self.location} already exists inside storage {self.storage}"
+
+
+class UploadError(StorageError):
+    """Error related to file upload process."""
+
+
+class LargeUploadError(UploadError):
+    """Storage cannot be initialized due to missing option."""
+
+    def __init__(self, actual_size: int, max_size: int):
+        self.actual_size = actual_size
+        self.max_size = max_size
+
+    def __str__(self):
+        return (
+            f"Upload size {self.actual_size} surpasses"
+            + f" max allowed size {self.max_size}"
+        )
+
+
+class UploadOutOfBoundError(LargeUploadError):
+    """Multipart upload exceeds expected size."""
+
+    def __str__(self):
+        return (
+            f"Upload size {self.actual_size} exceeds"
+            + f" expected size {self.max_size}"
+        )
+
+
+class UploadMismatchError(UploadError):
+    """Expected value of file attribute doesn't match the actual value."""
+
+    value_formatter = str
+
+    def __init__(self, attribute: str, actual: Any, expected: Any):
+        self.attribute = attribute
+        self.actual = actual
+        self.expected = expected
+
+    def __str__(self):
+        actual = self.value_formatter(self.actual)
+        expected = self.value_formatter(self.expected)
+        return (
+            f"Actual value of {self.attribute}({actual}) does not"
+            + f" match expected value({expected})"
+        )
+
+
+class UploadTypeMismatchError(UploadMismatchError):
+    """Expected value of content type doesn't match the actual value."""
+
+    def __init__(self, actual: Any, expected: Any):
+        super().__init__("content type", actual, expected)
+
+
+class UploadHashMismatchError(UploadMismatchError):
+    """Expected value of hash match the actual value."""
+
+    def __init__(self, actual: Any, expected: Any):
+        super().__init__("content hash", actual, expected)
+
+
+class UploadSizeMismatchError(UploadMismatchError):
+    """Expected value of upload size doesn't match the actual value."""
+
+    def __init__(self, actual: Any, expected: Any):
+        super().__init__("upload size", actual, expected)
+
+
+class WrongUploadTypeError(UploadError):
+    """Storage does not support given MIMEType."""
+
+    def __init__(self, content_type: str):
+        self.content_type = content_type
+
+    def __str__(self):
+        return f"Type {self.content_type} is not supported by storage"
+
+
+class LocationStrategyError(UploadError):
+    """Undefined name strategy."""
+
+    def __init__(self, strategy: str):
+        self.strategy = strategy
+
+    def __str__(self):
+        return f"Unknown name strategy {self.strategy}"
+
+
+class ExtrasError(StorageError):
+    """Wrong extras passed during upload."""
+
+    def __init__(self, extras: Any):
+        self.extras = extras
+
+    def __str__(self):
+        return f"Wrong extras: {self.extras}"
+
+
+class MissingExtrasError(ExtrasError):
+    """Wrong extras passed to storage method."""
+
+    def __init__(self, key: Any):
+        self.key = key
+
+    def __str__(self):
+        return f"Key {self.key} is missing from extras"
+
+
+class ContentError(UploadError):
+    """Storage cannot accept uploaded content."""
+
+    def __init__(self, storage: Storage, msg: str):
+        self.storage = storage
+        self.msg = msg
+
+    def __str__(self):
+        return f"{self.storage} rejected upload: {self.msg}"