file-keeper 0.0.1a0__tar.gz

file_keeper-0.0.1a0/PKG-INFO

@@ -0,0 +1,55 @@
+Metadata-Version: 2.2
+Name: file-keeper
+Version: 0.0.1a0
+Author-email: DataShades <datashades@linkdigital.com.au>, Sergey Motornyuk <sergey.motornyuk@linkdigital.com.au>
+Maintainer-email: DataShades <datashades@linkdigital.com.au>
+License: AGPL
+Project-URL: Homepage, https://github.com/DataShades/file-keeper
+Keywords: file-management,cloud,filesystem,DAL
+Classifier: Development Status :: 4 - Beta
+Classifier: License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: typing_extensions
+Requires-Dist: pluggy
+Requires-Dist: pytz
+Requires-Dist: python-magic
+Provides-Extra: test
+Requires-Dist: pytest-cov; extra == "test"
+Requires-Dist: pytest-faker; extra == "test"
+Requires-Dist: responses; extra == "test"
+Provides-Extra: docs
+Requires-Dist: mkdocs; extra == "docs"
+Requires-Dist: mkdocs-material; extra == "docs"
+Requires-Dist: pymdown-extensions; extra == "docs"
+Requires-Dist: mkdocstrings[python]; extra == "docs"
+Provides-Extra: dev
+Requires-Dist: pytest-cov; extra == "dev"
+Requires-Dist: pytest-faker; extra == "dev"
+Requires-Dist: freezegun; extra == "dev"
+Requires-Dist: responses; extra == "dev"
+Requires-Dist: mkdocs; extra == "dev"
+Requires-Dist: mkdocs-material; extra == "dev"
+Requires-Dist: pymdown-extensions; extra == "dev"
+Requires-Dist: mkdocstrings[python]; extra == "dev"
+Provides-Extra: s3
+Requires-Dist: boto3; extra == "s3"
+Requires-Dist: boto3-stubs[essential]; extra == "s3"
+Provides-Extra: gcs
+Requires-Dist: google-cloud-storage; extra == "gcs"
+Provides-Extra: opendal
+Requires-Dist: opendal; extra == "opendal"
+Provides-Extra: libcloud
+Requires-Dist: apache-libcloud; extra == "libcloud"
+Requires-Dist: cryptography; extra == "libcloud"
+
+# file-keeper
+
+Abstraction level for object storages.

file_keeper-0.0.1a0/README.md

@@ -0,0 +1,3 @@
+# file-keeper
+
+Abstraction level for object storages.

file_keeper-0.0.1a0/pyproject.toml

@@ -0,0 +1,211 @@
+[build-system]
+requires = [ "setuptools",]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.dynamic]
+version = {attr = "file_keeper.__version__"}
+
+[project]
+name = "file-keeper"
+dynamic = ["version"]
+description = ""
+classifiers = [
+            "Development Status :: 4 - Beta",
+            "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)",
+            "Programming Language :: Python :: 3.8",
+            "Programming Language :: Python :: 3.9",
+            "Programming Language :: Python :: 3.10",
+            "Programming Language :: Python :: 3.11",
+            "Programming Language :: Python :: 3.12",
+            "Programming Language :: Python :: 3.13",
+            "Programming Language :: Python :: 3.14",
+]
+keywords = [ "file-management", "cloud", "filesystem", "DAL" ]
+requires-python = ">= 3.8"
+dependencies = [ "typing_extensions", "pluggy", "pytz", "python-magic" ]
+authors = [
+    {name = "DataShades", email = "datashades@linkdigital.com.au"},
+    {name = "Sergey Motornyuk", email = "sergey.motornyuk@linkdigital.com.au"},
+]
+maintainers = [
+    {name = "DataShades", email = "datashades@linkdigital.com.au"},
+]
+
+[project.readme]
+file = "README.md"
+content-type = "text/markdown"
+
+[project.license]
+text = "AGPL"
+
+[project.urls]
+Homepage = "https://github.com/DataShades/file-keeper"
+
+[project.optional-dependencies]
+test = [ "pytest-cov", "pytest-faker", "responses" ]
+docs = [ "mkdocs", "mkdocs-material", "pymdown-extensions", "mkdocstrings[python]",]
+dev = [ "pytest-cov", "pytest-faker", "freezegun", "responses", "mkdocs", "mkdocs-material", "pymdown-extensions", "mkdocstrings[python]",]
+s3 = [ "boto3", "boto3-stubs[essential]"]
+gcs = [ "google-cloud-storage",]
+opendal = [ "opendal",]
+libcloud = [ "apache-libcloud", "cryptography",]
+
+[project.entry-points."file_keeper_ext"]
+default = "file_keeper.default"
+
+[tool.ruff]
+target-version = "py38"
+
+[tool.ruff.lint]
+select = [
+       "ANN0", # type annotations for function arguments
+       "B",  # likely bugs and design problems
+       "BLE",  # do not catch blind exception
+       "C4",  # better list/set/dict comprehensions
+       "C90",  # check McCabe complexity
+       "DTZ", # enforce timezone in date objects
+       "E",  # pycodestyle error
+       "W",  # pycodestyle warning
+       "F",  # pyflakes
+       "FA", # verify annotations from future
+       "G",  # format strings for logging statements
+       "N",  # naming conventions
+       "I",  # isort
+       "ICN", # import conventions
+       # "D1",  # require doc
+       "D2",  # doc formatting
+       "D4",  # doc convention
+       "PL",  # pylint
+       "PERF", # performance anti-patterns
+       "PT",  # pytest style
+       "PIE",  # misc lints
+       "RET",  # improvements for return statements
+       "RSE",  # improvements for rise statements
+       "S",  # security testing
+       "SIM",  # simplify code
+       "T10",  # debugging statements
+       "T20",  # print statements
+       "TID",  # tidier imports
+       "TRY",  # better exceptions
+       "UP",  # upgrade syntax for newer versions of the language
+]
+
+ignore = [
+       "RET503", # don't enforce return-None
+]
+
+[tool.ruff.lint.per-file-ignores]
+"tests/*" = ["S", "PL", "ANN", "D"]
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
+[tool.ruff.lint.isort]
+section-order = [
+              "future",
+              "standard-library",
+              "first-party",
+              "third-party",
+              "self",
+              "local-folder",
+]
+
+[tool.ruff.lint.isort.sections]
+self = ["file_keeper"]
+
+[tool.pytest.ini_options]
+addopts = "-p no:ckan"
+
+[tool.git-changelog]
+output = "CHANGELOG.md"
+convention = "conventional"
+parse-trailers = true
+
+[tool.pyright]
+pythonVersion = "3.8"
+include = ["."]
+exclude = [
+    # "**/test*",
+]
+strict = []
+
+strictParameterNoneValue = true
+# Check the meaning of rules here
+# https://github.com/microsoft/pyright/blob/main/docs/configuration.md
+reportFunctionMemberAccess = true # non-standard member accesses for functions
+reportMissingImports = true
+reportMissingModuleSource = true
+reportMissingTypeStubs = false
+reportImportCycles = false
+reportUnusedImport = true
+reportUnusedClass = true
+reportUnusedFunction = true
+reportUnusedVariable = true
+reportDuplicateImport = true
+reportOptionalSubscript = true
+reportOptionalMemberAccess = true
+reportOptionalCall = true
+reportOptionalIterable = true
+reportOptionalContextManager = true
+reportOptionalOperand = true
+reportTypedDictNotRequiredAccess = true
+reportConstantRedefinition = true
+reportIncompatibleMethodOverride = true
+reportIncompatibleVariableOverride = true
+reportOverlappingOverload = true
+reportUntypedFunctionDecorator = false
+reportUnknownParameterType = true
+reportUnknownArgumentType = false
+reportUnknownLambdaType = false
+reportUnknownMemberType = false
+reportMissingTypeArgument = true
+reportInvalidTypeVarUse = true
+reportCallInDefaultInitializer = true
+reportUnknownVariableType = true
+reportUntypedBaseClass = true
+reportUnnecessaryIsInstance = true
+reportUnnecessaryCast = true
+reportUnnecessaryComparison = true
+reportAssertAlwaysTrue = true
+reportSelfClsParameterName = true
+reportUnusedCallResult = false
+useLibraryCodeForTypes = true
+reportGeneralTypeIssues = true
+reportPropertyTypeMismatch = true
+reportWildcardImportFromLibrary = true
+reportUntypedClassDecorator = false
+reportUntypedNamedTuple = true
+reportPrivateUsage = true
+reportPrivateImportUsage = true
+reportInconsistentConstructor = true
+reportMissingSuperCall = false
+reportUninitializedInstanceVariable = true
+reportInvalidStringEscapeSequence = true
+reportMissingParameterType = true
+reportImplicitStringConcatenation = false
+reportUndefinedVariable = true
+reportUnboundVariable = true
+reportInvalidStubStatement = true
+reportIncompleteStub = true
+reportUnsupportedDunderAll = true
+reportUnusedCoroutine = true
+reportUnnecessaryTypeIgnoreComment = true
+reportMatchNotExhaustive = true
+
+[tool.coverage.run]
+branch = true
+omit = ["files_keeper/tests/*"]
+
+[tool.tox]
+requires = ["tox>=4"]
+env_list = ["3.13", "3.12", "3.11", "3.10", "3.9", "3.8"]
+
+[tool.tox.env_run_base]
+
+description = "run unit tests"
+deps = [
+    "pytest>=8",
+    "pytest-sugar"
+]
+commands = [["pytest", { replace = "posargs", default = ["tests"], extend = true }]]
+extras = ["test"]

file_keeper-0.0.1a0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

file_keeper-0.0.1a0/setup.py

@@ -0,0 +1,3 @@
+from setuptools import setup
+
+setup()

file_keeper-0.0.1a0/src/file_keeper/__init__.py

@@ -0,0 +1,48 @@
+__version__ = "0.0.1a0"
+
+from .core import exceptions as exc
+from .core.data import BaseData, FileData, MultipartData
+from .core.registry import Registry
+from .core.storage import (
+    Manager,
+    Reader,
+    Settings,
+    Storage,
+    Uploader,
+    adapters,
+    make_storage,
+)
+from .core.upload import Upload, make_upload
+from .core.utils import (
+    Capability,
+    HashingReader,
+    IterableBytesReader,
+    humanize_filesize,
+    is_supported_type,
+    parse_filesize,
+)
+from .ext import hookimpl  # must be the last line to avoid circular imports
+
+__all__ = [
+    "adapters",
+    "FileData",
+    "BaseData",
+    "MultipartData",
+    "is_supported_type",
+    "Registry",
+    "Capability",
+    "parse_filesize",
+    "humanize_filesize",
+    "IterableBytesReader",
+    "HashingReader",
+    "make_upload",
+    "Upload",
+    "exc",
+    "make_storage",
+    "Storage",
+    "Reader",
+    "Uploader",
+    "Manager",
+    "Settings",
+    "hookimpl",
+]

file_keeper-0.0.1a0/src/file_keeper/core/data.py

@@ -0,0 +1,107 @@
+"""Base abstract functionality of the extentsion.
+
+All classes required for specific storage implementations are defined
+here. Some utilities, like `make_storage` are also added to this module instead
+of `utils` to avoid import cycles.
+
+This module relies only on types, exceptions and utils to prevent import
+cycles.
+
+"""
+
+from __future__ import annotations
+
+import copy
+import dataclasses
+from typing import Any, Generic
+
+from typing_extensions import TypeVar
+
+from . import types
+
+TData = TypeVar("TData", bound=types.PData, default=Any)
+
+
+@dataclasses.dataclass
+class BaseData(Generic[TData]):
+    location: str
+    size: int = 0
+    content_type: str = ""
+    hash: str = ""
+    storage_data: dict[str, Any] = dataclasses.field(default_factory=dict)
+
+    _plain_keys = ["location", "size", "content_type", "hash"]
+    _complex_keys = ["storage_data"]
+
+    @classmethod
+    def from_dict(cls, record: dict[str, Any]):
+        return cls(
+            *[record[key] for key in cls._plain_keys],
+            *[copy.deepcopy(record[key]) for key in cls._complex_keys],
+        )
+
+    @classmethod
+    def from_object(cls, obj: TData):
+        return cls(
+            *[getattr(obj, key) for key in cls._plain_keys],
+            *[copy.deepcopy(getattr(obj, key)) for key in cls._complex_keys],
+        )
+
+    def into_object(self, obj: TData):
+        for key in self._plain_keys:
+            setattr(obj, key, getattr(self, key))
+
+        for key in self._complex_keys:
+            setattr(obj, key, copy.deepcopy(getattr(self, key)))
+
+        return obj
+
+
+@dataclasses.dataclass
+class FileData(BaseData[TData]):
+    """Information required by storage to operate the file.
+
+    Args:
+        location: filepath, filename or any other type of unique identifier
+        size: size of the file in bytes
+        content_type: MIMEtype of the file
+        hash: checksum of the file
+        storage_data: additional details set by storage adapter
+
+    Example:
+        ```
+        FileData(
+            "local/path.txt",
+            123,
+            "text/plain",
+            md5_of_content,
+        )
+        ```
+    """
+
+    content_type: str = "application/octet-stream"
+
+
+@dataclasses.dataclass
+class MultipartData(BaseData[TData]):
+    """Information required by storage to operate the incomplete upload.
+
+    Args:
+        location: filepath, filename or any other type of unique identifier
+        size: expected size of the file in bytes
+        content_type: expected MIMEtype of the file
+        hash: expected checksum of the file
+        storage_data: additional details set by storage adapter
+
+    Example:
+        ```
+        MultipartData(
+            "local/path.txt",
+            expected_size,
+            expected_content_type,
+            expected_hash,
+        )
+        ```
+    """
+
+    location: str = ""