tie-storage 0.0.1__tar.gz

tie_storage-0.0.1/PKG-INFO

@@ -0,0 +1,39 @@
+Metadata-Version: 2.4
+Name: tie-storage
+Version: 0.0.1
+Summary: TIE storage client library
+Author: ISPRAS TIE NLP team
+Author-email: modis@ispras.ru
+Maintainer: Vladimir Mayorov
+Maintainer-email: vmayorov@ispras.ru
+License: Apache Software License
+Classifier: Development Status :: 3 - Alpha
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: License :: OSI Approved :: Apache Software License
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: talisman-interfaces~=0.11.0
+Requires-Dist: fsspec
+Provides-Extra: s3
+Requires-Dist: boto3~=1.35.0; extra == "s3"
+Requires-Dist: botocore~=1.35.0; extra == "s3"
+Requires-Dist: s3fs; extra == "s3"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: license
+Dynamic: maintainer
+Dynamic: maintainer-email
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# tie-storage
+
+TIE storage client library providing a unified interface for local and S3-compatible storage.

tie_storage-0.0.1/README.md

@@ -0,0 +1,3 @@
+# tie-storage
+
+TIE storage client library providing a unified interface for local and S3-compatible storage.

tie_storage-0.0.1/VERSION

@@ -0,0 +1 @@
+0.0.1

tie_storage-0.0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

tie_storage-0.0.1/setup.py

@@ -0,0 +1,47 @@
+from setuptools import find_packages, setup
+
+with open("README.md", "r", encoding="utf-8") as f:
+    long_description = f.read()
+
+with open("VERSION", "r", encoding="utf-8") as f:
+    version = f.read()
+
+setup(
+    name='tie-storage',
+    version=version,
+    description='TIE storage client library',
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    author='ISPRAS TIE NLP team',
+    author_email='modis@ispras.ru',
+    maintainer='Vladimir Mayorov',
+    maintainer_email='vmayorov@ispras.ru',
+    packages=find_packages(include=['tie_storage', 'tie_storage.*']),
+    install_requires=[
+        'talisman-interfaces~=0.11.0',
+        'fsspec'
+    ],
+    extras_require={
+        's3': [
+            'boto3~=1.35.0',
+            'botocore~=1.35.0',
+            's3fs'
+        ]
+    },
+    entry_points={
+        'talisman.plugins': [
+            'storage = tie_storage'
+        ]
+    },
+    data_files=[('', ['VERSION'])],
+    python_requires='>=3.10',
+    license='Apache Software License',
+    classifiers=[
+        'Development Status :: 3 - Alpha',
+        'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.10',
+        'Intended Audience :: Developers',
+        'Operating System :: OS Independent',
+        'License :: OSI Approved :: Apache Software License'
+    ]
+)

tie_storage-0.0.1/tie_storage/__init__.py

@@ -0,0 +1,9 @@
+from .abstract import AbstractStorageClient
+from .configuration import STORAGE_CLIENTS
+from .datamodel import ObjectInfo
+
+__all__ = [
+    "AbstractStorageClient",
+    "ObjectInfo",
+    "STORAGE_CLIENTS"
+]

tie_storage-0.0.1/tie_storage/abstract.py

@@ -0,0 +1,129 @@
+import logging
+import os
+from io import BytesIO
+from pathlib import Path
+from typing import BinaryIO
+
+from fsspec import AbstractFileSystem
+
+from tie_storage.datamodel import ObjectInfo
+from tp_interfaces.abstract import AbstractConfigConstructableModel
+
+logger = logging.getLogger(__name__)
+
+
+class AbstractStorageClient(AbstractConfigConstructableModel):
+
+    def __init__(self, fs: AbstractFileSystem):
+        self._fs = fs
+
+    def _makedirs(self, path: str):
+        path = Path(path)
+        self._fs.makedirs(str(path.parent), exist_ok=True)
+
+    def _preprocess_storage_path(self, path: str, source: bool = False) -> str:
+        return path
+
+    def _postprocess_storage_path(self, path: str, source: bool = False) -> str:
+        return path
+
+    def upload_file(self, source: str, dest: str) -> str:
+        """Upload a local file to storage.
+
+        @param source: local file path
+        @param dest: storage destination path
+        @return: the storage file path
+        """
+        dest = self._preprocess_storage_path(dest)
+        self._makedirs(dest)
+        self._fs.put(source, dest)
+
+        logger.info(f"Uploaded {source} -> {dest}")
+        return self._postprocess_storage_path(dest)
+
+    def upload_fileobj(self, fileobj: BinaryIO, dest: str) -> str:
+        """Upload a file-like object to storage.
+
+        @param fileobj: file-like object to upload
+        @param dest: storage destination path
+        @return: the storage file path
+        """
+        dest = self._preprocess_storage_path(dest)
+        self._makedirs(dest)
+        with self._fs.open(dest, "wb") as f:
+            f.write(fileobj.read())
+
+        logger.info(f"Uploaded fileobj -> {dest}")
+        return self._postprocess_storage_path(dest)
+
+    def upload_directory(self, source_dir: str, dest: str, recursive: bool = False) -> str:
+        """Upload files from a local directory to storage.
+
+        @param source_dir: local directory path
+        @param dest: storage destination path
+        @param recursive: if True, upload subdirectories recursively
+        @return: the storage path to uploaded directory
+        """
+        preprocessed_dest = self._preprocess_storage_path(dest)
+
+        for entry in os.listdir(source_dir):
+            entry_path = os.path.join(source_dir, entry)
+            if os.path.isfile(entry_path):
+                self.upload_file(entry_path, f"{dest}/{entry}")
+            elif recursive and os.path.isdir(entry_path):
+                self.upload_directory(entry_path, f"{dest}/{entry}", recursive=True)
+
+        logger.info(f"Uploaded directory {source_dir} -> {dest}")
+        return self._postprocess_storage_path(preprocessed_dest)
+
+    def download_file(self, source: str, dest: str) -> str:
+        """Download a file from storage to local filesystem.
+
+        @param source: storage source path
+        @param dest: local destination file path
+        @return: the local absolute file path
+        """
+        dest_abspath = Path(dest).resolve()
+        os.makedirs(dest_abspath.parent, exist_ok=True)
+        self._fs.get(self._preprocess_storage_path(source, source=True), str(dest_abspath))
+        logger.info(f"Downloaded {source} -> {dest_abspath}")
+        return str(dest_abspath)
+
+    def download_fileobj(self, source: str) -> BytesIO:
+        """Download a file from storage as a BytesIO object.
+
+        @param source: storage source path
+        @return: BytesIO with file contents
+        """
+        data = BytesIO(self._fs.cat(self._preprocess_storage_path(source, source=True)))
+        logger.info(f"Downloaded {source} as fileobj")
+        return data
+
+    def download_to_dir(self, source: str, dest: str) -> str:
+        """Download a file from storage into a directory, preserving the filename.
+
+        @param source: storage source path
+        @param dest: local directory path to save the file in
+        @return: the local absolute file path
+        """
+        filename = os.path.basename(source)
+        dest = os.path.join(dest, filename)
+        return self.download_file(self._preprocess_storage_path(source, source=True), dest)
+
+    def list_objects(self, path: str) -> list[ObjectInfo]:
+        """List objects at the given path.
+
+        @param path: storage path to list
+        @return: list of ObjectInfo
+        """
+        entries = self._fs.ls(self._preprocess_storage_path(path), detail=True)
+        return [ObjectInfo(path=entry["name"], size=entry.get("size")) for entry in entries]
+
+    def remove(self, path: str) -> None:
+        """Remove a file or object from storage."""
+        self._fs.rm(self._preprocess_storage_path(path))
+        logger.info(f"Removed {path}")
+
+    def exists(self, path: str) -> bool:
+        """Check if a path exists in storage."""
+        return self._fs.exists(self._preprocess_storage_path(path))

tie_storage-0.0.1/tie_storage/configuration.py

@@ -0,0 +1,20 @@
+from typing import Type
+
+from tp_interfaces.abstract import ModelTypeFactory
+from .abstract import AbstractStorageClient
+
+
+def _get_local_client() -> Type[AbstractStorageClient]:
+    from .local import LocalStorageClient
+    return LocalStorageClient
+
+
+def _get_s3_client() -> Type[AbstractStorageClient]:
+    from .s3 import S3StorageClient
+    return S3StorageClient
+
+
+STORAGE_CLIENTS = ModelTypeFactory({
+    "local": _get_local_client,
+    "s3": _get_s3_client
+})

tie_storage-0.0.1/tie_storage/datamodel.py

@@ -0,0 +1,7 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class ObjectInfo:
+    path: str
+    size: int | None = None

tie_storage-0.0.1/tie_storage/helper.py

@@ -0,0 +1,14 @@
+from talisman_tools.plugin import StorageClientPlugins
+from tie_storage import AbstractStorageClient
+
+
+def storage_client_from_config(config: dict) -> AbstractStorageClient:
+    storage_plugin = config["plugin"]
+    storage_client = config["name"]
+    storage_client_factory = StorageClientPlugins.plugins[storage_plugin].get(storage_client)
+
+    if storage_client_factory is None:
+        raise ValueError(f"Unknown storage client: `{storage_client}`")
+
+    storage_client_config = config["config"]
+    return storage_client_factory().from_config(storage_client_config)

tie_storage-0.0.1/tie_storage/local.py

@@ -0,0 +1,30 @@
+import os
+
+from fsspec.implementations.local import LocalFileSystem
+from typing_extensions import Self
+
+from tie_storage.abstract import AbstractStorageClient
+
+
+class LocalStorageClient(AbstractStorageClient):
+    """Local filesystem storage client. Paths are resolved relative to base_path."""
+
+    def __init__(self, base_path: str):
+        self._base_path = os.path.abspath(base_path)
+        os.makedirs(self._base_path, exist_ok=True)
+        super().__init__(LocalFileSystem(auto_mkdir=True))
+
+    def _preprocess_storage_path(self, path: str, source: bool = False) -> str:
+        if source:
+            return path
+        return os.path.join(self._base_path, path)
+
+    @classmethod
+    def from_config(cls, config: dict) -> Self:
+        if "base_path" in config:
+            return cls(base_path=config["base_path"])
+        raise ValueError("Incorrect config")
+
+    @property
+    def base_path(self) -> str:
+        return self._base_path

tie_storage-0.0.1/tie_storage/s3.py

@@ -0,0 +1,101 @@
+import logging
+import os
+from enum import Enum
+from os.path import splitext
+from pathlib import Path
+
+from s3fs import S3FileSystem
+from typing_extensions import Self
+
+from tie_storage.abstract import AbstractStorageClient
+
+logger = logging.getLogger(__name__)
+
+
+class S3Envs(str, Enum):
+    S3_URL = ("S3_URL", "http://localhost:9000")
+    S3_ACCESS_KEY = ("S3_ACCESS_KEY", "minio")
+    S3_SECRET_KEY = ("S3_SECRET_KEY", "minio123")
+    S3_VERIFY_TLS = ("S3_VERIFY_TLS", "false")
+    S3_BUCKET = ("S3_BUCKET", None)
+
+    def __new__(cls, name: str, default_value: str):
+        obj = str.__new__(cls, name)
+        obj._value_ = name
+        obj._default_value_ = default_value
+        return obj
+
+    @property
+    def env(self) -> str:
+        return os.getenv(self._value_, self._default_value_)
+
+
+S3_PREFIX = "s3://"
+MAX_FILENAME_LENGTH = 255  # POSIX name limitation
+
+
+class S3StorageClient(AbstractStorageClient):
+    """S3-compatible storage client."""
+
+    def __init__(self, bucket: str | None = None):
+        fs = S3FileSystem(
+            key=S3Envs.S3_ACCESS_KEY.env,
+            secret=S3Envs.S3_SECRET_KEY.env,
+            client_kwargs={
+                'endpoint_url': S3Envs.S3_URL.env,
+                'verify': S3Envs.S3_VERIFY_TLS.env.lower() == "true"
+            }
+        )
+        super().__init__(fs)
+
+        env_bucket = S3Envs.S3_BUCKET.env
+        self._bucket = env_bucket if env_bucket else bucket
+        if self._bucket is None:
+            raise ValueError("`S3_BUCKET` is not set")
+
+        if not self._fs.exists(self._bucket):
+            logger.info(f"Creating bucket `{self._bucket}`")
+            self._fs.mkdir(self._bucket)
+        logger.info(f"S3StorageClient initialized. endpoint: `{S3Envs.S3_URL.env}`, bucket: `{self._bucket}`")
+
+    @property
+    def bucket(self) -> str:
+        return self._bucket
+
+    def _preprocess_storage_path(self, path: str, source: bool = False) -> str:
+        if path.startswith(S3_PREFIX):
+            path = path[len(S3_PREFIX):]
+
+        if source:
+            return path
+
+        if not path.startswith(f"{self._bucket}/") and path != self._bucket:
+            path = f"{self._bucket}/{path}" if path else self._bucket
+
+        path = Path(path)
+        file_name = _check_filename_length(path.name)
+        file_dir = str(path.parent)
+        return f"{file_dir}/{file_name}" if file_dir != "." else file_name
+
+    def _postprocess_storage_path(self, path: str, source: bool = False) -> str:
+        return f"{S3_PREFIX}{path}"
+
+    @classmethod
+    def from_config(cls, config: dict) -> Self:
+        return cls(bucket=config.get("bucket"))
+
+
+def _splitext_multi(path: str) -> tuple[str, str]:
+    """splitext that handles extensions with multiple dots (e.g. '.tar.gz')."""
+    parts = path.split(".")
+    if len(parts) > 2:
+        return ".".join(parts[:-2]), "." + ".".join(parts[-2:])
+    return splitext(path)
+
+
+def _check_filename_length(filename: str) -> str:
+    """Truncate filename if it exceeds POSIX name length limit."""
+    if len(filename) > MAX_FILENAME_LENGTH:
+        name, ext = _splitext_multi(filename)
+        filename = name[:MAX_FILENAME_LENGTH - len(ext)] + ext
+    return filename

tie_storage-0.0.1/tie_storage.egg-info/PKG-INFO

@@ -0,0 +1,39 @@
+Metadata-Version: 2.4
+Name: tie-storage
+Version: 0.0.1
+Summary: TIE storage client library
+Author: ISPRAS TIE NLP team
+Author-email: modis@ispras.ru
+Maintainer: Vladimir Mayorov
+Maintainer-email: vmayorov@ispras.ru
+License: Apache Software License
+Classifier: Development Status :: 3 - Alpha
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: License :: OSI Approved :: Apache Software License
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: talisman-interfaces~=0.11.0
+Requires-Dist: fsspec
+Provides-Extra: s3
+Requires-Dist: boto3~=1.35.0; extra == "s3"
+Requires-Dist: botocore~=1.35.0; extra == "s3"
+Requires-Dist: s3fs; extra == "s3"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: license
+Dynamic: maintainer
+Dynamic: maintainer-email
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# tie-storage
+
+TIE storage client library providing a unified interface for local and S3-compatible storage.

tie_storage-0.0.1/tie_storage.egg-info/SOURCES.txt

@@ -0,0 +1,16 @@
+README.md
+VERSION
+setup.py
+tie_storage/__init__.py
+tie_storage/abstract.py
+tie_storage/configuration.py
+tie_storage/datamodel.py
+tie_storage/helper.py
+tie_storage/local.py
+tie_storage/s3.py
+tie_storage.egg-info/PKG-INFO
+tie_storage.egg-info/SOURCES.txt
+tie_storage.egg-info/dependency_links.txt
+tie_storage.egg-info/entry_points.txt
+tie_storage.egg-info/requires.txt
+tie_storage.egg-info/top_level.txt

tie_storage-0.0.1/tie_storage.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

tie_storage-0.0.1/tie_storage.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[talisman.plugins]
+storage = tie_storage

tie_storage-0.0.1/tie_storage.egg-info/requires.txt

@@ -0,0 +1,7 @@
+talisman-interfaces~=0.11.0
+fsspec
+
+[s3]
+boto3~=1.35.0
+botocore~=1.35.0
+s3fs

tie_storage-0.0.1/tie_storage.egg-info/top_level.txt

@@ -0,0 +1 @@
+tie_storage