django-file-tools 0.1.0__tar.gz

django_file_tools-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 enricobarzetti
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

django_file_tools-0.1.0/PKG-INFO

@@ -0,0 +1,35 @@
+Metadata-Version: 2.1
+Name: django-file-tools
+Version: 0.1.0
+Summary: Tools for working with files in Django
+Home-page: https://github.com/enricobarzetti/django-file-tools
+License: MIT
+Author: Enrico Barzetti
+Requires-Python: >=3.9,<4.0
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: boto3 (>=1.34.12,<2.0.0)
+Requires-Dist: django (>=3.0)
+Requires-Dist: django-environ (>=0.11.2,<0.12.0)
+Requires-Dist: django-storages (>=1.14,<2.0)
+Requires-Dist: djangorestframework (>=3.12.0,<4.0.0)
+Description-Content-Type: text/markdown
+
+Run MinIO:
+
+`MINIO_ROOT_USER=admin MINIO_ROOT_PASSWORD=password ./minio server temp --address ":9000" --console-address ":9001"`
+
+Go to the MinIO console and generate access keys.  Set values in .env:
+
+```
+MINIO_ENDPOINT=http://127.0.0.1:9000
+MINIO_ACCESS_KEY=value
+MINIO_SECRET_ACCESS_KEY=value
+```
+
+When `MINIO_ENDPOINT` is set, MinIO will be used instead of S3.
+

django_file_tools-0.1.0/README.md

@@ -0,0 +1,13 @@
+Run MinIO:
+
+`MINIO_ROOT_USER=admin MINIO_ROOT_PASSWORD=password ./minio server temp --address ":9000" --console-address ":9001"`
+
+Go to the MinIO console and generate access keys.  Set values in .env:
+
+```
+MINIO_ENDPOINT=http://127.0.0.1:9000
+MINIO_ACCESS_KEY=value
+MINIO_SECRET_ACCESS_KEY=value
+```
+
+When `MINIO_ENDPOINT` is set, MinIO will be used instead of S3.

django_file_tools-0.1.0/django_file_tools/constants.py

@@ -0,0 +1 @@
+TEMP_MARKER = 'temp'

django_file_tools-0.1.0/django_file_tools/model_fields.py

@@ -0,0 +1,66 @@
+from pathlib import PurePath
+
+from django.db.models.fields import files
+
+from django_file_tools.constants import TEMP_MARKER
+from django_file_tools.s3 import get_file_without_prefix
+
+
+class FieldFile(files.FieldFile):
+    def copy(self, original, save=True, tags=None):
+        path = PurePath(original)
+        new_name = path.parts[-1]
+        new_name = self.field.generate_filename(self.instance, new_name)
+        self.name = self.storage.copy(original, new_name, max_length=self.field.max_length, tags=tags)
+        setattr(self.instance, self.field.name, self.name)
+
+        # Save the object because it has changed, unless save is False
+        if save:
+            self.instance.save()
+
+    def copy_from_other_bucket(self, bucket_name, key, prefix=None, save=True, tags=None):
+        new_name = get_file_without_prefix(key, prefix)
+        new_name = self.field.generate_filename(self.instance, new_name)
+        self.name = self.storage.copy_from_other_bucket(bucket_name, key, new_name, max_length=self.field.max_length, tags=tags)
+        setattr(self.instance, self.field.name, self.name)
+
+        # Save the object because it has changed, unless save is False
+        if save:
+            self.instance.save()
+
+    def replace(self, save=True, tags=None):
+        original = self.name
+        self.copy(original, save, tags)
+        self.storage.delete(original)
+
+    @property
+    def url(self):
+        self._require_file()
+        method_name = f'get_{self.field.name}_filename'
+        parameters = None
+        if hasattr(self.instance, method_name):
+            filename = getattr(self.instance, method_name)()
+            filename = self.storage.generate_filename(filename)
+            parameters = {'ResponseContentDisposition': f'attachment; filename={filename}'}
+        return self.storage.url(self.name, parameters=parameters)
+
+
+class FileField(files.FileField):
+    attr_class = FieldFile
+
+
+def copy_from_temp_storage(instance, tags=None):
+    # If tags is not passed then clear them
+    if tags is None:
+        tags = {}
+
+    save = False
+    for field in instance._meta.fields:
+        if isinstance(field, FileField):
+            file = getattr(instance, field.name)
+            if file.name != '':
+                if PurePath(file.name).parts[0].startswith(TEMP_MARKER):
+                    file.replace(save=False, tags=tags)
+                    save = True
+    if save:
+        instance.save()

django_file_tools-0.1.0/django_file_tools/pathers.py

@@ -0,0 +1,55 @@
+from functools import wraps
+from pathlib import PurePath
+
+
+class Pather:
+    """
+    class AnalyzeEditingInputFilePather(AnalyzeEditingFilesPather):
+        @staticmethod
+        def get_constructor_kwargs(instance):
+            return {'instance': instance}
+
+        def __init__(self, instance):
+            self.instance = instance
+
+        def base(self):
+            return PurePath(f'my_model/{self.instance.pk}/')
+
+        def input_dir(self):
+            return self.base() / PurePath('inputs')
+
+        def input_file(self, instance, filename):
+            return self.input_dir() / PurePath(filename)
+
+    @AnalyzeEditingInputFilePather.upload_to('input_file')
+    def location_input():
+        pass
+
+    class UploadedFile(models.Model):
+        content = FileField(blank=True, upload_to=location_input, max_length=1000)
+    """
+    @classmethod
+    def make_upload_to_callable(cls, method):
+        def f(instance, filename):
+            kwargs = cls.get_constructor_kwargs(instance)
+            pather = cls(**kwargs)
+            path = getattr(pather, method)(instance, filename)
+            return str(path)
+        return f
+
+    @classmethod
+    def upload_to(cls, method):
+        def decorator(h):
+            @wraps(h)
+            def f(instance, filename):
+                callable = cls.make_upload_to_callable(method)
+                return callable(instance, filename)
+            return f
+        return decorator
+
+    @staticmethod
+    def get_constructor_kwargs(instance):
+        raise NotImplementedError
+
+    def base(self):
+        return PurePath('/')

django_file_tools-0.1.0/django_file_tools/s3.py

@@ -0,0 +1,278 @@
+import errno
+import os
+import posixpath
+from datetime import date
+from pathlib import PurePath
+
+import boto3
+from botocore.exceptions import ClientError
+from django.conf import settings
+
+from django_file_tools.constants import TEMP_MARKER
+
+RETENTION = 'retention'
+EXPIRE_FAST = 'expire_fast'
+EXPIRE_SLOW = 'expire_slow'
+
+
+def get_client_resource():
+    client = boto3.client(
+        service_name='s3',
+        aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
+        aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
+        endpoint_url=settings.AWS_S3_ENDPOINT_URL,
+    )
+    resource = boto3.resource(
+        service_name='s3',
+        aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
+        aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
+        endpoint_url=settings.AWS_S3_ENDPOINT_URL,
+    )
+    return client, resource
+
+
+client, resource = get_client_resource()
+
+
+def reset_bucket(bucket):
+    bucket = resource.Bucket(bucket)
+    bucket.objects.all().delete()
+
+
+def bucket_exists(bucket_name):
+    try:
+        client.head_bucket(Bucket=bucket_name)
+        return True
+    except ClientError:
+        return False
+
+def assert_dir_exists(path):
+    try:
+        os.makedirs(path)
+    except OSError as e:
+        if e.errno != errno.EEXIST:
+            raise
+
+
+def download_dir(bucket, path, target):
+    # Handle missing / at end of prefix
+    if not path.endswith('/'):
+        path += '/'
+
+    paginator = client.get_paginator('list_objects_v2')
+    for result in paginator.paginate(Bucket=bucket, Prefix=path):
+        # Download each file individually
+        for key in result['Contents']:
+            # Calculate relative path
+            rel_path = key['Key'][len(path):]
+            # Skip paths ending in /
+            if not key['Key'].endswith('/'):
+                local_file_path = os.path.join(target, rel_path)
+                # Make sure directories exist
+                local_file_dir = os.path.dirname(local_file_path)
+                assert_dir_exists(local_file_dir)
+                client.download_file(bucket, key['Key'], local_file_path)
+
+
+def path_is_file_or_directory(bucket, path):
+    paginator = client.get_paginator('list_objects_v2')
+    total = 0
+    for result in paginator.paginate(Bucket=bucket, Prefix=path):
+        total += result['KeyCount']
+
+    if not path.endswith('/'):
+        path += '/'
+    total_with_slash = 0
+    for result in paginator.paginate(Bucket=bucket, Prefix=path):
+        total_with_slash += result['KeyCount']
+    if total == total_with_slash and total > 1:
+        return 'directory'
+    else:
+        return 'file'
+
+
+def download(bucket, path, target):
+    is_directory = path_is_file_or_directory(bucket, path) == 'directory'
+    if is_directory:
+        download_dir(bucket, path, target)
+        return (target, is_directory)
+    else:
+        assert_dir_exists(target)
+        filename = os.path.split(path)[1]
+        target_path = f'{target}/{filename}'
+        client.download_file(bucket, path, target_path)
+        return (target_path, is_directory)
+
+
+def normalize_prefix(prefix):
+    # The prefix needs to end with a slash, but if the root is empty, leave
+    # it.
+    if prefix and not prefix.endswith('/'):
+        prefix += '/'
+    return prefix
+
+
+def get_files_under_prefix(bucket_name, prefix):
+    def ls(bucket_name, prefix):
+        # Use a hash lookup instead of an array to prevent duplicate directories
+        directories = {}
+        files = []
+        paginator = client.get_paginator('list_objects')
+        pages = paginator.paginate(Bucket=bucket_name, Delimiter='/', Prefix=prefix)
+        for page in pages:
+            for entry in page.get('CommonPrefixes', ()):
+                key = posixpath.relpath(entry['Prefix'], prefix)
+                if not key in directories:
+                    directories[key] = True
+            for entry in page.get('Contents', ()):
+                files.append(posixpath.relpath(entry['Key'], prefix))
+        return directories.keys(), files
+
+    def collect_files(bucket_name, prefix, ret):
+        prefix = normalize_prefix(prefix)
+
+        directories, files = ls(bucket_name, prefix)
+        for file_ in files:
+            # S3 files named . are special files that are created when the "Create Folder" button is used on S3.  They
+            # show up when calling list_objects(), but will fail when head_object() is called on them.
+            if file_ == '.':
+                continue
+            ret.append(f'{prefix}{file_}')
+        for directory in directories:
+            ret = collect_files(bucket_name, f'{prefix}{directory}', ret)
+        return ret
+
+    return collect_files(bucket_name, prefix, [])
+
+
+def files_exist_in_prefix(bucket_name, prefix):
+    prefix = normalize_prefix(prefix)
+    paginator = client.get_paginator('list_objects')
+    pages = paginator.paginate(Bucket=bucket_name, Delimiter='/', Prefix=prefix)
+    page = next(iter(pages))
+    if 'Contents' in page:
+        return True
+    return False
+
+
+def file_exists(bucket_name, key):
+    try:
+        client.get_object(Bucket=bucket_name, Key=key)
+    except ClientError as ex:
+        if ex.response['Error']['Code'] == 'NoSuchKey':
+            return False
+        else:
+            raise
+    else:
+        return True
+
+
+def get_file_without_prefix(file_, prefix):
+    if prefix:
+        prefix = normalize_prefix(prefix)
+        return file_.split(prefix)[-1]
+    else:
+        return file_
+
+
+def get_s3_path(path):
+    path = PurePath(settings.AWS_STORAGE_BUCKET_NAME) / PurePath(path)
+    return f's3://{path}'
+
+
+def s3_read(key):
+    s3_object = client.get_object(Bucket=settings.AWS_STORAGE_BUCKET_NAME, Key=key)
+    body = s3_object['Body']
+    return body.read()
+
+
+def s3_write(key, content):
+    client.put_object(Bucket=settings.AWS_STORAGE_BUCKET_NAME, Key=key, Body=content)
+
+
+def s3_delete(key):
+    client.delete_object(Bucket=settings.AWS_STORAGE_BUCKET_NAME, Key=key)
+
+
+def set_tags(key, tags):
+    if tags is None:
+        return
+
+    tag_set = []
+    for k, v in tags.items():
+        tag_set.append({
+            'Key': k,
+            'Value': v,
+        })
+    client.put_object_tagging(
+        Bucket=settings.AWS_STORAGE_BUCKET_NAME,
+        Key=key,
+        Tagging={
+            'TagSet': tag_set
+        }
+    )
+
+
+def s3_temp_folder_cleanup():
+    def dir(bucket_name, prefix):
+        # Use a hash lookup instead of an array to prevent duplicate directories
+        directories = {}
+        paginator = client.get_paginator('list_objects')
+        pages = paginator.paginate(Bucket=bucket_name, Delimiter='/', Prefix=prefix)
+        for page in pages:
+            for entry in page.get('CommonPrefixes', ()):
+                key = posixpath.relpath(entry['Prefix'], prefix)
+                if not key in directories:
+                    directories[key] = True
+        return directories.keys()
+
+    delete_set = dict(Objects=[])
+    for prefix in dir(bucket_name=settings.AWS_STORAGE_BUCKET_NAME, prefix=TEMP_MARKER):
+        response = client.list_objects(Bucket=settings.AWS_STORAGE_BUCKET_NAME, Prefix=prefix, MaxKeys=1)
+        for entry in response.get('Contents', ()):
+            diff_in_hours = (date.today() - entry['LastModified']).total_seconds() / 3600
+            # If the first file in the folder (prefix) is over a day old, delete the whole folder
+            if diff_in_hours >= 24:
+                delete_set['Objects'].append(dict(Key=entry['Key']))
+                if len(delete_set['Objects']) >= 1000:
+                    client.delete_objects(Bucket=settings.AWS_STORAGE_BUCKET_NAME, Delete=delete_set)
+                    delete_set = dict(Objects=[])
+            break
+    if len(delete_set['Objects']) > 0:
+        client.delete_objects(Bucket=settings.AWS_STORAGE_BUCKET_NAME, Delete=delete_set)
+
+
+def s3_lifecycle_configuration():
+    client.put_bucket_lifecycle_configuration(
+        Bucket=settings.AWS_STORAGE_BUCKET_NAME,
+        LifecycleConfiguration={
+            'Rules': [
+                {
+                    'ID': 'Expire fast',
+                    'Filter': {
+                        'Tag': {
+                            'Key': RETENTION,
+                            'Value': EXPIRE_FAST,
+                        },
+                    },
+                    'Status': 'Enabled',
+                    'Expiration': {
+                        'Days': 1,
+                    },
+                },
+                {
+                    'ID': 'Expire slow',
+                    'Filter': {
+                        'Tag': {
+                            'Key': RETENTION,
+                            'Value': EXPIRE_SLOW,
+                        },
+                    },
+                    'Status': 'Enabled',
+                    'Expiration': {
+                        'Days': 180,
+                    },
+                },
+            ]
+        }
+    )

django_file_tools-0.1.0/django_file_tools/serializer_fields.py

@@ -0,0 +1,46 @@
+from django.core.files.storage import default_storage
+from rest_framework import fields
+from rest_framework import serializers
+
+from django_file_tools.s3 import bucket_exists
+from django_file_tools.s3 import file_exists
+from django_file_tools.s3 import files_exist_in_prefix
+
+
+class FileField(fields.FileField):
+    """Similar to the standard serializer FileField but takes strings as paths in the storage instead of file objects"""
+    default_error_messages = fields.FileField.default_error_messages
+    default_error_messages['does not exist'] = 'The file referenced does not exist'
+
+    def get_storage(self):
+        return default_storage
+
+    def to_internal_value(self, data):
+        storage = self.get_storage()
+        if not storage.exists(data):
+            self.fail('does not exist')
+        return data
+
+
+class FileFieldForModelSerializer(FileField):
+    def get_storage(self):
+        model = self.parent.Meta.model
+        return getattr(model, self.field_name).field.storage
+
+
+class S3PathField(fields.CharField):
+    def to_internal_value(self, value):
+        if value:
+            value = value.replace('s3://', '')
+            bucket_name, prefix = value.split('/', 1)
+            if not bucket_exists(bucket_name):
+                raise serializers.ValidationError(f'bucket {bucket_name} does not exist')
+
+            if_file_exists = file_exists(bucket_name, prefix)
+            if_folder_exists = files_exist_in_prefix(bucket_name, prefix)
+            if (not if_file_exists) and (not if_folder_exists):
+                if not prefix.endswith('/'):
+                    raise serializers.ValidationError(f'{value} does not exist')
+                else:
+                    raise serializers.ValidationError(f'{value} has no files')
+        return value

django_file_tools-0.1.0/django_file_tools/serializers.py

@@ -0,0 +1,11 @@
+import copy
+
+from django.db import models
+from rest_framework import serializers
+
+from . import serializer_fields
+
+
+class ModelSerializer(serializers.ModelSerializer):
+    serializer_field_mapping = copy.deepcopy(serializers.ModelSerializer.serializer_field_mapping)
+    serializer_field_mapping[models.FileField] = serializer_fields.FileFieldForModelSerializer

django_file_tools-0.1.0/django_file_tools/storage_backends.py

@@ -0,0 +1,61 @@
+from storages.backends.s3boto3 import S3Boto3Storage
+from storages.utils import clean_name
+
+
+class StorageCopyMixin:
+    def copy(self, original, new_name, max_length=None, tags=None):
+        new_name = self.get_available_name(new_name, max_length=max_length)
+        return self._copy(original, new_name, tags)
+
+    def copy_from_other_bucket(self, bucket_name, key, new_name, max_length=None, tags=None):
+        new_name = self.get_available_name(new_name, max_length=max_length)
+        return self._copy_from_other_bucket(bucket_name, key, new_name, tags)
+
+
+class S3Storage(StorageCopyMixin, S3Boto3Storage):
+    default_acl = 'private'
+    file_overwrite = False
+    custom_domain = False
+
+    @property
+    def s3_client(self):
+        return self.connection.meta.client
+
+    def set_tags(self, key, tags):
+        if tags is None:
+            return
+
+        tag_set = []
+        for k, v in tags.items():
+            tag_set.append({
+                'Key': k,
+                'Value': v,
+            })
+        self.s3_client.put_object_tagging(
+            Bucket=self.bucket_name,
+            Key=key,
+            Tagging={
+                'TagSet': tag_set
+            }
+        )
+
+    def _copy(self, original, new_name, tags=None):
+        normalized_original = self._normalize_name(clean_name(original))
+        normalized_new_name = self._normalize_name(clean_name(new_name))
+        copy_source = {
+            'Bucket': self.bucket_name,
+            'Key': normalized_original
+        }
+        self.bucket.copy(copy_source, normalized_new_name)
+        self.set_tags(normalized_new_name, tags)
+        return normalized_new_name
+
+    def _copy_from_other_bucket(self, bucket_name, key, new_name, tags=None):
+        normalized_new_name = self._normalize_name(clean_name(new_name))
+        copy_source = {
+            'Bucket': bucket_name,
+            'Key': key
+        }
+        self.bucket.copy(copy_source, normalized_new_name)
+        self.set_tags(normalized_new_name, tags)
+        return normalized_new_name

django_file_tools-0.1.0/django_file_tools/views.py

@@ -0,0 +1,58 @@
+import datetime
+
+from django.conf import settings
+from django.contrib.auth.decorators import login_required
+from django.http.response import Http404
+from django.http.response import JsonResponse
+
+from django_file_tools.model_fields import TEMP_MARKER
+from django_file_tools.s3 import EXPIRE_FAST
+from django_file_tools.s3 import RETENTION
+from django_file_tools.s3 import client
+
+
+@login_required
+def get_s3_signature(request):
+    service = 's3'
+    region = 'us-east-1'
+    t = datetime.datetime.utcnow()
+    algorithm = 'AWS4-HMAC-SHA256'
+    credential_scope = '/'.join([t.strftime('%Y%m%d'), region, service, 'aws4_request'])
+
+    name = request.GET.get('name')
+    if name is None:
+        raise Http404
+
+    def get_tag_xml(key, value):
+        return f"<Tagging><TagSet><Tag><Key>{key}</Key><Value>{value}</Value></Tag></TagSet></Tagging>"
+
+    conditions = [
+        {"x-amz-algorithm": algorithm},
+        {"x-amz-credential": credential_scope},
+        {"x-amz-date": t.isoformat()},
+        {"tagging": get_tag_xml(RETENTION, EXPIRE_FAST)},
+        {"success_action_status": "201"},
+        {"bucket": settings.AWS_STORAGE_BUCKET_NAME},
+        ["starts-with", "$key", TEMP_MARKER],
+    ]
+
+    fields = {
+        "x-amz-algorithm": algorithm,
+        "x-amz-credential": credential_scope,
+        "x-amz-date": t.isoformat(),
+        "tagging": get_tag_xml(RETENTION, EXPIRE_FAST),
+        "success_action_status": "201",
+    }
+
+    presigned = client.generate_presigned_post(
+        settings.AWS_STORAGE_BUCKET_NAME,
+        name,
+        Fields=fields,
+        Conditions=conditions,
+        ExpiresIn=7*24*60,
+    )
+
+    return JsonResponse({
+        'signature': presigned['fields'],
+        'postEndpoint': presigned['url'],
+    })

django_file_tools-0.1.0/pyproject.toml

@@ -0,0 +1,26 @@
+[tool.poetry]
+name = "django-file-tools"
+version = "0.1.0"
+description = "Tools for working with files in Django"
+authors = ["Enrico Barzetti"]
+readme = "README.md"
+homepage = "https://github.com/enricobarzetti/django-file-tools"
+license = "mit"
+packages = [
+    { include = "django_file_tools" },
+]
+
+[tool.poetry.dependencies]
+python = "^3.9"
+django = ">=3.0"
+django-storages = "^1.14"
+djangorestframework = "^3.12.0"
+boto3 = "^1.34.12"
+django-environ = "^0.11.2"
+
+[tool.poetry.group.dev.dependencies]
+isort = "^5.13.2"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"