thumbor-libs-blackhand 0.1.6__py3-none-any.whl → 0.5.0__py3-none-any.whl

thumbor_libs_blackhand/loaders/__init__.py

@@ -1,6 +1,4 @@
-#!/usr/bin/python
 # -*- coding: utf-8 -*-
-
 # Blackhand library for Thumbor
-# Licensed under the GNU/GPL license:
-# https://fsf.org/
+# Licensed under the MIT license:
+# http://www.opensource.org/licenses/mit-license

thumbor_libs_blackhand/loaders/mongodb_loader.py

@@ -1,11 +1,9 @@
-#!/usr/bin/python
 # -*- coding: utf-8 -*-
 # Blackhand library for Thumbor
-# Licensed under the GNU/GPL license:
-# https://fsf.org/
+# Licensed under the MIT license:
+# http://www.opensource.org/licenses/mit-license
 
 from pymongo.mongo_client import MongoClient
-from pymongo.server_api import ServerApi
 from bson.objectid import ObjectId
 from thumbor.loaders import LoaderResult
 import gridfs

thumbor_libs_blackhand/loaders/spec_http_or_specb_loader.py

@@ -1,8 +1,7 @@
-#!/usr/bin/python
 # -*- coding: utf-8 -*-
 # Blackhand library for Thumbor
-# Licensed under the GNU/GPL license:
-# https://fsf.org/
+# Licensed under the MIT license:
+# http://www.opensource.org/licenses/mit-license
 
 from thumbor.loaders import http_loader
 from thumbor_libs_blackhand.loaders import specb_file_fallback_file_loader

thumbor_libs_blackhand/loaders/specb_file_fallback_file_loader.py

@@ -1,8 +1,7 @@
-#!/usr/bin/python
 # -*- coding: utf-8 -*-
 # Blackhand library for Thumbor
-# Licensed under the GNU/GPL license:
-# https://fsf.org/
+# Licensed under the MIT license:
+# http://www.opensource.org/licenses/mit-license
 
 from os import fstat
 from datetime import datetime
@@ -30,7 +29,6 @@ async def load(context, path):
           result.error = LoaderResult.ERROR_NOT_FOUND
           result.successful = False
           return result
-        pass
 
     if not exists(file_path):
         file_path = unquote(file_path)

thumbor_libs_blackhand/metrics/__init__.py

@@ -1,6 +1,4 @@
-#!/usr/bin/python
 # -*- coding: utf-8 -*-
-
 # Blackhand library for Thumbor
-# Licensed under the GNU/GPL license:
-# https://fsf.org/
+# Licensed under the MIT license:
+# http://www.opensource.org/licenses/mit-license

thumbor_libs_blackhand/mongodb/pool_result_storage.py

@@ -0,0 +1,60 @@
+# Licensed under the MIT license:
+# http://www.opensource.org/licenses/mit-license
+# Copyright (c) 2020 Eka Cahya Pratama <ekapratama93@gmail.com>
+
+from motor.motor_tornado import MotorClient
+from pymongo import ASCENDING, DESCENDING
+from tornado.gen import convert_yielded
+
+
+class Singleton(type):
+    """
+    Define an Instance operation that lets clients access its unique
+    instance.
+    """
+
+    def __init__(cls, name, bases, attrs, **kwargs):
+        super().__init__(name, bases, attrs)
+        cls._instance = None
+
+    def __call__(cls, *args, **kwargs):
+        if cls._instance is None:
+            cls._instance = super().__call__(*args, **kwargs)
+        return cls._instance
+
+
+class MongoConnector(metaclass=Singleton):
+
+    def __init__(self,
+                 uri=None,
+                 host=None,
+                 port=None,
+                 db_name=None,
+                 col_name=None):
+        self.uri = uri
+        self.host = host
+        self.port = port
+        self.db_name = db_name
+        self.col_name = col_name
+        self.db_conn, self.col_conn = self.create_connection()
+        convert_yielded(self.ensure_index())
+
+    def create_connection(self):
+        if self.uri:
+            connection = MotorClient(self.uri)
+        else:
+            connection = MotorClient(self.host, self.port)
+
+        db_conn = connection[self.db_name]
+        col_conn = db_conn[self.col_name]
+
+        return db_conn, col_conn
+
+    async def ensure_index(self):
+        index_name = 'key_1_created_at_-1'
+        indexes = await self.col_conn.index_information()
+        if index_name not in indexes:
+            await self.col_conn.create_index(
+                [('path', ASCENDING), ('created_at', DESCENDING)],
+                name=index_name
+            )

thumbor_libs_blackhand/result_storages/__init__.py

@@ -0,0 +1,4 @@
+# -*- coding: utf-8 -*-
+# Blackhand library for Thumbor
+# Licensed under the MIT license:
+# http://www.opensource.org/licenses/mit-license

thumbor_libs_blackhand/result_storages/hybrid_result_storage.py

@@ -0,0 +1,213 @@
+# -*- coding: utf-8 -*-
+# Blackhand library for Thumbor
+# Licensed under the MIT license:
+# http://www.opensource.org/licenses/mit-license
+# libvips
+
+from  urllib import parse #, request; error
+#from motor.motor_tornado import MotorGridFSBucket
+#from pymongo.errors import PyMongoError
+from thumbor_libs_blackhand.mongodb.pool_result_storage import MongoConnector
+from datetime import datetime, timedelta
+from thumbor.result_storages import BaseStorage, ResultStorageResult
+from thumbor.engines import BaseEngine
+from thumbor.utils import logger
+from bson.binary import Binary
+from uuid import uuid4
+import pytz
+import re
+#import os
+
+
+class Storage(BaseStorage):
+
+    def __init__(self, context):
+        BaseStorage.__init__(self, context)
+        self.database, self.storage = self.__conn__()
+        super(Storage, self).__init__(context)
+
+
+    def __conn__(self):
+        '''Return the MongoDB params.
+        :returns: MongoDB DB and Collection
+        :rtype: pymongo.database.Database, pymongo.database.Collection
+        '''
+
+        password = parse.quote_plus(self.context.config.MONGO_RESULT_STORAGE_SERVER_PASSWORD)
+        user = parse.quote_plus(self.context.config.MONGO_RESULT_STORAGE_SERVER_USER)
+
+        if not self.context.config.MONGO_RESULT_STORAGE_SERVER_REPLICASET:
+          uric = ('mongodb://'+ user +':' + password + '@' + self.context.config.MONGO_RESULT_STORAGE_SERVER_HOSTS
+                  + '/?authSource=' + self.context.config.MONGO_RESULT_STORAGE_SERVER_DB)
+        else:
+          uric = ('mongodb://'+ user +':' + password + '@' + self.context.config.MONGO_RESULT_STORAGE_SERVER_HOSTS
+                  + '/?authSource=' + self.context.config.MONGO_RESULT_STORAGE_SERVER_DB
+                  + "&replicaSet=" + self.context.config.MONGO_RESULT_STORAGE_SERVER_REPLICASET
+                  + "&readPreference=" + self.context.config.MONGO_RESULT_STORAGE_SERVER_READ_PREFERENCE)
+
+        db_name = self.context.config.MONGO_RESULT_STORAGE_SERVER_DB
+        col_name = self.context.config.MONGO_RESULT_STORAGE_SERVER_COLLECTION
+
+        try:
+            uri = self.context.config.MONGO_RESULT_STORAGE_URI
+        except AttributeError:
+            uri=uric
+
+        try:
+            host = self.context.config.MONGO_RESULT_STORAGE_SERVER_HOST
+            port = self.context.config.MONGO_RESULT_STORAGE_SERVER_PORT
+        except AttributeError:
+            host=None
+            port=None
+
+        mongo_conn = MongoConnector(
+            db_name=db_name,
+            col_name=col_name,
+            uri=uri,
+            host=host,
+            port=port,
+        )
+
+        database = mongo_conn.db_conn
+        storage = mongo_conn.col_conn
+
+        return database, storage
+
+
+    @property
+    def is_auto_webp(self):
+        return self.context.config.AUTO_WEBP and self.context.request.accepts_webp
+
+    @property
+    def is_auto_webp(self):
+        '''Return if webp.
+        :return: If the file is a webp
+        :rettype: boolean
+        '''
+
+        return self.context.config.AUTO_WEBP \
+            and self.context.request.accepts_webp
+
+    def get_key_from_request(self):
+        '''Return a path key for the current request url.
+        :return: The storage key for the current url
+        :rettype: string
+        '''
+
+        path = f"result:{self.context.request.url}"
+
+        if self.is_auto_webp:
+            return f'{path}/webp'
+
+        return path
+
+
+    def get_max_age(self):
+
+        return self.context.config.RESULT_STORAGE_EXPIRATION_SECONDS
+
+    async def put(self, image_bytes):
+        key = self.get_key_from_request()
+        #max_age = self.get_max_age()
+        #result_ttl = self.get_max_age()
+        try:
+            CACHE_PATH = self.context.config.CACHE_PATH
+        except AttributeError:
+            raise
+        imgpath = ( datetime.now().strftime('%Y') + '/'
+                  + datetime.now().strftime('%m') + '/' + datetime.now().strftime('%d')
+                  + '/' + datetime.now().strftime('%H'))
+        mkpath = (CACHE_PATH + '/' + imgpath)
+        self.ensure_dir(mkpath)
+
+        #os.makedirs(mkpath)
+        ref_img = ''
+        ref_img = re.findall(r'/[a-zA-Z0-9]{24}(?:$|/)', key)
+        if ref_img:
+            ref_img2 = ref_img[0].replace('/','')
+        else:
+            ref_img2 = 'undef'
+
+
+        if self.context.config.get("MONGO_STORE_METADATA", False):
+            metadata = dict(self.context.headers)
+        else:
+            metadata = {}
+
+        cache_id = str(uuid4())
+        endingpath= mkpath + "/" + cache_id
+        doc = {
+            'path': key,
+            'created_at': datetime.utcnow(),
+            'data': "",
+            'metadata': metadata,
+            'content_type': BaseEngine.get_mimetype(image_bytes),
+            'ref_id': ref_img2,
+            'content_length' : len(image_bytes),
+            'cache_path': imgpath,
+            'cache_id' : cache_id
+            }
+        doc_cpm = dict(doc)
+
+
+        fichier = open(endingpath, "wb")
+        try:
+            fichier.write(Binary(image_bytes))
+        finally:
+            fichier.close()
+
+        await self.storage.insert_one(doc_cpm)
+        #return self.context.request.url
+        return key
+
+
+    async def get(self):
+
+        key = self.get_key_from_request()
+        logger.debug("[RESULT_STORAGE] image not found at %s", key)
+        try:
+            CACHE_PATH = self.context.config.CACHE_PATH
+        except AttributeError:
+            raise
+        age = datetime.utcnow() - timedelta(
+            seconds=self.get_max_age()
+        )
+        stored = await self.storage.find_one({
+            'path': key,
+            'created_at': {
+                '$gte': age
+            },
+        }, {
+            'ref_id': True,
+            'created_at': True,
+            'metadata': True,
+            'content_type': True,
+            'data' : True,
+            'content_length': True,
+            'cache_path' : True,
+            'cache_id' : True,
+        })
+
+
+        if not stored:
+            return None
+        metadata = stored['metadata']
+        metadata['LastModified'] = stored['created_at'].replace(
+            tzinfo=pytz.utc
+        )
+        metadata['Cache-Control'] = "max-age=60,public"
+        metadata['ContentLength'] = stored['content_length']
+        metadata['ContentType'] = stored['content_type']
+        cachefile = CACHE_PATH + '/' + stored['cache_path'] + "/" + stored['cache_id']
+
+        fichier = open(cachefile, "rb")
+        try:
+            tosend = fichier.read()
+        finally:
+            fichier.close()
+
+        return ResultStorageResult(
+            buffer=tosend,
+            metadata=metadata,
+            successful=True
+        )

thumbor_libs_blackhand/result_storages/mongodb_result_storage_v3.py

@@ -0,0 +1,196 @@
+# -*- coding: utf-8 -*-
+# Blackhand library for Thumbor
+# Licensed under the MIT license:
+# http://www.opensource.org/licenses/mit-license
+
+from  urllib import parse #, request; error
+#from motor.motor_tornado import MotorGridFSBucket
+#from pymongo.errors import PyMongoError
+from thumbor_libs_blackhand.mongodb.pool_result_storage import MongoConnector
+from datetime import datetime, timedelta
+from thumbor.result_storages import BaseStorage, ResultStorageResult
+from thumbor.engines import BaseEngine
+from thumbor.utils import logger
+from bson.binary import Binary
+import pytz
+import re
+
+
+class Storage(BaseStorage):
+
+    def __init__(self, context):
+        BaseStorage.__init__(self, context)
+        self.database, self.storage = self.__conn__()
+        super(Storage, self).__init__(context)
+
+
+
+    def __conn__(self):
+        '''Return the MongoDB params.
+        :returns: MongoDB DB and Collection
+        :rtype: pymongo.database.Database, pymongo.database.Collection
+        '''
+
+        password = parse.quote_plus(self.context.config.MONGO_RESULT_STORAGE_SERVER_PASSWORD)
+        user = parse.quote_plus(self.context.config.MONGO_RESULT_STORAGE_SERVER_USER)
+        if not self.context.config.MONGO_RESULT_STORAGE_SERVER_REPLICASET:
+          uric = ('mongodb://'+ user +':' + password + '@' + self.context.config.MONGO_RESULT_STORAGE_SERVER_HOSTS
+                  + '/?authSource=' + self.context.config.MONGO_RESULT_STORAGE_SERVER_DB)
+        else:
+          uric = ('mongodb://'+ user +':' + password + '@' + self.context.config.MONGO_RESULT_STORAGE_SERVER_HOSTS
+                  + '/?authSource=' + self.context.config.MONGO_RESULT_STORAGE_SERVER_DB
+                  + "&replicaSet=" + self.context.config.MONGO_RESULT_STORAGE_SERVER_REPLICASET
+                  + "&readPreference=" + self.context.config.MONGO_RESULT_STORAGE_SERVER_READ_PREFERENCE)
+
+        db_name = self.context.config.MONGO_RESULT_STORAGE_SERVER_DB
+        col_name = self.context.config.MONGO_RESULT_STORAGE_SERVER_COLLECTION
+        host = None
+        port = None
+
+        try:
+            uri = self.context.config.MONGO_RESULT_STORAGE_URI
+        except AttributeError:
+            uri=uric
+
+        try:
+            host = self.context.config.MONGO_RESULT_STORAGE_SERVER_HOST
+            port = self.context.config.MONGO_RESULT_STORAGE_SERVER_PORT
+        except AttributeError:
+            host=None
+            port=None
+
+        mongo_conn = MongoConnector(
+            db_name=db_name,
+            col_name=col_name,
+            uri=uri,
+            host=host,
+            port=port,
+        )
+
+        database = mongo_conn.db_conn
+        storage = mongo_conn.col_conn
+
+        return database, storage
+
+    @property
+    def is_auto_webp(self):
+        '''Return if webp.
+        :return: If the file is a webp
+        :rettype: boolean
+        '''
+
+        return self.context.config.AUTO_WEBP \
+            and self.context.request.accepts_webp
+
+    def get_key_from_request(self):
+        '''Return a path key for the current request url.
+        :return: The storage key for the current url
+        :rettype: string
+        '''
+
+        path = f"result:{self.context.request.url}"
+
+        if self.is_auto_webp:
+            return f'{path}/webp'
+
+        return path
+
+
+    def get_max_age(self):
+
+        return self.context.config.RESULT_STORAGE_EXPIRATION_SECONDS
+
+
+    async def put(self, image_bytes):
+        key = self.get_key_from_request()
+        #max_age = self.get_max_age()
+        #result_ttl = self.get_max_age()
+        ref_img = ''
+        ref_img = re.findall(r'/[a-zA-Z0-9]{24}(?:$|/)', key)
+        if ref_img:
+            ref_img2 = ref_img[0].replace('/','')
+        else:
+            ref_img2 = 'undef'
+
+
+        if self.context.config.get("MONGO_STORE_METADATA", False):
+            metadata = dict(self.context.headers)
+        else:
+            metadata = {}
+
+        doc = {
+            'path': key,
+            'created_at': datetime.utcnow(),
+            'data': Binary(image_bytes),
+            'metadata': metadata,
+            'content_type': BaseEngine.get_mimetype(image_bytes),
+            'ref_id': ref_img2,
+            'content_length' : len(image_bytes)
+            }
+        doc_cpm = dict(doc)
+
+        await self.storage.insert_one(doc_cpm)
+        #return self.context.request.url
+        return key
+
+    async def get(self):
+        key = self.get_key_from_request()
+        logger.debug("[RESULT_STORAGE] image not found at %s", key)
+
+
+        age = datetime.utcnow() - timedelta(
+            seconds=self.get_max_age()
+        )
+        stored = await self.storage.find_one({
+            'path': key,
+            'created_at': {
+                '$gte': age
+            },
+        }, {
+            'ref_id': True,
+            'created_at': True,
+            'metadata': True,
+            'content_type': True,
+            'data' : True,
+            'content_length': True,
+        })
+
+
+        if not stored:
+            return None
+        
+        try:
+            #self.context.config.MONGO_RESULT_STORAGE_MAXCACHESIZE
+            dedup = self.context.config.MONGO_RESULT_STORAGE_DEDUP
+        except:
+            dedup = False
+        
+        if not dedup:
+          logger.debug("Deduplication OFF")  
+        else:
+          filter={
+            'path': key
+          }
+          sort=list({
+            'created_at': -1
+          }.items())
+          skip=1
+          obj = self.storage.find(filter=filter, skip=skip)
+          async for doc in obj:
+            logger.info("Deduplication %s", key)
+            self.storage.delete_one({"_id": doc["_id"]})
+
+
+        metadata = stored['metadata']
+        metadata['LastModified'] = stored['created_at'].replace(
+            tzinfo=pytz.utc
+        )
+        metadata['Cache-Control'] = "max-age=60,public"
+        metadata['ContentLength'] = stored['content_length']
+        metadata['ContentType'] = stored['content_type']
+
+        return ResultStorageResult(
+            buffer=stored['data'],
+            metadata=metadata,
+            successful=True
+        )