thumbor-libs-blackhand 0.4.0__tar.gz → 0.5.1__tar.gz

thumbor_libs_blackhand-0.5.1/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2020 Blackhand.org
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

{thumbor_libs_blackhand-0.4.0 → thumbor_libs_blackhand-0.5.1}/PKG-INFO

@@ -1,20 +1,32 @@
-Metadata-Version: 1.1
+Metadata-Version: 2.4
 Name: thumbor_libs_blackhand
-Version: 0.4.0
+Version: 0.5.1
 Summary: libs thumbor
 Home-page: https://github.com/Bkhand/thumbor_libs_blackhand
 Author: Bertrand Thill
 Author-email: github@blackhand.org
 License: GNU
-Description: This module enable mongodb support and fallback for thumbor.
-        
 Keywords: thumbor,fallback,images,nfs,mongodb
-Platform: UNKNOWN
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Natural Language :: French
 Classifier: Operating System :: POSIX :: Linux
-Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Internet :: WWW/HTTP :: Dynamic Content
 Classifier: Topic :: Multimedia :: Graphics :: Presentation
+License-File: LICENSE
+Requires-Dist: thumbor>=7.7.0
+Requires-Dist: pymongo>=4.2.0
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: summary
+
+This module enable mongodb support and fallback for thumbor.

thumbor_libs_blackhand-0.5.1/README.md

@@ -0,0 +1,221 @@
+# thumbor_libs_blackhand
+Libs for thumbor 7+
+
+## Table of Contents
+1. [General](#General)
+2. [Loaders](#Loaders)
+3. [Storages](#Storages)
+4. [Result_storages](#Result_storages)
+5. [Url_signers](#Url_signers)
+6. [FAQs](#faqs)
+
+
+
+# General
+
+Collection de modules pour Thumbor 7+ Python3
+
+Ces libs ne sont pas destin�es a tourner en production (absence du stack de test)
+
+Test seulement.
+
+
+Environnement:
+```
+Thumbor 7.7.x
+Python  3.12
+MongoDB 7/8
+```
+
+# Loaders
+
+1. [spec_http_fallback_file_loader] (#spec_http_fallback_file_loader)
+2. [specb_http_or_specb_loader] (#specb_http_or_specb_loader)
+2. [mongodb_loader] (#mongodb_loader)
+
+## spec_http_fallback_file_loader
+
+Description: Loader de type file, avec un fallback sur un autre filesystem.
+
+Implementation: 
+```
+LOADER = thumbor_libs_blackhand.loaders.spec_http_fallback_file_loader
+PIC_LOADER_ROOT_PATH = #root path for file
+PIC_LOADER_FALLBACK_PATH = #fallback path for file
+PIC_LOADER_MAX_SIZE = #max size in bytes default 16777216
+```
+
+## specb_http_or_specb_loader
+
+Description: Loader de type spec_http_fallback_file_loader, avec un fallback sur du http/s http_loader.
+
+Implementation: 
+```
+LOADER = thumbor_libs_blackhand.loaders.specb_http_or_specb_loader
+PIC_LOADER_ROOT_PATH = #root path for file
+PIC_LOADER_FALLBACK_PATH = #fallback path for file
+PIC_LOADER_MAX_SIZE = #max size in bytes default 16777216
+
+#Ajouter les options additionnelles du LOADER http_loader standard
+```
+
+## mongodb_loader
+
+Description: Loader pour MongoDB/Gridfs.
+
+Implementation: 
+```
+LOADER = 'thumbor_libs_blackhand.loaders.mongodb_loader'
+MONGO_ORIGIN_DB = 'thumbor' # MongoDB loader database name
+MONGO_ORIGIN_COLLECTION = '<nom de la collection>' #host
+MONGO_ORIGIN_URI = 'url de connection vers mongoDB mongodb://'
+```
+
+Url type: 
+```
+https://thumbor-server.domain/[secret|unsafe]/[params]/XXXXXXXXXXXXXXXXXXXXXX[/.../..../.xxx  <= all is facultative after id ]
+where `XXXXXXXXXXXXXXXXXXXXXX` is a GridFS `file_id`
+```
+
+Note: avec utilisation de Varnish quelques modifs sont r�aliser
+```
+##### Configuration example for varnish (recv) with AUTO_WEBP ####
+if (req.http.Accept ~ "image/webp") {
+  set req.http.Accept = "image/webp";
+} else {
+  # not present, and we don't care about the rest
+  unset req.http.Accept;
+}
+```
+
+# storages
+
+## mongodb_webp_storage (LEGACY)
+
+Description: Stockage des images pour MongoDB/GridFS compatible avec la fonction auto_webp.
+
+Implementation: 
+```
+STORAGE = 'thumbor_libs_blackhand.storages.mongo_webp_storage'
+MONGO_STORAGE_DB = 'thumbor' # MongoDB storage server database name
+MONGO_STORAGE_DB = 'thumbor' # MongoDB storage server database name
+MONGO_STORAGE_COLLECTION = 'images' # MongoDB storage image collection
+
+```
+
+# Result_storages
+
+
+## mongodb_result_storage_V3 V4
+
+Description: Mise en cache des images pour MongoDB compatible avec la fonction auto_webp. Attention l'expiration doit �tre ger�e via un index TTL Mongo.
+
+Implementation: 
+```
+RESULT_STORAGE = 'thumbor_libs_blackhand.result_storages.mongo_result_storage_v3/4'
+MONGO_RESULT_STORAGE_SERVER_AUTH = "auth base in mongodb"
+MONGO_RESULT_STORAGE_SERVER_COLLECTION = "collection to store image & metadata"
+MONGO_RESULT_STORAGE_SERVER_DB = "base mongodb"
+MONGO_RESULT_STORAGE_SERVER_HOST = "host1,host2 ..."
+MONGO_RESULT_STORAGE_SERVER_PASSWORD = "password"
+MONGO_RESULT_STORAGE_SERVER_PORT = "27017"
+MONGO_RESULT_STORAGE_SERVER_READ_PREFERENCE = "secondaryPreferred"
+MONGO_RESULT_STORAGE_SERVER_REPLICASET = "name of replicaset"
+MONGO_RESULT_STORAGE_SERVER_USER = "user"
+```
+
+V4 seulement:
+```
+PRODUIT = ['','']
+```
+
+
+## hybrid_result_storage
+
+Description: Mise en cache des images pour MongoDB (metadata) + disk (binaire) compatible avec la fonction auto_webp. Attention l'expiration doit �tre ger�e via un index TTL Mongo - hors disque TODO.
+
+Implementation: 
+```
+RESULT_STORAGE = 'thumbor_libs_blackhand.result_storages.mongo_result_storage'
+MONGO_RESULT_STORAGE_SERVER_AUTH = "auth base in mongodb"
+MONGO_RESULT_STORAGE_SERVER_COLLECTION = "collection to store image & metadata"
+MONGO_RESULT_STORAGE_SERVER_DB = "base mongodb"
+MONGO_RESULT_STORAGE_SERVER_HOSTS = "host1,host2 ..."
+MONGO_RESULT_STORAGE_SERVER_PASSWORD = "password"
+MONGO_RESULT_STORAGE_SERVER_PORT = "27017"
+MONGO_RESULT_STORAGE_SERVER_READ_PREFERENCE = "secondaryPreferred"
+MONGO_RESULT_STORAGE_SERVER_REPLICASET = "name of replicaset"
+MONGO_RESULT_STORAGE_SERVER_USER = "user"
+CACHE_PATH = "path du storage du cache"
+```
+
+Options:
+```
+MONGO_STORE_METADATA = True
+```
+
+
+Note: avec utilisation de Varnish quelques modifs sont r�aliser
+
+Exemple: https://www.fastly.com/blog/test-new-encodings-fastly-including-webp
+
+```
+sub vcl_recv {
+  # Normalize Accept, we're only interested in webp right now.
+  # And only normalize for URLs we care about.
+  if (req.http.Accept && req.url ~ "(\.jpe?g|\.png)($|\?)") {
+    # So we don't have to keep using the above regex multiple times.
+    set req.http.X-Is-An-Image-URL = "yay";
+
+    # Test Le wep n'est pas acceptable
+    if (req.http.Accept ~ "image/webp[^,];q=0(\.0?0?0?)?[^0-9]($|[,;])") {
+      unset req.http.Accept;
+    }
+
+    # Le webp est acceptable
+    if (req.http.Accept ~ "image/webp") {
+      set req.http.Accept = "image/webp";
+    } else {
+      # Header non present
+      unset req.http.Accept;
+    }
+  }
+#FASTLY recv
+}
+
+sub vcl_miss {
+  # Si vous avez /foo/bar.jpeg, vous pouvez aussi avoir /foo/bar.webp
+
+  if (req.http.Accept ~ "image/webp" && req.http.X-Is-An-Image-URL) {
+    set bereq.url = regsuball(bereq.url, "(\.jpe?g|\.png)($|\?)", ".webp\2");
+  }
+#FASTLY miss
+}
+
+sub vcl_fetch {
+  if (req.http.X-Is-An-Image-URL) {
+    if (!beresp.http.Vary ~ "(^|\s|,)Accept($|\s|,)") {
+      if (beresp.http.Vary) {
+        set beresp.http.Vary = beresp.http.Vary ", Accept";
+      } else {
+         set beresp.http.Vary = "Accept";
+      }
+    }
+  }
+#FASTLY fetch
+}
+```
+
+# Url_signers
+
+## base64_hmac_sha1_trim
+
+Description: Url signers basique avec fonction trim.
+
+Implementation: 
+```
+URL_SIGNER = 'thumbor_libs_blackhand.url_signers.base64_hmac_sha1_trim'
+```
+
+# Metrics
+

thumbor_libs_blackhand-0.5.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

{thumbor_libs_blackhand-0.4.0 → thumbor_libs_blackhand-0.5.1}/setup.py

@@ -3,12 +3,13 @@
 # Licensed under the MIT license:
 # http://www.opensource.org/licenses/mit-license
 
+from setuptools import setup, find_packages
 
-from distutils.core import setup
+#from distutils.core import setup
 
 setup(
     name = "thumbor_libs_blackhand",
-    version = "0.4.0",
+    version = "0.5.1",
     description = "libs thumbor",
     author = "Bertrand Thill",
     author_email = "github@blackhand.org",
@@ -29,13 +30,13 @@ setup(
                    'License :: OSI Approved :: MIT License',
                    'Natural Language :: French',
                    'Operating System :: POSIX :: Linux',
-                   'Programming Language :: Python :: 3.9',
+                   'Programming Language :: Python :: 3.12',
                    'Topic :: Internet :: WWW/HTTP :: Dynamic Content',
                    'Topic :: Multimedia :: Graphics :: Presentation'
     ],
     package_dir = {"thumbor_libs_blackhand": "thumbor_libs_blackhand"},
-    install_requires=['thumbor>=7.1.0','pymongo>=4.2.0'],
+    install_requires=['thumbor>=7.7.0','pymongo>=4.2.0'],
     long_description = """\
 This module enable mongodb support and fallback for thumbor.
 """
-)
+)

thumbor_libs_blackhand-0.5.1/thumbor_libs_blackhand/result_storages/mongodb_result_storage_v5.py

@@ -0,0 +1,178 @@
+# -*- coding: utf-8 -*-
+# Blackhand library for Thumbor
+# Licensed under the MIT license:
+# http://www.opensource.org/licenses/mit-license
+
+from  urllib import parse #, request; error
+from thumbor_libs_blackhand.mongodb.pool_result_storage import MongoConnector
+from datetime import datetime, timedelta
+from thumbor.result_storages import BaseStorage, ResultStorageResult
+from thumbor.engines import BaseEngine
+from thumbor.utils import logger
+from bson.binary import Binary
+import pytz
+import re
+
+
+class Storage(BaseStorage):
+
+    def __init__(self, context):
+        BaseStorage.__init__(self, context)
+        self.database, self.storage = self.__conn__()
+        super(Storage, self).__init__(context)
+
+    def __conn__(self):
+        try:
+            uri = self.context.config.MONGO_RESULT_STORAGE_SERVER_URI
+            db_name=self.context.config.MONGO_RESULT_STORAGE_SERVER_DB
+            col_name=self.context.config.MONGO_RESULT_STORAGE_SERVER_COLLECTION
+        except AttributeError:
+            logger.error("[RESULT_STORAGE] Parametres databases incomplet")
+
+        mongo_conn = MongoConnector(
+            db_name=db_name,
+            col_name=col_name,
+            uri=uri,
+        )
+
+        database = mongo_conn.db_conn
+        storage = mongo_conn.col_conn
+
+        return database, storage
+
+
+    @property
+    def is_auto_webp(self):
+        '''Return if webp.
+        :return: If the file is a webp
+        :rettype: boolean
+        '''
+
+        return self.context.config.AUTO_WEBP \
+            and self.context.request.accepts_webp
+
+    def get_key_from_request(self):
+        '''Return a path key for the current request url.
+        :return: The storage key for the current url
+        :rettype: string
+        '''
+
+        path = f"result:{self.context.request.url}"
+
+        if self.is_auto_webp:
+            return f'{path}/webp'
+
+        return path
+
+
+    def get_max_age(self):
+
+        return self.context.config.RESULT_STORAGE_EXPIRATION_SECONDS
+
+
+    async def put(self, image_bytes):
+        key = self.get_key_from_request()
+        #max_age = self.get_max_age()
+        #result_ttl = self.get_max_age()
+        ref_img = ''
+        ref_img = re.findall(r'/[a-zA-Z0-9]{24}(?:$|/)', key)
+        if ref_img:
+            ref_img2 = ref_img[0].replace('/','')
+        else:
+            ref_img2 = 'undef'
+
+
+        if self.context.config.get("MONGO_STORE_METADATA", False):
+            metadata = dict(self.context.headers)
+        else:
+            metadata = {}
+
+        try:
+            listeProduit = self.context.config.PRODUIT
+            produit=''
+            for s in listeProduit:
+                if key.find(s) >= 1:
+                    produit=s.replace('/', '')
+                    break
+                else:
+                    produit="info"
+        except AttributeError:
+            produit = 'undef'
+
+        doc = {
+            'path': key,
+            'created_at': datetime.utcnow(),
+            'data': Binary(image_bytes),
+            'metadata': metadata,
+            'content_type': BaseEngine.get_mimetype(image_bytes),
+            'ref_id': ref_img2,
+            'content_length' : len(image_bytes),
+            'produit': produit
+            }
+        doc_cpm = dict(doc)
+
+        await self.storage.insert_one(doc_cpm)
+        #return self.context.request.url
+        return key
+
+    async def get(self):
+        key = self.get_key_from_request()
+        logger.debug("[RESULT_STORAGE] image not found at %s", key)
+
+        print(key)
+        age = datetime.utcnow() - timedelta(
+            seconds=self.get_max_age()
+        )
+        stored = await self.storage.find_one({
+            'path': key,
+            'created_at': {
+                '$gte': age
+            },
+        }, {
+            'ref_id': True,
+            'created_at': True,
+            'metadata': True,
+            'content_type': True,
+            'data' : True,
+            'content_length': True,
+        })
+
+
+        if not stored:
+            return None
+
+        try:
+            #self.context.config.MONGO_RESULT_STORAGE_MAXCACHESIZE
+            dedup = self.context.config.MONGO_RESULT_STORAGE_DEDUP
+        except:
+            dedup = False
+
+        if not dedup:
+          logger.debug("Deduplication OFF")
+        else:
+          filter={
+            'path': key
+          }
+          sort=list({
+            'created_at': -1
+          }.items())
+          skip=1
+          obj = self.storage.find(filter=filter, skip=skip)
+          async for doc in obj:
+            logger.info("Deduplication %s", key)
+            self.storage.delete_one({"_id": doc["_id"]})
+
+
+        metadata = stored['metadata']
+        metadata['LastModified'] = stored['created_at'].replace(
+            tzinfo=pytz.utc
+        )
+        metadata['Cache-Control'] = "max-age=60,public"
+        metadata['ContentLength'] = stored['content_length']
+        metadata['ContentType'] = stored['content_type']
+
+        return ResultStorageResult(
+            buffer=stored['data'],
+            metadata=metadata,
+            successful=True
+        )

thumbor_libs_blackhand-0.5.1/thumbor_libs_blackhand.egg-info/PKG-INFO

@@ -0,0 +1,32 @@
+Metadata-Version: 2.4
+Name: thumbor_libs_blackhand
+Version: 0.5.1
+Summary: libs thumbor
+Home-page: https://github.com/Bkhand/thumbor_libs_blackhand
+Author: Bertrand Thill
+Author-email: github@blackhand.org
+License: GNU
+Keywords: thumbor,fallback,images,nfs,mongodb
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Natural Language :: French
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Internet :: WWW/HTTP :: Dynamic Content
+Classifier: Topic :: Multimedia :: Graphics :: Presentation
+License-File: LICENSE
+Requires-Dist: thumbor>=7.7.0
+Requires-Dist: pymongo>=4.2.0
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: summary
+
+This module enable mongodb support and fallback for thumbor.

thumbor_libs_blackhand-0.5.1/thumbor_libs_blackhand.egg-info/SOURCES.txt

@@ -0,0 +1,23 @@
+LICENSE
+README.md
+setup.py
+thumbor_libs_blackhand.egg-info/PKG-INFO
+thumbor_libs_blackhand.egg-info/SOURCES.txt
+thumbor_libs_blackhand.egg-info/dependency_links.txt
+thumbor_libs_blackhand.egg-info/requires.txt
+thumbor_libs_blackhand.egg-info/top_level.txt
+thumbor_libs_blackhand/loaders/__init__.py
+thumbor_libs_blackhand/loaders/mongodb_loader.py
+thumbor_libs_blackhand/loaders/spec_http_or_specb_loader.py
+thumbor_libs_blackhand/loaders/specb_file_fallback_file_loader.py
+thumbor_libs_blackhand/metrics/__init__.py
+thumbor_libs_blackhand/mongodb/pool_result_storage.py
+thumbor_libs_blackhand/result_storages/__init__.py
+thumbor_libs_blackhand/result_storages/hybrid_result_storage.py
+thumbor_libs_blackhand/result_storages/mongodb_result_storage_v3.py
+thumbor_libs_blackhand/result_storages/mongodb_result_storage_v4.py
+thumbor_libs_blackhand/result_storages/mongodb_result_storage_v5.py
+thumbor_libs_blackhand/storages/__init__.py
+thumbor_libs_blackhand/storages/mongodb_webp_storage.py
+thumbor_libs_blackhand/url_signers/__init__.py
+thumbor_libs_blackhand/url_signers/base64_hmac_sha1_trim.py

thumbor_libs_blackhand-0.5.1/thumbor_libs_blackhand.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

thumbor_libs_blackhand-0.5.1/thumbor_libs_blackhand.egg-info/requires.txt

@@ -0,0 +1,2 @@
+thumbor>=7.7.0
+pymongo>=4.2.0

thumbor_libs_blackhand-0.5.1/thumbor_libs_blackhand.egg-info/top_level.txt

@@ -0,0 +1 @@
+thumbor_libs_blackhand