collective.xsendfile 2.0.0__py3-none-any.whl

collective/xsendfile/configure.zcml

@@ -0,0 +1,97 @@
+<configure
+        xmlns="http://namespaces.zope.org/zope"
+        xmlns:monkey="http://namespaces.plone.org/monkey"
+        xmlns:five="http://namespaces.zope.org/five"
+        xmlns:browser="http://namespaces.zope.org/browser"
+        xmlns:zcml="http://namespaces.zope.org/zcml"
+        xmlns:genericsetup="http://namespaces.zope.org/genericsetup"
+        i18n_domain="collective.xsendfile"
+        >
+
+    <five:registerPackage package="." initialize=".zope2.initialize"/>
+
+    <!-- Control panel -->
+    <include package="plone.registry"/>
+    <include package="plone.app.registry"/>
+    <browser:page
+            name="xsendfile-settings"
+            for="Products.CMFPlone.interfaces.IPloneSiteRoot"
+            class=".controlpanel.xsendfileSettingsControlPanel"
+            permission="cmf.ManagePortal"
+            />
+
+    <!-- Monkey patcher -->
+    <include package="collective.monkeypatcher"/>
+    <configure zcml:condition="installed plone.app.blob">
+      <monkey:patch
+              description="Add support for XSendfile"
+              class="plone.app.blob.field.BlobWrapper"
+              original="index_html"
+              replacement=".utils.plone_app_blob_field_BlobWrapper_index_html"
+              preserveOriginal="true"
+              />
+      <monkey:patch
+              description="Add support for XSendfile"
+              class="plone.app.blob.field.BlobWrapper"
+              original="getIterator"
+              replacement=".utils.plone_app_blob_field_BlobWrapper_getIterator"
+              preserveOriginal="true"
+              />
+    </configure>
+    <monkey:patch
+            description="Create blobstorage and internal folders/files with proper permissions for groups"
+            class="ZODB.blob.FilesystemHelper"
+            original="create"
+            replacement=".monkeypatch.FilesystemHelper_create"
+            preserveOriginal="true"
+            preconditions="ZODB-=5.2.1"
+            />
+    <monkey:patch
+            description="Create blobstorage and internal folders/files with proper permissions for groups"
+            class="ZODB.blob.FilesystemHelper"
+            original="isSecure"
+            replacement=".monkeypatch.FilesystemHelper_isSecure"
+            preserveOriginal="true"
+            preconditions="ZODB-=5.2.1"
+            />
+    <monkey:patch
+            description="Create blobstorage and internal folders/files with proper permissions for groups"
+            class="ZODB.blob.FilesystemHelper"
+            original="getPathForOID"
+            replacement=".monkeypatch.FilesystemHelper_getPathForOID"
+            preserveOriginal="true"
+            preconditions="ZODB-=5.2.1"
+            />
+    <configure zcml:condition="installed plone.namedfile">
+        <monkey:patch
+                description="Add support for XSendfile"
+                class="plone.namedfile.browser.Download"
+                original="__call__"
+                replacement=".utils.monkeypatch_plone_namedfile_browser_Download__call__"
+                preserveOriginal="true"
+                />
+        <monkey:patch
+                description="Add support for XSendfile"
+                class="plone.formwidget.namedfile.widget.Download"
+                original="__call__"
+                replacement=".utils.monkeypatch_plone_formwidget_namedfile_widget_download__call__"
+                preserveOriginal="true"
+                />
+        <monkey:patch
+            description="Add support for XSendfile for image scales"
+            class="plone.namedfile.scaling.ImageScale"
+            original="index_html"
+            replacement=".utils.plone_namedfile_scaling_ImageScale_index_html"
+            preserveOriginal="true"
+            />
+    </configure>
+
+    <genericsetup:registerProfile
+            name="default"
+            title="XSendFile support"
+            directory="profiles/default"
+            description="Installs the collective.xsendfile package"
+            provides="Products.GenericSetup.interfaces.EXTENSION"
+            />
+
+</configure>

collective/xsendfile/controlpanel.py

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+from collective.xsendfile.interfaces import IxsendfileSettings
+from collective.xsendfile.interfaces import _
+from plone.app.registry.browser import controlpanel
+
+import os
+
+if 'XSENDFILE_RESPONSEHEADER' in os.environ:
+    DESC = _(
+        'WARNING: xsendfile settings configured by environment. These settings '
+        'will be ignored.')
+else:
+    DESC = _(u'')
+
+
+class xsendfileSettingsEditForm(controlpanel.RegistryEditForm):
+    schema = IxsendfileSettings
+    label = _(u'xsendfile settings')
+
+    description = DESC
+
+    def updateFields(self):
+        super(xsendfileSettingsEditForm, self).updateFields()
+
+    def updateWidgets(self):
+        super(xsendfileSettingsEditForm, self).updateWidgets()
+
+
+class xsendfileSettingsControlPanel(controlpanel.ControlPanelFormWrapper):
+    form = xsendfileSettingsEditForm

collective/xsendfile/interfaces.py

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+from zope import schema
+from zope.i18nmessageid import MessageFactory
+from zope.interface import Interface
+from zope.schema.vocabulary import SimpleTerm
+from zope.schema.vocabulary import SimpleVocabulary
+
+_ = MessageFactory('collective.xsendfile')
+http_response_headers = SimpleVocabulary(
+    [
+        SimpleTerm(value=u'X-Sendfile', title=_(u'X-Sendfile (Apache, LHTTPd)')),
+        SimpleTerm(value=u'X-Accel-Redirect',
+                   title=_(u'X-Accel-Redirect (nginx, needs path rewriting)')),
+    ]
+)
+
+
+class IxsendfileSettings(Interface):
+    """Global xsendfile settings. This describes records stored in the
+    configuration registry and obtainable via plone.registry.
+    """
+
+    xsendfile_responseheader = schema.Choice(
+        title=_(u'HTTP Response Header'),
+        vocabulary=http_response_headers,
+        required=False,
+    )
+
+    xsendfile_enable_fallback = schema.Bool(
+        title=_(u'Enable fallback based on HTTP_X_FORWARDED_FOR proxy header'),
+        description=_(
+            u'Check if REQUEST header contains HTTP_X_FORWARDED_FOR and re-enable fallback (zope) file delivery if no proxy is detected.'),
+        default=True,
+        required=False,
+    )
+
+    xsendfile_pathregex_search = schema.TextLine(
+        title=_(u'Blob-path rewriting regex'),
+        description=_(
+            u'This regex will be used to modify the files path, in case you are using a different mountpoint on the xsendfile Server. If you are using nginx you have to prepend the id of the location you configured.'),
+        default=u'(.*)',
+        required=False,
+    )
+
+    xsendfile_pathregex_substitute = schema.TextLine(
+        title=_(u'Blob-path rewriting substitution'),
+        description=_(
+            u'This regex will be used to modify the files path, in case you are using a different mountpoint on the xsendfile Server. If you are using nginx you have to prepend the id of the location you configured.'),
+        default=u'\\1',
+        required=False,
+    )

collective/xsendfile/monkeypatch.py

@@ -0,0 +1,117 @@
+# -*- coding: utf-8 -*-
+"""
+
+    Monkey-patch ZODB to support blob storage FS permissios
+    which allows reading them by Apache.
+    http://stackoverflow.com/questions/6168566/collective-xsendfile-zodb-blobs-and-unix-file-permissions/6169177#6169177
+
+    ZODB was changed in 5.2.2 to no longer set permissions. As such, these patches aren't needed on later versions.
+    https://github.com/zopefoundation/ZODB/commit/12eaf5cbc379257ed4b1ed7eece386329796b560
+
+"""
+from ZODB import utils
+from ZODB import blob
+from ZODB.blob import LAYOUT_MARKER
+from ZODB.blob import log
+from importlib.metadata import version as distribution_version
+
+import itertools
+import logging
+import os
+import stat
+
+logger = logging.getLogger(__name__)
+
+
+def FilesystemHelper_create(self):
+    if not os.path.exists(self.base_dir):
+        os.makedirs(self.base_dir, 0o750)
+        log('Blob directory \'%s\' does not exist. '
+            'Created new directory.' % self.base_dir)
+    if not os.path.exists(self.temp_dir):
+        os.makedirs(self.temp_dir, 0o750)
+        log('Blob temporary directory \'%s\' does not exist. '
+            'Created new directory.' % self.temp_dir)
+
+    layout_marker_path = os.path.join(self.base_dir, LAYOUT_MARKER)
+    if not os.path.exists(layout_marker_path):
+        with open(layout_marker_path, 'w') as layout_marker:
+            layout_marker.write(self.layout_name)
+    else:
+        with open(layout_marker_path, 'r') as layout_marker:
+            layout = layout_marker.read().strip()
+        if layout != self.layout_name:
+            raise ValueError(
+                "Directory layout `%s` selected for blob directory %s, but "
+                "marker found for layout `%s`" %
+                (self.layout_name, self.base_dir, layout))
+
+def FilesystemHelper_isSecure(self, path):
+    """Ensure that (POSIX) path mode bits are 0750."""
+    return (os.stat(path).st_mode & 0o27) == 0
+
+def FilesystemHelper_getPathForOID(self, oid, create=False):
+    """Given an OID, return the path on the filesystem where
+    the blob data relating to that OID is stored.
+
+    If the create flag is given, the path is also created if it didn't
+    exist already.
+
+    """
+    # OIDs are numbers and sometimes passed around as integers. For our
+    # computations we rely on the 64-bit packed string representation.
+    if isinstance(oid, int):
+        oid = utils.p64(oid)
+
+    path = self.layout.oid_to_path(oid)
+    path = os.path.join(self.base_dir, path)
+
+    if create and not os.path.exists(path):
+        try:
+            os.makedirs(path, 0o750)
+        except OSError:
+            # We might have lost a race.  If so, the directory
+            # must exist now
+            logger.exception('Error creating blob directory.')
+            assert os.path.exists(path)
+    return path
+
+
+def rename_or_copy_blob(f1, f2, chmod=True):
+    """Try to rename f1 to f2, fallback to copy.
+
+    Under certain conditions a rename might not work, e.g. because the target
+    directory is on a different partition. In this case we try to copy the
+    data and remove the old file afterwards.
+
+    """
+    try:
+        os.rename(f1, f2)
+    except OSError:
+        logger.debug("Copied blob file %r to %r.", f1, f2)
+        file1 = open(f1, 'rb')
+        file2 = open(f2, 'wb')
+        try:
+            utils.cp(file1, file2)
+        finally:
+            file1.close()
+            file2.close()
+        blob.remove_committed(f1)
+
+    os.chmod(f2, stat.S_IREAD | stat.S_IRGRP)
+
+
+def _version_tuple(version):
+    parts = []
+    for part in version.split('.'):
+        digits = ''.join(itertools.takewhile(str.isdigit, part))
+        if not digits:
+            break
+        parts.append(int(digits))
+    return tuple(parts)
+
+
+if _version_tuple(distribution_version('ZODB')) < (5, 2, 2):
+    blob.rename_or_copy_blob = rename_or_copy_blob
+else:
+    logger.info('Not patching rename_or_copy_blob as ZODB >= 5.2.2')

collective/xsendfile/profiles/default/controlpanel.xml

@@ -0,0 +1,20 @@
+<?xml version="1.0"?>
+<object
+        name="portal_controlpanel"
+        xmlns:i18n="http://xml.zope.org/namespaces/i18n"
+        i18n:domain="plone"
+        purge="False">
+
+    <configlet
+            title="XSendFile Settings"
+            action_id="plone.app.event"
+            appId="plone.app.event"
+            category="Plone"
+            condition_expr=""
+            url_expr="string:${portal_url}/@@xsendfile-settings"
+            visible="True"
+            i18n:attributes="title">
+        <permission>Manage portal</permission>
+    </configlet>
+
+</object>

collective/xsendfile/profiles/default/metadata.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<metadata>
+    <dependencies>
+        <dependency>profile-plone.app.registry:default</dependency>
+    </dependencies>
+</metadata>
+
+

collective/xsendfile/profiles/default/registry.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0"?>
+<registry>
+    <records interface="collective.xsendfile.interfaces.IxsendfileSettings"/>
+</registry>

collective/xsendfile/testing.py

@@ -0,0 +1,60 @@
+from plone.app.testing import FunctionalTesting
+from plone.app.testing import IntegrationTesting
+from plone.app.testing import PLONE_FIXTURE
+from plone.app.testing import PloneSandboxLayer
+from plone.app.testing import TEST_USER_ID
+from plone.app.testing import TEST_USER_NAME
+from plone.app.testing import login
+from plone.app.testing import setRoles
+from plone.namedfile.file import NamedBlobFile
+from plone.namedfile.file import NamedBlobImage
+
+import collective.xsendfile
+import os
+
+
+def getData(filename):
+    """Return contents of a file from the tests directory."""
+    path = os.path.join(os.path.dirname(__file__), 'tests', filename)
+    with open(path, 'rb') as f:
+        return f.read()
+
+
+class TestLayer(PloneSandboxLayer):
+    defaultBases = (PLONE_FIXTURE,)
+
+    def setUpZope(self, app, configurationContext):
+        self.loadZCML(package=collective.xsendfile)
+        import plone.app.registry
+        self.loadZCML(package=plone.app.registry)
+
+    def setUpPloneSite(self, portal):
+        setRoles(portal, TEST_USER_ID, ['Manager'])
+        login(portal, TEST_USER_NAME)
+
+        data = getData('image.gif')
+        portal.invokeFactory(
+            'Image', id='image',
+            image=NamedBlobImage(data=data, filename='image.gif'),
+        )
+        portal.invokeFactory(
+            'File', id='file',
+            file=NamedBlobFile(data=data, filename='image.gif'),
+        )
+
+        # Pre-generate the default image scale so its blob is committed by
+        # the layer fixture. Otherwise the xsendfile patch on ImageScale
+        # bails out for uncommitted blobs and never sets X-SENDFILE.
+        portal['image'].unrestrictedTraverse('@@images').scale(
+            fieldname='image',
+        )
+
+
+FIXTURE = TestLayer()
+
+INTEGRATION_TESTING = IntegrationTesting(
+    bases=(FIXTURE,),
+    name='collective.xsendfile:Integration')
+FUNCTIONAL_TESTING = FunctionalTesting(
+    bases=(FIXTURE,),
+    name='collective.xsendfile:Functional')

collective/xsendfile/tests/test_blobs.py

@@ -0,0 +1,139 @@
+import os
+import unittest
+
+from collective.xsendfile.interfaces import IxsendfileSettings
+from collective.xsendfile.testing import INTEGRATION_TESTING
+from plone.app.testing import applyProfile
+from plone.registry.interfaces import IRegistry
+from zope.component import getUtility
+
+
+def clean_env():
+    for e in ['XSENDFILE_RESPONSEHEADER',
+              'XSENDFILE_PATHREGEX_SEARCH',
+              'XSENDFILE_PATHREGEX_SUBSTITUTE',
+              'XSENDFILE_ENABLE_FALLBACK']:
+        if e in os.environ:
+            del os.environ[e]
+
+
+class BaseTestCase(unittest.TestCase):
+    layer = INTEGRATION_TESTING
+
+    def setUp(self):
+        self.portal = self.layer['portal']
+        self.request = self.layer['request']
+
+    def tearDown(self):
+        clean_env()
+
+    def _download(self, content_id='file', field='file'):
+        view = self.portal[content_id].unrestrictedTraverse('@@download')
+        field_view = view.publishTraverse(self.request, field)
+        field_view()
+        return self.request.RESPONSE
+
+
+class DownloadTestCase(BaseTestCase):
+
+    def test_download(self):
+        os.environ['XSENDFILE_RESPONSEHEADER'] = 'X-SENDFILE'
+        self.request.set('HTTP_X_FORWARDED_FOR', '0.0.0.0')
+
+        response = self._download()
+        xsendfile = response.getHeader('X-SENDFILE')
+        self.assertIsNotNone(xsendfile)
+        self.assertTrue(os.path.isfile(xsendfile))
+
+    def test_download_with_filename(self):
+        os.environ['XSENDFILE_RESPONSEHEADER'] = 'X-SENDFILE'
+        self.request.set('HTTP_X_FORWARDED_FOR', '0.0.0.0')
+
+        view = self.portal['file'].unrestrictedTraverse('@@download')
+        field_view = view.publishTraverse(self.request, 'file')
+        filename = field_view.publishTraverse(self.request, 'image.gif')
+        filename()
+
+        xsendfile = self.request.RESPONSE.getHeader('X-SENDFILE')
+        self.assertIsNotNone(xsendfile)
+        self.assertTrue(os.path.isfile(xsendfile))
+
+    def test_substitute(self):
+        os.environ['XSENDFILE_RESPONSEHEADER'] = 'X-SENDFILE'
+        os.environ['XSENDFILE_PATHREGEX_SEARCH'] = r'(.*)'
+        os.environ['XSENDFILE_PATHREGEX_SUBSTITUTE'] = r'/xsendfile/\1'
+        self.request.set('HTTP_X_FORWARDED_FOR', '0.0.0.0')
+
+        response = self._download()
+        xsendfile = response.getHeader('X-SENDFILE')
+        self.assertIsNotNone(xsendfile)
+        self.assertIn('/xsendfile/', xsendfile)
+
+    def test_fallback(self):
+        os.environ['XSENDFILE_RESPONSEHEADER'] = 'X-SENDFILE'
+        os.environ['XSENDFILE_ENABLE_FALLBACK'] = 'True'
+
+        response = self._download()
+        self.assertIsNone(response.getHeader('X-SENDFILE'))
+
+    def test_not_configured(self):
+        response = self._download()
+        self.assertIsNone(response.getHeader('X-SENDFILE'))
+
+    def _image_scale(self):
+        scales = self.portal['image'].unrestrictedTraverse('@@images')
+        scale = scales.publishTraverse(self.request, 'image')
+        scale.index_html()
+        return self.request.RESPONSE
+
+    def test_image_scale(self):
+        os.environ['XSENDFILE_RESPONSEHEADER'] = 'X-SENDFILE'
+        self.request.set('HTTP_X_FORWARDED_FOR', '0.0.0.0')
+
+        response = self._image_scale()
+        xsendfile = response.getHeader('X-SENDFILE')
+        self.assertIsNotNone(xsendfile)
+        self.assertTrue(os.path.isfile(xsendfile))
+
+    def test_image_scale_not_configured(self):
+        self.request.set('HTTP_X_FORWARDED_FOR', '0.0.0.0')
+
+        response = self._image_scale()
+        self.assertIsNone(response.getHeader('X-SENDFILE'))
+
+    def _generate_image_scale(self, scalename):
+        # Traverse @@images/image/<scalename>. Unlike _image_scale (which
+        # serves the pre-generated default field scale), <scalename> has not
+        # been generated by the layer, so this both generates and serves it
+        # within the same, still-uncommitted transaction -- reproducing the
+        # on-the-fly generation an acceptance/prod first-access request hits.
+        scales = self.portal['image'].unrestrictedTraverse('@@images')
+        self.request['TraversalRequestNameStack'] = [scalename]
+        scale = scales.publishTraverse(self.request, 'image')
+        scale.index_html()
+        return self.request.RESPONSE
+
+    def test_image_scale_on_the_fly_generation(self):
+        # A scale generated on-the-fly is uncommitted while the generating
+        # request is still running, so Blob.committed() raises BlobError. The
+        # xsendfile ImageScale patch must detect that and fall back to
+        # streaming rather than propagating a 500. Before the fix this call
+        # raises ZODB.interfaces.BlobError: Uncommitted changes.
+        os.environ['XSENDFILE_RESPONSEHEADER'] = 'X-SENDFILE'
+        self.request.set('HTTP_X_FORWARDED_FOR', '0.0.0.0')
+
+        response = self._generate_image_scale('preview')
+        # Uncommitted blob -> no delegation, streamed by Zope instead.
+        self.assertIsNone(response.getHeader('X-SENDFILE'))
+
+    def test_registry(self):
+        self.request.set('HTTP_X_FORWARDED_FOR', '0.0.0.0')
+
+        applyProfile(self.portal, 'plone.app.registry:default')
+        applyProfile(self.portal, 'collective.xsendfile:default')
+        registry = getUtility(IRegistry)
+        settings = registry.forInterface(IxsendfileSettings)
+        settings.xsendfile_responseheader = 'X-Sendfile'
+
+        response = self._download()
+        self.assertIsNotNone(response.getHeader('X-Sendfile'))