plone.exportimport 1.0.0__py3-none-any.whl

plone/exportimport/__init__.py

@@ -0,0 +1,8 @@
+"""Plone Distribution support."""
+
+import logging
+
+
+PACKAGE_NAME = "plone.exportimport"
+
+logger = logging.getLogger(PACKAGE_NAME)

plone/exportimport/cli/__init__.py

@@ -0,0 +1,78 @@
+from plone import api
+from plone.exportimport.exporters import get_exporter
+from plone.exportimport.importers import get_importer
+from plone.exportimport.utils import cli as cli_helpers
+
+import argparse
+import sys
+import transaction
+
+
+CLI_SPEC = {
+    "exporter": {
+        "description": "Export Plone Site content",
+        "options": {
+            "zopeconf": "Path to zope.conf",
+            "site": "Plone site ID to export the content from",
+            "path": "Path to export the content",
+            "--include-revisions": "Include revision history",
+        },
+    },
+    "importer": {
+        "description": "Import content into a Plone Site",
+        "options": {
+            "zopeconf": "Path to zope.conf",
+            "site": "Plone site ID to import the content to",
+            "path": "Path to import the content from",
+        },
+    },
+}
+
+
+def _parse_args(description: str, options: dict, args: list):
+    parser = argparse.ArgumentParser(description=description)
+    for key, help in options.items():
+        if key.startswith("-"):
+            parser.add_argument(key, action="store_true", help=help)
+        else:
+            parser.add_argument(key, help=help)
+    namespace, _ = parser.parse_known_args(args[1:])
+    return namespace
+
+
+def exporter_cli(args=sys.argv):
+    """Export a Plone site."""
+    logger = cli_helpers.get_logger("Exporter")
+    exporter_cli = CLI_SPEC["exporter"]
+    # We get an argparse.Namespace instance.
+    namespace = _parse_args(exporter_cli["description"], exporter_cli["options"], args)
+    app = cli_helpers.get_app(namespace.zopeconf)
+    path = cli_helpers._process_path(namespace.path)
+    if not path:
+        logger.error(f"{namespace.path} does not exist, please create it first.")
+        sys.exit(1)
+    site = cli_helpers.get_site(app, namespace.site, logger)
+    with api.env.adopt_roles(["Manager"]):
+        results = get_exporter(site).export_site(path, options=namespace)
+    logger.info(f" Using path {path} to export content from Plone site at /{site.id}")
+    for item in results[1:]:
+        logger.info(f" Wrote {item}")
+
+
+def importer_cli(args=sys.argv):
+    """Import content to a Plone site."""
+    logger = cli_helpers.get_logger("Importer")
+    importer_cli = CLI_SPEC["importer"]
+    namespace = _parse_args(importer_cli["description"], importer_cli["options"], args)
+    app = cli_helpers.get_app(namespace.zopeconf)
+    path = cli_helpers._process_path(namespace.path)
+    if not path:
+        logger.error(f"{namespace.path} does not exist, aborting import.")
+        sys.exit(1)
+    site = cli_helpers.get_site(app, namespace.site, logger)
+    with api.env.adopt_roles(["Manager"]):
+        results = get_importer(site).import_site(path)
+    logger.info(f" Using path {path} to import content to Plone site at /{site.id}")
+    for item in results:
+        logger.info(f" - {item}")
+    transaction.commit()

plone/exportimport/configure.zcml

@@ -0,0 +1,21 @@
+<configure
+    xmlns="http://namespaces.zope.org/zope"
+    xmlns:i18n="http://namespaces.zope.org/i18n"
+    xmlns:zcml="http://namespaces.zope.org/zcml"
+    i18n_domain="plone"
+    >
+
+  <!-- Dependencies -->
+  <include package="Products.CMFCore" />
+  <include package="Products.CMFPlone" />
+  <include package="plone.namedfile" />
+  <include package="plone.app.dexterity" />
+  <include package="plone.restapi" />
+
+  <include package=".serializers" />
+  <include package=".deserializers" />
+  <include package=".exporters" />
+  <include package=".importers" />
+
+
+</configure>

plone/exportimport/deserializers/blob.py

@@ -0,0 +1,44 @@
+from pathlib import Path
+from plone.dexterity.interfaces import IDexterityContent
+from plone.exportimport import settings
+from plone.exportimport.interfaces import IExportImportRequestMarker
+from plone.exportimport.utils import path as path_utils
+from plone.namedfile.interfaces import INamedField
+from plone.restapi.deserializer.dxfields import DefaultFieldDeserializer
+from plone.restapi.interfaces import IFieldDeserializer
+from zope.component import adapter
+from zope.globalrequest import getRequest
+from zope.interface import implementer
+
+import codecs
+
+
+def load_blob(path: str) -> bytes:
+    """Load blob from fs and encode it as base64."""
+    request = getRequest()
+    content_import_path = Path(request[settings.IMPORT_PATH_KEY])
+    path = content_import_path / path
+    if not path.exists():
+        raise ValueError(f"Blob not found at {path}")
+    data = path_utils.encode_file_contents(path)
+    return codecs.decode(data, "base64")
+
+
+@adapter(INamedField, IDexterityContent, IExportImportRequestMarker)
+@implementer(IFieldDeserializer)
+class ExportImportNamedFieldDeserializer(DefaultFieldDeserializer):
+    def __call__(self, value):
+        result = None
+        blob_path = value.pop("blob_path", "")
+        content_type = value.get("content-type", "application/octet-stream")
+        filename = value.get("filename", None)
+        data = load_blob(blob_path)
+        # Convert if we have data
+        if data:
+            result = self.field._type(
+                data=data, contentType=content_type, filename=filename
+            )
+
+        # Always validate to check for required fields
+        self.field.validate(result)
+        return result

plone/exportimport/deserializers/blocks.py

@@ -0,0 +1,15 @@
+from plone.exportimport.interfaces import IExportImportRequestMarker
+from plone.restapi.behaviors import IBlocks
+from plone.restapi.deserializer.dxfields import DefaultFieldDeserializer
+from plone.restapi.interfaces import IFieldDeserializer
+from plone.schema import IJSONField
+from zope.component import adapter
+from zope.interface import implementer
+
+
+@implementer(IFieldDeserializer)
+@adapter(IJSONField, IBlocks, IExportImportRequestMarker)
+class ExportImportBlocksDeserializer(DefaultFieldDeserializer):
+    """We skip the subscribers that deserialize the blocks from the frontend.
+    We only need the raw data.
+    """

plone/exportimport/deserializers/configure.zcml

@@ -0,0 +1,4 @@
+<configure xmlns="http://namespaces.zope.org/zope">
+  <adapter factory=".blob.ExportImportNamedFieldDeserializer" />
+  <adapter factory=".blocks.ExportImportBlocksDeserializer" />
+</configure>

plone/exportimport/exporters/__init__.py

@@ -0,0 +1,85 @@
+from .base import BaseExporter
+from pathlib import Path
+from plone import api
+from plone.exportimport import interfaces
+from plone.exportimport import logger
+from plone.exportimport import PACKAGE_NAME
+from Products.CMFPlone.Portal import PloneSite
+from tempfile import mkdtemp
+from typing import Dict
+from typing import List
+from typing import Optional
+from zope.component import getAdapter
+from zope.component import hooks
+from zope.component import queryAdapter
+from zope.interface import implementer
+
+import argparse
+
+
+EXPORTER_NAMES = [
+    "plone.exporter.content",
+    "plone.exporter.principals",
+    "plone.exporter.redirects",
+    "plone.exporter.relations",
+    "plone.exporter.translations",
+    "plone.exporter.discussions",
+    "plone.exporter.portlets",
+]
+
+
+ExporterMapping = Dict[str, BaseExporter]
+
+
+@implementer(interfaces.IExporter)
+class Exporter:
+    """Export content from a Plone Site."""
+
+    exporters: ExporterMapping
+
+    def __init__(self, site):
+        self.site = site
+        self.exporters = self.all_exporters()
+
+    def all_exporters(self) -> ExporterMapping:
+        """Return all exporters."""
+        exporters = {}
+        for exporter_name in EXPORTER_NAMES:
+            exporter = queryAdapter(
+                self.site, interfaces.INamedExporter, name=exporter_name
+            )
+            if exporter:
+                exporters[exporter_name] = exporter
+        return exporters
+
+    @staticmethod
+    def _prepare_path(path: Optional[Path] = None) -> Path:
+        """Return a valid path to use for the export.
+
+        If base_path is not given, create a temporary directory to export
+        the content.
+        """
+        valid_path = path.is_dir() if path else False
+        if not valid_path:
+            path = Path(mkdtemp(prefix=PACKAGE_NAME))
+        return path
+
+    def export_site(
+        self, path: Optional[Path] = None, options: Optional[argparse.Namespace] = None
+    ) -> List[Path]:
+        """Export the given site to the filesystem."""
+        path = self._prepare_path(path)
+        paths: List[Path] = [path]
+        with hooks.site(self.site):
+            for exporter_name, exporter in self.exporters.items():
+                logger.debug(f"Exporting {self.site} with {exporter_name} to {path}")
+                new_paths = exporter.export_data(path, options=options)
+                paths.extend(new_paths)
+        return paths
+
+
+def get_exporter(site: PloneSite = None) -> Exporter:
+    """Get the exporter."""
+    if site is None:
+        site = api.portal.get()
+    return getAdapter(site, interfaces.IExporter)

plone/exportimport/exporters/base.py

@@ -0,0 +1,80 @@
+from pathlib import Path
+from plone.exportimport import types
+from plone.exportimport.utils import content as utils
+from plone.exportimport.utils import path as path_utils
+from Products.CMFPlone.Portal import PloneSite
+from typing import Any
+from typing import Callable
+from typing import List
+from typing import Optional
+from zope.globalrequest import getRequest
+
+import argparse
+import json
+
+
+class BaseExporter:
+    name: str
+    base_path: Path
+    errors: list = None
+    request: types.HTTPRequest = None
+    data_hooks: List[Callable] = None
+    obj_hooks: List[Callable] = None
+    options: Optional[argparse.Namespace] = None
+
+    def __init__(
+        self,
+        site: PloneSite = None,
+    ):
+        self.site = site
+        self.errors = []
+        self.request = getRequest()
+
+    def get_option(self, name, default=None):
+        return getattr(self.options, name, default)
+
+    @property
+    def filepath(self) -> Path:
+        """Filepath to be used during export."""
+        filename = f"{self.name}.json"
+        return self.base_path / filename
+
+    def _serializer(self, obj: Any) -> Callable:
+        """Serializer for object."""
+        return utils.get_serializer(obj, self.request)
+
+    def _dump(self, data: dict, filepath: Path = None) -> Path:
+        """Dump serialized data to disk."""
+        filepath = filepath if filepath else self.filepath
+        # Create container, if it does not exist
+        path_utils.get_parent_folder(filepath)
+        with open(filepath, "w") as fh:
+            json.dump(data, fh, indent=2, sort_keys=True)
+            # json.dump does not add a newline at the end of the file, so we
+            # explicitly do it.  Otherwise when you manually edit a file and
+            # you use an editor that respects the standard `.editorconfig`
+            # that we have in most Plone packages, you always get a diff because
+            # your editor has automatically added a newline at the end.
+            fh.write("\n")
+        return filepath
+
+    def dump(self) -> List[Path]:
+        """Serialize objects."""
+        return []
+
+    def export_data(
+        self,
+        base_path: Path,
+        data_hooks: List[Callable] = None,
+        obj_hooks: List[Callable] = None,
+        options: Optional[argparse.Namespace] = None,
+    ) -> List[Path]:
+        """Write data to filesystem."""
+        if not base_path.exists():
+            base_path.mkdir(parents=True)
+        self.base_path = base_path
+        self.data_hooks = self.data_hooks or data_hooks or []
+        self.obj_hooks = self.obj_hooks or obj_hooks or []
+        self.options = options
+        paths = self.dump()
+        return paths

plone/exportimport/exporters/configure.zcml

@@ -0,0 +1,61 @@
+<configure
+    xmlns="http://namespaces.zope.org/zope"
+    xmlns:zcml="http://namespaces.zope.org/zcml"
+    >
+  <!-- Main exporter -->
+  <adapter
+      factory=".Exporter"
+      for="plone.base.interfaces.siteroot.IPloneSiteRoot"
+      />
+
+  <!-- Exporters -->
+  <adapter
+      factory=".content.ContentExporter"
+      provides="plone.exportimport.interfaces.INamedExporter"
+      for="plone.base.interfaces.siteroot.IPloneSiteRoot"
+      name="plone.exporter.content"
+      />
+  <adapter
+      factory=".principals.PrincipalsExporter"
+      provides="plone.exportimport.interfaces.INamedExporter"
+      for="plone.base.interfaces.siteroot.IPloneSiteRoot"
+      name="plone.exporter.principals"
+      />
+  <adapter
+      factory=".redirects.RedirectsExporter"
+      provides="plone.exportimport.interfaces.INamedExporter"
+      for="plone.base.interfaces.siteroot.IPloneSiteRoot"
+      name="plone.exporter.redirects"
+      />
+  <adapter
+      factory=".relations.RelationsExporter"
+      provides="plone.exportimport.interfaces.INamedExporter"
+      for="plone.base.interfaces.siteroot.IPloneSiteRoot"
+      name="plone.exporter.relations"
+      />
+  <configure zcml:condition="installed plone.app.multilingual">
+    <adapter
+        factory=".translations.TranslationsExporter"
+        provides="plone.exportimport.interfaces.INamedExporter"
+        for="plone.base.interfaces.siteroot.IPloneSiteRoot"
+        name="plone.exporter.translations"
+        />
+  </configure>
+  <configure zcml:condition="installed plone.app.discussion">
+    <adapter
+        factory=".discussions.DiscussionsExporter"
+        provides="plone.exportimport.interfaces.INamedExporter"
+        for="plone.base.interfaces.siteroot.IPloneSiteRoot"
+        name="plone.exporter.discussions"
+        />
+  </configure>
+  <configure zcml:condition="installed plone.app.portlets">
+    <adapter
+        factory=".portlets.PortletsExporter"
+        provides="plone.exportimport.interfaces.INamedExporter"
+        for="plone.base.interfaces.siteroot.IPloneSiteRoot"
+        name="plone.exporter.portlets"
+        />
+  </configure>
+
+</configure>

plone/exportimport/exporters/content.py

@@ -0,0 +1,165 @@
+from .base import BaseExporter
+from pathlib import Path
+from plone import api
+from plone.base.interfaces import IPloneSiteRoot
+from plone.dexterity.content import DexterityContent
+from plone.exportimport import interfaces
+from plone.exportimport import logger
+from plone.exportimport import settings
+from plone.exportimport import types
+from plone.exportimport.interfaces import IExportImportRequestMarker
+from plone.exportimport.utils import content as content_utils
+from plone.exportimport.utils import request_provides
+from typing import Callable
+from typing import Generator
+from typing import List
+from typing import Optional
+from zope.interface import implementer
+
+import argparse
+
+
+@implementer(interfaces.INamedExporter)
+class ContentExporter(BaseExporter):
+    name: str = "content"
+    query: dict = None
+    filename_fmt: str = settings.EXPORT_CONTENT_FILEPATH
+    metadata: types.ExportImportMetadata = None
+    default_site_language: str = "en"
+
+    def all_objects(self) -> Generator:
+        """Return all objects to be serialized."""
+        query = self.query
+        catalog = api.portal.get_tool("portal_catalog")
+        if "object_provides" not in query:
+            query["object_provides"] = "plone.dexterity.interfaces.IDexterityContent"
+        brains = catalog.unrestrictedSearchResults(**query)
+        logger.info(f"Exporting {len(brains)}")
+        for index, brain in enumerate(brains, start=1):
+            try:
+                obj = brain.getObject()
+            except Exception:
+                brain_path = brain.getPath()
+                msg = f"Error getting object {brain_path} from brain"
+                self.errors.append({"path": brain_path, "message": msg})
+                logger.exception(msg, exc_info=True)
+            else:
+                yield obj
+
+            if not index % 100:
+                logger.info(f"Handled {index} items...")
+
+    def serialize(self, obj: DexterityContent) -> dict:
+        """Serialize object."""
+        obj_uid = content_utils.get_uid(obj)
+        obj_path = content_utils.get_obj_path(obj)
+        config = types.ExporterConfig(
+            site=self.site,
+            site_root_uid=self.site.UID(),
+            request=self.request,
+            serializer=self._serializer(obj),
+            logger_prefix=f"- {obj_path}:",
+        )
+        kwargs = {}
+        is_site_root = IPloneSiteRoot.providedBy(obj)
+        is_folderish = content_utils.is_folderish(obj)
+        if is_folderish:
+            if not is_site_root:
+                kwargs["include_items"] = False
+        # Process metadata
+        for helper in content_utils.metadata_helpers():
+            logger.debug(f"{config.logger_prefix} Updating metadata {helper.name}")
+            value = helper.func(obj, config)
+            if value is not None:
+                getattr(self.metadata, helper.name)[obj_uid] = value
+
+        # Apply object hooks
+        for func in self.obj_hooks:
+            logger.debug(
+                f"{config.logger_prefix} Running object hook {func.__name__} on object"
+            )
+            obj = func(obj, config)
+
+        # Serialize
+        data = config.serializer(**kwargs)
+
+        # Fix serialized data
+        for fixer in content_utils.fixers():
+            logger.debug(
+                f"{config.logger_prefix} Running {fixer.name} on serialized data"
+            )
+            data = fixer.func(data, obj, config)
+
+        # Enrich
+        for enricher in content_utils.enrichers(
+            include_revisions=self.get_option("include_revisions")
+        ):
+            logger.debug(f"{config.logger_prefix} Running {enricher.name}")
+            additional = enricher.func(obj, config)
+            if additional:
+                data.update(additional)
+
+        # Apply data hooks
+        for func in self.data_hooks:
+            logger.debug(
+                f"{config.logger_prefix} Running data hook {func.__name__} on payload"
+            )
+            data = func(data, obj, config)
+
+        # Cleanup data
+        for cleaner in content_utils.cleaners():
+            logger.debug(
+                f"{config.logger_prefix} Running {cleaner.name} on serialized data"
+            )
+            data = cleaner.func(data, config)
+        return data
+
+    def dump_one(self, obj: DexterityContent) -> Path:
+        """Serialize object and dump it to disk."""
+        obj_path = content_utils.get_obj_path(obj)
+        # Serialize object
+        data = self.serialize(obj)
+        base_path = self.base_path
+        filepath = base_path / self.filename_fmt.format(**data)
+        filepath = self._dump(data, filepath)
+        logger.debug(f"- {obj_path}: Wrote serialized data to {filepath}")
+        # Add to list of files
+        self.metadata._all_[obj_path] = str(filepath.relative_to(base_path))
+        return filepath
+
+    def dump_metadata(self) -> Path:
+        metadata = self.metadata.export()
+        filepath = self.base_path / "__metadata__.json"
+        return self._dump(metadata, filepath)
+
+    def dump(self) -> List[Path]:
+        """Serialize contents and dump them to disk."""
+        paths = []
+        with request_provides(self.request, IExportImportRequestMarker):
+            for obj in self.all_objects():
+                path = self.dump_one(obj)
+                if path:
+                    paths.append(path)
+            # Add list of blobs to serialization
+            paths.insert(0, self.dump_metadata())
+        return paths
+
+    def export_data(
+        self,
+        base_path: Path,
+        data_hooks: List[Callable] = None,
+        obj_hooks: List[Callable] = None,
+        query: Optional[dict] = None,
+        options: Optional[argparse.Namespace] = None,
+    ) -> List[Path]:
+        # Content in a subpath of base_path
+        base_path = base_path / self.name
+        query = query if query else {}
+        site = self.site
+        self.query = query if query else {"path": content_utils.get_obj_path(site)}
+        metadata = types.ExportImportMetadata()
+        self.metadata = metadata
+        self.request[settings.EXPORT_CONTENT_METADATA_KEY] = metadata
+        self.request[settings.EXPORT_PATH_KEY] = base_path
+        self.default_site_language = site.language
+        return super().export_data(base_path, data_hooks, obj_hooks, options=options)

plone/exportimport/exporters/discussions.py

@@ -0,0 +1,34 @@
+from .base import BaseExporter
+from pathlib import Path
+from plone.exportimport import interfaces
+from plone.exportimport import logger
+from typing import List
+from zope.interface import implementer
+
+
+try:
+    import plone.app.discussion  # noqa
+except ImportError:
+    HAS_DISCUSSION = False
+else:
+    HAS_DISCUSSION = True
+
+
+@implementer(interfaces.INamedExporter)
+class DiscussionsExporter(BaseExporter):
+    name: str = "discussions"
+
+    def dump(self) -> List[Path]:
+        """Serialize object and dump it to disk."""
+        if not HAS_DISCUSSION:
+            logger.debug("- Discussions: Skipping (plone.app.discussion not installed)")
+            return []
+
+        from plone.exportimport.utils import discussions as utils
+
+        discussions = utils.get_discussions()
+        filepath = self._dump(discussions, self.filepath)
+        logger.debug(
+            f"- Discussions: Wrote {len(discussions)} discussions to {filepath}"
+        )
+        return [filepath]

plone/exportimport/exporters/portlets.py

@@ -0,0 +1,19 @@
+from .base import BaseExporter
+from pathlib import Path
+from plone.exportimport import interfaces
+from plone.exportimport import logger
+from plone.exportimport.utils import portlets as utils
+from typing import List
+from zope.interface import implementer
+
+
+@implementer(interfaces.INamedExporter)
+class PortletsExporter(BaseExporter):
+    name: str = "portlets"
+
+    def dump(self) -> List[Path]:
+        """Serialize object and dump it to disk."""
+        portlets = utils.get_portlets()
+        filepath = self._dump(portlets, self.filepath)
+        logger.debug(f"- Portlets: Wrote {len(portlets)} portlets to {filepath}")
+        return [filepath]

plone/exportimport/exporters/principals.py

@@ -0,0 +1,24 @@
+from .base import BaseExporter
+from pathlib import Path
+from plone.exportimport import interfaces
+from plone.exportimport import logger
+from plone.exportimport.utils import principals as principals_utils
+from typing import List
+from zope.interface import implementer
+
+
+@implementer(interfaces.INamedExporter)
+class PrincipalsExporter(BaseExporter):
+    name: str = "principals"
+
+    def dump(self) -> List[Path]:
+        """Serialize object and dump it to disk."""
+        data = {
+            "groups": principals_utils.export_groups(),
+            "members": principals_utils.export_members(),
+        }
+        filepath = self._dump(data, self.filepath)
+        logger.debug(
+            f"- Principals: Wrote {len(data['groups'])} groups and {len(data['members'])} members to {filepath}"
+        )
+        return [filepath]

plone/exportimport/exporters/redirects.py

@@ -0,0 +1,19 @@
+from .base import BaseExporter
+from pathlib import Path
+from plone.exportimport import interfaces
+from plone.exportimport import logger
+from plone.exportimport.utils import redirects as utils
+from typing import List
+from zope.interface import implementer
+
+
+@implementer(interfaces.INamedExporter)
+class RedirectsExporter(BaseExporter):
+    name: str = "redirects"
+
+    def dump(self) -> List[Path]:
+        """Serialize object and dump it to disk."""
+        redirects = utils.get_redirects()
+        filepath = self._dump(redirects, self.filepath)
+        logger.debug(f"- Redirects: Wrote {len(redirects)} redirects to {filepath}")
+        return [filepath]