bookstack-file-exporter 0.0.1__py3-none-any.whl

bookstack_file_exporter/__main__.py

@@ -0,0 +1,16 @@
+import argparse
+import logging
+
+from bookstack_file_exporter import run
+from bookstack_file_exporter import run_args
+
+def main():
+    """run entrypoint"""
+    args: argparse.Namespace = run_args.get_args()
+    logging.basicConfig(format='%(asctime)s [%(levelname)s] %(message)s',
+                        level=run_args.get_log_level(args.log_level), datefmt='%Y-%m-%d %H:%M:%S')
+    run.exporter(args)
+
+
+if __name__ == '__main__':
+    main()

bookstack_file_exporter/archiver/archiver.py

@@ -0,0 +1,125 @@
+from typing import List, Dict, Union
+from datetime import datetime
+import logging
+
+from bookstack_file_exporter.exporter.node import Node
+from bookstack_file_exporter.archiver import util
+from bookstack_file_exporter.archiver.minio_archiver import MinioArchiver
+from bookstack_file_exporter.config_helper.remote import StorageProviderConfig
+
+log = logging.getLogger(__name__)
+
+_META_FILE_SUFFIX = "_meta.json"
+_TAR_SUFFIX = ".tar"
+_TAR_GZ_SUFFIX = ".tgz"
+
+_EXPORT_API_PATH = "export"
+
+_FILE_EXTENSION_MAP = {
+    "markdown": ".md",
+    "html": ".html",
+    "pdf": ".pdf",
+    "plaintext": ".txt",
+    "meta": _META_FILE_SUFFIX,
+    "tar": _TAR_SUFFIX,
+    "tgz": _TAR_GZ_SUFFIX
+}
+
+_DATE_STR_FORMAT = "%Y-%m-%d_%H-%M-%S"
+
+class Archiver:
+    """
+    Archiver pulls all the necessary files from upstream 
+    and then pushes them to the specified backup location(s)
+
+    Args:
+        :root_dir: str (required) = the base directory for 
+        which the archive .tgz will be placed.
+        :add_meta: bool (required) = whether or not to add 
+        metadata json files for each page, book, chapter, and/or shelve.
+        :base_page_url: str (required) = the full url and path to get page content.
+        :headers: Dict[str, str] (required) = the headers which include the Authorization to use
+
+    Returns:
+        Archiver instance with attributes that are 
+        accessible for use for file level archival and backup.
+    """
+    def __init__(self, base_dir: str, add_meta: Union[bool, None],
+                  base_page_url: str, headers: Dict[str, str]):
+        self.base_dir = base_dir
+        self.add_meta = add_meta
+        self.base_page_url = base_page_url
+        self._headers = headers
+        self._root_dir = self.generate_root_folder(self.base_dir)
+        # the tgz file will be name of
+        # parent export directory, bookstack-<timestamp>, and .tgz extension
+        self._archive_file = f"{self._root_dir}{_FILE_EXTENSION_MAP['tgz']}"
+        # name of intermediate tar file before gzip
+        self._tar_file = f"{self._root_dir}{_FILE_EXTENSION_MAP['tar']}"
+        # name of the base folder to use within the tgz archive
+        self._archive_base_path = self._root_dir.split("/")[-1]
+        # remote_system to function mapping
+        self._remote_exports = {'minio': self._archive_minio, 's3': self._archive_s3}
+
+    # create local tarball first
+    def archive(self, page_nodes: Dict[int, Node], export_formats: List[str]):
+        """create a .tgz of all page content"""
+        for _, page in page_nodes.items():
+            for ex_format in export_formats:
+                self._gather(page, ex_format)
+        self._gzip_tar()
+
+    # convert to bytes to be agnostic to end destination (future use case?)
+    def _gather(self, page_node: Node, export_format: str):
+        raw_data = self._get_data_format(page_node.id_, export_format)
+        self._gather_local(page_node.file_path, raw_data, export_format, page_node.meta)
+
+    def _gather_local(self, page_path: str, data: bytes,
+                      export_format: str, meta_data: Union[bytes, None]):
+        page_file_name = f"{self._archive_base_path}/" \
+        f"{page_path}{_FILE_EXTENSION_MAP[export_format]}"
+        util.write_bytes(self._tar_file, file_path=page_file_name, data=data)
+        if self.add_meta:
+            meta_file_name = f"{self._archive_base_path}/{page_path}{_FILE_EXTENSION_MAP['meta']}"
+            bytes_meta = util.get_json_bytes(meta_data)
+            util.write_bytes(self._tar_file, file_path=meta_file_name, data=bytes_meta)
+
+    # send to remote systems
+    def archive_remote(self, remote_targets: Dict[str, StorageProviderConfig]):
+        """for each target, do their respective tasks"""
+        if remote_targets:
+            for key, value in remote_targets.items():
+                self._remote_exports[key](value)
+
+    def _gzip_tar(self):
+        util.create_gzip(self._tar_file, self._archive_file)
+
+    def _archive_minio(self, config: StorageProviderConfig):
+        minio_archiver = MinioArchiver(config)
+        minio_archiver.upload_backup(self._archive_file)
+
+    def _archive_s3(self, config: StorageProviderConfig):
+        pass
+
+    def clean_up(self, clean_up_archive: Union[bool, None]):
+        """remove archive after sending to remote target"""
+        self._clean(clean_up_archive)
+
+    def _clean(self, clean_up_archive: Union[bool, None]):
+        # if user is uploading to object storage
+        # delete the local .tgz archive since we have it there already
+        if clean_up_archive:
+            util.remove_file(self._archive_file)
+
+    # convert page data to bytes
+    def _get_data_format(self, page_node_id: int, export_format: str) -> bytes:
+        url = self._get_export_url(node_id=page_node_id, export_format=export_format)
+        return util.get_byte_response(url=url, headers=self._headers)
+
+    def _get_export_url(self, node_id: int, export_format: str) -> str:
+        return f"{self.base_page_url}/{node_id}/{_EXPORT_API_PATH}/{export_format}"
+
+    @staticmethod
+    def generate_root_folder(base_folder_name: str) -> str:
+        """return base archive name"""
+        return base_folder_name + "_" + datetime.now().strftime(_DATE_STR_FORMAT)

bookstack_file_exporter/archiver/minio_archiver.py

@@ -0,0 +1,56 @@
+from typing import Union
+import logging
+
+from minio import Minio
+
+from bookstack_file_exporter.config_helper.remote import StorageProviderConfig
+
+log = logging.getLogger(__name__)
+
+class MinioArchiver:
+    """
+    Class to handle minio object upload and validations.
+    
+    Args:
+        config <StorageProviderConfig> = minio configuration
+        bucket <str> = upload bucket
+        path <str> (optional) = specify bucket path for upload
+
+    Returns:
+        MinioArchiver instance for archival use
+    """
+    def __init__(self, config: StorageProviderConfig):
+        self._client = Minio(
+            config.host,
+            access_key=config.access_key,
+            secret_key=config.secret_key,
+            region=config.region
+        )
+        self.bucket = config.bucket
+        self.path = self._generate_path(config.path)
+        self._validate_bucket()
+
+    def _validate_bucket(self):
+        if not self._client.bucket_exists(self.bucket):
+            raise ValueError(f"Given bucket does not exist: {self.bucket}")
+
+    def _generate_path(self, path_name: Union[str, None]) -> str:
+        if path_name:
+            if path_name[-1] == '/':
+                return path_name[:-1]
+            return path_name
+        return ""
+
+    def upload_backup(self, local_file_path: str):
+        """upload archive file to minio bucket"""
+        # this will be the name of the object to upload
+        # only get the file name not path
+        # we are going to use path provided by user for object storage
+        file_name = local_file_path.split("/")[-1]
+        if self.path:
+            object_path = f"{self.path}/{file_name}"
+        else:
+            object_path = file_name
+        result = self._client.fput_object(self.bucket, object_path, local_file_path)
+        log.info("""Created object: %s with tag: %s and version-id: %s""",
+                 result.object_name, result.etag, result.version_id)

bookstack_file_exporter/archiver/util.py

@@ -0,0 +1,43 @@
+from typing import Dict, Union
+import json
+import os
+import logging
+import tarfile
+import shutil
+from io import BytesIO
+import gzip
+
+from bookstack_file_exporter.common import util
+
+log = logging.getLogger(__name__)
+
+def get_byte_response(url: str, headers: Dict[str, str]) -> bytes:
+    """get byte response from http request"""
+    response = util.http_get_request(url=url, headers=headers)
+    return response.content
+
+def write_bytes(base_tar_dir: str, file_path: str, data: bytes):
+    """append byte data to tar file"""
+    with tarfile.open(base_tar_dir, "a") as tar:
+        data_obj = BytesIO(data)
+        tar_info = tarfile.TarInfo(name=file_path)
+        tar_info.size = data_obj.getbuffer().nbytes
+        log.debug("Adding file: %s with size: %d bytes to tar file", tar_info.name, tar_info.size)
+        tar.addfile(tar_info, fileobj=data_obj)
+
+def get_json_bytes(data: Dict[str, Union[str, int]]) -> bytes:
+    """dump dict to json file"""
+    return json.dumps(data, indent=4).encode('utf-8')
+
+# set as function in case we want to do checks or final actions later
+def remove_file(file_path: str):
+    """remove a file"""
+    os.remove(file_path)
+
+def create_gzip(tar_file: str, gzip_file: str, remove_old: bool = True):
+    """create a gzip of an existing tar file and remove it"""
+    with open(tar_file, 'rb') as f_in:
+        with gzip.open(gzip_file, 'wb') as f_out:
+            shutil.copyfileobj(f_in, f_out)
+    if remove_old:
+        remove_file(tar_file)

bookstack_file_exporter/common/util.py

@@ -0,0 +1,32 @@
+import logging
+from typing import Tuple, Dict
+import requests
+from requests.adapters import HTTPAdapter, Retry
+
+log = logging.getLogger(__name__)
+
+def http_get_request(url: str, headers: Dict[str, str], timeout: int = 30) -> requests.Response:
+    """make http requests and return response object"""
+    verify, url_prefix = should_verify(url)
+    try:
+        with requests.Session() as session:
+            # {backoff factor} * (2 ** ({number of previous retries}))
+            # {raise_on_status} if status falls in status_forcelist range
+            #  and retries have been exhausted.
+            # {status_force_list} 429 is supposed to be included
+            retries = Retry(total=3,
+                            backoff_factor=0.5,
+                            raise_on_status=True,
+                            status_forcelist=[ 500, 502, 503, 504 ])
+            session.mount(url_prefix, HTTPAdapter(max_retries=retries))
+            response = session.get(url, headers=headers, verify=verify, timeout=timeout)
+    except Exception as req_err:
+        log.error("Failed to make request for %s", url)
+        raise req_err
+    return response
+
+def should_verify(url: str) -> Tuple[bool, str]:
+    """check if http or https"""
+    if url.startswith("https://"):
+        return (True, "https://")
+    return (False, "http://")

bookstack_file_exporter/config_helper/config_helper.py

@@ -0,0 +1,200 @@
+import os
+import argparse
+from typing import Dict, Tuple
+import logging
+
+import yaml
+
+from bookstack_file_exporter.config_helper import models
+from bookstack_file_exporter.config_helper.remote import StorageProviderConfig
+
+log = logging.getLogger(__name__)
+
+
+_DEFAULT_HEADERS = {
+    'Content-Type': 'application/json; charset=utf-8'
+}
+
+_API_PATHS = {
+    "shelves": "api/shelves",
+    "books": "api/books",
+    "chapters": "api/chapters",
+    "pages": "api/pages"
+}
+
+_UNASSIGNED_BOOKS_DIR = "unassigned/"
+
+_BASE_DIR_NAME = "bookstack_export"
+
+_BOOKSTACK_TOKEN_FIELD ='BOOKSTACK_TOKEN_ID'
+_BOOKSTACK_TOKEN_SECRET_FIELD='BOOKSTACK_TOKEN_SECRET'
+_MINIO_ACCESS_KEY_FIELD='MINIO_ACCESS_KEY'
+_MINIO_SECRET_KEY_FIELD='MINIO_SECRET_KEY'
+
+## Normalize config from cli or from config file
+class ConfigNode:
+    """
+    Get Run Configuration from YAML file and normalize the data in an accessible object
+
+    Args:
+        Arg parse from user input
+
+    Returns:
+        ConfigNode object with attributes that are 
+        accessible for use for further downstream processes
+
+    Raises:
+        YAMLError: if provided configuration file is not valid YAML
+
+        ValueError: if improper arguments are given from user
+    """
+    def __init__(self, args: argparse.Namespace):
+        self.unassigned_book_dir = _UNASSIGNED_BOOKS_DIR
+        self.user_inputs = self._generate_config(args.config_file)
+        self._base_dir_name = self._set_base_dir(args.output_dir)
+        self._token_id, self._token_secret = self._generate_credentials()
+        self._headers = self._generate_headers()
+        self._urls = self._generate_urls()
+        self._minio_access_key = ""
+        self._minio_secret_key = ""
+        self._object_storage_config = self._generate_remote_config()
+
+    def _generate_config(self, config_file: str) -> models.UserInput:
+        if not os.path.isfile(config_file):
+            raise FileNotFoundError(config_file)
+        with open(config_file, "r", encoding="utf-8") as yaml_stream:
+            try:
+                yaml_input = yaml.safe_load(yaml_stream)
+            except Exception as load_err:
+                # log here to make it easier to identify the issue
+                log.error("Failed to load yaml configuration file")
+                raise load_err
+        try:
+            user_inputs = models.UserInput(**yaml_input)
+        except Exception as err:
+            # log here to make it easier to identify the issue
+            log.error("Yaml configuration failed schema validation")
+            raise err
+        return user_inputs
+
+    def _generate_credentials(self) -> Tuple[str, str]:
+        # if user provided credentials in config file, load them
+        token_id = ""
+        token_secret = ""
+        if self.user_inputs.credentials:
+            token_id = self.user_inputs.credentials.token_id
+            token_secret = self.user_inputs.credentials.token_secret
+
+        # check to see if env var is specified, if so, it takes precedence
+        token_id = self._check_var(_BOOKSTACK_TOKEN_FIELD, token_id)
+        token_secret = self._check_var(_BOOKSTACK_TOKEN_SECRET_FIELD, token_secret)
+        return token_id, token_secret
+
+    def _generate_remote_config(self) -> Dict[str, StorageProviderConfig]:
+        object_config = {}
+        # check for optional minio credentials if configuration is set in yaml configuration file
+        if self.user_inputs.minio_config:
+            minio_access_key = self._check_var(_MINIO_ACCESS_KEY_FIELD,
+                                               self.user_inputs.minio_config.access_key)
+            minio_secret_key = self._check_var(_MINIO_SECRET_KEY_FIELD,
+                                               self.user_inputs.minio_config.secret_key)
+            object_config["minio"] = StorageProviderConfig(minio_access_key,
+                                     minio_secret_key, self.user_inputs.minio_config.bucket,
+                                     host=self.user_inputs.minio_config.host,
+                                     path=self.user_inputs.minio_config.path,
+                                     region=self.user_inputs.minio_config.region)
+        return object_config
+
+    def _generate_headers(self) -> Dict[str, str]:
+        headers = {}
+        # add additional_headers provided by user
+        if self.user_inputs.additional_headers:
+            for key, value in self.user_inputs.additional_headers.items():
+                headers[key] = value
+
+        # add default headers
+        for key, value in _DEFAULT_HEADERS.items():
+            # do not override if user added one already with same key
+            if key not in headers:
+                headers[key] = value
+
+        # do not override user provided one
+        if 'Authorization' not in headers:
+            headers['Authorization'] = f"Token {self._token_id}:{self._token_secret}"
+        return headers
+
+    def _generate_urls(self) -> Dict[str, str]:
+        urls = {}
+        # remove trailing slash
+        host = self.user_inputs.host
+        if host[-1] == '/':
+            host = host[:-1]
+        # check to see if http protocol is defined
+        if "http" not in self.user_inputs.host:
+            # use https by default
+            url_prefix = "https://"
+        else:
+            url_prefix = ""
+        for key, value in _API_PATHS.items():
+            urls[key] = f"{url_prefix}{self.user_inputs.host}/{value}"
+        return urls
+
+    def _set_base_dir(self, cmd_output_dir: str) -> str:
+        output_dir = self.user_inputs.output_path
+        # override if command line specified
+        if cmd_output_dir:
+            log.debug("Output directory overwritten by command line option")
+            output_dir = cmd_output_dir
+        # check if user provided an output path
+        if output_dir:
+            # detect trailing slash
+            # normalize to no trailing slash for later consistency
+            if output_dir[-1] == '/':
+                base_dir = f"{output_dir}{_BASE_DIR_NAME}"
+            else:
+                base_dir = f"{output_dir}/{_BASE_DIR_NAME}"
+        else:
+            base_dir = _BASE_DIR_NAME
+        return base_dir
+
+    @property
+    def headers(self) -> Dict[str, str]:
+        """get generated headers"""
+        return self._headers
+
+    @property
+    def urls(self) -> Dict[str, str]:
+        """get generated urls"""
+        return self._urls
+
+    @property
+    def base_dir_name(self) -> str:
+        """get base dir of output target"""
+        return self._base_dir_name
+
+    @property
+    def object_storage_config(self) -> Dict[str, StorageProviderConfig]:
+        """return remote storage configuration"""
+        return self._object_storage_config
+
+    @staticmethod
+    def _check_var(env_key: str, default_val: str) -> str:
+        """
+        :param: env_key = the environment variable to check
+        :param: default_val = the default value if any to set if env variable not set
+        
+        :return: env_key if present or default_val if not
+        :throws: ValueError if both parameters are empty.
+        """
+        env_value = os.environ.get(env_key, "")
+        # env value takes precedence
+        if env_value:
+            log.debug("""env key: %s specified.
+                       Will override configuration file value if set.""", env_key)
+            return env_value
+        # check for optional inputs, if env and input is missing
+        if not env_value and not default_val:
+            raise ValueError(f"""{env_key} is not specified in env and is
+                              missing from configuration - at least one should be set""")
+        # fall back to configuration file value if present
+        return default_val

bookstack_file_exporter/config_helper/models.py

@@ -0,0 +1,29 @@
+from typing import Dict, Literal, List, Optional
+from pydantic import BaseModel
+
+# pylint: disable=R0903
+
+class MinioConfig(BaseModel):
+    """YAML schema for minio configuration"""
+    host: str
+    access_key: Optional[str] = None
+    secret_key: Optional[str] = None
+    bucket: str
+    path: Optional[str] = None
+    region: str
+
+class BookstackAccess(BaseModel):
+    """YAML schema for bookstack access credentials"""
+    token_id: str
+    token_secret: str
+
+class UserInput(BaseModel):
+    """YAML schema for user provided configuration file"""
+    host: str
+    additional_headers: Optional[Dict[str, str]] = None
+    credentials: Optional[BookstackAccess] = None
+    formats: List[Literal["markdown", "html", "pdf", "plaintext"]]
+    output_path: Optional[str] = None
+    export_meta: Optional[bool] = None
+    minio_config: Optional[MinioConfig] = None
+    clean_up: Optional[bool] = None

bookstack_file_exporter/config_helper/remote.py

@@ -0,0 +1,29 @@
+from typing import Union
+
+## convenience class
+## able to work for minio, s3, etc.
+class StorageProviderConfig:
+    """
+    Convenience class to get dot notation for remote object storage
+    configuration access.
+    
+    Args:
+        access_key <str> = required token id
+        secret_key <str> = required secret token
+        bucket <str> = bucket to upload
+        host <str> (optional) = if provider requires a host/url
+        path <str> (optional) = specify bucket path for upload
+        region <str> (optional) = if provider requires region
+
+    Returns:
+        StorageProviderConfig instance for dot notation access
+    """
+    def __init__(self, access_key: str, secret_key: str, bucket: str,
+                 host: Union[str, None]=None, path: Union[str, None]=None,
+                 region: Union[str, None]=None):
+        self.host = host
+        self.access_key = access_key
+        self.secret_key = secret_key
+        self.bucket = bucket
+        self.path = path
+        self.region = region

bookstack_file_exporter/exporter/exporter.py

@@ -0,0 +1,144 @@
+from typing import Dict, List
+import logging
+
+from bookstack_file_exporter.exporter import util
+from bookstack_file_exporter.exporter.node import Node
+
+log = logging.getLogger(__name__)
+
+class NodeExporter():
+    """
+    NodeExporter class provides an interface to help create 
+    Bookstack resources/nodes (pages, books, etc) and their relationships.
+
+    Raises:
+
+    ValueError if data returned from bookstack api is empty or not in desired format.
+    """
+    def __init__(self, api_urls: Dict[str, str], headers: Dict[str,str]):
+        self.api_urls = api_urls
+        self.headers = headers
+
+    def get_all_shelves(self) -> Dict[int, Node]:
+        """
+        Function to get all shelf Node instances 
+        :returns: Dict[int, Node] for all shelf nodes
+        """
+        base_url = self.api_urls["shelves"]
+        all_parents: List[int] = util.get_all_ids(base_url, self.headers)
+        if not all_parents:
+            log.warning("No shelves found in given Bookstack instance")
+            return {}
+        return self._get_parents(base_url, all_parents)
+
+    def _get_parents(self, base_url: str, parent_ids: List[int],
+                      path_prefix: str = "") -> Dict[int, Node]:
+        parent_nodes = {}
+        for parent_id in parent_ids:
+            parent_url = f"{base_url}/{parent_id}"
+            parent_data = util.get_json_response(url=parent_url, headers=self.headers)
+            parent_nodes[parent_id] = Node(parent_data, path_prefix=path_prefix)
+        return parent_nodes
+
+    def get_chapter_nodes(self, book_nodes: Dict[int, Node]) -> Dict[int, Node]:
+        """ get chapter nodes """
+        # Chapters are treated a little differently
+        # They are under books like pages but have their own children
+        # i.e. not a terminal node
+        base_url = self.api_urls["chapters"]
+        all_chapters: List[int] = util.get_all_ids(base_url, self.headers)
+        if not all_chapters:
+            log.debug("No chapters found in given Bookstack instance")
+            return {}
+        return self._get_chapters(base_url, all_chapters, book_nodes)
+
+    def _get_chapters(self, base_url: str, all_chapters: List[int],
+                       book_nodes: Dict[int, Node]) -> Dict[int, Node]:
+        chapter_nodes = {}
+        for chapter_id in all_chapters:
+            chapter_url = f"{base_url}/{chapter_id}"
+            chapter_data = util.get_json_response(url=chapter_url, headers=self.headers)
+            book_id = chapter_data['book_id']
+            chapter_nodes[chapter_id] = Node(chapter_data, book_nodes[book_id])
+        return chapter_nodes
+
+    def get_child_nodes(self, resource_type: str, parent_nodes: Dict[int, Node],
+                        filter_empty: bool = True) -> Dict[int, Node]:
+        """get child nodes from a book/chapter/shelf"""
+        base_url = self.api_urls[resource_type]
+        return self._get_children(base_url, parent_nodes, filter_empty)
+
+    def _get_children(self, base_url: str, parent_nodes: Dict[int, Node],
+                       filter_empty: bool) -> Dict[int, Node]:
+        child_nodes = {}
+        for _, parent in parent_nodes.items():
+            if parent.children:
+                for child in parent.children:
+                    child_id = child['id']
+                    child_url = f"{base_url}/{child_id}"
+                    child_data = util.get_json_response(url=child_url, headers=self.headers)
+                    child_node = Node(child_data, parent)
+                    if filter_empty:
+                        if not child_node.empty:
+                            child_nodes[child_id] = child_node
+                    else:
+                        child_nodes[child_id] = child_node
+        return child_nodes
+
+    def get_unassigned_books(self, existing_resources: Dict[int, Node],
+                              path_prefix: str) -> Dict[int, Node]:
+        """get books not under a shelf"""
+        base_url = self.api_urls["books"]
+        all_resources: List[int] = util.get_all_ids(url=base_url, headers=self.headers)
+        unassigned = []
+        # get all existing ones and compare against current known resources
+        for resource_id in all_resources:
+            if resource_id not in existing_resources:
+                unassigned.append(resource_id)
+        if not unassigned:
+            return {}
+        # books with no shelf treated like a parent resource
+        return self._get_parents(base_url, unassigned, path_prefix)
+
+    # convenience function
+    def get_all_books(self, shelve_nodes: Dict[int, Node], unassigned_dir: str) -> Dict[int, Node]:
+        """get all books"""
+        book_nodes = {}
+        # get books in shelves
+        if shelve_nodes:
+            book_nodes = self.get_child_nodes("books", shelve_nodes)
+        # books with no shelve assignment
+        # default will be put in "unassigned" directory relative to backup dir
+        books_no_shelf = self.get_unassigned_books(book_nodes, unassigned_dir)
+
+        # add new book nodes to map
+        # these should not already be present in map
+        # since we started with shelves first and then moved our way down.
+        if books_no_shelf:
+            for key, value in books_no_shelf.items():
+                book_nodes[key] = value
+
+        return book_nodes
+
+    # convenience function
+    def get_all_pages(self, book_nodes: Dict[int, Node]) -> Dict[int, Node]:
+        """get all pages and their content"""
+        ## pages
+        page_nodes = {}
+        if book_nodes:
+            page_nodes: Dict[int, Node] = self.get_child_nodes("pages", book_nodes)
+        ## chapters (if exists)
+        # chapter nodes are treated a little differently
+        # chapters are children under books
+        chapter_nodes: Dict[int, Node] = self.get_chapter_nodes(book_nodes)
+        # add chapter node pages
+        # replace existing page node if found with proper chapter parent
+        if chapter_nodes:
+            page_chapter_nodes: Dict[int, Node] = self.get_child_nodes("pages", chapter_nodes)
+            ## since we filter empty, check if there is any content
+            ## add all chapter pages to existing page nodes
+            if page_chapter_nodes:
+                for key, value in page_chapter_nodes.items():
+                    page_nodes[key] = value
+        return page_nodes
+        

bookstack_file_exporter/exporter/node.py

@@ -0,0 +1,79 @@
+from typing import Dict, Union, List
+
+# shelves --> 'books'
+# books --> 'content'
+# chapters --> 'pages'
+_CHILD_KEYS = ['books', 'contents', 'pages']
+
+_NULL_PAGE_NAME = "New Page"
+
+class Node():
+    """
+    Node class provides an interface to create bookstack child/parent 
+    relationships for resources like pages, books, chapters, and shelves.
+
+    Args:
+        metadata: Dict[str, Union[str, int]] (required) 
+        = The metadata of the resource from bookstack api
+        parent: Union['Node', None] (optional) 
+        = The parent resource if any, parent/children are also of the same class 'Node'.
+        path_prefix: Union[str, None] (optional) 
+        = This appends a relative 'root' directory to the child resource path/file_name. 
+            It is mainly used to prepend a shelve level 
+            directory for books that are not assigned or under any shelf.
+
+    Returns:
+        Node instance to help create and reference bookstack child/parent 
+        relationships for resources like pages, books, chapters, and shelves.
+
+    """
+    def __init__(self, meta: Dict[str, Union[str, int]],
+                 parent: Union['Node', None] = None, path_prefix: str = ""):
+        self.meta = meta
+        self._parent = parent
+        self._path_prefix = path_prefix
+        # for convenience/usage for exporter
+        self.name: str = self.meta['slug']
+        self.id_: int = self.meta['id']
+        self._display_name = self.meta['name']
+        # children
+        self._children = self._get_children()
+        # if parent
+        self._file_path = self._get_file_path()
+
+    def _get_file_path(self) -> str:
+        if self._parent:
+            return f"{self._parent.file_path}/{self.name}"
+        return ""
+
+    def _get_children(self) -> List[Dict[str, Union[str, int]]]:
+        children = []
+        # find first match
+        for match in _CHILD_KEYS:
+            if match in self.meta:
+                children = self.meta[match]
+                break
+        return children
+
+    @property
+    def file_path(self):
+        """get the base file path"""
+        # check to see if parent exists
+        if not self._file_path:
+            # return base path + name if no parent
+            return f"{self._path_prefix}{self.name}"
+        # if parent exists
+        # return the combined path
+        return f"{self._path_prefix}{self._file_path}"
+
+    @property
+    def children(self):
+        """return all children of a book/chapter/shelf"""
+        return self._children
+
+    @property
+    def empty(self):
+        """return True if page node lacks content"""
+        if not self.name and self._display_name == _NULL_PAGE_NAME:
+            return True
+        return False

bookstack_file_exporter/exporter/util.py

@@ -0,0 +1,17 @@
+from typing import Dict, Union, List
+import logging
+from bookstack_file_exporter.common import util
+
+log = logging.getLogger(__name__)
+
+def get_json_response(url: str, headers: Dict[str, str]) -> List[Dict[str, Union[str,int]]]:
+    """get http response data in json format"""
+    response =  util.http_get_request(url=url, headers=headers)
+    return response.json()
+
+def get_all_ids(url: str, headers: Dict[str, str]) -> List[int]:
+    """get all ids for a bookstack resource"""
+    ids_api_meta = get_json_response(url=url, headers=headers)
+    if ids_api_meta:
+        return [item['id'] for item in ids_api_meta['data']]
+    return []

bookstack_file_exporter/run.py

@@ -0,0 +1,53 @@
+import argparse
+import sys
+import logging
+from typing import Dict
+
+from bookstack_file_exporter.config_helper.config_helper import ConfigNode
+from bookstack_file_exporter.exporter.node import Node
+from bookstack_file_exporter.exporter.exporter import NodeExporter
+from bookstack_file_exporter.archiver.archiver import Archiver
+
+log = logging.getLogger(__name__)
+
+def exporter(args: argparse.Namespace):
+    """export bookstack nodes and archive locally and/or remotely"""
+    ## get configuration from helper
+    config = ConfigNode(args)
+
+    ## convenience vars
+    bookstack_headers = config.headers
+    api_urls = config.urls
+    export_formats = config.user_inputs.formats
+    unassigned_dir = config.unassigned_book_dir
+    page_base_url = config.urls['pages']
+    base_export_dir = config.base_dir_name
+
+    #### Export Data #####
+    # need to implement pagination for apis
+    log.info("Beginning export")
+
+    ## Use exporter class to get all the resources (pages, books, etc.) and their relationships
+    export_helper = NodeExporter(api_urls, bookstack_headers)
+    ## shelves
+    shelve_nodes: Dict[int, Node] = export_helper.get_all_shelves()
+    ## books
+    book_nodes: Dict[int, Node] = export_helper.get_all_books(shelve_nodes, unassigned_dir)
+    ## pages
+    page_nodes: Dict[int, Node] = export_helper.get_all_pages(book_nodes)
+    if not page_nodes:
+        log.warning("No page data available from given Bookstack instance. Nothing to archive")
+        sys.exit(0)
+    log.info("Beginning archive")
+    ## start archive ##
+    archive: Archiver = Archiver(base_export_dir, config.user_inputs.export_meta,
+                                 page_base_url, bookstack_headers)
+    # create tar
+    archive.archive(page_nodes, export_formats)
+    # archive to remote targets
+    archive.archive_remote(config.object_storage_config)
+    # if remote target is specified and clean is true
+    # clean up the .tgz archive since it is already uploaded
+    archive.clean_up(config.user_inputs.clean_up)
+
+    log.info("Completed run")

bookstack_file_exporter/run_args.py

@@ -0,0 +1,36 @@
+import argparse
+import logging
+
+LOG_LEVEL = {
+    'debug': logging.DEBUG,
+    'info': logging.INFO,
+    'warning': logging.WARNING,
+    'error': logging.ERROR
+}
+
+def get_log_level(log_level:str) -> int:
+    """return log level int"""
+    return LOG_LEVEL.get(log_level)
+
+def get_args() -> argparse.Namespace:
+    """return user cmd line options"""
+    parser = argparse.ArgumentParser(description='BookStack File Exporter')
+    parser.add_argument('-c',
+                    '--config-file',
+                    type=str,
+                    default="data/config.yml",
+                    help='''Provide a configuration file (full or relative path).
+                     See README for more details''')
+    parser.add_argument('-o',
+                    '--output-dir',
+                    type=str,
+                    default="",
+                    help='''Optional, specify an output directory.
+                     This can also be specified in the config.yml file''')
+    parser.add_argument('-v',
+                    '--log-level',
+                    type=str.lower,
+                    default='info',
+                    help='Set verbosity level for logging.',
+                    choices=LOG_LEVEL.keys())
+    return parser.parse_args()

bookstack_file_exporter-0.0.1.dist-info/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 homeylab
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

bookstack_file_exporter-0.0.1.dist-info/METADATA

@@ -0,0 +1,251 @@
+Metadata-Version: 2.1
+Name: bookstack-file-exporter
+Version: 0.0.1
+Summary: An exporter written in python to export all documents from a bookstack instance in different formats
+Home-page: https://github.com/homeylab/bookstack-file-exporter
+Author: pchang388
+License: MIT License
+Keywords: bookstack,exporter
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: Pyyaml >=6.0.1
+Requires-Dist: Pydantic >=2.3.0
+Requires-Dist: requests >=2.31.0
+Requires-Dist: minio >=7.1.16
+
+# bookstack-file-exporter
+
+_This is project is still under active development. Functionality is there and is relatively stable at this time._
+
+This tool provides a way to export Bookstack pages in a folder-tree layout locally with an option to push to remote object storage locations.
+
+This small project was mainly created to run as a cron job in k8s but works anywhere. This would allow me to export my docs in markdown, or other formats like pdf. I use Bookstack's markdown editor as default instead of WYSIWYG editor and this makes my notes portable anywhere even if offline.
+
+The main use case is to backup all docs in a folder-tree format to cover the scenarios:
+
+1. Offline copy wanted.
+2. Back up at a file level as an accessory or alternative to disk and volume backups.
+3. Share docs with another person to keep locally.
+4. Migrate to Markdown documenting for simplicity.
+5. Provide an easy way to do automated file backups locally, in docker, or kubernetes.
+
+Supported backup formats are
+
+1. local
+2. minio
+3. s3 (Not Yet Implemented)
+
+Backups are exported in `.tgz` format and generated based off timestamp. Export names will be in the format: `%Y-%m-%d_%H-%M-%S` (Year-Month-Day_Hour-Minute-Second). *Files are first pulled locally to create the tarball and then can be sent to object storage if needed*. Example file name: `bookstack_export_2023-09-22_07-19-54.tgz`.
+
+This script can be run directly via cli as a pip module.
+```
+# if you already have python bin directory in your path
+bookstack-file-exporter -c <path_to_config_file>
+
+# using pip
+python -m bookstack_file_exporter -c <path_to_config_file>
+```
+
+## Using This Application
+
+### Run via Pip
+Note: This application is tested and developed on Python `3.11.X`. It will probably work for >= `3.8` but is recommended to install (or set up a venv) a `3.11.X` version.
+
+```bash
+python -m pip install bookstack-file-exporter
+
+# if you already have python bin directory in your path
+bookstack-file-exporter -c <path_to_config_file>
+
+# using pip
+python -m bookstack_file_exporter -c <path_to_config_file>
+```
+Command line options:
+| option | required | description |
+| ------ | -------- | ----------- |
+|`-c`, `--config-file`|True|Relative or Absolute path to a valid configuration file. This configuration file is checked against a schema for validation.|
+|`-v`, `--log-level` |False, default: info|Provide a valid log level: info, debug, warning, error.|
+
+### Run Via Docker
+Example
+```bash
+docker run \
+    --user ${USER_ID}:${USER_GID} \
+	-v $(pwd)/local/config.yml:/export/config/config.yml:ro \
+	-v $(pwd)/bkps:/export/dump \
+	bookstack-file-exporter:0.0.1
+```
+Required Options:
+| option | description |
+| `config.yml` file mount | Provide a valid configuration file. Specified in example as read only: `-v ${CURDIR}/local/config.yml:/export/config/config.yml:ro`, `${USER_LOCAL_PATH}:${STATIC_DOCKER_PATH}` |
+| `dump` file mount | Directory to place exports. Specified in example: `-v ${CURDIR}/bkps:/export/dump`, `${USER_LOCAL_PATH}:${STATIC_DOCKER_PATH}` |
+
+Tokens and other options can be specified, example:
+```bash
+# '-e' flag for env vars
+# --user flag to override the uid/gid for created files
+docker run \
+	-e LOG_LEVEL='debug' \
+    -e BOOKSTACK_TOKEN_ID='xyz' \
+    -e BOOKSTACK_TOKEN_SECRET='xyz' \
+	--user 1000:1000 \
+	-v $(pwd)/local/config.yml:/export/config/config.yml:ro \
+	-v $(pwd):/export/dump \
+	bookstack-file-exporter:0.0.1
+```
+
+### Authentication
+**Note visibility of pages is based on user**, so use a user that has access to pages you want to back up
+
+Ref: [https://demo.bookstackapp.com/api/docs#authentication](https://demo.bookstackapp.com/api/docs#authentication)
+
+Provide a tokenId and a tokenSecret as environment variables or directly in the configuration file.
+- `BOOKSTACK_TOKEN_ID`
+- `BOOKSTACK_TOKEN_SECRET`
+
+For object storage authentication, find the relevant sections further down in this document.
+
+### Configuration file
+See below for an example and explanation. Optionally, look at `examples/` folder for more. 
+
+Schema and values are checked so ensure proper settings are provided.
+```
+# if http/https not specified, defaults to https
+# if you put http here, it will try verify=false, to not check certs
+host: "https://bookstack.yourdomain.com"
+
+# You could optionally set the bookstack token_id and token_secret here instead of env
+# If env variable is also supplied, env variable will take precedence
+credentials:
+    token_id: ""
+    token_secret: ""
+
+# additional headers to add, examples below
+additional_headers:
+  test: "test"
+  test2: "test2"
+  User-Agent: "test-agent"
+
+# supported formats from bookstack below
+# valid formats: markdown, html, pdf, plaintext
+# you can specify one or as many as you'd like
+formats:
+  - markdown
+  - html
+  - pdf
+  - plaintext
+
+# optional minio configuration
+# If not required, you should omit/comment out the section
+# You can specify env vars instead for access and secret key
+# See Minio Backups section of this doc for more info on required fields
+minio_config:
+  host: "minio.yourdomain.com"
+  access_key: ""
+  secret_key: ""
+  region: "us-east-1"
+  bucket: "mybucket"
+  path: "bookstack/backups"
+
+# output directory for the exported archive
+# relative or full path
+# User who runs the command should have access to write and create sub folders in this directory
+# optional, if not provided, will use current run directory by default
+output_path: "bkps/"
+
+# optional export of metadata about the page in a json file
+# this metadata contains general information about the page
+# like: last update, owner, revision count, etc.
+# omit this or set to false if not needed
+export_meta: true
+
+# optional if using object storage targets
+# After uploading to object storage targets, choose to clean up local files
+# delete the archive from local filesystem
+# will not be cleaned up if set to false or omitted
+clean_up: true
+```
+
+### Backup Behavior
+We will use slug names (from Bookstack API) by default, as such certain characters like `!`, `/` will be ignored and spaces replaced.
+
+All sub directories will be created as required during the export process.
+
+```
+Shelves --> Books --> Chapters --> Pages
+
+## Example
+kafka
+---> controller
+    ---> settings
+        ---> logs (chapter)
+            ---> retention.md
+            ---> compression.pdf
+            ---> something.html
+            ---> other.txt
+        ---> optional
+        ---> main
+    ---> deploy
+---> broker
+    ---> settings
+    ---> deploy
+---> schema-registry
+    ---> protobuf
+    ---> settings
+```
+
+Books without a shelf will be put in a shelve folder named `unassigned`.
+
+Empty/New Pages will be ignored since they have not been modified yet from creation and are empty but also do not have a valid slug. Example:
+```
+{
+    ...
+    "name": "New Page",
+    "slug": "",
+    ...
+}
+```
+
+You may notice some directories (books) and/or files (pages) in the archive have a random string at the end, example - `nKA`: `user-and-group-management-nKA`. This is expected and is because there were resources with the same name created in another shelve and bookstack adds a string at the end to ensure uniqueness.
+
+### Minio Backups
+When specifying `minio_config` in the configuration file, these fields are required in the file:
+```
+# a host/ip + port combination is also allowed
+# example: "minio.yourdomain.com:8443"
+host: "minio.yourdomain.com"
+
+# this is required since minio api appears to require it
+# set to the region your bucket resides in
+# if unsure, try "us-east-1" first
+region: "us-east-1"
+
+# bucket to upload to
+bucket "mybucket"
+```
+
+These fields are optional:
+```
+# access key for the minio instance
+# optionally set as env variable instead
+access_key: ""
+
+# secret key for the minio instance
+# optionally set as env variable instead
+secret_key: ""
+
+# the path of the backup
+# in example below, the exported archive will appear in: `<bucket_name>:/bookstack/backups/bookstack-<timestamp>.tgz`
+path: "bookstack/backups"
+```
+
+As mentioned you can optionally set access and secret key as env variables. If both are specified, env variable will take precedence.
+- `MINIO_ACCESS_KEY`
+- `MINIO_SECRET_KEY`
+
+## Future Items
+1. Be able to pull media/photos locally and place in their respective page folders for a more complete file level backup.
+2. Include the exporter in a maintained helm chart as an optional deployment. The helm chart is [here](https://github.com/homeylab/helm-charts/tree/main/charts/bookstack).
+3. Export S3 or more options.

bookstack_file_exporter-0.0.1.dist-info/RECORD

@@ -0,0 +1,24 @@
+bookstack_file_exporter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+bookstack_file_exporter/__main__.py,sha256=2jPiBcxzY7pPM26wKrVAKgNmwjfnG3KjrdruwbTb1AE,442
+bookstack_file_exporter/run.py,sha256=-8eNYa5YJSTYVR1lbplMKUBudFoLYsN9WS_PjD7fSA8,2045
+bookstack_file_exporter/run_args.py,sha256=a79JqV7pmA47vTdE-2IhVX-xeeYiJKRH4RClrDMOfPg,1248
+bookstack_file_exporter/archiver/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+bookstack_file_exporter/archiver/archiver.py,sha256=VYRRjXOs7D2LxaMhhG_wsmLzAJEfK7uVifCpdkyvVxQ,5226
+bookstack_file_exporter/archiver/minio_archiver.py,sha256=jpddpNE6WMKaYiJs-jcEZF4o0MUcTVLrTkjFJKyH0eA,1923
+bookstack_file_exporter/archiver/util.py,sha256=sb9h0GrGzUahkEniNQ1C4C-ofFzwMvXIrdosm-spVKk,1480
+bookstack_file_exporter/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+bookstack_file_exporter/common/util.py,sha256=2S_BIx05eu_mjKSrbcvNVYEVCS0yheIEcatU0n7Njkg,1333
+bookstack_file_exporter/config_helper/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+bookstack_file_exporter/config_helper/config_helper.py,sha256=INSCPT0GyYcS87VnSLMqBgYzB3OEX7moyTCgNM5Us1g,7688
+bookstack_file_exporter/config_helper/models.py,sha256=StoywdzjCNJjqaI7CQm9d1GBAdEwUCx7qfWfASo_ezg,898
+bookstack_file_exporter/config_helper/remote.py,sha256=ssGekxW9tN91BFuGuKLIgeHKHRbMNtZjiM4LEuesxcE,1013
+bookstack_file_exporter/exporter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+bookstack_file_exporter/exporter/exporter.py,sha256=5KA4h1smdD4EhVk6-cfnDNUWp25Ab0x28sqX8jlL1HU,6324
+bookstack_file_exporter/exporter/node.py,sha256=-7YOUtVdnLPO9-uy3rRWiV1J9Ivnqfxe1FflYW-ZE_k,2708
+bookstack_file_exporter/exporter/util.py,sha256=_En8NGDcUldfuaPW3wrf8RSX6dj_1kNTk4LCIDG1F-8,640
+bookstack_file_exporter-0.0.1.dist-info/LICENSE,sha256=ToZ-JOFE6-SiD4z5P0cA21eBwTvEWxlBcKAGWMXbM5o,1065
+bookstack_file_exporter-0.0.1.dist-info/METADATA,sha256=ZCawbU4fraw6Ni3e-i1pkLenWIzj1D-91HGd0hR39JU,9069
+bookstack_file_exporter-0.0.1.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
+bookstack_file_exporter-0.0.1.dist-info/entry_points.txt,sha256=0-7syMTwEqR4OFDVTbNLVa0OWRJk801CWVEqcZasPWo,82
+bookstack_file_exporter-0.0.1.dist-info/top_level.txt,sha256=o_iIJ9azW-HvDMZdAEKgpTAc3SVYA3z9npCxUYo5Xtw,24
+bookstack_file_exporter-0.0.1.dist-info/RECORD,,

bookstack_file_exporter-0.0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.41.2)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

bookstack_file_exporter-0.0.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+bookstack-file-exporter = bookstack_file_exporter.__main__:main

bookstack_file_exporter-0.0.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+bookstack_file_exporter