cds2md 0.0.0__py3-none-any.whl

cds2md/__init__.py

@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: BSD-2-Clause
+# Copyright jdknight
+
+__version__ = '0.0.0'

cds2md/__main__.py

@@ -0,0 +1,42 @@
+# SPDX-License-Identifier: BSD-2-Clause
+# Copyright jdknight
+
+from cds2md.engine import engine
+from cds2md.logger import logger
+from cds2md.opts import process_options
+
+
+def main() -> None:
+    """
+    publish helper mainline
+
+    The mainline for cds2md's publisher helper utility.
+    """
+
+    # process command line/environment options
+    opts = process_options()
+
+    # if running a debugging, output the configuration used
+    if opts.debug:
+        debug_opts = vars(opts).copy()
+
+        # sanitize options which may expose token/etc. information
+        if debug_opts.get('confluence_token'):
+            debug_opts['confluence_token'] = '(set)'  # noqa: S105
+        if debug_opts.get('header'):
+            debug_opts['header'] = '(set)'
+        if debug_opts.get('headers'):
+            debug_opts['headers'] = '(set)'
+
+        # pretty paths
+        debug_opts['cache'] = debug_opts['cache'].as_posix() + '/'
+        debug_opts['output'] = debug_opts['output'].as_posix() + '/'
+
+        logger.debug(f'configuration: {debug_opts}')
+
+    # run the utility
+    engine(opts)
+
+
+if __name__ == '__main__':
+    main()  # pragma: no cover

cds2md/cache.py

@@ -0,0 +1,41 @@
+# SPDX-License-Identifier: BSD-2-Clause
+# Copyright jdknight
+
+from argparse import Namespace
+from dataclasses import dataclass
+from hashlib import sha1
+
+
+@dataclass
+class AttachmentMetaCache:
+    id: int
+    page_id: int
+    size: int
+    title: str
+    type: str
+    url: str
+    version: int
+
+
+@dataclass
+class PageMetaCache:
+    id: int
+    version: int
+    parent_id: int | None = None
+
+
+def cache_key(opts: Namespace) -> str:
+    """
+    generate a cache string
+
+    Build a cache string based on the configured runtime options. This will
+    be used to populate a cache path based on the Confluence URL and space.
+
+    Args:
+        opts: the runtime options
+
+    Returns:
+        the cache string
+    """
+    value = f'{opts.confluence_url}--{opts.confluence_space_key}'
+    return sha1(value.encode()).hexdigest()  # noqa: S324

cds2md/convert.py

@@ -0,0 +1,210 @@
+# SPDX-License-Identifier: BSD-2-Clause
+# Copyright jdknight
+
+from argparse import Namespace
+from bs4 import BeautifulSoup
+from cds2md.cache import cache_key
+from cds2md.defs import CACHE_DB
+from cds2md.logger import logger
+from cds2md.translator import Translator
+from cds2md.translator import TranslatorContext
+from dataclasses import dataclass
+from pathlib import Path
+from slugify import slugify
+import json
+import shutil
+
+
+@dataclass
+class AttachmentCache:
+    id: int
+    name: str
+    version: int
+    target_path: Path | None = None
+
+
+@dataclass
+class PageCache:
+    attachments: list[AttachmentCache]
+    body: str
+    id: int
+    title: str
+    version: int
+    parent_id: int | None = None
+    target_path: Path | None = None
+
+
+def convert(opts: Namespace) -> bool:
+    # ensure cache directory is prepared
+    cache_dir = opts.cache / cache_key(opts)
+    cache_db = cache_dir / CACHE_DB
+    if not cache_db.is_file():
+        logger.error('no cache database found')
+        return False
+
+    # extract cached fetch data to be processed
+    pages_cache = []
+    pageid2cache = {}
+
+    with cache_db.open() as f:
+        pages = json.load(f)['pages']
+
+        for page_id_str, page_data in pages.items():
+            page_atachments = page_data['attachments']
+            page_id = int(page_id_str)
+            page_parent = page_data['parent']
+            page_version = page_data['version']
+            page_cache_dir = cache_dir / page_id_str
+            page_cache_filename = f'{page_id}-{page_version}.json'
+            page_cache_file = page_cache_dir / page_cache_filename
+
+            attachments = []
+            for attachment_name, attachment_data in page_atachments.items():
+                attachment_id = attachment_data['id']
+                attachment_version = attachment_data['version']
+
+                attachment_cache = AttachmentCache(
+                    id=attachment_id,
+                    name=attachment_name,
+                    version=attachment_version,
+                )
+                attachments.append(attachment_cache)
+
+            with page_cache_file.open() as f2:
+                page_cache_data = json.load(f2)
+                page_body = page_cache_data['body']
+                page_title = page_cache_data['title']
+
+                page_cache = PageCache(
+                    attachments=attachments,
+                    body=page_body,
+                    id=page_id,
+                    parent_id=page_parent,
+                    title=page_title,
+                    version=page_version,
+                )
+                pageid2cache[page_id] = page_cache
+                pages_cache.append(page_cache)
+
+    # determine the paths for each page cache to be stored
+    attachment2target: dict[tuple[str, str], str] = {}
+    title2target: dict[str, str] = {}
+    reserved_attachment_targets = set()
+
+    for page_cache in pages_cache:
+        title_slug = slugify(page_cache.title)
+        md_filename = f'{title_slug}.md'
+
+        # if a flat setup, place the page at the root of the output
+        if opts.flat:
+            page_path = opts.output / md_filename
+        # if using hierarchy hints, determine based with parent data
+        else:
+            path_parts = [
+                md_filename,
+            ]
+
+            target_parent = page_cache.parent_id
+            while target_parent:
+                parent = pageid2cache.get(target_parent)
+                assert parent
+                parent_slug = slugify(parent.title)
+                path_parts.append(parent_slug)
+                target_parent = parent.parent_id
+
+            page_path = opts.output
+            for s in reversed(path_parts):
+                page_path = page_path / s
+
+        page_cache.target_path = page_path
+
+        root_path = page_path.relative_to(opts.output)
+        page_full_slug = root_path.as_posix()
+        title2target[page_cache.title] = page_full_slug
+
+        for attachment_cache in page_cache.attachments:
+            attachment_pathname = Path(attachment_cache.name)
+            attachment_name = slugify(attachment_pathname.stem)
+            attachment_ext = slugify(attachment_pathname.suffix)
+
+            attachment_suffix = 1
+            while True:
+                sfx = attachment_suffix if attachment_suffix > 1 else ''
+                attachment_filename = f'{attachment_name}{sfx}.{attachment_ext}'
+                attachment_path = page_path.parent / attachment_filename
+
+                # check if we have a conflicting attachment
+                if attachment_path in reserved_attachment_targets:
+                    attachment_suffix += 1
+                    continue
+
+                break
+
+            attachment_cache.target_path = attachment_path
+            reserved_attachment_targets.add(attachment_path)
+
+            attachment_ref = (page_full_slug, attachment_cache.name)
+
+            attachment_root_path = attachment_path.relative_to(opts.output)
+            attachment_full_slug = attachment_root_path.as_posix()
+            attachment2target[attachment_ref] = attachment_full_slug
+
+    # process each page
+    ctx = TranslatorContext(
+        attachment2target=attachment2target,
+        opts=opts,
+        title2target=title2target,
+    )
+
+    for page_cache in pages_cache:
+        assert page_cache.target_path
+        doc_path = page_cache.target_path.as_posix()
+
+        # parse the page's body
+        logger.debug(f'processing body of page: {doc_path}')
+        soup = BeautifulSoup(page_cache.body, 'html.parser')
+
+        # translate the parsed content into a markdown output
+        translator = Translator(ctx, page_cache.title)
+        logger.debug(f'walking page: {doc_path}')
+        translator.walk(soup)
+
+        # build the final markdown document contents
+        final_body = ''.join(translator.output).strip() + '\n'
+
+        # write the markdown content to its output file
+        assert page_cache.target_path
+        md_file = page_cache.target_path
+        md_file.parent.mkdir(parents=True, exist_ok=True)
+        logger.debug(f'writing to markdown file: {md_file.as_posix()}')
+
+        if md_file.is_dir():
+            logger.error('cannot write markdown into path '
+                        f'that is a directory: {md_file.as_posix()}')
+            return False
+
+        md_file.write_text(final_body)
+
+        # process each attachment
+        if page_cache.attachments:
+            logger.debug(f'processing attachments for page: {doc_path}')
+            page_id = page_cache.id
+            attachment_cache_dir = cache_dir / f'{page_id}' / 'attachments'
+
+            for attachment_cache in page_cache.attachments:
+                attachment_id = attachment_cache.id
+                attachment_version = attachment_cache.version
+                attachment_fname = f'{attachment_id}-{attachment_version}.dat'
+                attachment_data = attachment_cache_dir / attachment_fname
+
+                assert attachment_cache.target_path
+                attachment_folder = attachment_cache.target_path.parent
+                attachment_folder.mkdir(parents=True, exist_ok=True)
+
+                attachment_desc = attachment_cache.target_path.as_posix()
+                logger.debug(f'copying attachment (source): {attachment_data}')
+                logger.debug(f'copying attachment (target): {attachment_desc}')
+                shutil.copy2(attachment_data, attachment_cache.target_path)
+
+    return True
+

cds2md/defs.py

@@ -0,0 +1,27 @@
+# SPDX-License-Identifier: BSD-2-Clause
+# Copyright jdknight
+
+from enum import StrEnum
+
+
+# name of the cache database for a cached dataset
+CACHE_DB = 'cache.json'
+
+# name of the cache directory
+CACHE_DIRNAME = '.cds2md-cache'
+
+
+class Action(StrEnum):
+    ALL = 'all'
+    CONNECTION_CHECK = 'connection-check'
+    FETCH = 'fetch'
+    PROCESS = 'process'
+
+
+class AdmonitionType(StrEnum):
+    DEFAULT = 'default'
+    DOCUSAURUS = 'docusaurus'
+    GITHUB = 'github'
+    GITLAB = 'gitlab'
+    MYST  = 'myst'
+    PLAIN = 'plain'

cds2md/engine.py

@@ -0,0 +1,103 @@
+# SPDX-License-Identifier: BSD-2-Clause
+# Copyright jdknight
+
+from argparse import Namespace
+from cds2md import __version__ as cds2md_version
+from cds2md.defs import Action
+from cds2md.convert import convert
+from cds2md.fetch import fetch
+from cds2md.logger import logger
+from cds2md.validate_connection import validate_connection
+from cds2md.validate_space import validate_space
+import httpx
+import ssl
+
+
+def engine(opts: Namespace) -> None:
+    # perform any action requiring remote work
+    if opts.action in [Action.ALL, Action.CONNECTION_CHECK, Action.FETCH]:
+        perform_web_action(opts)
+
+    # perform process action
+    if opts.action in [Action.ALL, Action.PROCESS]:
+        process(opts)
+
+
+def perform_web_action(opts: Namespace) -> None:
+    # prepare an http client for requests
+    timeout = httpx.Timeout(
+        connect=opts.timeout,
+        read=15.0,
+        write=5.0,
+        pool=5.0,
+    )
+
+    headers: dict[str, str] = {}
+    headers |= opts.headers
+    headers |= {
+        # will only process json
+        'Accept': 'application/json',
+        # configure confluence pat to perform queries with
+        'Authorization': f'Bearer {opts.confluence_token}',
+        # user agent for our interactions
+        'User-Agent': f'Cds2md/{cds2md_version}',
+        # ignore csrf
+        'X-Atlassian-Token': 'no-check',
+    }
+
+    # setup ssl context
+    ssl_context = ssl.create_default_context()
+    if opts.ca_certificate:
+        ssl_context.load_verify_locations(cafile=opts.ca_certificate)
+    elif opts.ignore_tls:
+        ssl_context = False  # type: ignore[assignment]
+
+    with httpx.Client(
+                headers=headers,
+                timeout=timeout,
+                verify=ssl_context,
+            ) as client:
+
+        # perform connection check action
+        if opts.action in [Action.ALL, Action.CONNECTION_CHECK]:
+            connection_check(opts, client)
+
+        # perform data fetch action
+        if opts.action in [Action.ALL, Action.FETCH]:
+            fetch_attempt(opts, client)
+
+
+def connection_check(opts: Namespace, client: httpx.Client) -> None:
+    logger.info('validating availability of confluence instance...')
+    logger.debug(f'instance: {opts.confluence_url}')
+
+    if not validate_connection(opts, client):
+        raise SystemExit(3)
+
+    logger.info('confluence instance appears to be online')
+
+
+def fetch_attempt(opts: Namespace, client: httpx.Client) -> None:
+    logger.info('validating the confluence space can be found...')
+    logger.debug(f'space-key: {opts.confluence_space_key}')
+
+    # verify the space exists
+    if not validate_space(opts, client):
+        raise SystemExit(4)
+
+    logger.info('confluence space has been detected')
+
+    # try to pull space content
+    logger.info('attempting to fetch content...')
+
+    if not fetch(opts, client):
+        raise SystemExit(5)
+
+
+def process(opts: Namespace) -> None:
+    logger.info('converting cached data into markdown files')
+
+    if not convert(opts):
+        raise SystemExit(3)
+
+    logger.info('conversion completed')