cds2md 0.0.0__tar.gz

cds2md-0.0.0/LICENSE

@@ -0,0 +1,22 @@
+Copyright 2026 jdknight
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

cds2md-0.0.0/PKG-INFO

@@ -0,0 +1,130 @@
+Metadata-Version: 2.4
+Name: cds2md
+Version: 0.0.0
+Summary: Confluence Data Center to Markdown Utility
+Author: James Knight
+Author-email: git@jdknight.me
+Maintainer-email: James Knight <git@jdknight.me>
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-Expression: BSD-2-Clause
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Python :: 3.15
+Classifier: Topic :: Documentation
+License-File: LICENSE
+Requires-Dist: beautifulsoup4
+Requires-Dist: httpx
+Requires-Dist: python-slugify
+Project-URL: Download, https://pypi.python.org/pypi/cds2md
+Project-URL: Homepage, https://github.com/jdknight/cds2md
+Project-URL: Source, https://github.com/jdknight/cds2md
+
+# CDS2MD
+
+[![pip Version](https://badgen.net/pypi/v/cds2md?label=PyPI)](https://pypi.python.org/pypi/cds2md)
+[![Build Status](https://github.com/jdknight/cds2md/actions/workflows/build.yml/badge.svg)](https://github.com/jdknight/cds2md/actions/workflows/build.yml)
+
+## Overview
+
+This utility can be used to convert Confluence® Data Center pages to Markdown.
+Atlassian's Confluence Data Center is [scheduled for end-of-life in 2029][eol].
+This tool can be used in an attempt migrate data from Confluence Data Center
+instances to another location.
+
+## Requirements
+
+* [Python][python] 3.11+
+
+## Installation
+
+This tool can be installed using [pip][pip] or [pipx][pipx]:
+
+```shell
+pipx install cds2md
+ (or)
+pip install -U cds2md
+ (or)
+python -m pip install -U cds2md
+```
+
+# Usage
+
+This tool can be invoked from a command line using:
+
+```none
+cds2md --help
+ (or)
+python -m cds2md --help
+```
+
+A minimal example run is as follows:
+
+```
+export CDS2MD_CONFLUENCE_TOKEN=TOKEN
+export CDS2MD_CONFLUENCE_SPACE_KEY=MYSPACE
+export CDS2MD_CONFLUENCE_URL=https://wiki.example.com/
+cds2md
+```
+
+This will fetch page information from a Confluence space and store this data
+into a local cache folder (`.cds2md-cache/`). Once all pages are fetched,
+content will be converted to Markdown and placed in the output directory
+(`output/`).
+
+# Options
+
+The following is a list of available options:
+
+```
+positional arguments:
+  {all,connection-check,fetch,process}
+                        action to perform (default: all)
+
+options:
+  -h, --help            show this help message and exit
+  --ca-certificate CA_CERTIFICATE
+                        ca certificate to use when communicating with Confluence
+  --cache CACHE         path to store fetch cached data
+  --confluence-space-key CONFLUENCE_SPACE_KEY
+                        space to query for pages
+  --confluence-token CONFLUENCE_TOKEN
+                        token used to query the Confluence instance
+  --confluence-token-file CONFLUENCE_TOKEN_FILE
+                        file which holds a Confluence token to use
+  --confluence-token-stdin
+                        pass in the token via the standard input stream
+  --confluence-url CONFLUENCE_URL
+                        url of the Confluence data center instance to interact
+                        with
+  --debug               enabling debugging logs
+  --development         configure to operate in a development state
+  --header, -H HEADER   http headers to include in requests
+  --ignore-tls          ignore all TLS checks when communicating with Confluence
+  --flat                ignore any Confluence-detected hierarchy structure
+  --output OUTPUT       path to output markdown content into
+  --timeout TIMEOUT     timeout for connection REST requests to Confluence
+  --tweak-admonition-type {default,docusaurus,github,gitlab,myst,plain}
+                        admonition formatting to use (default: default)
+  --tweak-html-images   force all images to be generated as html-based images
+  --tweak-html-tables   force all tables to be generated as html-based tables
+  --version             show program's version number and exit
+  --wrap WRAP           hint to the number of characters to wrap to
+```
+
+----
+
+The CDS2MD project is unaffiliated with Atlassian.\
+Atlassian is a registered trademark of Atlassian Pty Ltd.\
+Confluence is a registered trademark of Atlassian Pty Ltd.
+
+
+[eol]: https://www.atlassian.com/licensing/data-center-end-of-life
+[python]: https://www.python.org/
+

cds2md-0.0.0/README.md

@@ -0,0 +1,101 @@
+# CDS2MD
+
+[![pip Version](https://badgen.net/pypi/v/cds2md?label=PyPI)](https://pypi.python.org/pypi/cds2md)
+[![Build Status](https://github.com/jdknight/cds2md/actions/workflows/build.yml/badge.svg)](https://github.com/jdknight/cds2md/actions/workflows/build.yml)
+
+## Overview
+
+This utility can be used to convert Confluence® Data Center pages to Markdown.
+Atlassian's Confluence Data Center is [scheduled for end-of-life in 2029][eol].
+This tool can be used in an attempt migrate data from Confluence Data Center
+instances to another location.
+
+## Requirements
+
+* [Python][python] 3.11+
+
+## Installation
+
+This tool can be installed using [pip][pip] or [pipx][pipx]:
+
+```shell
+pipx install cds2md
+ (or)
+pip install -U cds2md
+ (or)
+python -m pip install -U cds2md
+```
+
+# Usage
+
+This tool can be invoked from a command line using:
+
+```none
+cds2md --help
+ (or)
+python -m cds2md --help
+```
+
+A minimal example run is as follows:
+
+```
+export CDS2MD_CONFLUENCE_TOKEN=TOKEN
+export CDS2MD_CONFLUENCE_SPACE_KEY=MYSPACE
+export CDS2MD_CONFLUENCE_URL=https://wiki.example.com/
+cds2md
+```
+
+This will fetch page information from a Confluence space and store this data
+into a local cache folder (`.cds2md-cache/`). Once all pages are fetched,
+content will be converted to Markdown and placed in the output directory
+(`output/`).
+
+# Options
+
+The following is a list of available options:
+
+```
+positional arguments:
+  {all,connection-check,fetch,process}
+                        action to perform (default: all)
+
+options:
+  -h, --help            show this help message and exit
+  --ca-certificate CA_CERTIFICATE
+                        ca certificate to use when communicating with Confluence
+  --cache CACHE         path to store fetch cached data
+  --confluence-space-key CONFLUENCE_SPACE_KEY
+                        space to query for pages
+  --confluence-token CONFLUENCE_TOKEN
+                        token used to query the Confluence instance
+  --confluence-token-file CONFLUENCE_TOKEN_FILE
+                        file which holds a Confluence token to use
+  --confluence-token-stdin
+                        pass in the token via the standard input stream
+  --confluence-url CONFLUENCE_URL
+                        url of the Confluence data center instance to interact
+                        with
+  --debug               enabling debugging logs
+  --development         configure to operate in a development state
+  --header, -H HEADER   http headers to include in requests
+  --ignore-tls          ignore all TLS checks when communicating with Confluence
+  --flat                ignore any Confluence-detected hierarchy structure
+  --output OUTPUT       path to output markdown content into
+  --timeout TIMEOUT     timeout for connection REST requests to Confluence
+  --tweak-admonition-type {default,docusaurus,github,gitlab,myst,plain}
+                        admonition formatting to use (default: default)
+  --tweak-html-images   force all images to be generated as html-based images
+  --tweak-html-tables   force all tables to be generated as html-based tables
+  --version             show program's version number and exit
+  --wrap WRAP           hint to the number of characters to wrap to
+```
+
+----
+
+The CDS2MD project is unaffiliated with Atlassian.\
+Atlassian is a registered trademark of Atlassian Pty Ltd.\
+Confluence is a registered trademark of Atlassian Pty Ltd.
+
+
+[eol]: https://www.atlassian.com/licensing/data-center-end-of-life
+[python]: https://www.python.org/

cds2md-0.0.0/cds2md/__init__.py

@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: BSD-2-Clause
+# Copyright jdknight
+
+__version__ = '0.0.0'

cds2md-0.0.0/cds2md/__main__.py

@@ -0,0 +1,42 @@
+# SPDX-License-Identifier: BSD-2-Clause
+# Copyright jdknight
+
+from cds2md.engine import engine
+from cds2md.logger import logger
+from cds2md.opts import process_options
+
+
+def main() -> None:
+    """
+    publish helper mainline
+
+    The mainline for cds2md's publisher helper utility.
+    """
+
+    # process command line/environment options
+    opts = process_options()
+
+    # if running a debugging, output the configuration used
+    if opts.debug:
+        debug_opts = vars(opts).copy()
+
+        # sanitize options which may expose token/etc. information
+        if debug_opts.get('confluence_token'):
+            debug_opts['confluence_token'] = '(set)'  # noqa: S105
+        if debug_opts.get('header'):
+            debug_opts['header'] = '(set)'
+        if debug_opts.get('headers'):
+            debug_opts['headers'] = '(set)'
+
+        # pretty paths
+        debug_opts['cache'] = debug_opts['cache'].as_posix() + '/'
+        debug_opts['output'] = debug_opts['output'].as_posix() + '/'
+
+        logger.debug(f'configuration: {debug_opts}')
+
+    # run the utility
+    engine(opts)
+
+
+if __name__ == '__main__':
+    main()  # pragma: no cover

cds2md-0.0.0/cds2md/cache.py

@@ -0,0 +1,41 @@
+# SPDX-License-Identifier: BSD-2-Clause
+# Copyright jdknight
+
+from argparse import Namespace
+from dataclasses import dataclass
+from hashlib import sha1
+
+
+@dataclass
+class AttachmentMetaCache:
+    id: int
+    page_id: int
+    size: int
+    title: str
+    type: str
+    url: str
+    version: int
+
+
+@dataclass
+class PageMetaCache:
+    id: int
+    version: int
+    parent_id: int | None = None
+
+
+def cache_key(opts: Namespace) -> str:
+    """
+    generate a cache string
+
+    Build a cache string based on the configured runtime options. This will
+    be used to populate a cache path based on the Confluence URL and space.
+
+    Args:
+        opts: the runtime options
+
+    Returns:
+        the cache string
+    """
+    value = f'{opts.confluence_url}--{opts.confluence_space_key}'
+    return sha1(value.encode()).hexdigest()  # noqa: S324

cds2md-0.0.0/cds2md/convert.py

@@ -0,0 +1,210 @@
+# SPDX-License-Identifier: BSD-2-Clause
+# Copyright jdknight
+
+from argparse import Namespace
+from bs4 import BeautifulSoup
+from cds2md.cache import cache_key
+from cds2md.defs import CACHE_DB
+from cds2md.logger import logger
+from cds2md.translator import Translator
+from cds2md.translator import TranslatorContext
+from dataclasses import dataclass
+from pathlib import Path
+from slugify import slugify
+import json
+import shutil
+
+
+@dataclass
+class AttachmentCache:
+    id: int
+    name: str
+    version: int
+    target_path: Path | None = None
+
+
+@dataclass
+class PageCache:
+    attachments: list[AttachmentCache]
+    body: str
+    id: int
+    title: str
+    version: int
+    parent_id: int | None = None
+    target_path: Path | None = None
+
+
+def convert(opts: Namespace) -> bool:
+    # ensure cache directory is prepared
+    cache_dir = opts.cache / cache_key(opts)
+    cache_db = cache_dir / CACHE_DB
+    if not cache_db.is_file():
+        logger.error('no cache database found')
+        return False
+
+    # extract cached fetch data to be processed
+    pages_cache = []
+    pageid2cache = {}
+
+    with cache_db.open() as f:
+        pages = json.load(f)['pages']
+
+        for page_id_str, page_data in pages.items():
+            page_atachments = page_data['attachments']
+            page_id = int(page_id_str)
+            page_parent = page_data['parent']
+            page_version = page_data['version']
+            page_cache_dir = cache_dir / page_id_str
+            page_cache_filename = f'{page_id}-{page_version}.json'
+            page_cache_file = page_cache_dir / page_cache_filename
+
+            attachments = []
+            for attachment_name, attachment_data in page_atachments.items():
+                attachment_id = attachment_data['id']
+                attachment_version = attachment_data['version']
+
+                attachment_cache = AttachmentCache(
+                    id=attachment_id,
+                    name=attachment_name,
+                    version=attachment_version,
+                )
+                attachments.append(attachment_cache)
+
+            with page_cache_file.open() as f2:
+                page_cache_data = json.load(f2)
+                page_body = page_cache_data['body']
+                page_title = page_cache_data['title']
+
+                page_cache = PageCache(
+                    attachments=attachments,
+                    body=page_body,
+                    id=page_id,
+                    parent_id=page_parent,
+                    title=page_title,
+                    version=page_version,
+                )
+                pageid2cache[page_id] = page_cache
+                pages_cache.append(page_cache)
+
+    # determine the paths for each page cache to be stored
+    attachment2target: dict[tuple[str, str], str] = {}
+    title2target: dict[str, str] = {}
+    reserved_attachment_targets = set()
+
+    for page_cache in pages_cache:
+        title_slug = slugify(page_cache.title)
+        md_filename = f'{title_slug}.md'
+
+        # if a flat setup, place the page at the root of the output
+        if opts.flat:
+            page_path = opts.output / md_filename
+        # if using hierarchy hints, determine based with parent data
+        else:
+            path_parts = [
+                md_filename,
+            ]
+
+            target_parent = page_cache.parent_id
+            while target_parent:
+                parent = pageid2cache.get(target_parent)
+                assert parent
+                parent_slug = slugify(parent.title)
+                path_parts.append(parent_slug)
+                target_parent = parent.parent_id
+
+            page_path = opts.output
+            for s in reversed(path_parts):
+                page_path = page_path / s
+
+        page_cache.target_path = page_path
+
+        root_path = page_path.relative_to(opts.output)
+        page_full_slug = root_path.as_posix()
+        title2target[page_cache.title] = page_full_slug
+
+        for attachment_cache in page_cache.attachments:
+            attachment_pathname = Path(attachment_cache.name)
+            attachment_name = slugify(attachment_pathname.stem)
+            attachment_ext = slugify(attachment_pathname.suffix)
+
+            attachment_suffix = 1
+            while True:
+                sfx = attachment_suffix if attachment_suffix > 1 else ''
+                attachment_filename = f'{attachment_name}{sfx}.{attachment_ext}'
+                attachment_path = page_path.parent / attachment_filename
+
+                # check if we have a conflicting attachment
+                if attachment_path in reserved_attachment_targets:
+                    attachment_suffix += 1
+                    continue
+
+                break
+
+            attachment_cache.target_path = attachment_path
+            reserved_attachment_targets.add(attachment_path)
+
+            attachment_ref = (page_full_slug, attachment_cache.name)
+
+            attachment_root_path = attachment_path.relative_to(opts.output)
+            attachment_full_slug = attachment_root_path.as_posix()
+            attachment2target[attachment_ref] = attachment_full_slug
+
+    # process each page
+    ctx = TranslatorContext(
+        attachment2target=attachment2target,
+        opts=opts,
+        title2target=title2target,
+    )
+
+    for page_cache in pages_cache:
+        assert page_cache.target_path
+        doc_path = page_cache.target_path.as_posix()
+
+        # parse the page's body
+        logger.debug(f'processing body of page: {doc_path}')
+        soup = BeautifulSoup(page_cache.body, 'html.parser')
+
+        # translate the parsed content into a markdown output
+        translator = Translator(ctx, page_cache.title)
+        logger.debug(f'walking page: {doc_path}')
+        translator.walk(soup)
+
+        # build the final markdown document contents
+        final_body = ''.join(translator.output).strip() + '\n'
+
+        # write the markdown content to its output file
+        assert page_cache.target_path
+        md_file = page_cache.target_path
+        md_file.parent.mkdir(parents=True, exist_ok=True)
+        logger.debug(f'writing to markdown file: {md_file.as_posix()}')
+
+        if md_file.is_dir():
+            logger.error('cannot write markdown into path '
+                        f'that is a directory: {md_file.as_posix()}')
+            return False
+
+        md_file.write_text(final_body)
+
+        # process each attachment
+        if page_cache.attachments:
+            logger.debug(f'processing attachments for page: {doc_path}')
+            page_id = page_cache.id
+            attachment_cache_dir = cache_dir / f'{page_id}' / 'attachments'
+
+            for attachment_cache in page_cache.attachments:
+                attachment_id = attachment_cache.id
+                attachment_version = attachment_cache.version
+                attachment_fname = f'{attachment_id}-{attachment_version}.dat'
+                attachment_data = attachment_cache_dir / attachment_fname
+
+                assert attachment_cache.target_path
+                attachment_folder = attachment_cache.target_path.parent
+                attachment_folder.mkdir(parents=True, exist_ok=True)
+
+                attachment_desc = attachment_cache.target_path.as_posix()
+                logger.debug(f'copying attachment (source): {attachment_data}')
+                logger.debug(f'copying attachment (target): {attachment_desc}')
+                shutil.copy2(attachment_data, attachment_cache.target_path)
+
+    return True
+

cds2md-0.0.0/cds2md/defs.py

@@ -0,0 +1,27 @@
+# SPDX-License-Identifier: BSD-2-Clause
+# Copyright jdknight
+
+from enum import StrEnum
+
+
+# name of the cache database for a cached dataset
+CACHE_DB = 'cache.json'
+
+# name of the cache directory
+CACHE_DIRNAME = '.cds2md-cache'
+
+
+class Action(StrEnum):
+    ALL = 'all'
+    CONNECTION_CHECK = 'connection-check'
+    FETCH = 'fetch'
+    PROCESS = 'process'
+
+
+class AdmonitionType(StrEnum):
+    DEFAULT = 'default'
+    DOCUSAURUS = 'docusaurus'
+    GITHUB = 'github'
+    GITLAB = 'gitlab'
+    MYST  = 'myst'
+    PLAIN = 'plain'

cds2md-0.0.0/cds2md/engine.py

@@ -0,0 +1,103 @@
+# SPDX-License-Identifier: BSD-2-Clause
+# Copyright jdknight
+
+from argparse import Namespace
+from cds2md import __version__ as cds2md_version
+from cds2md.defs import Action
+from cds2md.convert import convert
+from cds2md.fetch import fetch
+from cds2md.logger import logger
+from cds2md.validate_connection import validate_connection
+from cds2md.validate_space import validate_space
+import httpx
+import ssl
+
+
+def engine(opts: Namespace) -> None:
+    # perform any action requiring remote work
+    if opts.action in [Action.ALL, Action.CONNECTION_CHECK, Action.FETCH]:
+        perform_web_action(opts)
+
+    # perform process action
+    if opts.action in [Action.ALL, Action.PROCESS]:
+        process(opts)
+
+
+def perform_web_action(opts: Namespace) -> None:
+    # prepare an http client for requests
+    timeout = httpx.Timeout(
+        connect=opts.timeout,
+        read=15.0,
+        write=5.0,
+        pool=5.0,
+    )
+
+    headers: dict[str, str] = {}
+    headers |= opts.headers
+    headers |= {
+        # will only process json
+        'Accept': 'application/json',
+        # configure confluence pat to perform queries with
+        'Authorization': f'Bearer {opts.confluence_token}',
+        # user agent for our interactions
+        'User-Agent': f'Cds2md/{cds2md_version}',
+        # ignore csrf
+        'X-Atlassian-Token': 'no-check',
+    }
+
+    # setup ssl context
+    ssl_context = ssl.create_default_context()
+    if opts.ca_certificate:
+        ssl_context.load_verify_locations(cafile=opts.ca_certificate)
+    elif opts.ignore_tls:
+        ssl_context = False  # type: ignore[assignment]
+
+    with httpx.Client(
+                headers=headers,
+                timeout=timeout,
+                verify=ssl_context,
+            ) as client:
+
+        # perform connection check action
+        if opts.action in [Action.ALL, Action.CONNECTION_CHECK]:
+            connection_check(opts, client)
+
+        # perform data fetch action
+        if opts.action in [Action.ALL, Action.FETCH]:
+            fetch_attempt(opts, client)
+
+
+def connection_check(opts: Namespace, client: httpx.Client) -> None:
+    logger.info('validating availability of confluence instance...')
+    logger.debug(f'instance: {opts.confluence_url}')
+
+    if not validate_connection(opts, client):
+        raise SystemExit(3)
+
+    logger.info('confluence instance appears to be online')
+
+
+def fetch_attempt(opts: Namespace, client: httpx.Client) -> None:
+    logger.info('validating the confluence space can be found...')
+    logger.debug(f'space-key: {opts.confluence_space_key}')
+
+    # verify the space exists
+    if not validate_space(opts, client):
+        raise SystemExit(4)
+
+    logger.info('confluence space has been detected')
+
+    # try to pull space content
+    logger.info('attempting to fetch content...')
+
+    if not fetch(opts, client):
+        raise SystemExit(5)
+
+
+def process(opts: Namespace) -> None:
+    logger.info('converting cached data into markdown files')
+
+    if not convert(opts):
+        raise SystemExit(3)
+
+    logger.info('conversion completed')