cal-docs-server 3.0.0b1__py3-none-any.whl

cal_docs_server/index_docs.py

@@ -0,0 +1,449 @@
+# ----------------------------------------------------------------------------------------
+#   index_docs
+#   ----------
+#
+#   Scans the document directories and produces an index
+#
+#   License
+#   -------
+#   MIT License - Copyright 2025-2026 Cyber Assessment Labs
+#
+#   Authors
+#   -------
+#   bena
+#
+#   Version History
+#   ---------------
+#   Mar 2024 - Created
+#   Dec 2025 - New version 2
+# ----------------------------------------------------------------------------------------
+
+# ----------------------------------------------------------------------------------------
+#   Imports
+# ----------------------------------------------------------------------------------------
+
+import os
+import re
+import sys
+from typing import Any
+from typing import TypedDict
+from typing import cast
+import json5
+import yaml
+from . import async_file
+
+# ----------------------------------------------------------------------------------------
+#   Types
+# ----------------------------------------------------------------------------------------
+
+IndexList = list["IndexItemDict"]
+
+
+class IndexItemDict(TypedDict):
+    name: str
+    directory_name: str
+    description: str
+    logo_ref: str | None
+    versions: list[IndexVersionItem]
+    latest_version_dir: str | None
+
+
+class IndexVersionItem(TypedDict):
+    version: str
+    directory_name: str
+    url: str
+    download_url: str
+
+
+class DocInfoFormat(TypedDict):
+    name: str
+    description: str
+    noindex: bool
+
+
+# ----------------------------------------------------------------------------------------
+#   Functions
+# ----------------------------------------------------------------------------------------
+
+
+# ----------------------------------------------------------------------------------------
+async def make_index(root_directory: str) -> IndexList:
+    """
+    Scans the `root_directory` of the documents and produces an Index list.
+    """
+
+    index_list: IndexList = []
+
+    all_dirs = await _list_directories(root_directory)
+    base_dirs = await _filter_non_versioned_dirs(all_dirs)
+
+    # Find any "orphaned" versioned directories (versioned dirs without a base dir)
+    orphaned_bases = await _find_orphaned_version_bases(all_dirs, base_dirs)
+
+    # Combine base_dirs with orphaned bases
+    all_base_dirs = list(set(base_dirs + orphaned_bases))
+    all_base_dirs.sort()
+
+    for dir in all_base_dirs:
+        # First get the versions for this package
+        versions = await _get_package_versions(root_directory, dir, all_dirs)
+
+        # Get package info (will use latest version dir if versions exist)
+        index_item = await _get_package_info(root_directory, dir, versions)
+        if index_item:
+            index_item["versions"] = versions
+            # Find the latest proper version (non-beta)
+            index_item["latest_version_dir"] = _get_latest_proper_version_dir(versions)
+            index_list.append(index_item)
+
+    return index_list
+
+
+# ----------------------------------------------------------------------------------------
+def get_version_redirects(index_list: IndexList) -> dict[str, str]:
+    """
+    Creates a mapping from unversioned package names to their latest version directories.
+    Returns a dict like {"calrep-sdk": "calrep-sdk-6.2.2"}
+    """
+    redirects: dict[str, str] = {}
+    for item in index_list:
+        if item["latest_version_dir"]:
+            # Only create redirect if there's a versioned directory to redirect to
+            redirects[item["directory_name"]] = item["latest_version_dir"]
+    return redirects
+
+
+# ----------------------------------------------------------------------------------------
+#   Private Functions
+# ----------------------------------------------------------------------------------------
+
+
+# ----------------------------------------------------------------------------------------
+async def _list_directories(directory: str) -> list[str]:
+    """
+    Returns a (non-recursive) list of all directories within `directory`. Does not
+    include directories starting with `.`. Returns empty list on error.
+    """
+
+    try:
+        all_entries: list[str] = os.listdir(directory)
+    except OSError as e:
+        print(f"Error listing directory {directory}: {e}", file=sys.stderr)
+        return []
+
+    just_dirs: list[str] = []
+
+    for entry in all_entries:
+        if not entry.startswith("."):
+            if os.path.isdir(os.path.join(directory, entry)):
+                just_dirs.append(entry)
+
+    return just_dirs
+
+
+# ----------------------------------------------------------------------------------------
+async def _filter_non_versioned_dirs(directories: list[str]) -> list[str]:
+    """
+    Takes a list of dir names and returns a new list that contains only base (ie not
+    versioned) names. Specifically directories that dont end with something like `-1.2.3`
+    """
+
+    regex1 = re.compile(r"-\d+[.\-+]?$")
+    regex2 = re.compile(r"-\d+[.\-+].*$")
+
+    base_names: list[str] = []
+    for name in directories:
+        if re.search(regex1, name) or re.search(regex2, name):
+            continue
+        base_names.append(name)
+
+    base_names.sort()
+
+    return base_names
+
+
+# ----------------------------------------------------------------------------------------
+async def _find_orphaned_version_bases(
+    all_dirs: list[str], base_dirs: list[str]
+) -> list[str]:
+    """
+    Finds "orphaned" versioned directories - directories that have version numbers but
+    no corresponding base directory. For example, if we have "pkg-1.0.0" and "pkg-2.0.0"
+    but no "pkg" directory, this returns ["pkg"] so we can create an index entry for it.
+    """
+    regex1 = re.compile(r"-\d+[.\-+]?$")
+    regex2 = re.compile(r"-\d+[.\-+].*$")
+
+    orphaned_bases: set[str] = set()
+
+    for dir_name in all_dirs:
+        # Check if this looks like a versioned directory
+        if re.search(regex1, dir_name) or re.search(regex2, dir_name):
+            # Extract the base name by finding the last "-" followed by version number
+            # Use the same regex to find where the version starts
+            match = re.search(r"-\d+[.\-+]", dir_name)
+            if match:
+                # Get everything before the version part
+                base_name = dir_name[: match.start()]
+
+                # Check if this base name exists in base_dirs
+                if base_name not in base_dirs:
+                    orphaned_bases.add(base_name)
+
+    return sorted(list(orphaned_bases))
+
+
+# ----------------------------------------------------------------------------------------
+async def _get_package_info(
+    root_dir: str, dir: str, versions: list[IndexVersionItem]
+) -> IndexItemDict | None:
+    """
+    Fills out an IndexItemDict for the package at `root_dir/dir`. This will read the
+    description file if it exists.
+    If versions exist, it will use the latest proper version directory for metadata.
+    Otherwise it will use the unversioned base directory.
+    Note: This does not fill in the `versions` field
+    """
+
+    try:
+        # If we have versions, use the latest proper version for metadata
+        latest_version_dir = _get_latest_proper_version_dir(versions)
+        if latest_version_dir:
+            # Use the latest proper version directory for metadata
+            metadata_dir = os.path.join(root_dir, latest_version_dir)
+        else:
+            # No versions or no proper versions, use the base unversioned directory
+            metadata_dir = os.path.join(root_dir, dir)
+
+        index_html_file_name = os.path.join(metadata_dir, "index.html")
+        index_md_file_name = os.path.join(metadata_dir, "index.md")
+
+        if os.path.isfile(index_html_file_name) or os.path.isfile(index_md_file_name):
+            # Is a package so fill out an item.
+            logo_ref = await _get_logo_ref(metadata_dir)
+
+            index_item: IndexItemDict = {
+                "name": dir,
+                "directory_name": dir,
+                "description": "",
+                "logo_ref": logo_ref,
+                "versions": [],
+                "latest_version_dir": None,
+            }
+
+            try:
+                if doc_info := await _read_docinfo_file(metadata_dir):
+                    # There was a docinfo file, so we can add some more details for the item
+                    if doc_info["name"]:
+                        index_item["name"] = doc_info["name"]
+                    if doc_info["description"]:
+                        index_item["description"] = doc_info["description"]
+
+                    if doc_info["noindex"]:
+                        # Special case, we have been asked to not index this. So lets bail.
+                        return None
+            except Exception:
+                # If there was any error processing docinfo file we just skip it
+                print(
+                    f"Failed trying to process docinfo file for {metadata_dir}",
+                    file=sys.stderr,
+                )
+                pass
+
+            return index_item
+
+    except Exception:
+        # If we otherwise failed to process this directory then also just return a blank.
+        print(f"Failed to process package dir {dir}")
+        pass
+
+    return None
+
+
+# ----------------------------------------------------------------------------------------
+async def _read_docinfo_file(dir_path: str) -> DocInfoFormat | None:
+    """
+    Looks in `dir_path` for a document info file which can be one of the following files:
+     - "docinfo.json"
+     - "docinfo.json5"
+     - "docinfo.yml"
+     - "docinfo.yaml"
+
+    If it exists then it will be parsed and the info returned, otherwise a None will be
+    returned.
+    """
+
+    NAMES_LIST = [
+        ("docinfo.json", "json"),
+        ("docinfo.json5", "json"),
+        ("docinfo.yml", "yaml"),
+        ("docinfo.yaml", "yaml"),
+    ]
+
+    data: dict[str, Any] | None = None
+    for file_name, type in NAMES_LIST:
+        file_path = os.path.join(dir_path, file_name)
+        if os.path.isfile(file_path):
+            text = await _read_text_file(file_path)
+
+            if type == "json":
+                data = cast("dict[str, Any]", json5.loads(text))
+                break
+            elif type == "yaml":
+                data = yaml.safe_load(text)
+                break
+            else:
+                raise RuntimeError("Internal Error. Invalid value in constant list")
+
+    if data:
+        # We got the data from one of the files. So take the values we are interested in
+        # if they exist.
+        try:
+            info: DocInfoFormat = {
+                "name": data.get("name", ""),
+                "description": data.get("description", ""),
+                "noindex": bool(data.get("noindex", False)),
+            }
+
+            # Successfully got info so return it.
+            return info
+        except Exception:
+            # This is an unexpected format, so just disregard it.
+            pass
+
+    return None
+
+
+# ----------------------------------------------------------------------------------------
+async def _read_text_file(file_path: str) -> str:
+    """
+    Reads the contents of the text file at `file_path`
+    """
+
+    async with async_file.open_text(file_path, "r") as f:
+        text = await f.read()
+    return text
+
+
+# ----------------------------------------------------------------------------------------
+def _split_to_parts(string: str) -> list[str | int]:
+    """
+    Splits string into a list of parts where each part is either a string or a number
+    """
+    return [
+        int(text) if text.isdigit() else text.lower()
+        for text in re.split(r"(\d+)", string)
+    ]
+
+
+# ----------------------------------------------------------------------------------------
+def _is_proper_version(version_str: str) -> bool:
+    """
+    Checks if a version string is a proper semantic version (e.g., "1.2.3").
+    Returns False for beta, alpha, rc, or other pre-release versions.
+    """
+    # A proper version should only contain digits, dots, and optionally dashes followed by digits
+    # We want to exclude versions with letters like "b1", "alpha", "rc", etc.
+    # Also exclude versions with "+" which typically indicate build metadata
+    if "+" in version_str:
+        return False
+
+    # Check for common pre-release indicators
+    lower_version = version_str.lower()
+    if any(
+        indicator in lower_version
+        for indicator in ["alpha", "beta", "rc", "dev", "pre"]
+    ):
+        return False
+
+    # Check for "b" followed by digits (e.g., "1.2.0b1")
+    if re.search(r"[a-zA-Z]", version_str):
+        return False
+
+    return True
+
+
+# ----------------------------------------------------------------------------------------
+def _get_latest_proper_version_dir(versions: list[IndexVersionItem]) -> str | None:
+    """
+    Finds the latest proper version (non-beta, non-alpha, etc.) from the versions list.
+    Returns the directory_name of that version, or None if no proper versions exist.
+    """
+    # Filter to only proper versions
+    proper_versions = [v for v in versions if _is_proper_version(v["version"])]
+
+    if not proper_versions:
+        return None
+
+    # Sort by version number (highest first)
+    proper_versions.sort(key=lambda x: _split_to_parts(x["version"]), reverse=True)
+
+    # Return the directory of the highest version
+    return proper_versions[0]["directory_name"]
+
+
+# ----------------------------------------------------------------------------------------
+async def _get_package_versions(
+    root_dir: str, package_name: str, folder_names: list[str]
+) -> list[IndexVersionItem]:
+    """
+    Looks for any versioned folders for `package_name` and returns them in the list.
+    """
+
+    versions: list[IndexVersionItem] = []
+
+    for dir_name in folder_names:
+        if dir_name.startswith(package_name + "-"):
+            extra = dir_name[len(package_name) + 1 :]
+            version_info: IndexVersionItem = {
+                "version": extra,
+                "directory_name": dir_name,
+                "url": f"/{dir_name}/",
+                "download_url": f"/api/download/{package_name}/{extra}",
+            }
+            # Check to see if its got a noindex flag
+            if not await _does_dir_have_noindex(os.path.join(root_dir, dir_name)):
+                versions.append(version_info)
+
+    versions.sort(key=lambda x: _split_to_parts(x["version"]), reverse=True)
+    return versions
+
+
+# ----------------------------------------------------------------------------------------
+async def _does_dir_have_noindex(dir: str) -> bool:
+    """
+    Checks in the directory for the docinfo file and zees if it has the noindex flag.
+    """
+
+    info = await _read_docinfo_file(dir)
+    if info and info.get("noindex", ""):
+        return True
+    return False
+
+
+# ----------------------------------------------------------------------------------------
+async def _get_logo_ref(dir: str) -> str | None:
+    """
+    Looks for the file `docinfo.png` in the directory and if it exists returns an image
+    source ref for it. Otherwise None
+    """
+
+    logo_file = os.path.join(dir, "docinfo.png")
+    if os.path.isfile(logo_file):
+        base_name = os.path.basename(dir)
+        ref = f"{base_name}/docinfo.png"
+        return ref
+
+    logo_file = os.path.join(dir, "docinfo.jpg")
+    if os.path.isfile(logo_file):
+        base_name = os.path.basename(dir)
+        ref = f"{base_name}/docinfo.jpg"
+        return ref
+
+    logo_file = os.path.join(dir, "docinfo.svg")
+    if os.path.isfile(logo_file):
+        base_name = os.path.basename(dir)
+        ref = f"{base_name}/docinfo.svg"
+        return ref
+
+    return None

cal_docs_server/main.py

@@ -0,0 +1,191 @@
+# ----------------------------------------------------------------------------------------
+#   main
+#   ----
+#
+#   Starting point for the asyncio
+#
+#   License
+#   -------
+#   MIT License - Copyright 2025-2026 Cyber Assessment Labs
+#
+#   Authors
+#   -------
+#   bena
+#
+#   Version History
+#   ---------------
+#   Mar 2024 - Created
+#   Dec 2025 - New version 2
+# ----------------------------------------------------------------------------------------
+
+# ----------------------------------------------------------------------------------------
+#   Imports
+# ----------------------------------------------------------------------------------------
+
+import argparse
+import asyncio
+import json
+import os
+import sys
+from typing import Any
+from . import index_docs
+from . import render_index
+from . import version
+from . import web_server
+
+# ----------------------------------------------------------------------------------------
+#   Constants
+# ----------------------------------------------------------------------------------------
+
+DEFAULT_CONFIG_DISPLAY = "~/.config/cal-docs-server/config.json"
+DEFAULT_CONFIG_PATH = os.path.expanduser(DEFAULT_CONFIG_DISPLAY)
+
+# ----------------------------------------------------------------------------------------
+#   Functions
+# ----------------------------------------------------------------------------------------
+
+
+# ----------------------------------------------------------------------------------------
+def _load_config(config_path: str | None) -> dict[str, Any]:
+    """
+    Loads configuration from a JSON file.
+    If config_path is None, tries to load from DEFAULT_CONFIG_PATH.
+    Returns empty dict if file doesn't exist (only for default path).
+    """
+
+    path = config_path or DEFAULT_CONFIG_PATH
+    is_explicit = config_path is not None
+
+    if not os.path.isfile(path):
+        if is_explicit:
+            print(f"Error: Config file not found: {path}", file=sys.stderr)
+            sys.exit(1)
+        return {}
+
+    try:
+        with open(path) as f:
+            config: dict[str, Any] = json.load(f)
+        return config
+    except json.JSONDecodeError as e:
+        print(f"Error: Invalid JSON in config file {path}: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+# ----------------------------------------------------------------------------------------
+async def main(argv: list[str]) -> int:
+    """
+    Main function
+    """
+
+    parser = argparse.ArgumentParser(
+        prog="cal-docs-server",
+        description=(
+            "CAL Documentation Server - A lightweight web server for hosting "
+            "versioned documentation with automatic indexing, markdown support, "
+            "and a REST API for programmatic access and uploads."
+        ),
+        epilog=(
+            "Configuration file: Options can be specified in a JSON config file. The"
+            f" server looks for {DEFAULT_CONFIG_DISPLAY} by default. Use --config to"
+            " specify a different file. CLI arguments override config file values."
+        ),
+    )
+    parser.add_argument(
+        "--version", action="version", version=f"cal-docs-server {version.VERSION_STR}"
+    )
+    parser.add_argument(
+        "--config",
+        "-c",
+        metavar="PATH",
+        help=(
+            f"path to JSON config file (default: {DEFAULT_CONFIG_DISPLAY}). "
+            "Config file options: docs, port, name, tokens, base_url"
+        ),
+    )
+    parser.add_argument(
+        "--port",
+        "-p",
+        type=int,
+        metavar="PORT",
+        help="port to listen on (default: 80)",
+    )
+    parser.add_argument(
+        "--docs",
+        "-d",
+        metavar="PATH",
+        help="path to documentation root directory (required)",
+    )
+    parser.add_argument(
+        "--name",
+        "-n",
+        metavar="NAME",
+        help="server name displayed on index page (default: Documents Server)",
+    )
+    parser.add_argument(
+        "--json",
+        "-j",
+        action="store_true",
+        help="output documentation index as JSON to stdout instead of running server",
+    )
+    parser.add_argument(
+        "--html",
+        "-H",
+        action="store_true",
+        help="output index page HTML to stdout instead of running server",
+    )
+    parser.add_argument(
+        "--tokens",
+        "-t",
+        metavar="PATH",
+        help="path to JSON file containing API tokens for upload authentication",
+    )
+    parser.add_argument(
+        "--base-url",
+        "-b",
+        metavar="URL",
+        help="base URL for links in API responses (e.g., https://docs.example.com)",
+    )
+    args = parser.parse_args(argv)
+
+    # Load config file (default or specified)
+    config = _load_config(args.config)
+
+    # Merge config with CLI args (CLI takes precedence)
+    # Map config keys to their defaults
+    docs = args.docs or config.get("docs")
+    port = args.port if args.port is not None else config.get("port", 80)
+    name = args.name or config.get("name", "Documents Server")
+    tokens = args.tokens or config.get("tokens")
+    base_url = args.base_url or config.get("base_url")
+
+    # Expand ~ in paths
+    if docs:
+        docs = os.path.expanduser(docs)
+    if tokens:
+        tokens = os.path.expanduser(tokens)
+
+    # Validate required options
+    if not docs:
+        parser.error("--docs is required (either via CLI or config file)")
+
+    if args.json:
+        index_list = await index_docs.make_index(docs)
+        index_json = json.dumps(index_list, indent=2)
+        print(index_json)
+        return 0
+    elif args.html:
+        index_html = await render_index.render_index(docs, server_name=name)
+        print(index_html)
+        return 0
+
+    await web_server.launch_web_server(
+        root_directory=docs,
+        port=port,
+        server_name=name,
+        tokens_file=tokens,
+        base_url=base_url,
+    )
+
+    # Continue until terminated
+    while True:
+        await asyncio.sleep(1)