clear-skies-doc-builder 2.0.1__py3-none-any.whl → 2.0.3__py3-none-any.whl

{clear_skies_doc_builder-2.0.1.dist-info → clear_skies_doc_builder-2.0.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: clear-skies-doc-builder
-Version: 2.0.1
+Version: 2.0.3
 Summary: The docbuilder for all 'official' clearskies plugins (as well as the main clearskies docs)
 Project-URL: repository, https://github.com/clearskies-py/docs
 Project-URL: issues, https://github.com/clearskies-py/docs/issues

clear_skies_doc_builder-2.0.3.dist-info/RECORD

@@ -0,0 +1,38 @@
+clearskies_doc_builder/__init__.py,sha256=AFQzZ9HwIxPyLpsUhkSDRh_6hm8-5H4RpAelTShCDTc,1300
+clearskies_doc_builder/build_callable.py,sha256=9_HrytRA5MeGpuUKRFgmlK5Mb1WKt6TfWlCOi1aS-Pc,983
+clearskies_doc_builder/prepare_doc_space.py,sha256=eGDsjWaVx0gwCU9mZkrmZTsUcgeod-4PBNqSuPdfwQw,745
+clearskies_doc_builder/backends/__init__.py,sha256=q5jpy8xfZ4SbGQ1T30q4mp4h346HaMivvmNqtgTMQxw,303
+clearskies_doc_builder/backends/attribute_backend.py,sha256=0vlnREe3TcYjrmBFvAlTFeWzQ-LzT__wVJG7inWQLzM,3581
+clearskies_doc_builder/backends/class_backend.py,sha256=6jOKzUDLYIQofCxno3Y9L_FiKT9x2MsPOe3c1HJ2k2M,4031
+clearskies_doc_builder/backends/module_backend.py,sha256=hJMeJtgInaZfXnFpxtCuRA05jBZTpMhVrR8Se3uul9Y,5718
+clearskies_doc_builder/backends/python.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+clearskies_doc_builder/builders/__init__.py,sha256=ndAK4-imR9vhl7CS425ASYi46FN5-eR9Dr6bKxCn5lA,292
+clearskies_doc_builder/builders/builder.py,sha256=3aQ_UTKJFDDUCNHKhwn1sr74CKR3YCe7xuWRX528eAw,6133
+clearskies_doc_builder/builders/module.py,sha256=IrSb5ZKyuPzsK5UXSX-UFTEx5ENH05VEOeYuZ2rnDxY,3286
+clearskies_doc_builder/builders/single_class.py,sha256=3JZBVtt9GToPFoSIlggWbalRFbfayLpLVb9-2-1ymdY,3454
+clearskies_doc_builder/builders/single_class_to_section.py,sha256=uEyawxW4cIdavBVnK2TBiIqywUgj3IxmUcS16E81cgI,1754
+clearskies_doc_builder/columns/__init__.py,sha256=--cof8kFPwlZncM3MBQYkvfq8TejEeI0B99vTlEVCkU,566
+clearskies_doc_builder/columns/any.py,sha256=5kjwEz4-t7MH9sFyycWxEcDdDmKZsb3wfX1Nmvc6QI4,998
+clearskies_doc_builder/columns/attribute.py,sha256=2KGLIxCCbjc2yPZ8fnXqUjyq9pV2Rg164XA2ZquT0Lo,971
+clearskies_doc_builder/columns/attributes.py,sha256=Qrk7n2rXX-yAItwuap5NKZv13I2wQl6dv9Qym3IJVQo,966
+clearskies_doc_builder/columns/base_classes.py,sha256=GmVrAyfaxX9tT2WUYvz9kCkzHDdVxE2eSatzUYCsmRQ,893
+clearskies_doc_builder/columns/class_column.py,sha256=lLPUONBsLVNrmpsL_gcJ6El-PG8t0Cky21DanGT_Jfs,965
+clearskies_doc_builder/columns/method.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+clearskies_doc_builder/columns/module.py,sha256=LJ22plawtxJC_M8nhqUSIZVhka7cB0GD_u-35LjvOJA,1007
+clearskies_doc_builder/columns/module_classes.py,sha256=4CsXcku0hHVZ0V26u1wzfHYXcHgntFo9hmj91Wn5QlU,728
+clearskies_doc_builder/configs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+clearskies_doc_builder/models/__init__.py,sha256=SxTzmTPpBHrqzO3nef4r3fBZ_C3F7UdyOaxmOoMTIcM,402
+clearskies_doc_builder/models/arg.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+clearskies_doc_builder/models/attribute.py,sha256=13SQbGCQ7O1fQAWKeCDIkpYtlVsrrqxb93spiabMjk4,385
+clearskies_doc_builder/models/attribute_reference.py,sha256=U7ISNKfq-45X54ZbvnTMDZVMyai9cQU_gpsPhVzy5JM,189
+clearskies_doc_builder/models/class_model.py,sha256=KqZ3im8OxOAAPj7bgVM8v2-n-RRoDaFCqpYDhBUpn78,899
+clearskies_doc_builder/models/class_reference.py,sha256=GWcPc9OMmDe6nuFx-o3Rt0uQ_PRUe5dHMyFKOUQ4I2M,185
+clearskies_doc_builder/models/method.py,sha256=rdZkG-Dx9_h0aIRLneh_jCixrykbWfCnoDfE1bhHavA,493
+clearskies_doc_builder/models/method_reference.py,sha256=U4YOpRLotyEp6G0Y5OHORdo8mZOnjUzOgqOYCFBPDUw,177
+clearskies_doc_builder/models/module.py,sha256=5JUF2LUKTpVsYlviT7T_qgLdMApGtS4qIUloCEzJ-uo,573
+clearskies_doc_builder/models/module_reference.py,sha256=-zHnrkP6JR7j2XFuW94uGlPENI3ZshxgBu4yY9ayzSg,177
+clearskies_doc_builder/models/property.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+clear_skies_doc_builder-2.0.3.dist-info/METADATA,sha256=2dHuWzKFnDWHbevWGNM0utGKiFcssuNQzedJATY9ngQ,3075
+clear_skies_doc_builder-2.0.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+clear_skies_doc_builder-2.0.3.dist-info/licenses/LICENSE,sha256=pgH32-xrgpIDf6WIBUA1X19o-sIHIXgToCxo-tBWDdw,1071
+clear_skies_doc_builder-2.0.3.dist-info/RECORD,,

clearskies_doc_builder/__init__.py

@@ -0,0 +1,35 @@
+import json
+import pathlib
+import sys
+
+import clearskies
+
+from clearskies_doc_builder import backends, models
+from clearskies_doc_builder.build_callable import build_callable
+
+
+def build(build_file_string: str) -> None:
+    # We assume a folder structure here where the repo root contains a `src/` folder and a `docs/python` folder.
+    # `build_file_string` should contain the absolute path to the file that kicked this off, which should
+    # live in the `docs/python` folder.  This comes in as a string, which we convert to a path.
+    # We then also need to calculate the path to the `src/` folder and add that to our
+    # python path.  We do this because we want to import the clearskies module in question, since the python
+    # code is where all of our documentation lives.
+
+    doc_python_path = pathlib.Path(build_file_string).parents[0]
+    project_root = doc_python_path.parents[1]
+    sys.path.append(str(project_root / "src"))
+
+    config_file = open(str(doc_python_path / "config.json"), "r")
+    config = json.loads(config_file.read())
+    config_file.close()
+
+    cli = clearskies.contexts.Cli(
+        build_callable,
+        modules=[models, backends],
+        bindings={
+            "config": config,
+            "project_root": project_root / "docs",
+        },
+    )
+    cli()

clearskies_doc_builder/backends/__init__.py

@@ -0,0 +1,9 @@
+from clearskies_doc_builder.backends.attribute_backend import AttributeBackend
+from clearskies_doc_builder.backends.class_backend import ClassBackend
+from clearskies_doc_builder.backends.module_backend import ModuleBackend
+
+__all__ = [
+    "AttributeBackend",
+    "ClassBackend",
+    "ModuleBackend",
+]

clearskies_doc_builder/backends/attribute_backend.py

@@ -0,0 +1,91 @@
+import inspect
+from typing import Any
+
+import clearskies
+import clearskies.column
+import clearskies.model
+import clearskies.query
+
+from clearskies_doc_builder.backends.module_backend import ModuleBackend
+
+
+class AttributeBackend(ModuleBackend):
+    _search_functions = {
+        "id": lambda attribute, name, value: id(attribute) == int(value),  # type: ignore
+        "name": lambda attribute, name, value: name == value,  # type: ignore
+        "type": lambda attribute, name, value: attribute.__class__ == value,  # type: ignore
+    }
+
+    def records(
+        self, query: clearskies.query.Query, next_page_data: dict[str, str | int] | None = None
+    ) -> list[dict[str, Any]]:
+        """
+        Return a list of records that match the given query configuration.
+
+        next_page_data is used to return data to the caller.  Pass in an empty dictionary, and it will be populated
+        with the data needed to return the next page of results.  If it is still an empty dictionary when returned,
+        then there is no additional data.
+        """
+        disallowed = ["joins", "selects", "group_by"]
+        for attribute_name in disallowed:
+            if getattr(query, attribute_name):
+                raise ValueError(f"The AttributeBackend received {attribute_name} in a query but doesn't support this.")
+
+        for condition in query.conditions:
+            if condition.operator != "=":
+                raise ValueError("The AttributeBackend only supports searching with the equals operator")
+
+        if "parent_class" not in query.conditions_by_column:
+            raise ValueError("When searching for attributes you must include a condition on 'parent_class'")
+
+        parent_class = query.conditions_by_column["parent_class"][0].values[0]
+        matching_attributes = []
+        for name in dir(parent_class):
+            attribute = getattr(parent_class, name)
+            matches = True
+            for condition in query.conditions:
+                if condition.column_name not in self._search_functions:
+                    continue
+                if not self._search_functions[condition.column_name](attribute, name, condition.values[0]):  # type: ignore
+                    matches = False
+
+            if not matches:
+                continue
+            matching_attributes.append(self.unpack(attribute, name, parent_class))
+
+        return self.paginate(matching_attributes, query)
+
+    def unpack(self, attribute: Any, name: str, parent_class: type) -> dict[str, Any]:  # type: ignore
+        all_args = []
+        args = []
+        kwargs = []
+        defaults: dict[str, Any] = {}
+        argdata = None
+        try:
+            argdata = inspect.getfullargspec(attribute)
+        except:
+            pass
+
+        if argdata:
+            nargs = len(argdata.args)
+            nkwargs = len(argdata.defaults) if argdata.defaults else 0
+            npargs = nargs - nkwargs
+            all_args = argdata.args
+            kwargs = all_args[nargs - nkwargs :]
+            args = all_args[:nkwargs]
+            defaults = {}
+            if argdata.defaults:
+                defaults = {argdata.args[index + npargs]: default for (index, default) in enumerate(argdata.defaults)}
+
+        return {
+            "id": id(attribute),
+            "name": name,
+            "type": attribute.__class__,
+            "doc": attribute.__doc__,
+            "parent_class": parent_class,
+            "attribute": attribute,
+            "all_args": all_args,
+            "args": args,
+            "kwargs": kwargs,
+            "defaults": defaults,
+        }

clearskies_doc_builder/backends/class_backend.py

@@ -0,0 +1,99 @@
+import importlib
+import inspect
+from types import ModuleType
+from typing import Any
+
+import clearskies
+import clearskies.column
+import clearskies.model
+import clearskies.query
+
+from clearskies_doc_builder.backends.module_backend import ModuleBackend
+
+
+class ClassBackend(ModuleBackend):
+    _search_functions = {
+        "id": lambda module, value: id(module) == int(value),
+        "source_file": lambda module, value: (module.__file__ if hasattr(module, "__file__") else "") == value,
+    }
+
+    def records(
+        self, query: clearskies.query.Query, next_page_data: dict[str, str | int] | None = None
+    ) -> list[dict[str, Any]]:
+        """
+        Return a list of records that match the given query configuration.
+
+        next_page_data is used to return data to the caller.  Pass in an empty dictionary, and it will be populated
+        with the data needed to return the next page of results.  If it is still an empty dictionary when returned,
+        then there is no additional data.
+        """
+        disallowed = ["joins", "selects", "group_by"]
+        for attribute_name in disallowed:
+            if getattr(query, attribute_name):
+                raise ValueError(f"The ClassBackend received {attribute_name} in a query but doesn't support this.")
+
+        for condition in query.conditions:
+            if condition.operator != "=":
+                raise ValueError("The ClassBackend only supports searching with the equals operator")
+
+        if "import_path" in query.conditions_by_column:
+            import_path = query.conditions_by_column["import_path"][0].values[0]
+            path_parts = import_path.split(".")
+            if len(path_parts) < 2:
+                raise ValueError(
+                    'In order to search for classes by import path you must provide the module and class name, e.g. `classes.find("import_path=clearskies.Endpoint")`'
+                )
+            class_name = path_parts[-1]
+            module_path = ".".join(path_parts[0:-1])
+            module = importlib.import_module(module_path)
+            if not hasattr(module, class_name):
+                raise ValueError(f"Module {import_path} has no class named {class_name}")
+            Class = getattr(module, class_name)
+            if not inspect.isclass(Class):
+                raise ValueError(
+                    f"I was asked to import the class named '{import_path}' but this doesn't actually reference a class"
+                )
+            return [self.unpack(Class, module)]
+
+        if "module" not in query.conditions_by_column:
+            raise ValueError(
+                "When searching for classes you must include a condition on either 'module' or 'import_path'"
+            )
+
+        parent_module = query.conditions_by_column["module"][0].values[0]
+        matching_classes = []
+        for name in dir(parent_module):
+            attribute = getattr(parent_module, name)
+            if not inspect.isclass(attribute):
+                continue
+
+            matches = True
+            for condition in query.conditions:
+                if condition.column_name not in self._search_functions:
+                    continue
+                if not self._search_functions[condition.column_name](attribute, condition.values[0]):
+                    matches = False
+
+            if not matches:
+                continue
+            matching_classes.append(self.unpack(attribute, parent_module))
+
+        return self.paginate(matching_classes, query)
+
+    def unpack(self, Class: type, module: ModuleType) -> dict[str, Any]:  # type: ignore
+        source_file = ""
+        try:
+            # this fails for built ins
+            source_file = inspect.getfile(Class)
+        except TypeError:
+            pass
+
+        return {
+            "id": id(Class),
+            "import_path": Class.__module__ + "." + Class.__name__,
+            "name": Class.__name__,
+            "source_file": source_file,
+            "doc": Class.__doc__,
+            "module": module,
+            "type": Class,
+        }

clearskies_doc_builder/backends/module_backend.py

@@ -0,0 +1,131 @@
+import importlib
+import sys
+from types import ModuleType
+from typing import Any, Callable
+
+import clearskies
+import clearskies.column
+import clearskies.model
+import clearskies.query
+from clearskies.autodoc.schema import Integer as AutoDocInteger
+from clearskies.autodoc.schema import Schema as AutoDocSchema
+
+
+class ModuleBackend(clearskies.backends.Backend):
+    _search_functions = {
+        "id": lambda module, value: id(module) == int(value),
+        "is_builtin": lambda module, value: (0 if hasattr(module, "__file__") else 1) == int(value),
+        "source_file": lambda module, value: (module.__file__ if hasattr(module, "__file__") else "") == value,
+        "module": lambda module, value: id(module) == id(value),
+    }
+
+    def update(self, id: int | str, data: dict[str, Any], model: clearskies.model.Model) -> dict[str, Any]:
+        """Update the record with the given id with the information from the data dictionary."""
+        raise Exception(f"The {self.__class__.__name__} only supports read operations: update is not allowed")
+
+    def create(self, data: dict[str, Any], model: clearskies.model.Model) -> dict[str, Any]:
+        """Create a record with the information from the data dictionary."""
+        raise Exception(f"The {self.__class__.__name__} only supports read operations: create is not allowed")
+
+    def delete(self, id: int | str, model: clearskies.model.Model) -> bool:
+        """Delete the record with the given id."""
+        raise Exception(f"The {self.__class__.__name__} only supports read operations: delete is not allowed")
+
+    def count(self, query: clearskies.query.Query) -> int:
+        """Return the number of records which match the given query configuration."""
+        return len(self.records(query))
+
+    def records(
+        self, query: clearskies.query.Query, next_page_data: dict[str, str | int] | None = None
+    ) -> list[dict[str, Any]]:
+        """
+        Return a list of records that match the given query configuration.
+
+        next_page_data is used to return data to the caller.  Pass in an empty dictionary, and it will be populated
+        with the data needed to return the next page of results.  If it is still an empty dictionary when returned,
+        then there is no additional data.
+        """
+        disallowed = ["joins", "selects", "group_by"]
+        for attribute_name in disallowed:
+            if getattr(query, attribute_name):
+                raise ValueError(f"The ModuleBackend received {attribute_name} in a query but doesn't support this.")
+
+        for condition in query.conditions:
+            if condition.operator != "=":
+                raise ValueError("The ModuleBackend only supports searching with the equals operator")
+
+        module_name_condition = query.conditions_by_column.get("import_path", query.conditions_by_column.get("name"))
+        if module_name_condition:
+            module_name = module_name_condition[0].values[0]
+            module = importlib.import_module(module_name)
+            return [self.unpack(module)]
+
+        matching_modules = []
+        module_names = set(sys.modules) & set(globals())
+        for module_name in module_names:
+            if module_name not in sys.modules:
+                continue
+
+            module = sys.modules[module_name]
+            matches = True
+            for condition in query.conditions:
+                if condition.column_name not in self._search_functions:
+                    continue
+                if not self._search_functions[condition.column_name](module, condition.values[0]):
+                    matches = False
+
+            if not matches:
+                continue
+
+            matching_modules.append(self.unpack(module))
+
+        return self.paginate(matching_modules, query)
+
+    def unpack(self, module: ModuleType) -> dict[str, Any]:
+        return {
+            "id": id(module),
+            "import_path": module.__name__,
+            "name": module.__name__,
+            "is_builtin": not hasattr(module, "__file__"),
+            "source_file": module.__file__ if hasattr(module, "__file__") else "",
+            "doc": module.__doc__,
+            "module": module,
+        }
+
+    def paginate(self, records, query):
+        return records
+
+    def validate_pagination_data(self, data: dict[str, Any], case_mapping: Callable) -> str:
+        extra_keys = set(data.keys()) - set(self.allowed_pagination_keys())
+        if len(extra_keys):
+            key_name = case_mapping("start")
+            return "Invalid pagination key(s): '" + "','".join(extra_keys) + f"'.  Only '{key_name}' is allowed"
+        if "start" not in data:
+            key_name = case_mapping("start")
+            return f"You must specify '{key_name}' when setting pagination"
+        start = data["start"]
+        try:
+            start = int(start)
+        except:
+            key_name = case_mapping("start")
+            return f"Invalid pagination data: '{key_name}' must be a number"
+        return ""
+
+    def allowed_pagination_keys(self) -> list[str]:
+        return ["start"]
+
+    def documentation_pagination_next_page_response(self, case_mapping: Callable[[str], str]) -> list[Any]:
+        return [AutoDocInteger(case_mapping("start"), example=0)]
+
+    def documentation_pagination_next_page_example(self, case_mapping: Callable[[str], str]) -> dict[str, Any]:
+        return {case_mapping("start"): 0}
+
+    def documentation_pagination_parameters(
+        self, case_mapping: Callable[[str], str]
+    ) -> list[tuple[AutoDocSchema, str]]:
+        return [
+            (
+                AutoDocInteger(case_mapping("start"), example=0),
+                "The zero-indexed record number to start listing results from",
+            )
+        ]

clearskies_doc_builder/build_callable.py

@@ -0,0 +1,24 @@
+from typing import Any
+
+from clearskies_doc_builder import models
+from clearskies_doc_builder.prepare_doc_space import prepare_doc_space
+
+
+def build_callable(modules: models.Module, classes: models.Class, config: dict[str, Any], project_root: str):
+    doc_root = prepare_doc_space(project_root)
+    nav_order_parent_count = {}
+
+    for index, branch in enumerate(config["tree"]):
+        nav_order_title_tracker = branch.get("parent", branch["title"])
+        if nav_order_title_tracker not in nav_order_parent_count:
+            nav_order_parent_count[nav_order_title_tracker] = 0
+        nav_order_parent_count[nav_order_title_tracker] += 1
+        builder_class = classes.find("import_path=" + branch["builder"]).type
+        builder = builder_class(
+            branch,
+            modules,
+            classes,
+            doc_root,
+            nav_order=nav_order_parent_count[nav_order_title_tracker] if branch.get("parent") else index + 2,
+        )
+        builder.build()

clearskies_doc_builder/builders/__init__.py

@@ -0,0 +1,9 @@
+from clearskies_doc_builder.builders.module import Module
+from clearskies_doc_builder.builders.single_class import SingleClass
+from clearskies_doc_builder.builders.single_class_to_section import SingleClassToSection
+
+__all__ = [
+    "Module",
+    "SingleClass",
+    "SingleClassToSection",
+]

clearskies_doc_builder/builders/builder.py

@@ -0,0 +1,136 @@
+import pathlib
+import re
+import tokenize
+
+
+class Builder:
+    _attribute_cache: dict[str, dict[str, str]] = {}
+
+    def __init__(self, branch, modules, classes, doc_root, nav_order):
+        self.modules = modules
+        self.classes = classes
+        self.doc_root = pathlib.Path(doc_root)
+        self.title = branch["title"]
+        self.source = branch["source"]
+        self.nav_order = nav_order
+        self._attribute_cache = {}
+        self.args_to_additional_attributes_map = {}
+
+    def make_index_from_class_overview(self, title_snake_case, source_class, section_folder_path):
+        filename = "index"
+        section_folder_path.mkdir(exist_ok=True)
+
+        doc = self.build_header(self.title, filename, title_snake_case, None, self.nav_order, True)
+        (elevator_pitch, overview) = self.parse_overview_doc(
+            self.raw_docblock_to_md(source_class.doc).lstrip("\n").lstrip(" ")
+        )
+        doc += f"\n\n# {self.title}\n\n{elevator_pitch}\n\n## Overview\n\n{overview}"
+
+        output_file = section_folder_path / f"{filename}.md"
+        with output_file.open(mode="w") as doc_file:
+            doc_file.write(doc)
+
+    def parse_overview_doc(self, overview_doc):
+        parts = overview_doc.lstrip("\n").split("\n", 1)
+        if len(parts) < 2:
+            return (parts[0], "")
+        return (parts[0], parts[1].lstrip(" "))
+
+    def extract_attribute_docs(self, source_class, argument_names, additional_attribute_sources=[]):
+        """
+        Fetch the docblocks for class arguments.
+
+        Sadly, python doesn't support docblocks on class arguments.  I only discovered this after writing all
+        the docblocs this way.  Still, I don't want to move my docblocs, because puttig them on arguments is
+        legitimately the place where they make the most sense.  So, we have to use the python parsing capabilities
+        built into python in order to extract them ourselves.  Very exciting... :cry:
+
+        We substantially simplify this process (in a way that hopefully works) by setting stringent requirements
+        for how our docblocks need to be defined.  The docblock must come before the argument and they must be
+        at the top of the class.  In addition, when we are called, we are provided with a list of all argument
+        names so that we are looking for a specific list of things rather than searching more generically for
+        a series of documented arguments.  So, we're looking for a pattern of:
+
+         1. tokenize.STRING
+         2. tokenize.NEWLINE
+         3. tokenize.NAME
+
+        This will probably match a bunch of things, which is where our list of argument names comes in.
+        Also, we'll only use the first combination of these things we find, which menas that attribute definitions
+        must be at the top of the file. This will help us avoid getting confused by variable definitions with
+        matching names later in the class.
+        """
+        # built in classes (which we will reach with our iterative approach) don't have a source file.
+        if not source_class.source_file:
+            return {}
+
+        # we will iterate over base classes, and these often get re-used, so let's keep a cache
+        if source_class.source_file in self._attribute_cache:
+            return self._attribute_cache[source_class.source_file]
+
+        doc_strings = {}
+        with open(source_class.source_file, "r") as fp:
+            # so this is both very simple and, hopefully, not prone to failure. The tokenization information that comes back from the
+            # parser is surprisingly generic and vague.  However, we are looking for something
+            last_string = ""
+            for token_type, token_string, (srow, scol), (erow, ecol), line_content in tokenize.generate_tokens(
+                fp.readline
+            ):
+                if token_type == tokenize.STRING:
+                    last_string = token_string
+                    continue
+                if token_type == tokenize.NEWLINE:
+                    continue
+                if token_type != tokenize.NAME:
+                    last_string = ""
+                    continue
+                if not last_string or token_string not in argument_names:
+                    continue
+                doc_strings[token_string] = last_string
+
+        # and let's repeat this for any base classes just to make sure we don't miss anything.  Often attributes are defined in
+        # bases and we want to use those docs if we don't have them.
+        for base_class in source_class.base_classes:
+            doc_strings = {
+                **self.extract_attribute_docs(base_class, argument_names),
+                **doc_strings,
+            }
+
+        for additional_source_class in additional_attribute_sources:
+            doc_strings = {
+                **self.extract_attribute_docs(additional_source_class, argument_names),
+                **doc_strings,
+            }
+
+        self._attribute_cache[source_class.source_file] = doc_strings
+        return doc_strings
+
+    def build_header(self, title, filename, section_name, parent, nav_order, has_children):
+        permalink = "/docs/" + (f"{section_name}/" if section_name else "") + f"{filename}.html"
+        header = f"""---
+layout: default
+title: {title}
+permalink: {permalink}
+nav_order: {nav_order}
+"""
+        if parent:
+            header += f"parent: {parent}\n"
+        if has_children:
+            header += "has_children: true\n"
+        header += "---"
+        return header
+
+    def raw_docblock_to_md(self, docblock):
+        return re.sub(r"\n    ", "\n", docblock)
+
+    def default_args(self):
+        default_args = {}
+        for key, value in self.args_to_additional_attributes_map.items():
+            parts = value.split(".")
+            import_path = ".".join(parts[:-1])
+            attribute_name = parts[-1]
+            source_class = self.classes.find(f"import_path={import_path}")
+            doc = source_class.attributes.where(f"name={attribute_name}").first().doc
+            if doc:
+                default_args[key] = self.raw_docblock_to_md(doc)
+        return default_args

clearskies_doc_builder/builders/module.py

@@ -0,0 +1,69 @@
+from collections import OrderedDict
+from typing import Any
+
+import clearskies
+
+from .builder import Builder
+
+
+class Module(Builder):
+    def __init__(self, branch, modules, classes, doc_root, nav_order):
+        super().__init__(branch, modules, classes, doc_root, nav_order)
+        self.class_list = branch["classes"]
+        self.args_to_additional_attributes_map = branch.get("args_to_additional_attributes_map", {})
+
+    def build(self):
+        title_snake_case = clearskies.functional.string.title_case_to_snake_case(self.title.replace(" ", "")).replace(
+            "_", "-"
+        )
+        section_folder_path = self.doc_root / title_snake_case
+        source_class = self.classes.find(f"import_path={self.source}")
+        self.make_index_from_class_overview(title_snake_case, source_class, section_folder_path)
+
+        default_args = self.default_args()
+
+        nav_order = 0
+        for class_name in self.class_list:
+            nav_order += 1
+            source_class = self.classes.find(f"import_path={class_name}")
+            title = source_class.name
+            filename = clearskies.functional.string.title_case_to_snake_case(source_class.name).replace("_", "-")
+            class_doc = self.build_header(source_class.name, filename, title_snake_case, self.title, nav_order, False)
+            (elevator_pitch, overview) = self.parse_overview_doc(
+                self.raw_docblock_to_md(source_class.doc).lstrip("\n").lstrip(" ")
+            )
+            class_doc += f"\n\n# {title}\n\n{elevator_pitch}\n\n"
+            main_doc = "## Overview\n\n{overview}\n\n"
+            table_of_contents = " 1. [Overview](#overview)\n"
+
+            # Find the documentation for all of our init args.
+            arguments: dict[str, Any] = OrderedDict()
+            for arg in source_class.init.all_args:
+                if arg == "self":
+                    continue
+                arguments[arg] = {
+                    "required": arg not in source_class.init.kwargs,
+                    "doc": default_args.get(arg, ""),
+                }
+
+            # for various reasons, it's easier to extract docs for all the arguments at once:
+            docs = self.extract_attribute_docs(source_class, list(arguments.keys()))
+            for arg, doc in docs.items():
+                # you would think that we would only get arguments that belong to our class, but this isn't the case
+                # because the processing caches results from parent classes, and we don't always use all attributes
+                # available from all our parents.
+                if arg not in arguments:
+                    continue
+                arguments[arg]["doc"] = doc
+
+            for index, arg in enumerate(arguments.keys()):
+                arg_data = arguments[arg]
+                table_of_contents += f" {index + 2}. [{arg}](#{arg})\n"
+                main_doc += f"## {arg}\n**" + ("Required" if arg_data["required"] else "Optional") + "**\n\n"
+                main_doc += self.raw_docblock_to_md(arg_data["doc"].replace('"""', "")) + "\n\n"
+
+            class_doc += f"{table_of_contents}\n{main_doc}"
+
+            output_file = section_folder_path / f"{filename}.md"
+            with output_file.open(mode="w") as doc_file:
+                doc_file.write(class_doc)

clearskies_doc_builder/builders/single_class.py

@@ -0,0 +1,79 @@
+from collections import OrderedDict
+from typing import Any
+
+import clearskies
+
+from .builder import Builder
+
+
+class SingleClass(Builder):
+    def __init__(self, branch, modules, classes, doc_root, nav_order):
+        super().__init__(branch, modules, classes, doc_root, nav_order)
+        self.additional_attribute_sources = branch.get("additional_attribute_sources", [])
+        self.args_to_additional_attributes_map = branch.get("args_to_additional_attributes_map", {})
+        self.parent = branch.get("parent", False)
+
+    def build(self):
+        section_name = (
+            clearskies.functional.string.title_case_to_snake_case(self.parent if self.parent else self.title)
+            .replace("_", "-")
+            .replace(" ", "")
+        )
+        section_folder_path = self.doc_root / section_name
+        section_folder_path.mkdir(exist_ok=True)
+        source_class = self.classes.find(f"import_path={self.source}")
+
+        title_snake_case = clearskies.functional.string.title_case_to_snake_case(self.title.replace(" ", "")).replace(
+            "_", "-"
+        )
+        class_doc = self.build_header(self.title, title_snake_case, section_name, self.parent, self.nav_order, False)
+        (elevator_pitch, overview) = self.parse_overview_doc(
+            self.raw_docblock_to_md(source_class.doc).lstrip("\n").lstrip(" ")
+        )
+        class_doc += f"\n\n# {self.title}\n\n{elevator_pitch}\n\n"
+        main_doc = f"## Overview\n\n{overview}\n\n"
+        table_of_contents = " 1. [Overview](#overview)\n"
+
+        default_args = self.default_args()
+
+        # Find the documentation for all of our init args.
+        arguments: dict[str, Any] = OrderedDict()
+        for arg in source_class.init.all_args:
+            if arg == "self":
+                continue
+            arguments[arg] = {
+                "required": arg not in source_class.init.kwargs,
+                "doc": default_args.get(arg, ""),
+            }
+
+        # for various reasons, it's easier to extract docs for all the arguments at once:
+        docs = self.extract_attribute_docs(
+            source_class,
+            list(arguments.keys()),
+            additional_attribute_sources=[
+                self.classes.find(f"import_path={source}") for source in self.additional_attribute_sources
+            ],
+        )
+        for arg, doc in docs.items():
+            # you would think that we would only get arguments that belong to our class, but this isn't the case
+            # because the processing caches results from parent classes, and we don't always use all attributes
+            # available from all our parents.
+            if arg not in arguments:
+                continue
+            arguments[arg]["doc"] = doc
+
+        for index, arg in enumerate(arguments.keys()):
+            arg_data = arguments[arg]
+            table_of_contents += f" {index + 2}. [{arg}](#{arg})\n"
+            main_doc += f"## {arg}\n**" + ("Required" if arg_data["required"] else "Optional") + "**\n\n"
+            main_doc += self.raw_docblock_to_md(arg_data["doc"].replace('"""', "")) + "\n\n"
+
+        class_doc += f"{table_of_contents}\n{main_doc}"
+
+        output_file = section_folder_path / (
+            "index.md"
+            if not self.parent
+            else clearskies.functional.string.title_case_to_snake_case(self.title.replace(" ", "")) + ".md"
+        )
+        with output_file.open(mode="w") as doc_file:
+            doc_file.write(class_doc)

clearskies_doc_builder/builders/single_class_to_section.py

@@ -0,0 +1,39 @@
+import re
+
+import clearskies
+
+from .builder import Builder
+
+
+class SingleClassToSection(Builder):
+    def __init__(self, branch, modules, classes, doc_root, nav_order):
+        super().__init__(branch, modules, classes, doc_root, nav_order)
+        self.docs = branch["docs"]
+
+    def build(self):
+        section_name = clearskies.functional.string.title_case_to_snake_case(self.title).replace("_", "-")
+        section_folder_path = self.doc_root / section_name
+        source_class = self.classes.find(f"import_path={self.source}")
+        self.make_index_from_class_overview(section_name, source_class, section_folder_path)
+
+        for index, doc_data in enumerate(self.docs):
+            title = doc_data["title"]
+            title_snake_case = clearskies.functional.string.title_case_to_snake_case(title.replace(" ", "")).replace(
+                "_", "-"
+            )
+            doc = self.build_header(title, title_snake_case, section_name, self.title, index + 1, False)
+            doc += f"\n\n# {title}\n\n"
+            table_of_contents = ""
+            attribute_docs = ""
+
+            for index, attribute_name in enumerate(doc_data["attributes"]):
+                attribute = source_class.attributes.find(f"name={attribute_name}")
+                table_of_contents += f" {index + 1}. [{attribute_name}]({title_snake_case}.html#{attribute_name})\n"
+                attribute_docs += f"\n\n## {attribute_name}\n\n"
+                attribute_docs += re.sub("\n    ", "\n", self.raw_docblock_to_md(attribute.doc))
+
+            doc += f"{table_of_contents}{attribute_docs}"
+
+            output_file = section_folder_path / f"{title_snake_case}.md"
+            with output_file.open(mode="w") as doc_file:
+                doc_file.write(doc)

clearskies_doc_builder/columns/__init__.py

@@ -0,0 +1,17 @@
+from clearskies_doc_builder.columns.any import Any
+from clearskies_doc_builder.columns.attribute import Attribute
+from clearskies_doc_builder.columns.attributes import Attributes
+from clearskies_doc_builder.columns.base_classes import BaseClasses
+from clearskies_doc_builder.columns.class_column import Class
+from clearskies_doc_builder.columns.module import Module
+from clearskies_doc_builder.columns.module_classes import ModuleClasses
+
+__all__ = [
+    "Any",
+    "Attribute",
+    "Attributes",
+    "BaseClasses",
+    "Class",
+    "Module",
+    "ModuleClasses",
+]

clearskies_doc_builder/columns/any.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Self, overload
+from typing import Any as AnyType
+
+import clearskies
+
+if TYPE_CHECKING:
+    from clearskies import Model
+
+
+class Any(clearskies.Column):
+    is_writeable = clearskies.configs.boolean.Boolean(default=False)
+    _descriptor_config_map = None
+
+    def __init__(self):
+        super().__init__()
+
+    @overload
+    def __get__(self, instance: None, cls: type[Model]) -> Self:
+        pass
+
+    @overload
+    def __get__(self, instance: Model, cls: type[Model]) -> AnyType:
+        pass
+
+    def __get__(self, instance, cls):
+        return super().__get__(instance, cls)
+
+    def __set__(self, instance: Model, value: AnyType) -> None:
+        instance._next_data[self.name] = value
+
+    def from_backend(self, value):
+        return value
+
+    def to_backend(self, data: dict[str, Any]) -> dict[str, Any]:
+        if self.name not in data:
+            return data
+        return {**data, self.name: data[self.name]}

clearskies_doc_builder/columns/attribute.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+from typing import Callable
+
+import clearskies
+
+
+class Attribute(clearskies.columns.HasMany):
+    def __init__(
+        self,
+        child_model_class,
+        readable_child_column_names: list[str] = [],
+        filter: Callable | None = None,
+    ):
+        self.filter = filter
+        super().__init__(
+            child_model_class,
+            foreign_column_name="parent_class",
+            readable_child_column_names=readable_child_column_names,
+        )
+
+    def __get__(self, model, cls):
+        if model is None:
+            self.model_class = cls
+            return self  # type:  ignore
+
+        # this makes sure we're initialized
+        if "name" not in self._config:
+            model.get_columns()
+
+        attributes = self.child_model.where(self.child_model_class.parent_class.equals(model.type))
+        if self.filter:
+            return list(filter(self.filter, attributes))[0]
+
+        return attributes[0]

clearskies_doc_builder/columns/attributes.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+from typing import Callable
+
+import clearskies
+
+
+class Attributes(clearskies.columns.HasMany):
+    def __init__(
+        self,
+        child_model_class,
+        readable_child_column_names: list[str] = [],
+        filter: Callable | None = None,
+    ):
+        self.filter = filter
+        super().__init__(
+            child_model_class,
+            foreign_column_name="parent_class",
+            readable_child_column_names=readable_child_column_names,
+        )
+
+    def __get__(self, model, cls):
+        if model is None:
+            self.model_class = cls
+            return self  # type:  ignore
+
+        # this makes sure we're initialized
+        if "name" not in self._config:
+            model.get_columns()
+
+        attributes = self.child_model.where(self.child_model_class.parent_class.equals(model.type))
+        if self.filter:
+            return list(filter(self.filter, attributes))
+
+        return attributes

clearskies_doc_builder/columns/base_classes.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+import clearskies
+
+
+class BaseClasses(clearskies.columns.HasMany):
+    def __init__(
+        self,
+        readable_child_column_names: list[str] = [],
+    ):
+        self.foreign_column_name = "type"
+        self.readable_child_column_names = readable_child_column_names
+
+    def finalize_configuration(self, model_class, name) -> None:
+        self.child_model_class = model_class
+        super().finalize_configuration(model_class, name)
+
+    def __get__(self, model, cls):
+        if model is None:
+            self.model_class = cls
+            return self  # type:  ignore
+
+        # this makes sure we're initialized
+        if "name" not in self._config:
+            model.get_columns()
+
+        bases = []
+        for cls in model.type.__bases__:
+            bases.append(model.model(model.backend.unpack(cls, model.module)))
+
+        return bases

clearskies_doc_builder/columns/class_column.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Self, overload
+
+import clearskies
+
+if TYPE_CHECKING:
+    from clearskies import Model
+
+
+class Class(clearskies.Column):
+    is_writeable = clearskies.configs.boolean.Boolean(default=False)
+    _descriptor_config_map = None
+
+    def __init__(self):
+        super().__init__()
+
+    @overload
+    def __get__(self, instance: None, cls: type[Model]) -> Self:
+        pass
+
+    @overload
+    def __get__(self, instance: Model, cls: type[Model]) -> type:
+        pass
+
+    def __get__(self, instance, cls):
+        return super().__get__(instance, cls)
+
+    def __set__(self, instance: Model, value: type) -> None:
+        instance._next_data[self.name] = value
+
+    def from_backend(self, value):
+        return value
+
+    def to_backend(self, data: dict[str, Any]) -> dict[str, Any]:
+        if self.name not in data:
+            return data
+        return {**data, self.name: data[self.name]}

clearskies_doc_builder/columns/module.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from types import ModuleType
+from typing import TYPE_CHECKING, Any, Self, overload
+
+import clearskies
+
+if TYPE_CHECKING:
+    from clearskies import Model
+
+
+class Module(clearskies.Column):
+    is_writeable = clearskies.configs.boolean.Boolean(default=False)
+    _descriptor_config_map = None
+
+    def __init__(self):
+        super().__init__()
+
+    @overload
+    def __get__(self, instance: None, cls: type[Model]) -> Self:
+        pass
+
+    @overload
+    def __get__(self, instance: Model, cls: type[Model]) -> ModuleType:
+        pass
+
+    def __get__(self, instance, cls):
+        return super().__get__(instance, cls)
+
+    def __set__(self, instance: Model, value: ModuleType) -> None:
+        instance._next_data[self.name] = value
+
+    def from_backend(self, value):
+        return value
+
+    def to_backend(self, data: dict[str, Any]) -> dict[str, Any]:
+        if self.name not in data:
+            return data
+        return {**data, self.name: data[self.name]}

clearskies_doc_builder/columns/module_classes.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+import clearskies
+
+
+class ModuleClasses(clearskies.columns.HasMany):
+    def __init__(
+        self,
+        child_model_class,
+        readable_child_column_names: list[str] = [],
+    ):
+        super().__init__(
+            child_model_class,
+            foreign_column_name="module",
+            readable_child_column_names=readable_child_column_names,
+        )
+
+    def __get__(self, model, cls):
+        if model is None:
+            self.model_class = cls
+            return self  # type:  ignore
+
+        # this makes sure we're initialized
+        if "name" not in self._config:
+            model.get_columns()
+
+        return self.child_model.where(cls.module.equals(model.module))

clearskies_doc_builder/models/__init__.py

@@ -0,0 +1,13 @@
+from clearskies_doc_builder.models.attribute import Attribute
+from clearskies_doc_builder.models.class_model import Class
+from clearskies_doc_builder.models.class_reference import ClassReference
+from clearskies_doc_builder.models.method import Method
+from clearskies_doc_builder.models.module import Module
+
+__all__ = [
+    "Attribute",
+    "Class",
+    "ClassReference",
+    "Method",
+    "Module",
+]

clearskies_doc_builder/models/attribute.py

@@ -0,0 +1,15 @@
+import clearskies
+
+from clearskies_doc_builder import backends, columns
+
+
+class Attribute(clearskies.Model):
+    id_column_name = "id"
+    backend = backends.AttributeBackend()
+
+    id = clearskies.columns.Integer()
+    name = clearskies.columns.String()
+    type = columns.Class()
+    doc = clearskies.columns.String()
+    attribute = columns.Any()
+    parent_class = columns.Class()

clearskies_doc_builder/models/attribute_reference.py

@@ -0,0 +1,8 @@
+from clearskies.model import ModelClassReference
+
+from . import attribute
+
+
+class AttributeReference(ModelClassReference):
+    def get_model_class(self):
+        return attribute.Attribute

clearskies_doc_builder/models/class_model.py

@@ -0,0 +1,26 @@
+import clearskies
+
+from clearskies_doc_builder import backends, columns
+
+from .attribute_reference import AttributeReference
+from .method_reference import MethodReference
+
+
+class Class(clearskies.Model):
+    id_column_name = "id"
+    backend = backends.ClassBackend()
+
+    id = clearskies.columns.Integer()
+    type = columns.Any()
+    source_file = clearskies.columns.String()
+    import_path = clearskies.columns.String()
+    name = clearskies.columns.String(is_searchable=False)
+    doc = clearskies.columns.String(is_searchable=False)
+    module = columns.Module()
+    base_classes = columns.BaseClasses()
+    attributes = columns.Attributes(AttributeReference)
+    methods = columns.Attributes(AttributeReference, filter=lambda attribute: callable(attribute.attribute))
+    init = columns.Attribute(
+        MethodReference,
+        filter=lambda attribute: attribute.name == "__init__",
+    )

clearskies_doc_builder/models/class_reference.py

@@ -0,0 +1,8 @@
+from clearskies.model import ModelClassReference
+
+from . import class_model
+
+
+class ClassReference(ModelClassReference):
+    def get_model_class(self):
+        return class_model.Class

clearskies_doc_builder/models/method.py

@@ -0,0 +1,20 @@
+import clearskies
+
+from clearskies_doc_builder import backends, columns
+
+
+class Method(clearskies.Model):
+    id_column_name = "id"
+    backend = backends.AttributeBackend()
+
+    id = clearskies.columns.Integer()
+    name = clearskies.columns.String()
+    type = columns.Class()
+    doc = clearskies.columns.String()
+    attribute = columns.Any()
+    parent_class = columns.Class()
+
+    args = columns.Any()
+    kwargs = columns.Any()
+    all_args = columns.Any()
+    defaults = columns.Any()

clearskies_doc_builder/models/method_reference.py

@@ -0,0 +1,8 @@
+from clearskies.model import ModelClassReference
+
+from . import method
+
+
+class MethodReference(ModelClassReference):
+    def get_model_class(self):
+        return method.Method

clearskies_doc_builder/models/module.py

@@ -0,0 +1,19 @@
+import clearskies
+
+from clearskies_doc_builder import backends, columns
+
+from . import class_reference
+
+
+class Module(clearskies.Model):
+    id_column_name = "id"
+    backend = backends.ModuleBackend()
+
+    id = clearskies.columns.Integer()
+    import_path = clearskies.columns.String()
+    source_file = clearskies.columns.String()
+    is_builtin = clearskies.columns.Boolean()
+    name = clearskies.columns.String()
+    doc = clearskies.columns.String(is_searchable=False)
+    module = columns.Module()
+    classes = columns.ModuleClasses(class_reference.ClassReference)

clearskies_doc_builder/models/module_reference.py

@@ -0,0 +1,8 @@
+from clearskies.model import ModelClassReference
+
+from . import module
+
+
+class ModuleReference(ModelClassReference):
+    def get_model_class(self):
+        return module.Module

clearskies_doc_builder/prepare_doc_space.py

@@ -0,0 +1,25 @@
+import pathlib
+import shutil
+
+
+def prepare_doc_space(project_root):
+    project_path = pathlib.Path(project_root)
+    build_path = project_path / "build"
+    doc_path = build_path / "docs"
+    jekyll_path = project_path / "jekyll"
+    includes_path = jekyll_path / "_includes"
+
+    if doc_path.is_dir():
+        shutil.rmtree(doc_path)
+    build_path.mkdir(parents=True, exist_ok=True)
+
+    for file in jekyll_path.glob("*"):
+        if not file.is_file():
+            continue
+        shutil.copy2(str(file), str(build_path / file.name))
+
+    shutil.copytree(str(jekyll_path / "docs"), str(build_path / "docs"))
+    if includes_path.is_dir():
+        shutil.copytree(str(includes_path), str(build_path / "_includes"))
+
+    return str(doc_path)

clear_skies_doc_builder-2.0.1.dist-info/RECORD

@@ -1,4 +0,0 @@
-clear_skies_doc_builder-2.0.1.dist-info/METADATA,sha256=lulMc1B5pmXTvq2qrhrAAUVhsnBqyNy2wcbpAX4rPDo,3075
-clear_skies_doc_builder-2.0.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-clear_skies_doc_builder-2.0.1.dist-info/licenses/LICENSE,sha256=pgH32-xrgpIDf6WIBUA1X19o-sIHIXgToCxo-tBWDdw,1071
-clear_skies_doc_builder-2.0.1.dist-info/RECORD,,