plain-parser 0.1.0__py3-none-any.whl

plain_parser/plain_file.py

@@ -0,0 +1,861 @@
+import io
+import os
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Iterable, Sequence, cast
+from urllib.parse import urlparse
+
+import frontmatter
+import mistletoe
+import mistletoe.block_token
+from liquid2 import DictLoader, Environment
+from mistletoe.block_token import List, Paragraph, Quote
+from mistletoe.markdown_renderer import Fragment, MarkdownRenderer
+from mistletoe.span_token import Emphasis, Link, RawText, SpanToken, Strong
+from mistletoe.token import Token
+from mistletoe.utils import traverse
+
+from plain_parser import concept_utils, loaders, plain_spec
+from plain_parser.exceptions import (
+    ImportedModuleWithFunctionalitiesError,
+    MissingFunctionalitiesError,
+    ModuleDoesNotExistError,
+    PlainSyntaxError,
+    UnsupportedBase64Content,
+)
+from plain_parser.liquid_nodes import Plain2CodeIncludeTag, Plain2CodeLoaderMixin
+from plain_parser.loaders import find_large_base64_blob
+
+RESOURCE_MARKER = "[resource]"
+
+IMPORT_DIRECTIVE = "import"
+REQUIRES_DIRECTIVE = "requires"
+REQUIRED_CONCEPTS_DIRECTIVE = "required_concepts"
+EXPORTED_CONCEPTS_DIRECTIVE = "exported_concepts"
+
+PLAIN_SOURCE_FILE_EXTENSION = ".plain"
+
+PLAIN_SOURCE_TEMPLATE = {
+    plain_spec.DEFINITIONS: None,
+    plain_spec.NON_FUNCTIONAL_REQUIREMENTS: None,
+    plain_spec.TEST_REQUIREMENTS: None,
+}
+
+
+@dataclass
+class PlainFileParseResult:
+    plain_source: dict
+    plain_source_obj: frontmatter.Post
+    required_modules: list[str]
+    required_concepts: list[str]
+
+
+class PlainRenderer(MarkdownRenderer):
+    def render_link(self, token: Link) -> Iterable[Fragment]:
+        yield from self.embed_span(
+            Fragment(f"`{RESOURCE_MARKER}"),
+            cast("Iterable[SpanToken]", token.children),
+            Fragment("`"),
+        )
+
+
+def get_filename_from_module_name(module_name: str) -> str:
+    return f"{module_name}{PLAIN_SOURCE_FILE_EXTENSION}"
+
+
+def get_module_name_from_filename(filename: str) -> str:
+    return filename.replace(PLAIN_SOURCE_FILE_EXTENSION, "")
+
+
+def remove_quotes(token):
+    # If the token has no children, there's nothing to remove.
+    if not hasattr(token, "children") or token.children is None:
+        return
+
+    # Convert children to a list for easy filtering.
+    children_list = list(token.children)
+
+    # Build a filtered list that excludes Quote tokens.
+    new_children = []
+    for child in children_list:
+        if not isinstance(child, Quote):
+            # Recursively remove quotes in any nested children.
+            remove_quotes(child)
+            new_children.append(child)
+
+    # Convert the filtered list back to a tuple (or a list, if you prefer).
+    # Mistletoe tokens often expect a tuple, so tuple is safest.
+    token.children = tuple(new_children)
+
+
+def check_section_for_linked_resources(section):
+    linked_resources = []
+    for link in traverse(section, klass=Link):
+        parsed_url = urlparse(link.node.target)
+        if parsed_url.scheme != "" or os.path.isabs(link.node.target):
+            raise PlainSyntaxError(
+                f"Plain syntax error: Only relative links are allowed (text: {link.node.children[0].content}, target: {link.node.target})."
+            )
+
+        if not os.path.exists(link.node.target):
+            raise PlainSyntaxError(f"Plain syntax error: Link {link.node.target} does not exist.")
+
+        if not os.path.isfile(link.node.target):
+            raise PlainSyntaxError(f"Plain syntax error: Link {link.node.target} must be a file.")
+
+        if len(link.node.children) != 1:
+            raise PlainSyntaxError(f"Plain syntax error: Link must have text specified (link: {link.node.target}).")
+
+        linked_resource_file_extension = os.path.splitext(os.path.basename(link.node.target))[1]
+        if linked_resource_file_extension == ".plain":
+            raise PlainSyntaxError(
+                f"Referenced resource '{link.node.target}' is a .plain file. "
+                f"Referencing .plain files through Linked Resources is not supported. "
+                "Please use the import section to include definitions or implementation/test requirements from another module."
+            )
+
+        linked_resources.append({"text": link.node.children[0].content, "target": link.node.target})
+
+    if linked_resources:
+        section.linked_resources = linked_resources
+
+
+def check_for_linked_resources(plain_source):
+    for specification_heading in plain_spec.ALLOWED_SPECIFICATION_HEADINGS:
+        if specification_heading in plain_source and hasattr(plain_source[specification_heading], "children"):
+            for requirement in plain_source[specification_heading].children:
+                check_section_for_linked_resources(requirement)
+
+                if hasattr(requirement, plain_spec.ACCEPTANCE_TESTS):
+                    for acceptance_test in requirement.acceptance_tests:
+                        check_section_for_linked_resources(acceptance_test)
+
+
+def process_section_code_variables(section, code_variables):
+    if "markdown" not in section:
+        return
+
+    code_variables_array = []
+    for key, value in code_variables.items():
+        if key not in section["markdown"]:
+            continue
+
+        variable_name = next(iter(value))
+        section["markdown"] = section["markdown"].replace(key, f"{{{{ {variable_name} }}}}")
+
+        code_variables_array.append({"name": variable_name, "value": value[variable_name]})
+
+    if code_variables_array:
+        section["code_variables"] = code_variables_array
+
+
+def process_code_variables(plain_source, code_variables):
+    for specification_heading in plain_spec.ALLOWED_SPECIFICATION_HEADINGS:
+        if specification_heading in plain_source:
+            for requirement in plain_source[specification_heading]:
+                process_section_code_variables(requirement, code_variables)
+
+
+def has_functional_specs_section(plain_source) -> bool:
+    """Whether the module declares a ***functional specs*** section (even if it is empty)."""
+    return plain_spec.FUNCTIONAL_REQUIREMENTS in plain_source
+
+
+def count_functionalities(plain_source) -> int:
+    """Number of functionalities under the ***functional specs*** section (0 if absent or empty)."""
+    section = plain_source.get(plain_spec.FUNCTIONAL_REQUIREMENTS)
+    return len(section.children) if section is not None else 0
+
+
+def validate_functionalities_have_implementation_reqs(plain_source, module_name, has_requires=False) -> None:
+    """Raise if the module has functionalities but no implementation reqs are specified.
+
+    ``has_requires`` indicates whether the module depends on another module via ``requires``.
+    Unlike ``import``, ``requires`` does not inherit implementation reqs, so a hint is added to
+    steer users who expected inheritance.
+    """
+    implementation_reqs = plain_source[plain_spec.NON_FUNCTIONAL_REQUIREMENTS]
+    has_implementation_reqs = (
+        implementation_reqs is not None
+        and hasattr(implementation_reqs, "children")
+        and len(implementation_reqs.children) > 0
+    )
+    if not has_implementation_reqs:
+        message = (
+            f"Plain syntax error: Module '{module_name}' defines functionalities but specifies no "
+            f"{plain_spec.NON_FUNCTIONAL_REQUIREMENTS}. At least one implementation req is required."
+        )
+        if has_requires:
+            message += (
+                " Implementation reqs are not inherited through 'requires' — "
+                "add them to this module, or 'import' a module that provides them."
+            )
+        raise PlainSyntaxError(message)
+
+
+def _is_acceptance_test_heading(token) -> tuple[bool, str | None]:
+    """
+    Check if token is an 'Acceptance test:' heading.
+
+    This method is going to evaluate to True when:
+
+    - The token is a Paragraph
+    - The paragraph has 1 child
+    - The child is an Emphasis (*** ... *** marker)
+    - The Emphasis has 1 child (the raw text content)
+    - The child of the Emphasis is a Strong
+    - The Strong has 1 child
+    - The child of the Strong is a RawText
+    - The RawText content is equal to "Acceptance test:"
+
+    This is expected structure for the acceptance test heading.
+    """
+    if not isinstance(token, Paragraph):
+        return False, None
+
+    # Check for the specific structure, descending one level at a time. mistletoe types
+    # ``children`` as ``Optional[Iterable[Token]]``, so cast each level to a concrete
+    # sequence before indexing/len.
+    children = cast("Sequence[Token]", token.children)
+    if len(children) != 1 or not isinstance(children[0], Emphasis):
+        return False, None
+
+    emphasis_children = cast("Sequence[Token]", children[0].children)
+    if len(emphasis_children) != 1 or not isinstance(emphasis_children[0], Strong):
+        return False, None
+
+    strong_children = cast("Sequence[Token]", emphasis_children[0].children)
+    if len(strong_children) != 1 or not isinstance(strong_children[0], RawText):
+        return False, None
+
+    # Check the actual text content
+    content = strong_children[0].content.strip()
+    if content == plain_spec.ACCEPTANCE_TEST_HEADING:
+        return True, None
+    problem = f"Syntax error at line {token.line_number}: Invalid acceptance test heading (`{content}`). Expected: `{plain_spec.ACCEPTANCE_TEST_HEADING}.`"
+    return False, problem
+
+
+def _find_acceptance_test_heading(token):
+    """
+    Recursively search a token tree for an `acceptance tests` heading.
+
+    Returns the heading token if one is found anywhere in the subtree, otherwise None.
+    Used to detect acceptance tests nested under sections other than functional specs.
+    """
+    is_acceptance_test_heading, _ = _is_acceptance_test_heading(token)
+    if is_acceptance_test_heading:
+        return token
+
+    if getattr(token, "children", None):
+        for child in token.children:
+            found = _find_acceptance_test_heading(child)
+            if found is not None:
+                return found
+
+    return None
+
+
+def _process_single_acceptance_test_requirement(functional_requirement: mistletoe.block_token.ListItem):
+    """
+    Process a single functionality to extract acceptance tests.
+
+    Expected functional_requirement properties:
+    - Is a list item
+    - If acceptance tests are specified, it has 3 children:
+        - List item element with functionality instructions/text
+        - Paragraph with `***Acceptance test:***` heading
+        - List of acceptance tests
+    - If acceptance tests are not specified, it has 1 child:
+        - List item element with functionality instructions/text
+    """
+    new_children: list = []
+    functional_requirement_children = iter(functional_requirement.children or ())
+    acceptance_tests_found_already = False
+
+    for functional_requirement_child in functional_requirement_children:
+        is_acceptance_test_heading, acceptance_test_heading_problem = _is_acceptance_test_heading(
+            functional_requirement_child
+        )
+        if acceptance_test_heading_problem:
+            # Handle the case when the heading is not valid. This case includes cases such as:
+            # - Writing `acceptance test` instead of `acceptance tests` (or any other syntax diffs).
+            # - Instead of specifying `acceptance tests` below the functionality, creator of the plain file
+            #   might have specified some other building block (e.g. `implementation reqs`)
+            raise PlainSyntaxError(f"Plain syntax error: {acceptance_test_heading_problem}")
+
+        if is_acceptance_test_heading:
+            if acceptance_tests_found_already:
+                # Handle edge case of duplicated ***acceptance tests*** heading
+                raise PlainSyntaxError(
+                    f"Plain syntax error: Syntax error at line {functional_requirement_child.line_number}: Duplicate 'acceptance tests' heading found within the same functionality. Only one block of acceptance tests is allowed per functionality."
+                )
+
+            try:
+                # If there is an acceptance test heading, the next token should be a list, with children being list items
+                next_token = next(functional_requirement_children)
+                if isinstance(next_token, List):
+                    # Found valid acceptance test list -> Assign it property acceptance_tests and append all list items to it
+                    if not hasattr(functional_requirement, plain_spec.ACCEPTANCE_TESTS):
+                        functional_requirement.acceptance_tests = []
+                    for list_item in next_token.children:
+                        functional_requirement.acceptance_tests.append(list_item)
+                    acceptance_tests_found_already = True
+                else:
+                    # Not followed by a list, keep both tokens
+                    new_children.append(functional_requirement_child)
+                    new_children.append(next_token)
+            except StopIteration:
+                # No next token, keep this one
+                new_children.append(functional_requirement_child)
+        else:
+            # Regular token, keep it
+            new_children.append(functional_requirement_child)
+
+    # Assign the children property to all the children of the functionality from previous, with exception
+    # of those we parsed as acceptance tests
+    container_type = cast(type, type(functional_requirement.children))
+    functional_requirement.children = container_type(new_children)
+
+
+def process_acceptance_tests(plain_source):
+    # Early returns for cases without functionalities
+    if plain_spec.FUNCTIONAL_REQUIREMENTS not in plain_source:
+        return
+    frs = plain_source[plain_spec.FUNCTIONAL_REQUIREMENTS]
+    if not hasattr(frs, "children"):
+        return
+
+    # Process each functionality
+    for functional_requirement in frs.children:
+        if not hasattr(functional_requirement, "children"):
+            continue
+
+        # Process each requirement to extract acceptance tests
+        _process_single_acceptance_test_requirement(functional_requirement)
+
+
+def get_raw_text(token):
+    if isinstance(token, RawText):
+        yield token.content
+    elif hasattr(token, "children"):
+        if token.children is None:
+            return
+        for child in token.children:
+            yield from get_raw_text(child)
+    elif hasattr(token, "content"):
+        yield token.content
+    else:
+        raise Exception(f"Unknown token type: {type(token)}")
+
+
+def marshall_plain_source(input_plain_source):
+    plain_source = {}
+    with PlainRenderer() as renderer:
+        if "ID" in input_plain_source:
+            plain_source["ID"] = input_plain_source["ID"]
+
+        if "Heading" in input_plain_source:
+            plain_source["Heading"] = input_plain_source["Heading"]
+
+        for specification_heading in plain_spec.ALLOWED_SPECIFICATION_HEADINGS:
+            if specification_heading in input_plain_source and hasattr(
+                input_plain_source[specification_heading], "children"
+            ):
+                list_of_requirements = []
+                for requirement in input_plain_source[specification_heading].children:
+                    requirement_section = {"markdown": renderer.render(requirement).strip()}
+                    if hasattr(requirement, "linked_resources"):
+                        requirement_section["linked_resources"] = requirement.linked_resources
+
+                    if hasattr(requirement, plain_spec.ACCEPTANCE_TESTS):
+                        requirement_section[plain_spec.ACCEPTANCE_TESTS] = []
+                        for acceptance_test in requirement.acceptance_tests:
+                            acceptance_test_section = {"markdown": renderer.render(acceptance_test).strip()}
+                            if hasattr(acceptance_test, "linked_resources"):
+                                acceptance_test_section["linked_resources"] = acceptance_test.linked_resources
+                            requirement_section[plain_spec.ACCEPTANCE_TESTS].append(acceptance_test_section)
+
+                    list_of_requirements.append(requirement_section)
+
+                plain_source[specification_heading] = list_of_requirements
+
+    return plain_source
+
+
+class Plain2CodeDictLoader(Plain2CodeLoaderMixin, DictLoader):
+    pass
+
+
+def render_plain_source(plain_source, loaded_templates, code_variables):
+    env = Environment(loader=Plain2CodeDictLoader(loaded_templates))
+    env.tags["include"] = Plain2CodeIncludeTag(env)
+    env.filters["code_variable"] = plain_spec.code_variable_liquid_filter
+    env.filters["prohibited_chars"] = plain_spec.prohibited_chars_liquid_filter
+
+    template = env.from_string(plain_source)
+
+    return template.render(code_variables=code_variables)
+
+
+def process_imports(
+    plain_source: dict,
+    imports: list[str],
+    code_variables: dict,
+    template_dirs: list[str],
+    imported_modules: list[str],
+    modules_trace: list[str],
+) -> list[str]:
+    required_concepts = list[str]()
+    for module_name in imports:
+        if module_name in modules_trace:
+            raise PlainSyntaxError(f"Plain syntax error: Circular import detected: {module_name}.")
+
+        if module_name in imported_modules:
+            continue
+
+        plain_file_parse_result = parse_plain_file(
+            module_name, code_variables, template_dirs, imported_modules, modules_trace
+        )
+
+        if has_functional_specs_section(plain_file_parse_result.plain_source):
+            raise ImportedModuleWithFunctionalitiesError(
+                f"Module '{module_name}' is imported but contains functional specs. "
+                f"Imported modules may only provide definitions, implementation reqs, and test reqs — "
+                f"use 'requires' instead of 'import' if this module's functionalities should be built on."
+            )
+
+        for specification_heading in plain_file_parse_result.plain_source:
+            if specification_heading not in plain_spec.ALLOWED_IMPORT_SPECIFICATION_HEADINGS:
+                raise PlainSyntaxError(
+                    f"Plain syntax error: Invalid specification heading (`{specification_heading}`). Allowed headings: {', '.join(plain_spec.ALLOWED_IMPORT_SPECIFICATION_HEADINGS)}"
+                )
+
+            if plain_source[specification_heading] is None:
+                plain_source[specification_heading] = plain_file_parse_result.plain_source[specification_heading]
+            elif plain_file_parse_result.plain_source[specification_heading] is not None:
+                plain_source[specification_heading].children.extend(
+                    plain_file_parse_result.plain_source[specification_heading].children
+                )
+
+        if REQUIRED_CONCEPTS_DIRECTIVE in plain_file_parse_result.plain_source_obj.metadata:
+            for item in plain_file_parse_result.plain_source_obj.metadata[REQUIRED_CONCEPTS_DIRECTIVE]:
+                assert isinstance(
+                    item, str
+                ), f"Syntax error: Invalid {REQUIRED_CONCEPTS_DIRECTIVE} metadata. Expected a string."
+                new_concepts, _ = concept_utils.extract_concepts_from_definition(item)
+                required_concepts.extend(new_concepts)
+
+            required_concepts.extend(plain_file_parse_result.required_concepts)
+
+        imported_modules.append(module_name)
+
+    return required_concepts
+
+
+def normalize_line_endings(plain_source_text: str) -> str:
+    return plain_source_text.replace("\r\n", "\n").replace("\r", "\n")
+
+
+def restore_stripped_lines(plain_source_text: str, content: str) -> str:
+    stripped_source = plain_source_text.rstrip()
+    if not content or not stripped_source.endswith(content):
+        return content
+
+    stripped_line_count = stripped_source[: len(stripped_source) - len(content)].count("\n")
+    return "\n" * stripped_line_count + content
+
+
+def read_plain_source_metadata(plain_source_text):
+    try:
+        plain_source_obj = frontmatter.loads(plain_source_text)
+    except Exception as e:
+        raise PlainSyntaxError(f"Plain syntax error: Invalid frontmatter: {e}")
+
+    for directive in [EXPORTED_CONCEPTS_DIRECTIVE, REQUIRED_CONCEPTS_DIRECTIVE]:
+        if directive in plain_source_obj.metadata:
+            assert isinstance(
+                plain_source_obj.metadata[directive], list
+            ), f"Syntax error: Invalid {directive} metadata. Expected a list."
+            prepared_metadata = []
+            for item in plain_source_obj.metadata[directive]:
+                if isinstance(item, dict):
+                    for k, v in item.items():
+                        prepared_metadata.append(f"- {k}: {v}")
+                elif isinstance(item, str):
+                    prepared_metadata.append(f"- {item}")
+                else:
+                    raise PlainSyntaxError(
+                        f"Plain syntax error: Invalid {directive} metadata. Expected a dictionary or a string."
+                    )
+
+            plain_source_obj.metadata[directive] = prepared_metadata
+
+    return plain_source_obj
+
+
+def parse_plain_source(  # noqa: C901
+    plain_source_text: str,
+    code_variables: dict,
+    template_dirs: list[str],
+    imported_modules: list[str],
+    modules_trace: list[str],
+) -> PlainFileParseResult:
+    plain_source_text = normalize_line_endings(plain_source_text)
+
+    plain_source_obj = read_plain_source_metadata(plain_source_text)
+
+    plain_source = PLAIN_SOURCE_TEMPLATE.copy()
+
+    if IMPORT_DIRECTIVE in plain_source_obj.metadata:
+        required_concepts = process_imports(
+            plain_source,
+            plain_source_obj.metadata[IMPORT_DIRECTIVE],
+            code_variables,
+            template_dirs,
+            imported_modules,
+            modules_trace,
+        )
+    else:
+        required_concepts = list[str]()
+
+    [_, loaded_templates] = loaders.get_loaded_templates(template_dirs, plain_source_text)
+
+    plain_source_content = restore_stripped_lines(plain_source_text, plain_source_obj.content)
+
+    plain_source_full_text = render_plain_source(plain_source_content, loaded_templates, code_variables)
+
+    plain_file = mistletoe.Document(io.StringIO(plain_source_full_text))
+
+    remove_quotes(plain_file)
+
+    current_specification_heading = None
+    processed_specification_headings = set[str]()
+    for token in plain_file.children:
+        token_text = "".join(get_raw_text(token)).strip()
+        if isinstance(token, Paragraph):
+            paragraph_children = cast("Sequence[Token]", token.children)
+            emphasis_children = (
+                cast("Sequence[Token]", paragraph_children[0].children)
+                if paragraph_children and isinstance(paragraph_children[0], Emphasis)
+                else ()
+            )
+            if not (
+                len(paragraph_children) == 1
+                and isinstance(paragraph_children[0], Emphasis)
+                and len(emphasis_children) == 1
+                and isinstance(emphasis_children[0], Strong)
+            ):
+
+                raise PlainSyntaxError(
+                    f"Plain syntax error: Syntax error at line {token.line_number}: Invalid specification (`{token_text}`)"
+                )
+
+            strong_children = cast("Sequence[Token]", emphasis_children[0].children)
+            specification_heading = cast(RawText, strong_children[0]).content
+
+            if specification_heading == plain_spec.ACCEPTANCE_TEST_HEADING:
+                raise PlainSyntaxError(
+                    f"Plain syntax error: Syntax error at line {token.line_number}: {plain_spec.ACCEPTANCE_TEST_HEADING} heading should be nested under specific functional spec."
+                )
+
+            if specification_heading not in plain_spec.ALLOWED_SPECIFICATION_HEADINGS:
+                raise PlainSyntaxError(
+                    f"Plain syntax error: Syntax error at line {token.line_number}: Invalid specification heading (`{specification_heading}`). Allowed headings: {', '.join(plain_spec.ALLOWED_SPECIFICATION_HEADINGS)}"
+                )
+
+            if (
+                specification_heading == plain_spec.FUNCTIONAL_REQUIREMENTS
+                and specification_heading not in plain_source
+            ):
+                plain_source[specification_heading] = None
+
+            if specification_heading in processed_specification_headings:
+                raise PlainSyntaxError(
+                    f"Plain syntax error: Syntax error at line {token.line_number}: Duplicate specification heading (`{specification_heading}`)"
+                )
+
+            if specification_heading == plain_spec.DEFINITIONS and current_specification_heading is not None:
+                raise PlainSyntaxError(
+                    f"Plain syntax error: Syntax error at line {token.line_number}: Definitions specification must be the first specification in the section  (`{token_text}`)"
+                )
+
+            current_specification_heading = specification_heading
+            if plain_source[current_specification_heading] is None:
+                plain_source[current_specification_heading] = mistletoe.Document("")
+
+            processed_specification_headings.add(current_specification_heading)
+        elif isinstance(token, List):
+            if current_specification_heading is None:
+                raise PlainSyntaxError(
+                    f"Plain syntax error: Syntax error at line {token.line_number}: Missing specification heading (`{token_text}`)"
+                )
+            plain_source[current_specification_heading].children.extend(token.children)
+
+        else:
+            raise PlainSyntaxError(
+                f"Plain syntax error: Syntax error at line {token.line_number}: Invalid source structure  (`{token_text}`)"
+            )
+
+    # Acceptance tests are only allowed nested under functional specs. Reject them when
+    # they are nested under any other section (e.g. definitions, implementation reqs, test reqs).
+    for non_functional_heading in (
+        plain_spec.DEFINITIONS,
+        plain_spec.NON_FUNCTIONAL_REQUIREMENTS,
+        plain_spec.TEST_REQUIREMENTS,
+    ):
+        section = plain_source.get(non_functional_heading)
+        if section is None:
+            continue
+        nested_acceptance_test_heading = _find_acceptance_test_heading(section)
+        if nested_acceptance_test_heading is not None:
+            raise PlainSyntaxError(
+                f"Plain syntax error: Syntax error at line {nested_acceptance_test_heading.line_number}: {plain_spec.ACCEPTANCE_TEST_HEADING} heading should be nested under specific functional spec."
+            )
+
+    defined_concepts = set[str]()
+    if plain_source[plain_spec.DEFINITIONS] is not None:
+        with PlainRenderer() as renderer:
+            for token in plain_source[plain_spec.DEFINITIONS].children:
+                rendered_token = renderer.render(token)
+                new_concepts, _ = concept_utils.extract_concepts_from_definition(rendered_token)
+                defined_concepts.update(new_concepts)
+                for concept in new_concepts:
+                    if concept in required_concepts:
+                        required_concepts.remove(concept)
+
+    if EXPORTED_CONCEPTS_DIRECTIVE in plain_source_obj.metadata:
+        module_name = modules_trace[-1] if modules_trace else None
+        subject = f"Module '{module_name}'" if module_name else "Module"
+
+        for entry in plain_source_obj.metadata[EXPORTED_CONCEPTS_DIRECTIVE]:
+            exported_concepts, errors = concept_utils.extract_concepts_from_definition(entry)
+            if errors:
+                raise PlainSyntaxError(
+                    f"Plain syntax error: {subject} exports an invalid concept (`{entry}`). "
+                    f"Expected a concept token, e.g. :ConceptName:."
+                )
+
+            for exported_concept in exported_concepts:
+                if exported_concept in concept_utils.DEFAULT_CONCEPTS:
+                    raise PlainSyntaxError(
+                        f"Plain syntax error: {subject} cannot export default concept {exported_concept}. "
+                        f"Only user-defined concepts can be exported."
+                    )
+
+                if exported_concept not in defined_concepts:
+                    message = f"Plain syntax error: {subject} exports undefined concept {exported_concept}."
+                    if REQUIRES_DIRECTIVE in plain_source_obj.metadata:
+                        message += (
+                            " An exported concept must be defined in this module's definitions or in an "
+                            "imported module; a concept inherited through requires cannot be re-exported."
+                        )
+                    raise PlainSyntaxError(message)
+
+    required_modules = []
+    if REQUIRES_DIRECTIVE in plain_source_obj.metadata:
+        required_modules = plain_source_obj.metadata[REQUIRES_DIRECTIVE]
+
+    return PlainFileParseResult(
+        plain_source=plain_source,
+        plain_source_obj=plain_source_obj,
+        required_modules=required_modules,
+        required_concepts=required_concepts,
+    )
+
+
+def read_module_plain_source(module_name: str, template_dirs: list[str]) -> str:
+    plain_source_text = loaders.open_from(template_dirs, module_name + PLAIN_SOURCE_FILE_EXTENSION)
+    if plain_source_text is None:
+        raise ModuleDoesNotExistError(f"Module does not exist ({module_name}).")
+
+    blob = find_large_base64_blob(plain_source_text)
+    if blob is not None:
+        raise UnsupportedBase64Content(
+            f"Module '{module_name}' contains a base64-encoded blob ({len(blob)} characters) "
+            "inlined in the specification. This is not supported."
+            "Remove the base64 data from the .plain file or if necessary,"
+            "include the binary file path in the specification."
+        )
+
+    return plain_source_text
+
+
+def parse_plain_file(
+    module_name: str,
+    code_variables: dict,
+    template_dirs: list[str],
+    imported_modules: list[str],
+    modules_trace: list[str],
+) -> PlainFileParseResult:  # noqa: C901
+    plain_source_text = read_module_plain_source(module_name, template_dirs)
+
+    return parse_plain_source(
+        plain_source_text,
+        code_variables,
+        template_dirs,
+        imported_modules,
+        modules_trace + [module_name],
+    )
+
+
+def process_required_modules(
+    required_modules: list[str],
+    code_variables: dict,
+    template_dirs: list[str],
+    all_required_modules: list[str],
+    modules_trace: list[str],
+) -> list[mistletoe.block_token.token]:
+    exported_definitions = list[mistletoe.block_token.token]()
+    for module_name in required_modules:
+        if module_name in modules_trace:
+            raise PlainSyntaxError(f"Plain syntax error: Circular required module detected: {module_name}.")
+
+        if len(all_required_modules) > 0 and module_name == all_required_modules[-1]:
+            continue
+
+        plain_file_parse_result = parse_plain_file(
+            module_name, code_variables, template_dirs, imported_modules=[], modules_trace=[]
+        )
+
+        if len(plain_file_parse_result.required_modules) == 0:
+            if len(all_required_modules) > 0:
+                # For now we require that there is fixed order how required modules are dependent.
+                # In the future we will support the cases where required modules can be rendered independently
+                # and then merged (somehow).
+                raise PlainSyntaxError(
+                    f"Plain syntax error: There must be a fixed order how required modules are dependent ({module_name})."
+                )
+        else:
+            process_required_modules(
+                plain_file_parse_result.required_modules,
+                code_variables,
+                template_dirs,
+                all_required_modules,
+                modules_trace + [module_name],
+            )
+
+        if EXPORTED_CONCEPTS_DIRECTIVE in plain_file_parse_result.plain_source_obj.metadata:
+            exported_concepts = list[str]()
+            for concept in plain_file_parse_result.plain_source_obj.metadata[EXPORTED_CONCEPTS_DIRECTIVE]:
+                exported_concepts.extend(concept_utils.extract_concepts_from_definition(concept)[0])
+
+            with PlainRenderer() as renderer:
+                for exported_concept in exported_concepts:
+                    exported_definitions.extend(
+                        concept_utils.find_concept_definitions_in_plain_source(
+                            exported_concept, plain_file_parse_result.plain_source, renderer
+                        )
+                    )
+
+        all_required_modules.append(module_name)
+
+    return exported_definitions
+
+
+def process_exported_definitions(plain_source: dict, exported_definitions: list[mistletoe.block_token.token]) -> None:
+    if len(exported_definitions) == 0:
+        return
+
+    if plain_source[plain_spec.DEFINITIONS] is None:
+        plain_source[plain_spec.DEFINITIONS] = mistletoe.Document("")
+
+    with PlainRenderer() as renderer:
+        for exported_definition in exported_definitions:
+            add_defintion = True
+
+            exported_rendered_definition = renderer.render(exported_definition).strip()
+            for definition in plain_source[plain_spec.DEFINITIONS].children:
+                rendered_definition = renderer.render(definition).strip()
+                if exported_rendered_definition == rendered_definition:
+                    add_defintion = False
+                    break
+
+            if add_defintion:
+                plain_source[plain_spec.DEFINITIONS].children.append(exported_definition)
+
+
+def plain_file_parser(  # noqa: C901
+    plain_source_file_name: str,
+    template_dirs: list[str],
+) -> tuple[str, dict, list[str]]:
+    # code_variables are used to pass code variables to the plain source
+    # they need to be passed as an argument to the function because they populated when liquid templating is applied
+    # and we need to pass them to the marshalled_plain_source_tree after it's rendered
+    plain_source_file_path = Path(plain_source_file_name)
+    if plain_source_file_path.suffix != PLAIN_SOURCE_FILE_EXTENSION:
+        raise PlainSyntaxError(
+            f"Plain syntax error: Invalid plain file extension: {plain_source_file_path.suffix}. Expected: {PLAIN_SOURCE_FILE_EXTENSION}."
+        )
+
+    module_name = (
+        plain_source_file_path.stem
+        if plain_source_file_path.is_absolute()
+        else plain_source_file_path.with_suffix("").as_posix()
+    )
+
+    code_variables = {}
+
+    plain_file_parse_result = parse_plain_file(
+        module_name,
+        code_variables,
+        template_dirs,
+        imported_modules=[],
+        modules_trace=[],
+    )
+
+    if len(plain_file_parse_result.required_concepts) > 0:
+        missing_required_concepts_msg = "Missing required concepts: "
+        missing_required_concepts_msg += ", ".join(plain_file_parse_result.required_concepts)
+        raise PlainSyntaxError(
+            f"Plain syntax error: Not all required concepts were defined. {missing_required_concepts_msg}."
+        )
+
+    if not has_functional_specs_section(plain_file_parse_result.plain_source):
+        # No ***functional specs*** section at all: valid as an import, but not renderable. Usage error.
+        raise MissingFunctionalitiesError(
+            f"Module '{module_name}' does not have any functionality specified. "
+            f"At least one functionality is required for rendering."
+        )
+
+    if count_functionalities(plain_file_parse_result.plain_source) == 0:
+        # ***functional specs*** section present but empty: invalid in every role. Syntax error.
+        raise PlainSyntaxError(
+            f"Plain syntax error: Module '{module_name}' has an empty "
+            f"'{plain_spec.FUNCTIONAL_REQUIREMENTS}' section. At least one functionality must be specified."
+        )
+
+    validate_functionalities_have_implementation_reqs(
+        plain_file_parse_result.plain_source,
+        module_name,
+        has_requires=bool(plain_file_parse_result.required_modules),
+    )
+
+    exported_definitions = process_required_modules(
+        plain_file_parse_result.required_modules,
+        code_variables={},
+        template_dirs=template_dirs,
+        all_required_modules=[],
+        modules_trace=[],
+    )
+
+    process_exported_definitions(plain_file_parse_result.plain_source, exported_definitions)
+
+    process_acceptance_tests(plain_file_parse_result.plain_source)
+
+    check_for_linked_resources(plain_file_parse_result.plain_source)
+
+    marshalled_plain_source = marshall_plain_source(plain_file_parse_result.plain_source)
+
+    process_code_variables(marshalled_plain_source, code_variables)
+
+    validation_errors = concept_utils.validate_concepts(marshalled_plain_source)
+    if len(validation_errors) > 0:
+        errors_msg = "\n".join(validation_errors)
+        msg = f"Found {len(validation_errors)} errors in the plain file:\n{errors_msg}"
+        raise PlainSyntaxError(f"Plain syntax error: {msg}")
+
+    if plain_spec.DEFINITIONS in marshalled_plain_source:
+        concept_utils.sort_definitions(marshalled_plain_source[plain_spec.DEFINITIONS])
+
+    return module_name, marshalled_plain_source, plain_file_parse_result.required_modules