plain-parser 0.1.0__py3-none-any.whl

plain_parser/__init__.py

@@ -0,0 +1,27 @@
+"""Parser for ***plain specification files."""
+
+from plain_parser import concept_utils, exceptions, loaders, plain_file, plain_spec
+from plain_parser.plain_file import (
+    PlainFileParseResult,
+    get_filename_from_module_name,
+    get_module_name_from_filename,
+    marshall_plain_source,
+    parse_plain_file,
+    parse_plain_source,
+    plain_file_parser,
+)
+
+__all__ = [
+    "PlainFileParseResult",
+    "concept_utils",
+    "exceptions",
+    "get_filename_from_module_name",
+    "get_module_name_from_filename",
+    "loaders",
+    "marshall_plain_source",
+    "parse_plain_file",
+    "parse_plain_source",
+    "plain_file",
+    "plain_file_parser",
+    "plain_spec",
+]

plain_parser/concept_utils.py

@@ -0,0 +1,221 @@
+import re
+from copy import deepcopy
+from typing import Optional
+
+import mistletoe.block_token
+import networkx as nx
+
+from plain_parser import plain_spec
+from plain_parser.exceptions import PlainSyntaxError
+
+# These are reserved concept names that are pre-defined and must not be redefined in the definitions section.
+DEFAULT_CONCEPTS = {
+    ":ConformanceTests:",
+    ":UnitTests:",
+    ":AcceptanceTest:",
+    ":AcceptanceTests:",
+    ":Implementation:",
+    ":plainDefinitions:",
+    ":plainImplementationReqs:",
+    ":plainFunctionality:",
+    ":plainTestReqs:",
+    ":plainImplementationCode:",
+}
+
+
+def extract_concepts_from_definition(text: str) -> tuple[list[str], list[str]]:
+    pattern = r"-\s(:[^\:]+:)(?:,\s*:[^\:]+:)*"
+    match = re.search(pattern, text)
+
+    if not match:
+        return list(), [
+            f"Syntax error: Invalid definition specification text: {text}. Should start with `- :<concept>:` (where <concept> is any string that does not contain `:`)."
+        ]
+
+    concept_candidates = re.findall(r":[^\:]+:", match.group())
+    valid_concepts = list[str]()
+    concept_re = r":[+\-\.0-9A-Z_a-z]+:"
+    errors = list[str]()
+    for candidate in concept_candidates:
+        if re.match(concept_re, candidate):
+            valid_concepts.append(candidate)
+        else:
+            errors.append(
+                f"Syntax error: Invalid concept: {candidate}. Should contain only letters, numbers, hyphens and dots."
+            )
+
+    return valid_concepts, errors
+
+
+def extract_concepts_from_spec_text(text: str) -> str:
+    return re.findall(r":[+\-\.0-9A-Z_a-z]+:", text)
+
+
+def collect_concepts(plain_source) -> tuple[set[str], list[str]]:
+    if plain_spec.DEFINITIONS not in plain_source:
+        return set(), list()
+
+    concepts = list[str]()
+    errors = list[str]()
+    for definition in plain_source[plain_spec.DEFINITIONS]:
+        spec_text = definition["markdown"]
+        new_concepts, new_errors = extract_concepts_from_definition(spec_text)
+        errors.extend(new_errors)
+        concepts.extend(new_concepts)
+
+    return concepts, errors
+
+
+def validate_concepts_in_spec(spec: dict, concepts: set[str], spec_group: str) -> list[str]:
+    spec_text = spec["markdown"]
+    used_concepts = extract_concepts_from_spec_text(spec_text)
+    errors = []
+    for used_concept in used_concepts:
+        if used_concept not in concepts and used_concept not in DEFAULT_CONCEPTS:
+            errors.append(
+                f"Syntax error: Concept {used_concept} is not defined in the definitions. {spec_group}: {spec_text}"
+            )
+
+    return errors
+
+
+def validate_concepts_in_spec_group(spec_group, plain_source, concepts: set[str]) -> list[str]:
+    if spec_group not in plain_source:
+        return []
+
+    errors = []
+    for spec in plain_source[spec_group]:
+        errors.extend(validate_concepts_in_spec(spec, concepts, spec_group))
+
+    return errors
+
+
+def validate_concepts(marshalled_plain_source) -> list[str]:
+    errors = list[str]()
+    concepts = deepcopy(DEFAULT_CONCEPTS)
+    tree_based_concepts = deepcopy(DEFAULT_CONCEPTS)
+
+    new_concepts, new_errors = collect_concepts(marshalled_plain_source)
+    errors.extend(new_errors)
+
+    unique_new_concepts = set(new_concepts)
+    intersection = unique_new_concepts.intersection(concepts)
+
+    if len(unique_new_concepts) < len(new_concepts) or (len(intersection) > 0):
+        misused_concepts = {concept for concept in new_concepts if new_concepts.count(concept) > 1}
+        misused_concepts.update(intersection)
+        errors.append(f"Syntax error: Concepts were defined multiple times: {', '.join(misused_concepts)} .")
+
+    concepts.update(unique_new_concepts)
+    tmp_tree_based_concepts = tree_based_concepts.union(unique_new_concepts)
+
+    for spec_group in [
+        plain_spec.NON_FUNCTIONAL_REQUIREMENTS,
+        plain_spec.TEST_REQUIREMENTS,
+        plain_spec.FUNCTIONAL_REQUIREMENTS,
+        plain_spec.DEFINITIONS,
+    ]:
+        errors.extend(validate_concepts_in_spec_group(spec_group, marshalled_plain_source, tmp_tree_based_concepts))
+
+    if plain_spec.FUNCTIONAL_REQUIREMENTS in marshalled_plain_source:
+        for func_spec in marshalled_plain_source[plain_spec.FUNCTIONAL_REQUIREMENTS]:
+            if "acceptance_tests" not in func_spec:
+                continue
+
+            acceptance_tests = func_spec["acceptance_tests"]
+            for spec in acceptance_tests:
+                errors.extend(validate_concepts_in_spec(spec, tmp_tree_based_concepts, plain_spec.ACCEPTANCE_TESTS))
+
+    return errors
+
+
+def find_concept_definitions_in_plain_source(
+    concept: str,
+    plain_source: dict,
+    renderer,
+) -> list[mistletoe.block_token.token]:
+    definitions = list[mistletoe.block_token.token]()
+    if plain_source[plain_spec.DEFINITIONS] is None:
+        return definitions
+
+    for definition in plain_source[plain_spec.DEFINITIONS].children:
+        rendered_definition = renderer.render(definition)
+        defined_concepts, _ = extract_concepts_from_definition(rendered_definition)
+        if concept in defined_concepts:
+            definitions.append(definition)
+            used_concepts = [
+                used_concept
+                for used_concept in extract_concepts_from_spec_text(rendered_definition)
+                if used_concept != concept and used_concept not in DEFAULT_CONCEPTS
+            ]
+            for used_concept in used_concepts:
+                definitions.extend(find_concept_definitions_in_plain_source(used_concept, plain_source, renderer))
+
+    return definitions
+
+
+def build_adjacency_list(definitions: Optional[list[dict]]) -> tuple[dict, dict]:
+    adjacency_list = dict[str, list[str]]()
+    concept_definitions = dict[str, dict]()
+    if definitions is None or len(definitions) == 0:
+        return adjacency_list, concept_definitions
+
+    for definition in definitions:
+        def_text = definition["markdown"]
+        new_concepts, _ = extract_concepts_from_definition(def_text)
+        used_concepts = [
+            used_concept
+            for used_concept in extract_concepts_from_spec_text(def_text)
+            if used_concept not in new_concepts and used_concept not in DEFAULT_CONCEPTS
+        ]
+        used_concepts = set(used_concepts)
+
+        for new_concept in new_concepts:
+            if new_concept not in adjacency_list:
+                adjacency_list[new_concept] = []
+
+        for used_concept in used_concepts:
+            if used_concept not in adjacency_list:
+                adjacency_list[used_concept] = []
+
+            adjacency_list[used_concept].extend(new_concepts)
+
+        for new_concept in new_concepts:
+            concept_definitions[new_concept] = definition
+
+    return adjacency_list, concept_definitions
+
+
+def sort_definitions(definitions: list[dict]) -> list[dict]:
+    if len(definitions) <= 1:
+        return
+
+    adjacency_list, concept_definitions = build_adjacency_list(definitions)
+
+    concept_graph = nx.DiGraph()
+    for node, neighbors in adjacency_list.items():
+        for neighbour in neighbors:
+            concept_graph.add_edge(node, neighbour)
+        if not neighbors:
+            concept_graph.add_node(node)
+
+    if not nx.is_directed_acyclic_graph(concept_graph):
+        msg = "Found cycles in the concept graph. Cycles are not allowed."
+        all_cycles = list(nx.simple_cycles(concept_graph))
+
+        for cycle in all_cycles:
+            cyclic_definitions = []
+            cyclic_definitions.append(concept_definitions[cycle[0]]["markdown"])
+            cyclic_definitions.append(concept_definitions[cycle[1]]["markdown"])
+            msg += "Cyclic definitons:\n"
+            msg += "\n".join(cyclic_definitions)
+            msg += "\n"
+
+        raise PlainSyntaxError(f"Plain syntax error: {msg}")
+
+    order = list(nx.topological_sort(concept_graph))
+    if len(order) > 0:
+        definition_to_order_idx = {
+            plain_spec.hash_text(str(concept_definitions[concept])): idx for idx, concept in enumerate(order)
+        }
+        definitions.sort(key=lambda definition: definition_to_order_idx[plain_spec.hash_text(str(definition))])

plain_parser/exceptions.py

@@ -0,0 +1,39 @@
+class PlainSyntaxError(Exception):
+    pass
+
+
+class UnsupportedBase64Content(Exception):
+    pass
+
+
+class UnsupportedResourceType(Exception):
+    pass
+
+
+class InvalidFridArgument(Exception):
+    pass
+
+
+class InvalidLiquidVariableName(Exception):
+    pass
+
+
+class ModuleDoesNotExistError(Exception):
+    pass
+
+
+class MissingFunctionalitiesError(Exception):
+    """Raised when a module to be rendered has no functionalities specified at all."""
+
+    pass
+
+
+class ImportedModuleWithFunctionalitiesError(Exception):
+    """Raised when a module brought in via ``import`` contains functional specs.
+
+    This is a usage error, not a syntax error: the module is syntactically valid
+    and would be fine as a render target, but functionalities are not allowed in
+    the import role.
+    """
+
+    pass

plain_parser/liquid_nodes.py

@@ -0,0 +1,109 @@
+from pathlib import Path
+from typing import Mapping, Sequence, TextIO
+
+from liquid2 import Environment, RenderContext, Template, TemplateNotFoundError
+from liquid2.builtin import IncludeTag
+from liquid2.builtin.tags.include_tag import IncludeNode
+
+
+class Plain2CodeIncludeNode(IncludeNode):
+    def render_to_output(self, context: RenderContext, buffer: TextIO) -> int:
+        """Render the node to the output buffer."""
+        name = self.name.evaluate(context)
+        whitespaces = 0
+        is_comment = False
+        i = self.token.start
+        while self.token.source[i] != "\n" and i >= 0:
+            if self.token.source[i] == " ":
+                whitespaces += 1
+            elif self.token.source[i] == ">":
+                is_comment = True
+                break
+            else:
+                whitespaces = 0
+            i -= 1
+
+        if is_comment:
+            return buffer.write(str(self))
+
+        try:
+            template = context.env.get_template(str(name), context=context, tag=self.tag, whitespaces=whitespaces)
+        except TemplateNotFoundError as err:
+            wrapped = TemplateNotFoundError(f"Template not found: {err}")
+            wrapped.token = self.name.token
+            wrapped.template_name = context.template.full_name()
+            raise wrapped from err
+
+        namespace: dict[str, object] = dict(arg.evaluate(context) for arg in self.args)
+
+        character_count = 0
+
+        with context.extend(namespace, template=template):
+            if self.var:
+                val = self.var.evaluate(context)
+                key = self.alias or template.name.split(".")[0]
+
+                if isinstance(val, Sequence) and not isinstance(val, str):
+                    context.raise_for_loop_limit(len(val))
+                    for itm in val:
+                        namespace[key] = itm
+                        character_count += template.render_with_context(context, buffer, partial=True)
+                else:
+                    namespace[key] = val
+                    character_count = template.render_with_context(context, buffer, partial=True)
+            else:
+
+                character_count = template.render_with_context(context, buffer, partial=True)
+
+        return character_count
+
+
+class Plain2CodeIncludeTag(IncludeTag):
+    node_class = Plain2CodeIncludeNode
+
+
+class Plain2CodeLoaderMixin:
+    def __init__(self, *args, **kwargs):
+        if not hasattr(self, "get_source"):
+            raise NotImplementedError("Class must implement get_source")
+        super().__init__(*args, **kwargs)
+
+    def load(
+        self,
+        env: Environment,
+        name: str,
+        *,
+        globals: Mapping[str, object] | None = None,
+        context: RenderContext | None = None,
+        **kwargs: object,
+    ) -> Template:
+        """
+        Find and parse template source code.
+
+        Args:
+            env: The `Environment` attempting to load the template source text.
+            name: A name or identifier for a template's source text.
+            globals: A mapping of render context variables attached to the
+                resulting template.
+            context: An optional render context that can be used to narrow the template
+                source search space.
+            kwargs: Arbitrary arguments that can be used to narrow the template source
+                search space.
+        """
+        source, full_name, uptodate, matter = self.get_source(env, name, context=context, **kwargs)
+        whitespaces = kwargs.get("whitespaces", 0)
+        assert isinstance(whitespaces, int)
+        source = source.rstrip().replace("\n", "\n" + " " * whitespaces)
+
+        path = Path(full_name)
+
+        template = env.from_string(
+            source,
+            name=path.name,
+            path=path,
+            globals=globals,
+            overlay_data=matter,
+        )
+
+        template.uptodate = uptodate
+        return template

plain_parser/loaders.py

@@ -0,0 +1,101 @@
+import os
+import re
+from typing import Optional
+
+from liquid2 import Environment, FileSystemLoader, StrictUndefined
+from liquid2.exceptions import UndefinedError
+
+from plain_parser import plain_spec
+from plain_parser.exceptions import UnsupportedBase64Content, UnsupportedResourceType
+from plain_parser.liquid_nodes import Plain2CodeIncludeTag, Plain2CodeLoaderMixin
+
+MAX_BASE64_BLOB_LENGTH = 8192
+
+# Matches a long contiguous base64 / base64url run, optionally preceded by a data: URI header.
+_BASE64_BLOB_PATTERN = re.compile(
+    r"(?:data:[\w.+-]+/[\w.+-]+;base64,)?[A-Za-z0-9+/_-]{%d,}={0,2}" % MAX_BASE64_BLOB_LENGTH
+)
+
+
+def find_large_base64_blob(text: str) -> Optional[str]:
+    """Return the first contiguous base64 blob at or above the threshold, or None."""
+    match = _BASE64_BLOB_PATTERN.search(text)
+    return match.group(0) if match else None
+
+
+def open_from(dirs, file_name):
+    for dir in dirs:
+        full_file_name = os.path.join(dir, file_name)
+        if not os.path.isfile(full_file_name):
+            continue
+
+        with open(full_file_name, "rb") as f:
+            content = f.read()
+        return content.decode("utf-8")
+
+    return None
+
+
+def load_linked_resources(template_dirs: list[str], resources_list, module_name: str):
+    linked_resources = {}
+
+    for resource in resources_list:
+        file_name = resource["target"]
+        if file_name in linked_resources:
+            continue
+
+        try:
+            content = open_from(template_dirs, file_name)
+        except UnicodeDecodeError:
+            raise UnsupportedResourceType(
+                f"Referenced resource '{file_name}' in module '{module_name}' is a binary file. "
+                f"Only text files (e.g. .md, .txt, .json, .yaml) can be referenced from a .plain file."
+            )
+
+        if content is None:
+            raise FileNotFoundError(f"Resource file '{file_name}' referenced in module '{module_name}' not found.")
+
+        blob = find_large_base64_blob(content)
+        if blob is not None:
+            raise UnsupportedBase64Content(
+                f"Referenced resource '{file_name}' in module '{module_name}' contains a large "
+                f"base64-encoded blob ({len(blob)} characters), such as an embedded image. Inline "
+                "base64 data is not supported. Remove the data from the resource. "
+                "If the data should be used by the end software, "
+                "save the data to a separate file and include the file path in the specification without it being a reference file."
+            )
+
+        linked_resources[file_name] = content
+
+    return linked_resources
+
+
+class TrackingFileSystemLoader(Plain2CodeLoaderMixin, FileSystemLoader):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.loaded_templates = {}
+
+    def get_source(self, env, template_name, **kwargs):
+        source = super().get_source(env, template_name, **kwargs)
+        self.loaded_templates[template_name] = source.source
+        return source
+
+
+def get_loaded_templates(source_path, plain_source):
+    # Render the plain source with Liquid templating engine
+    # to identify the templates that are being loaded
+
+    liquid_loader = TrackingFileSystemLoader(source_path)
+    liquid_env = Environment(loader=liquid_loader, undefined=StrictUndefined)
+    liquid_env.tags["include"] = Plain2CodeIncludeTag(liquid_env)
+
+    liquid_env.filters["code_variable"] = plain_spec.code_variable_liquid_filter
+    liquid_env.filters["prohibited_chars"] = plain_spec.prohibited_chars_liquid_filter
+
+    plain_source_template = liquid_env.from_string(plain_source)
+    try:
+        plain_source = plain_source_template.render()
+    except UndefinedError as e:
+        raise Exception(f"Undefined liquid variable: {str(e)}")
+
+    return plain_source, liquid_loader.loaded_templates