strictdoc 0.0.49a7__tar.gz → 0.0.50a1__tar.gz

{strictdoc-0.0.49a7 → strictdoc-0.0.50a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: strictdoc
-Version: 0.0.49a7
+Version: 0.0.50a1
 Summary: StrictDoc is open-source software for technical documentation and requirements management.
 Project-URL: Changelog, https://github.com/strictdoc-project/strictdoc/blob/main/CHANGELOG.md
 Project-URL: Homepage, https://strictdoc.readthedocs.io/en/stable/
@@ -32,6 +32,7 @@ Requires-Dist: pygments==2.*,>=2.10.0
 Requires-Dist: python-datauri==0.*,>=0.2.9
 Requires-Dist: python-multipart
 Requires-Dist: reqif==0.*,>=0.0.39
+Requires-Dist: requests
 Requires-Dist: selenium
 Requires-Dist: setuptools
 Requires-Dist: spdx-tools

{strictdoc-0.0.49a7 → strictdoc-0.0.50a1}/pyproject.toml

@@ -97,6 +97,8 @@ dependencies = [
     # HTML2PDF dependencies
     "selenium",
     "webdriver-manager",
+    # requests is used by StrictDoc's custom HTML2PDF_HTTPClient.
+    "requests",
 ]
 # @sdoc[/SDOC-SRS-89]
 

{strictdoc-0.0.49a7 → strictdoc-0.0.50a1}/strictdoc/__init__.py

@@ -1,6 +1,6 @@
 from strictdoc.core.environment import SDocRuntimeEnvironment
 
-__version__ = "0.0.49a7"
+__version__ = "0.0.50a1"
 
 
 environment = SDocRuntimeEnvironment(__file__)

{strictdoc-0.0.49a7 → strictdoc-0.0.50a1}/strictdoc/backend/sdoc/free_text_reader.py

@@ -17,7 +17,7 @@ class SDFreeTextReader:
             use_regexp_group=True,
         )
 
-        parse_context = ParseContext()
+        parse_context = ParseContext(file_path)
 
         document = meta_model.model_from_str(input_string, file_name=file_path)
         parse_context.document_reference.set_document(document)

{strictdoc-0.0.49a7 → strictdoc-0.0.50a1}/strictdoc/backend/sdoc/include_reader.py

@@ -20,9 +20,7 @@ class SDIncludeReader:
         )
         assert isinstance(parse_context, ParseContext)
 
-        processor = SDocParsingProcessor(
-            parse_context=parse_context, delegate=None
-        )
+        processor = SDocParsingProcessor(parse_context=parse_context)
         meta_model.register_obj_processors(processor.get_default_processors())
 
         section = meta_model.model_from_str(input_string, file_name=file_path)

{strictdoc-0.0.49a7 → strictdoc-0.0.50a1}/strictdoc/backend/sdoc/models/document.py

@@ -6,6 +6,7 @@ from strictdoc.backend.sdoc.models.document_bibliography import (
 from strictdoc.backend.sdoc.models.document_config import DocumentConfig
 from strictdoc.backend.sdoc.models.document_grammar import DocumentGrammar
 from strictdoc.backend.sdoc.models.document_view import DocumentView
+from strictdoc.backend.sdoc.models.fragment_from_file import FragmentFromFile
 from strictdoc.backend.sdoc.models.free_text import FreeText
 from strictdoc.core.document_meta import DocumentMeta
 from strictdoc.helpers.auto_described import auto_described
@@ -77,6 +78,10 @@ class SDocDocument:  # pylint: disable=too-many-instance-attributes
         task_list = list(self.section_contents)
         while len(task_list) > 0:
             section_or_requirement = task_list.pop(0)
+            if isinstance(section_or_requirement, FragmentFromFile):
+                if section_or_requirement.has_any_requirements():
+                    return True
+                continue
             if section_or_requirement.is_requirement:
                 return True
             assert section_or_requirement.is_section, section_or_requirement

strictdoc-0.0.50a1/strictdoc/backend/sdoc/models/fragment_from_file.py

@@ -0,0 +1,41 @@
+from typing import List, Optional
+
+from strictdoc.backend.sdoc.models.fragment import Fragment
+from strictdoc.helpers.auto_described import auto_described
+
+
+@auto_described
+class FragmentFromFile:
+    def __init__(
+        self,
+        parent,
+        file,
+    ):
+        self.parent = parent
+        self.file = file
+
+        self.ng_level = None
+        self.ng_has_requirements = False
+        self.resolved_fragment: Optional[Fragment] = None
+
+    @property
+    def document(self):
+        raise NotImplementedError
+
+    def has_any_requirements(self) -> bool:
+        task_list = list(self.section_contents)
+        while len(task_list) > 0:
+            section_or_requirement = task_list.pop(0)
+            if isinstance(section_or_requirement, FragmentFromFile):
+                if section_or_requirement.has_any_requirements():
+                    return True
+            if section_or_requirement.is_requirement:
+                return True
+            assert section_or_requirement.is_section, section_or_requirement
+            task_list.extend(section_or_requirement.section_contents)
+        return False
+
+    @property
+    def section_contents(self) -> List:
+        assert self.resolved_fragment is not None, self.resolved_fragment
+        return self.resolved_fragment.section_contents

strictdoc-0.0.50a1/strictdoc/backend/sdoc/pickle_cache.py

@@ -0,0 +1,39 @@
+import hashlib
+import os
+import tempfile
+
+from strictdoc.helpers.md5 import get_file_md5
+
+
+class PickleCache:
+    @staticmethod
+    def get_cached_file_path(file_path: str, path_to_output_root: str):
+        path_to_tmp_dir = tempfile.gettempdir()
+
+        full_path_to_file = (
+            file_path
+            if os.path.abspath(file_path)
+            else os.path.abspath(file_path)
+        )
+
+        file_md5 = get_file_md5(file_path)
+
+        # File name contains an MD5 hash of its full path to ensure the
+        # uniqueness of the cached items. Additionally, the unique file name
+        # contains a full path to the output root to prevent collisions
+        # between StrictDoc invocations running against the same set of SDoc
+        # files in parallel.
+        unique_identifier = path_to_output_root + full_path_to_file
+        unique_identifier_md5 = hashlib.md5(
+            unique_identifier.encode("utf-8")
+        ).hexdigest()
+        file_name = os.path.basename(full_path_to_file)
+        file_name += "_" + unique_identifier_md5 + "_" + file_md5
+
+        path_to_cached_file = os.path.join(
+            path_to_tmp_dir,
+            "strictdoc_cache",
+            file_name,
+        )
+
+        return path_to_cached_file

strictdoc-0.0.50a1/strictdoc/backend/sdoc/post_processing/sdoc_fragment_post_processor.py

@@ -0,0 +1,39 @@
+import os
+
+from strictdoc.backend.sdoc.include_reader import SDIncludeReader
+from strictdoc.backend.sdoc.models.document import SDocDocument
+from strictdoc.backend.sdoc.models.fragment import Fragment
+from strictdoc.backend.sdoc.models.fragment_from_file import FragmentFromFile
+from strictdoc.backend.sdoc.processor import ParseContext
+
+
+class SDocFragmentPostProcessor:
+    @staticmethod
+    def process_document(_: SDocDocument, parse_context: ParseContext):
+        fragment_from_file_: FragmentFromFile
+        for fragment_from_file_ in parse_context.fragments_from_files:
+            assert isinstance(
+                fragment_from_file_, FragmentFromFile
+            ), fragment_from_file_
+            fragment: Fragment = SDocFragmentPostProcessor.parse_fragment(
+                fragment_from_file_, parse_context
+            )
+            fragment_from_file_.resolved_fragment = fragment
+
+    @staticmethod
+    def parse_fragment(
+        fragment_from_file: FragmentFromFile, parse_context: ParseContext
+    ):
+        path_to_fragment = (
+            os.path.join(
+                parse_context.path_to_sdoc_dir, fragment_from_file.file
+            )
+            if parse_context.path_to_sdoc_dir is not None
+            else fragment_from_file.file
+        )
+        reader = SDIncludeReader()
+        fragment = reader.read_from_file(
+            file_path=path_to_fragment, context=parse_context
+        )
+        assert isinstance(fragment, Fragment)
+        return fragment

{strictdoc-0.0.49a7 → strictdoc-0.0.50a1}/strictdoc/backend/sdoc/processor.py

@@ -1,4 +1,5 @@
-from typing import Optional
+import os.path
+from typing import List, Optional
 
 from textx import get_location, get_model
 
@@ -20,7 +21,13 @@ from strictdoc.helpers.exception import StrictDocException
 
 
 class ParseContext:
-    def __init__(self):
+    def __init__(self, path_to_sdoc_file: Optional[str]):
+        self.path_to_sdoc_file: Optional[str] = path_to_sdoc_file
+        self.path_to_sdoc_dir: Optional[str] = None
+        if path_to_sdoc_file is not None:
+            assert os.path.isfile(path_to_sdoc_file), path_to_sdoc_file
+            self.path_to_sdoc_dir = os.path.dirname(path_to_sdoc_file)
+
         self.document_reference: DocumentReference = DocumentReference()
         self.document_config: Optional[DocumentConfig] = None
         self.document_grammar: DocumentGrammar = DocumentGrammar.create_default(
@@ -31,14 +38,17 @@ class ParseContext:
         self.uses_old_refs_field: bool = False
         self.at_least_one_relations_field: bool = False
 
+        # FIXME: Plain list of all fragments found in the document.
+        self.fragments_from_files: List = []
+
 
 class SDocParsingProcessor:
-    def __init__(self, parse_context: ParseContext, delegate):
+    def __init__(self, parse_context: ParseContext):
         self.parse_context: ParseContext = parse_context
-        self.delegate = delegate
 
     def process_document(self, document: SDocDocument):
         document.grammar = self.parse_context.document_grammar
+        self.parse_context.document = document
 
     def get_default_processors(self):
         return {
@@ -47,7 +57,7 @@ class SDocParsingProcessor:
             "DocumentGrammar": self.process_document_grammar,
             "DocumentView": self.process_document_view,
             "SDocSection": self.process_section,
-            "FragmentFromFile": self.process_include,
+            "FragmentFromFile": self.process_fragment_from_file,
             "CompositeRequirement": self.process_composite_requirement,
             "SDocNode": self.process_requirement,
             "FreeText": self.process_free_text,
@@ -98,25 +108,15 @@ class SDocParsingProcessor:
         self._resolve_parents(section)
         assert section.ng_level > 0
 
-    def process_include(self, include: FragmentFromFile):
-        self._resolve_parents(include)
-        self.parse_context.current_include_parent = include.parent
-
-        assert self.delegate is not None
-        fragment = self.delegate(include, self.parse_context)
-
-        parent_section_contents = include.parent.section_contents
-        index = parent_section_contents.index(include)
-        before = parent_section_contents[:index]
-        index = (
-            index + 1
-        )  # black and flake8 fight to put a space before ':' otherwise
-        after = parent_section_contents[index:]
-
-        include.parent.section_contents = []
-        include.parent.section_contents.extend(before)
-        include.parent.section_contents.extend(fragment.section_contents)
-        include.parent.section_contents.extend(after)
+    def process_fragment_from_file(self, fragment_from_file: FragmentFromFile):
+        assert isinstance(
+            fragment_from_file, FragmentFromFile
+        ), fragment_from_file
+
+        self._resolve_parents(fragment_from_file)
+        self.parse_context.current_include_parent = fragment_from_file.parent
+
+        self.parse_context.fragments_from_files.append(fragment_from_file)
 
     def process_composite_requirement(
         self, composite_requirement: CompositeRequirement

{strictdoc-0.0.49a7 → strictdoc-0.0.50a1}/strictdoc/backend/sdoc/reader.py

@@ -1,23 +1,23 @@
-import hashlib
 import os.path
 import sys
-import tempfile
 import traceback
 from pathlib import Path
+from typing import Tuple
 
 from textx import metamodel_from_str
 
 from strictdoc.backend.sdoc.error_handling import StrictDocSemanticError
 from strictdoc.backend.sdoc.grammar.grammar_builder import SDocGrammarBuilder
-from strictdoc.backend.sdoc.include_reader import SDIncludeReader
 from strictdoc.backend.sdoc.models.constants import DOCUMENT_MODELS
 from strictdoc.backend.sdoc.models.document import SDocDocument
-from strictdoc.backend.sdoc.models.fragment import Fragment
+from strictdoc.backend.sdoc.pickle_cache import PickleCache
+from strictdoc.backend.sdoc.post_processing.sdoc_fragment_post_processor import (
+    SDocFragmentPostProcessor,
+)
 from strictdoc.backend.sdoc.processor import ParseContext, SDocParsingProcessor
 from strictdoc.backend.sdoc.validations.sdoc_validator import SDocValidator
 from strictdoc.helpers.cast import assert_cast
 from strictdoc.helpers.exception import StrictDocException
-from strictdoc.helpers.md5 import get_file_md5
 from strictdoc.helpers.pickle import pickle_dump, pickle_load
 from strictdoc.helpers.textx import drop_textx_meta
 
@@ -34,10 +34,8 @@ class SDReader:
             use_regexp_group=True,
         )
 
-        parse_context = ParseContext()
-        processor = SDocParsingProcessor(
-            parse_context=parse_context, delegate=SDReader.parse_include
-        )
+        parse_context = ParseContext(path_to_sdoc_file=file_path)
+        processor = SDocParsingProcessor(parse_context=parse_context)
         meta_model.register_obj_processors(processor.get_default_processors())
 
         document: SDocDocument = meta_model.model_from_str(
@@ -69,17 +67,18 @@ class SDReader:
         return document, parse_context
 
     @staticmethod
-    def read(input_string, file_path=None):
-        document, _ = SDReader._read(input_string, file_path)
-        SDocValidator.validate_document(document)
+    def read(input_string, file_path=None) -> SDocDocument:
+        document, _ = SDReader.read_with_parse_context(input_string, file_path)
+        return document
 
-        # HACK:
-        # ProcessPoolExecutor doesn't work because of non-picklable parts
-        # of textx. The offending fields are stripped down because they
-        # are not used anyway.
-        drop_textx_meta(document)
+    @staticmethod
+    def read_with_parse_context(
+        input_string, file_path=None
+    ) -> Tuple[SDocDocument, ParseContext]:
+        document, parse_context = SDReader._read(input_string, file_path)
+        SDocValidator.validate_document(document)
 
-        return document
+        return document, parse_context
 
     def read_from_file(self, file_path: str) -> SDocDocument:
         """
@@ -87,32 +86,8 @@ class SDReader:
         object.
         """
 
-        path_to_tmp_dir = tempfile.gettempdir()
-
-        full_path_to_file = (
-            file_path
-            if os.path.abspath(file_path)
-            else os.path.abspath(file_path)
-        )
-
-        file_md5 = get_file_md5(file_path)
-
-        # File name contains an MD5 hash of its full path to ensure the
-        # uniqueness of the cached items. Additionally, the unique file name
-        # contains a full path to the output root to prevent collisions
-        # between StrictDoc invocations running against the same set of SDoc
-        # files in parallel.
-        unique_identifier = self.path_to_output_root + full_path_to_file
-        unique_identifier_md5 = hashlib.md5(
-            unique_identifier.encode("utf-8")
-        ).hexdigest()
-        file_name = os.path.basename(full_path_to_file)
-        file_name += "_" + unique_identifier_md5 + "_" + file_md5
-
-        path_to_cached_file = os.path.join(
-            path_to_tmp_dir,
-            "strictdoc_cache",
-            file_name,
+        path_to_cached_file = PickleCache.get_cached_file_path(
+            file_path, self.path_to_output_root
         )
         if os.path.isfile(path_to_cached_file):
             with open(path_to_cached_file, "rb") as cache_file:
@@ -126,7 +101,17 @@ class SDReader:
             sdoc_content = file.read()
 
         try:
-            sdoc = self.read(sdoc_content, file_path=file_path)
+            sdoc, parse_context = self.read_with_parse_context(
+                sdoc_content, file_path=file_path
+            )
+
+            SDocFragmentPostProcessor.process_document(sdoc, parse_context)
+
+            # HACK:
+            # ProcessPoolExecutor doesn't work because of non-picklable parts
+            # of textx. The offending fields are stripped down because they
+            # are not used anyway.
+            drop_textx_meta(sdoc)
 
             sdoc_pickled = pickle_dump(sdoc)
             with open(path_to_cached_file, "wb") as cache_file:
@@ -150,12 +135,3 @@ class SDReader:
             # TODO: when --debug is provided
             traceback.print_exc()
             sys.exit(1)
-
-    @staticmethod
-    def parse_include(include, parse_context):
-        reader = SDIncludeReader()
-        fragment = reader.read_from_file(
-            file_path=include.file, context=parse_context
-        )
-        assert isinstance(fragment, Fragment)
-        return fragment

{strictdoc-0.0.49a7 → strictdoc-0.0.50a1}/strictdoc/backend/sdoc/writer.py

@@ -1,5 +1,7 @@
+import os.path
 from enum import Enum
-from typing import List, Optional
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
 
 from strictdoc.backend.sdoc.models.anchor import Anchor
 from strictdoc.backend.sdoc.models.document import SDocDocument
@@ -9,6 +11,8 @@ from strictdoc.backend.sdoc.models.document_bibliography import (
 from strictdoc.backend.sdoc.models.document_config import DocumentConfig
 from strictdoc.backend.sdoc.models.document_grammar import DocumentGrammar
 from strictdoc.backend.sdoc.models.document_view import DefaultViewElement
+from strictdoc.backend.sdoc.models.fragment import Fragment
+from strictdoc.backend.sdoc.models.fragment_from_file import FragmentFromFile
 from strictdoc.backend.sdoc.models.inline_link import InlineLink
 from strictdoc.backend.sdoc.models.node import CompositeRequirement, SDocNode
 from strictdoc.backend.sdoc.models.reference import (
@@ -30,6 +34,7 @@ from strictdoc.backend.sdoc.models.type_system import (
     RequirementFieldType,
 )
 from strictdoc.core.document_iterator import DocumentCachingIterator
+from strictdoc.core.document_meta import DocumentMeta
 from strictdoc.helpers.cast import assert_cast
 
 
@@ -43,7 +48,37 @@ class SDWriter:
     def __init__(self):
         pass
 
+    def write_to_file(self, document: SDocDocument):
+        document_content, fragments_dict = self.write_with_fragments(document)
+
+        document_meta: DocumentMeta = assert_cast(document.meta, DocumentMeta)
+
+        with open(
+            document_meta.input_doc_full_path, "w", encoding="utf8"
+        ) as output_file:
+            output_file.write(document_content)
+
+        path_to_output_file_dir = os.path.dirname(
+            document_meta.input_doc_full_path
+        )
+        Path(path_to_output_file_dir).mkdir(parents=True, exist_ok=True)
+
+        for fragment_path_, fragment_content_ in fragments_dict.items():
+            path_to_output_fragment = os.path.join(
+                path_to_output_file_dir, fragment_path_
+            )
+            with open(path_to_output_fragment, "w", encoding="utf8") as file_:
+                file_.write(fragment_content_)
+
     def write(self, document: SDocDocument):
+        document_output, _ = self.write_with_fragments(document)
+        return document_output
+
+    def write_with_fragments(
+        self, document: SDocDocument
+    ) -> Tuple[str, Dict[str, str]]:
+        fragments_dict: Dict[str, str] = {}
+
         document_iterator = DocumentCachingIterator(document)
         output = ""
 
@@ -201,8 +236,50 @@ class SDWriter:
         for free_text in document.free_texts:
             output += "\n"
             output += self._print_free_text(free_text)
+
+        output += self._print_body(
+            document, document, document_iterator, fragments_dict
+        )
+
+        return output, fragments_dict
+
+    def _print_body(
+        self,
+        root_node: [SDocDocument, Fragment],
+        document: SDocDocument,
+        document_iterator: DocumentCachingIterator,
+        fragments_dict,
+    ):
+        assert isinstance(root_node, (SDocDocument, Fragment)), root_node
+        assert isinstance(
+            document_iterator, DocumentCachingIterator
+        ), document_iterator
+
+        output = ""
+
         closing_tags = []
-        for content_node in document_iterator.all_content():
+        for content_node in document_iterator.all_content(
+            root_node, print_fragments=False
+        ):
+            if isinstance(content_node, FragmentFromFile):
+                fragment_from_file: FragmentFromFile = assert_cast(
+                    content_node, FragmentFromFile
+                )
+                output += "\n"
+                output += self._print_fragment_from_file(fragment_from_file)
+
+                assert fragment_from_file.resolved_fragment is not None
+                fragment_content = self._print_fragment(
+                    fragment_from_file.resolved_fragment,
+                    document,
+                    document_iterator,
+                    fragments_dict,
+                )
+
+                fragments_dict[fragment_from_file.file] = fragment_content
+
+                continue
+
             while (
                 len(closing_tags) > 0
                 and content_node.ng_level <= closing_tags[-1][1]
@@ -215,7 +292,7 @@ class SDWriter:
             if not content_node.ng_whitelisted:
                 continue
 
-            if isinstance(content_node, SDocSection):
+            elif isinstance(content_node, SDocSection):
                 output += self._print_section(content_node, document)
                 closing_tags.append((TAG.SECTION, content_node.ng_level))
             elif isinstance(content_node, SDocNode):
@@ -240,6 +317,33 @@ class SDWriter:
 
         return output
 
+    def _print_fragment(
+        self,
+        fragment: Fragment,
+        document: SDocDocument,
+        document_iterator: DocumentCachingIterator,
+        fragments_dict,
+    ):
+        output = "[FRAGMENT]\n"
+
+        output += self._print_body(
+            fragment, document, document_iterator, fragments_dict
+        )
+
+        return output
+
+    def _print_fragment_from_file(self, fragment_from_file: FragmentFromFile):
+        assert isinstance(fragment_from_file, FragmentFromFile)
+        output = ""
+        output += "[FRAGMENT_FROM_FILE]"
+        output += "\n"
+
+        output += "FILE: "
+        output += fragment_from_file.file
+        output += "\n"
+
+        return output
+
     def _print_section(self, section: SDocSection, document: SDocDocument):
         assert isinstance(section, SDocSection)
         output = ""

{strictdoc-0.0.49a7 → strictdoc-0.0.50a1}/strictdoc/core/actions/passthrough_action.py

@@ -36,7 +36,7 @@ class PassthroughAction:
             else os.path.join(os.getcwd(), "output", "sdoc")
         )
         for document in traceability_index.document_tree.document_list:
-            output = writer.write(document)
+            output, fragments_dict = writer.write_with_fragments(document)
 
             path_to_output_file_dir = os.path.join(
                 output_dir, document.meta.input_doc_dir_rel_path
@@ -48,3 +48,12 @@ class PassthroughAction:
             path_to_output_file += ".sdoc"
             with open(path_to_output_file, "w", encoding="utf8") as file:
                 file.write(output)
+
+            for fragment_path_, fragment_content_ in fragments_dict.items():
+                path_to_output_fragment = os.path.join(
+                    path_to_output_file_dir, fragment_path_
+                )
+                with open(
+                    path_to_output_fragment, "w", encoding="utf8"
+                ) as file_:
+                    file_.write(fragment_content_)