senfd 0.1.0__py3-none-any.whl

senfd/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.4.1"

senfd/cli.py

@@ -0,0 +1,81 @@
+"""
+Command-Line Interface
+======================
+
+Produces organized and semantically enriched ``.json`` documents from
+"figure-documents".
+"""
+
+from argparse import ArgumentParser, Namespace
+from pathlib import Path
+
+import senfd
+import senfd.schema
+from senfd.documents import CategorizedFigureDocument
+
+
+def parse_args() -> Namespace:
+    """Return command-line arguments"""
+
+    parser = ArgumentParser(description="Semantically organize and enrich figures")
+    parser.add_argument(
+        "document", nargs="*", type=Path, help="path to one or more .json document(s)"
+    )
+    parser.add_argument(
+        "--output",
+        type=Path,
+        help="directory where the output will be saved",
+        default=Path("output"),
+    )
+    parser.add_argument(
+        "--skip-validate",
+        action="store_true",
+        help="skip post-parse validation",
+    )
+    parser.add_argument(
+        "--dump-schema",
+        action="store_true",
+        help=f"dump schema({senfd.schema.FILENAME}) and exit",
+    )
+    parser.add_argument(
+        "--version",
+        action="store_true",
+        help="print the version and exit",
+    )
+    args = parser.parse_args()
+    if not args.document and not args.dump_schema and not args.version:
+        parser.error("the following arguments are required: document")
+
+    return args
+
+
+def main() -> int:
+    """Command-line entrypoint"""
+
+    args = parse_args()
+    args.output.mkdir(parents=True, exist_ok=True)
+
+    if args.version:
+        print(senfd.__version__)
+        return 0
+
+    if args.dump_schema:
+        CategorizedFigureDocument.to_schema_file(
+            args.output / CategorizedFigureDocument.FILENAME_SCHEMA
+        )
+        return 0
+
+    for path in args.document:
+        document = CategorizedFigureDocument.from_figure_document_file(path)
+        document.to_json_file(
+            args.output
+            / Path(path.name).with_suffix(CategorizedFigureDocument.SUFFIX_JSON).name
+        )
+
+        if args.skip_validate:
+            continue
+
+        if not document.is_valid():
+            print("Document is invalid, see above for details.")
+
+    return 0

senfd/documents.py

@@ -0,0 +1,228 @@
+"""
+Documents
+=========
+
+The following classes model documents at different stages of parsing / processing an
+NVMe specification document from a raw extract into something semantically rich.
+
+"""
+
+import inspect
+import json
+import re
+from pathlib import Path
+from typing import ClassVar, Dict, List, Optional
+
+from jinja2 import Environment, PackageLoader, select_autoescape
+from pydantic import BaseModel, Field, ValidationError
+
+import senfd.figures
+import senfd.tables
+
+TRANSLATION_TABLE: Dict[int, str] = str.maketrans(
+    {
+        "–": "-",  # en dash
+        "—": "-",  # em dash
+        "‘": "'",  # left single quotation mark
+        "’": "'",  # right single quotation mark
+        "“": '"',  # left double quotation mark
+        "”": '"',  # right double quotation mark
+        "…": "...",  # ellipsis
+        "é": "e",  # e with acute accent
+        "á": "a",  # a with acute accent
+        "í": "i",  # i with acute accent
+        "ó": "o",  # o with acute accent
+        "ú": "u",  # u with acute accent
+        "ü": "u",  # u with diaeresis
+        "ñ": "n",  # n with tilde
+        "ç": "c",  # c with cedilla
+    }
+)
+
+
+class DocumentMeta(BaseModel):
+    version: str = senfd.__version__
+    stem: str = Field(default_factory=str)
+
+
+class Document(BaseModel):
+    """
+    Base document - providing functionality for describing and persisting documents
+
+    The intent here is that all Documents should be representable as JSON and HTML
+    """
+
+    SUFFIX_JSON: ClassVar[str] = ".document.json"
+    SUFFIX_HTML: ClassVar[str] = ".document.html"
+
+    FILENAME_SCHEMA: ClassVar[str] = "document.schema.json"
+    FILENAME_HTML_TEMPLATE: ClassVar[str] = "document.html.jinja2"
+
+    meta: DocumentMeta = Field(default_factory=DocumentMeta)
+
+    @classmethod
+    def to_schema_file(cls, path: Optional[Path] = None):
+        """Writes the document JSON schema to file at the given 'path'"""
+
+        if path is None:
+            path = Path.cwd() / cls.FILENAME_HTML_TEMPLATE
+
+        with path.open("w") as schema_file:
+            json.dump(cls.schema(), schema_file, indent=4)
+
+    def get_json_filename(self):
+        return f"{self.meta.stem}.{self.SUFFIX_JSON}"
+
+    def to_json(self) -> str:
+        """Returns the document as a JSON-formated string"""
+
+        return self.model_dump_json(indent=4)
+
+    def to_json_file(self, path: Optional[Path] = None):
+        """Writes the document, formated as JSON, to file at the given 'path'"""
+
+        if path is None:
+            path = (Path.cwd() / self.meta.stem).with_suffix(self.SUFFIX_JSON)
+        path.write_text(self.to_json())
+
+    def get_html_filename(self):
+        return f"{self.meta.stem}.{self.SUFFIX_HTML}"
+
+    def to_html(self) -> str:
+        """Returns the document as a HTML-formatted string"""
+
+        env = Environment(
+            loader=PackageLoader("senfd", "templates"),
+            autoescape=select_autoescape(["html", "xml"]),
+        )
+        template = env.get_template(self.FILENAME_HTML_TEMPLATES)
+
+        return template.render(document=self)
+
+    def to_html_file(self, path: Optional[Path] = None):
+        """Writes the document to HTML-formatted file"""
+
+        if path is None:
+            path = (Path.cwd() / self.meta.stem).with_suffix(self.SUFFIX_HTML)
+        path.write_text(self.to_html())
+
+    def is_valid(self):
+        """Returns True when validator raises no exceptions, False otherwise"""
+
+        try:
+            self.validate(self.dict())
+        except ValidationError as e:
+            print(e)
+            return False
+
+        return True
+
+
+class FigureDocument(Document):
+
+    SUFFIX_JSON: ClassVar[str] = ".figure.document.json"
+    SUFFIX_HTML: ClassVar[str] = ".figure.document.html"
+
+    FILENAME_SCHEMA: ClassVar[str] = "figure.document.schema.json"
+    FILENAME_HTML_TEMPLATE: ClassVar[str] = "figure.document.html.jinja2"
+
+    figures: List[senfd.figures.Figure] = Field(default_factory=list)
+
+
+class CategorizedFigureDocumentMeta(DocumentMeta):
+    tabular: List[int] = Field(default_factory=list)
+    nontabular: List[senfd.figures.Figure] = Field(default_factory=list)
+    uncategorized: List[senfd.figures.Figure] = Field(default_factory=list)
+
+
+class CategorizedFigureDocument(Document):
+
+    SUFFIX_JSON: ClassVar[str] = ".categorized.figure.document.json"
+    SUFFIX_HTML: ClassVar[str] = ".categorized.figure.document.html"
+
+    FILENAME_SCHEMA: ClassVar[str] = "categorized.figure.document.schema.json"
+    FILENAME_HTML_TEMPLATE: ClassVar[str] = "categorized.figure.document.html.jinja2"
+
+    meta: CategorizedFigureDocumentMeta = Field(
+        default_factory=CategorizedFigureDocumentMeta
+    )
+    acronyms: List[senfd.figures.Acronyms] = Field(default_factory=list)
+    io_controller_command_set_support: List[
+        senfd.figures.IoControllerCommandSetSupport
+    ] = Field(default_factory=list)
+    command_set_opcodes: List[senfd.figures.CommandSetOpcodes] = Field(
+        default_factory=list
+    )
+    command_support_requirements: List[senfd.figures.CommandSupportRequirements] = (
+        Field(default_factory=list)
+    )
+    command_sqe_dword: List[senfd.figures.CommandSqeDword] = Field(default_factory=list)
+    command_sqe_dwords: List[senfd.figures.CommandSqeDwords] = Field(
+        default_factory=list
+    )
+    command_sqe_data_pointer: List[senfd.figures.CommandSqeDataPointer] = Field(
+        default_factory=list
+    )
+    command_cqe_dword: List[senfd.figures.CommandCqeDword] = Field(default_factory=list)
+    command_specific_status_values: List[senfd.figures.CommandSpecificStatusValues] = (
+        Field(default_factory=list)
+    )
+    general_command_status_values: List[senfd.figures.GeneralCommandStatusValues] = (
+        Field(default_factory=list)
+    )
+    cns_values: List[senfd.figures.CnsValues] = Field(default_factory=list)
+    feature_support: List[senfd.figures.FeatureSupport] = Field(default_factory=list)
+    feature_identifiers: List[senfd.figures.FeatureIdentifiers] = Field(
+        default_factory=list
+    )
+    log_page_identifiers: List[senfd.figures.LogPageIdentifiers] = Field(
+        default_factory=list
+    )
+    offset: List[senfd.figures.Offset] = Field(default_factory=list)
+    property_definition: List[senfd.figures.PropertyDefinition] = Field(
+        default_factory=list
+    )
+
+    @classmethod
+    def from_figure_document_file(cls, path: Path):
+        """Instantiate an 'organized' Document from a 'figure' document"""
+
+        def get_figure_enriching_classes():
+            """
+            To avoid manually crafting a list of classes, this function
+            introspectively examines this module for applicable
+            classes with "REGEX_FIGURE_DESCRIPTION" class attribute.
+            """
+            return [
+                cls
+                for _, cls in inspect.getmembers(senfd.figures, inspect.isclass)
+                if issubclass(cls, senfd.figures.FromFigureDescriptionMatch)
+                and hasattr(cls, "REGEX_FIGURE_DESCRIPTION")
+            ]
+
+        figure_document = senfd.documents.FigureDocument.parse_file(path)
+
+        document = cls()
+
+        figure_organizers = get_figure_enriching_classes()
+        for figure in figure_document.figures:
+            if not figure.table:
+                document.meta.nontabular.append(figure)
+                continue
+
+            match = None
+            description = figure.description.translate(TRANSLATION_TABLE)
+            for candidate in figure_organizers:
+                match = re.match(
+                    candidate.REGEX_FIGURE_DESCRIPTION, description, flags=re.IGNORECASE
+                )
+                if match:
+                    obj = candidate.from_figure_description(figure, match)
+                    obj.into_document(document)
+                    document.meta.tabular.append(obj.figure_nr)
+                    break
+
+            if not match:
+                document.meta.uncategorized.append(figure)
+
+        return document

senfd/figures.py

@@ -0,0 +1,179 @@
+"""
+Figures
+=======
+
+The following classses model the figures found in documents.
+"""
+
+import re
+from typing import ClassVar, Optional
+
+from pydantic import BaseModel
+
+import senfd.tables
+from senfd.utils import pascal_to_snake
+
+
+class Figure(BaseModel):
+    """
+    A figure as captioned in a table of figures in the NVMe Specification Documents.
+
+    This is a minimally enriched figure representation. The intent is that via regular
+    expressions, it should be possible to construct instances when feeding it with
+    the text from a table of figures or a table that conventionally contains a figure
+    caption in the first row.
+
+    The page number is only retrievable from table of figure captions and thus optional.
+    The table data is similarly represented in a form with minimal enrichment.
+    """
+
+    REGEX_TABLE_OF_FIGURES: ClassVar[str] = (
+        r"^(?P<caption>Figure\s+(?P<figure_nr>\d+)\s*:"
+        r"\s*(?P<description>.*?))(?P<page_nr>\d+)?$"
+    )
+    REGEX_TABLE_ROW: ClassVar[str] = (
+        r"^(?P<caption>Figure\s+(?P<figure_nr>\d+)\s*:" r"\s*(?P<description>.*?))$"
+    )
+    figure_nr: int  # Figure as numbered in the specification document
+    caption: str  # The entire figure caption
+    description: str  # The part of figure caption without the "Fig X:" prefix
+
+    page_nr: Optional[int] = None
+    table: Optional[senfd.tables.Table] = None
+
+    @classmethod
+    def from_regex(cls, regex, text):
+        match = re.match(regex, text)
+        if not match:
+            return None
+
+        data = {
+            "figure_nr": int(match.group("figure_nr")),
+            "caption": match.group("caption").strip(),
+            "description": match.group("description").strip(),
+            "table": None,
+            "page_nr": None,
+        }
+        if "page_nr" in match.groupdict():
+            data["page_nr"] = int(match.group("page_nr"))
+
+        return cls(**data)
+
+
+class FromFigureDescriptionMatch(Figure):
+
+    @classmethod
+    def from_figure_description(cls, figure, match):
+        shared = set(figure.dict().keys()).intersection(set(match.groupdict().keys()))
+        if shared:
+            raise RuntimeError(f"cls({cls.__name__}) has overlap({shared})")
+
+        data = figure.dict()
+        mdict = match.groupdict()
+        if mdict:
+            data.update(mdict if mdict else {})
+
+        return cls(**data)
+
+    def into_document(self, document):
+        key = pascal_to_snake(self.__class__.__name__)
+        getattr(document, key).append(self)
+
+
+class Acronyms(FromFigureDescriptionMatch):
+    REGEX_FIGURE_DESCRIPTION: ClassVar[str] = r".*Acronym\s+(definitions|Descriptions)"
+
+
+class IoControllerCommandSetSupport(FromFigureDescriptionMatch):
+    REGEX_FIGURE_DESCRIPTION: ClassVar[str] = (
+        r".*-\s+(?P<command_set_name>.*)Command\s+Set\s+Support"
+    )
+    command_set_name: str
+
+
+class CnsValues(FromFigureDescriptionMatch):
+    """Enrichment of a: 'Command Dword N' table"""
+
+    REGEX_FIGURE_DESCRIPTION: ClassVar[str] = r".*CNS\s+Values.*"
+
+
+class CommandSupportRequirements(FromFigureDescriptionMatch):
+    REGEX_FIGURE_DESCRIPTION: ClassVar[str] = (
+        r"\s*(?P<command_span>.*)\s+Command\s*Support\s*Requirements.*"
+    )
+    command_span: str
+
+
+class CommandSqeDataPointer(FromFigureDescriptionMatch):
+    REGEX_FIGURE_DESCRIPTION: ClassVar[str] = (
+        r"(?P<command_name>[\w\s]+)\s+-\s+Data\s+Pointer"
+    )
+    command_name: str
+
+
+class CommandSqeDwords(FromFigureDescriptionMatch):
+    REGEX_FIGURE_DESCRIPTION: ClassVar[str] = (
+        r"(?P<command_name>[\w\s]+)\s*-\s*Command\s*Dword\s*"
+        r"(?P<command_dword_lower>\d+)"
+        r".*and.*?\s(?P<command_dword_upper>\d+)"
+    )
+    command_name: str
+    command_dword_lower: str
+    command_dword_upper: Optional[str]
+
+
+class CommandSqeDword(FromFigureDescriptionMatch):
+    REGEX_FIGURE_DESCRIPTION: ClassVar[str] = (
+        r"(?P<command_name>[\w\s]+)\s*-\s*Command\s*Dword\s*(?P<command_dword>\d+).*?"
+    )
+    command_name: str
+    command_dword: str
+
+
+class CommandCqeDword(FromFigureDescriptionMatch):
+    REGEX_FIGURE_DESCRIPTION: ClassVar[str] = (
+        r"(?P<command_name>[\w\s]+)\s+-\s+"
+        r"Completion\sQueue\sEntry\sDword\s(?P<command_dword>\d+)"
+    )
+    command_name: str
+    command_dword: str
+
+
+class CommandSetOpcodes(FromFigureDescriptionMatch):
+    REGEX_FIGURE_DESCRIPTION: ClassVar[str] = (
+        r"Opcodes\sfor\s(?P<command_set_name>.*)\sCommands"
+    )
+    command_set_name: str
+
+
+class GeneralCommandStatusValues(FromFigureDescriptionMatch):
+    REGEX_FIGURE_DESCRIPTION: ClassVar[str] = (
+        r".*(Generic|General)\s+Command\s+Status\s+Values.*"
+    )
+
+
+class CommandSpecificStatusValues(FromFigureDescriptionMatch):
+    REGEX_FIGURE_DESCRIPTION: ClassVar[str] = (
+        r"(?P<command_name>[\w\s]+)\s+-\s+Command\s+Specific\s+Status\s+Values"
+    )
+    command_name: str
+
+
+class FeatureIdentifiers(FromFigureDescriptionMatch):
+    REGEX_FIGURE_DESCRIPTION: ClassVar[str] = r".*Feature\s*Identifiers.*"
+
+
+class FeatureSupport(FromFigureDescriptionMatch):
+    REGEX_FIGURE_DESCRIPTION: ClassVar[str] = r".*Feature\s*Support.*"
+
+
+class LogPageIdentifiers(FromFigureDescriptionMatch):
+    REGEX_FIGURE_DESCRIPTION: ClassVar[str] = r".*Log\s+Page\s+Identifiers.*"
+
+
+class Offset(FromFigureDescriptionMatch):
+    REGEX_FIGURE_DESCRIPTION: ClassVar[str] = r".*offset"
+
+
+class PropertyDefinition(FromFigureDescriptionMatch):
+    REGEX_FIGURE_DESCRIPTION: ClassVar[str] = r".*Property Definition.*"

senfd/schema/__init__.py

@@ -0,0 +1 @@
+FILENAME = "enriched.figures.schema.json"

senfd/schema/checker.py

@@ -0,0 +1,21 @@
+"""
+
+"""
+
+import importlib.resources as pkg_resources
+import json
+from typing import Any, Dict
+
+import senfd.schema
+
+
+def get_schema() -> Dict[str, Any]:
+    """
+    Load the JSON Schema for the stage2 document from the package as a dict
+
+    Returns:
+        Dict[str, Any]: The JSON Schema as a dictionary.
+    """
+
+    with pkg_resources.open_text(senfd.schema, "enhanced.schema.json") as content:
+        return json.load(content)

senfd/schema/enriched.figures.schema.json

@@ -0,0 +1,110 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "properties": {
+    "path": {
+      "type": "string"
+    },
+    "figures": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "figure_nr": {
+            "type": "integer"
+          },
+          "page_nr": {
+            "type": "integer"
+          },
+          "title": {
+            "type": "string"
+          },
+          "description": {
+            "type": "string"
+          },
+          "table": {
+            "anyOf": [
+              {
+                "type": "null"
+              },
+              {
+                "type": "array",
+                "items": {
+                  "type": "array",
+                  "items": {
+                    "type": "object",
+                    "properties": {
+                      "text": {
+                        "type": "string"
+                      },
+                      "tables": {
+                        "type": "array",
+                        "items": {
+                          "type": "array",
+                          "items": {
+                            "type": "array",
+                            "items": {
+                              "type": "object",
+                              "properties": {
+                                "text": {
+                                  "type": "string"
+                                },
+                                "tables": {
+                                  "type": "array",
+                                  "items": {
+                                    "type": "array",
+                                    "items": {
+                                      "type": "array",
+                                      "items": {
+                                        "type": "object",
+                                        "properties": {
+                                          "text": {
+                                            "type": "string"
+                                          },
+                                          "tables": {
+                                            "type": "array"
+                                          }
+                                        },
+                                        "required": [
+                                          "tables",
+                                          "text"
+                                        ]
+                                      }
+                                    }
+                                  }
+                                }
+                              },
+                              "required": [
+                                "tables",
+                                "text"
+                              ]
+                            }
+                          }
+                        }
+                      }
+                    },
+                    "required": [
+                      "tables",
+                      "text"
+                    ]
+                  }
+                }
+              }
+            ]
+          }
+        },
+        "required": [
+          "description",
+          "figure_nr",
+          "page_nr",
+          "table",
+          "title"
+        ]
+      }
+    }
+  },
+  "required": [
+    "figures",
+    "path"
+  ]
+}