novelWriter 2.4.3__py3-none-any.whl → 2.5b1__py3-none-any.whl

novelwriter/core/tokenizer.py

@@ -24,18 +24,19 @@ along with this program. If not, see <https://www.gnu.org/licenses/>.
 """
 from __future__ import annotations
 
-import re
 import json
 import logging
+import re
 
 from abc import ABC, abstractmethod
-from time import time
-from pathlib import Path
 from functools import partial
+from pathlib import Path
+from time import time
 
 from PyQt5.QtCore import QCoreApplication, QRegularExpression
+from PyQt5.QtGui import QFont
 
-from novelwriter.common import formatTimeStamp, numberToRoman, checkInt
+from novelwriter.common import checkInt, formatTimeStamp, numberToRoman
 from novelwriter.constants import (
     nwHeadFmt, nwKeyWords, nwLabels, nwRegEx, nwShortcode, nwUnicode, trConst
 )
@@ -48,6 +49,10 @@ logger = logging.getLogger(__name__)
 ESCAPES = {r"\*": "*", r"\~": "~", r"\_": "_", r"\[": "[", r"\]": "]", r"\ ": ""}
 RX_ESC = re.compile("|".join([re.escape(k) for k in ESCAPES.keys()]), flags=re.DOTALL)
 
+T_Formats = list[tuple[int, int, str]]
+T_Comment = tuple[str, T_Formats]
+T_Token = tuple[int, int, str, T_Formats, int]
+
 
 def stripEscape(text: str) -> str:
     """Strip escaped Markdown characters from paragraph text."""
@@ -80,6 +85,8 @@ class Tokenizer(ABC):
     FMT_SUP_E = 12  # End superscript
     FMT_SUB_B = 13  # Begin subscript
     FMT_SUB_E = 14  # End subscript
+    FMT_FNOTE = 15  # Footnote marker
+    FMT_STRIP = 16  # Strip the format code
 
     # Block Type
     T_EMPTY    = 1   # Empty line (new paragraph)
@@ -111,45 +118,53 @@ class Tokenizer(ABC):
 
     # Lookups
     L_HEADINGS = [T_TITLE, T_HEAD1, T_HEAD2, T_HEAD3, T_HEAD4]
+    L_SKIP_INDENT = [T_TITLE, T_HEAD1, T_HEAD2, T_HEAD2, T_HEAD3, T_HEAD4, T_SEP, T_SKIP]
+    L_SUMMARY = [T_SYNOPSIS, T_SHORT]
 
     def __init__(self, project: NWProject) -> None:
 
         self._project = project
 
         # Data Variables
-        self._text   = ""    # The raw text to be tokenized
-        self._handle = None  # The item handle currently being processed
-        self._result = ""    # The result of the last document
+        self._text   = ""     # The raw text to be tokenized
+        self._handle = None   # The item handle currently being processed
+        self._result = ""     # The result of the last document
+        self._keepMD = False  # Whether to keep the markdown text
 
-        self._keepMarkdown = False  # Whether to keep the markdown text
-        self._allMarkdown  = []     # The result novelWriter markdown of all documents
+        # Tokens and Meta Data (Per Document)
+        self._tokens: list[T_Token] = []
+        self._footnotes: dict[str, T_Comment] = {}
 
-        # Processed Tokens and Meta Data
-        self._tokens: list[tuple[int, int, str, list[tuple[int, int]], int]] = []
+        # Tokens and Meta Data (Per Instance)
         self._counts: dict[str, int] = {}
         self._outline: dict[str, str] = {}
+        self._markdown: list[str] = []
 
         # User Settings
-        self._textFont     = "Serif"  # Output text font
-        self._textSize     = 11       # Output text size
-        self._textFixed    = False    # Fixed width text
+        self._textFont     = QFont("Serif", 11)  # Output text font
         self._lineHeight   = 1.15     # Line height in units of em
         self._blockIndent  = 4.00     # Block indent in units of em
+        self._firstIndent  = False    # Enable first line indent
+        self._firstWidth   = 1.40     # First line indent in units of em
+        self._indentFirst  = False    # Indent first paragraph
         self._doJustify    = False    # Justify text
         self._doBodyText   = True     # Include body text
         self._doSynopsis   = False    # Also process synopsis comments
         self._doComments   = False    # Also process comments
         self._doKeywords   = False    # Also process keywords like tags and references
         self._skipKeywords = set()    # Keywords to ignore
+        self._keepBreaks   = True     # Keep line breaks in paragraphs
 
         # Margins
-        self._marginTitle = (1.000, 0.500)
-        self._marginHead1 = (1.000, 0.500)
-        self._marginHead2 = (0.834, 0.500)
-        self._marginHead3 = (0.584, 0.500)
-        self._marginHead4 = (0.584, 0.500)
+        self._marginTitle = (1.417, 0.500)
+        self._marginHead1 = (1.417, 0.500)
+        self._marginHead2 = (1.668, 0.500)
+        self._marginHead3 = (1.168, 0.500)
+        self._marginHead4 = (1.168, 0.500)
         self._marginText  = (0.000, 0.584)
         self._marginMeta  = (0.000, 0.584)
+        self._marginFoot  = (1.417, 0.467)
+        self._marginSep   = (1.168, 1.168)
 
         # Title Formats
         self._fmtTitle   = nwHeadFmt.TITLE  # Formatting for titles
@@ -205,6 +220,9 @@ class Tokenizer(ABC):
             nwShortcode.SUP_O:    self.FMT_SUP_B, nwShortcode.SUP_C:    self.FMT_SUP_E,
             nwShortcode.SUB_O:    self.FMT_SUB_B, nwShortcode.SUB_C:    self.FMT_SUB_E,
         }
+        self._shortCodeVals = {
+            nwShortcode.FOOTNOTE_B: self.FMT_FNOTE,
+        }
 
         return
 
@@ -220,7 +238,7 @@ class Tokenizer(ABC):
     @property
     def allMarkdown(self) -> list[str]:
         """The combined novelWriter Markdown text."""
-        return self._allMarkdown
+        return self._markdown
 
     @property
     def textStats(self) -> dict[str, int]:
@@ -298,11 +316,9 @@ class Tokenizer(ABC):
         )
         return
 
-    def setFont(self, family: str, size: int, isFixed: bool = False) -> None:
+    def setFont(self, font: QFont) -> None:
         """Set the build font."""
-        self._textFont = family
-        self._textSize = round(int(size))
-        self._textFixed = isFixed
+        self._textFont = font
         return
 
     def setLineHeight(self, height: float) -> None:
@@ -315,6 +331,15 @@ class Tokenizer(ABC):
         self._blockIndent = min(max(float(indent), 0.0), 10.0)
         return
 
+    def setFirstLineIndent(self, state: bool, indent: float, first: bool) -> None:
+        """Set first line indent and whether to also indent first
+        paragraph after a heading.
+        """
+        self._firstIndent = state
+        self._firstWidth = indent
+        self._indentFirst = first
+        return
+
     def setJustify(self, state: bool) -> None:
         """Enable or disable text justification."""
         self._doJustify = state
@@ -355,6 +380,11 @@ class Tokenizer(ABC):
         self._marginMeta = (float(upper), float(lower))
         return
 
+    def setSeparatorMargins(self, upper: float, lower: float) -> None:
+        """Set the upper and lower meta text margin."""
+        self._marginSep = (float(upper), float(lower))
+        return
+
     def setLinkHeadings(self, state: bool) -> None:
         """Enable or disable adding an anchor before headings."""
         self._linkHeadings = state
@@ -385,9 +415,14 @@ class Tokenizer(ABC):
         self._skipKeywords = set(x.lower().strip() for x in keywords.split(","))
         return
 
+    def setKeepLineBreaks(self, state: bool) -> None:
+        """Keep line breaks in paragraphs."""
+        self._keepBreaks = state
+        return
+
     def setKeepMarkdown(self, state: bool) -> None:
         """Keep original markdown during build."""
-        self._keepMarkdown = state
+        self._keepMD = state
         return
 
     ##
@@ -417,8 +452,8 @@ class Tokenizer(ABC):
             self._tokens.append((
                 self.T_TITLE, 1, title, [], textAlign
             ))
-            if self._keepMarkdown:
-                self._allMarkdown.append(f"#! {title}\n\n")
+            if self._keepMD:
+                self._markdown.append(f"#! {title}\n\n")
 
         return
 
@@ -466,22 +501,23 @@ class Tokenizer(ABC):
           4: The internal formatting map of the text, self.FMT_*
           5: The style of the block, self.A_*
         """
-        self._tokens = []
         if self._isNovel:
             self._hFormatter.setHandle(self._handle)
 
         nHead = 0
         breakNext = False
         tmpMarkdown = []
+        tHandle = self._handle or ""
+        tokens: list[T_Token] = []
         for aLine in self._text.splitlines():
             sLine = aLine.strip().lower()
 
             # Check for blank lines
             if len(sLine) == 0:
-                self._tokens.append((
+                tokens.append((
                     self.T_EMPTY, nHead, "", [], self.A_NONE
                 ))
-                if self._keepMarkdown:
+                if self._keepMD:
                     tmpMarkdown.append("\n")
 
                 continue
@@ -507,7 +543,7 @@ class Tokenizer(ABC):
                     continue
 
                 elif sLine == "[vspace]":
-                    self._tokens.append(
+                    tokens.append(
                         (self.T_SKIP, nHead, "", [], sAlign)
                     )
                     continue
@@ -515,11 +551,11 @@ class Tokenizer(ABC):
                 elif sLine.startswith("[vspace:") and sLine.endswith("]"):
                     nSkip = checkInt(sLine[8:-1], 0)
                     if nSkip >= 1:
-                        self._tokens.append(
+                        tokens.append(
                             (self.T_SKIP, nHead, "", [], sAlign)
                         )
                     if nSkip > 1:
-                        self._tokens += (nSkip - 1) * [
+                        tokens += (nSkip - 1) * [
                             (self.T_SKIP, nHead, "", [], self.A_NONE)
                         ]
                     continue
@@ -533,24 +569,32 @@ class Tokenizer(ABC):
                 if aLine.startswith("%~"):
                     continue
 
-                cStyle, cText, _ = processComment(aLine)
+                cStyle, cKey, cText, _, _ = processComment(aLine)
                 if cStyle == nwComment.SYNOPSIS:
-                    self._tokens.append((
-                        self.T_SYNOPSIS, nHead, cText, [], sAlign
+                    tLine, tFmt = self._extractFormats(cText)
+                    tokens.append((
+                        self.T_SYNOPSIS, nHead, tLine, tFmt, sAlign
                     ))
-                    if self._doSynopsis and self._keepMarkdown:
+                    if self._doSynopsis and self._keepMD:
                         tmpMarkdown.append(f"{aLine}\n")
                 elif cStyle == nwComment.SHORT:
-                    self._tokens.append((
-                        self.T_SHORT, nHead, cText, [], sAlign
+                    tLine, tFmt = self._extractFormats(cText)
+                    tokens.append((
+                        self.T_SHORT, nHead, tLine, tFmt, sAlign
                     ))
-                    if self._doSynopsis and self._keepMarkdown:
+                    if self._doSynopsis and self._keepMD:
+                        tmpMarkdown.append(f"{aLine}\n")
+                elif cStyle == nwComment.FOOTNOTE:
+                    tLine, tFmt = self._extractFormats(cText, skip=self.FMT_FNOTE)
+                    self._footnotes[f"{tHandle}:{cKey}"] = (tLine, tFmt)
+                    if self._keepMD:
                         tmpMarkdown.append(f"{aLine}\n")
                 else:
-                    self._tokens.append((
-                        self.T_COMMENT, nHead, cText, [], sAlign
+                    tLine, tFmt = self._extractFormats(cText)
+                    tokens.append((
+                        self.T_COMMENT, nHead, tLine, tFmt, sAlign
                     ))
-                    if self._doComments and self._keepMarkdown:
+                    if self._doComments and self._keepMD:
                         tmpMarkdown.append(f"{aLine}\n")
 
             elif aLine.startswith("@"):
@@ -560,11 +604,14 @@ class Tokenizer(ABC):
                 # are automatically skipped.
 
                 valid, bits, _ = self._project.index.scanThis(aLine)
-                if valid and bits and bits[0] not in self._skipKeywords:
-                    self._tokens.append((
+                if (
+                    valid and bits and bits[0] in nwLabels.KEY_NAME
+                    and bits[0] not in self._skipKeywords
+                ):
+                    tokens.append((
                         self.T_KEYWORD, nHead, aLine[1:].strip(), [], sAlign
                     ))
-                    if self._doKeywords and self._keepMarkdown:
+                    if self._doKeywords and self._keepMD:
                         tmpMarkdown.append(f"{aLine}\n")
 
             elif aLine.startswith(("# ", "#! ")):
@@ -597,10 +644,10 @@ class Tokenizer(ABC):
                         self._hFormatter.resetAll()
                     self._noSep = True
 
-                self._tokens.append((
+                tokens.append((
                     tType, nHead, tText, [], tStyle
                 ))
-                if self._keepMarkdown:
+                if self._keepMD:
                     tmpMarkdown.append(f"{aLine}\n")
 
             elif aLine.startswith(("## ", "##! ")):
@@ -632,10 +679,10 @@ class Tokenizer(ABC):
                     self._hFormatter.resetScene()
                     self._noSep = True
 
-                self._tokens.append((
+                tokens.append((
                     tType, nHead, tText, [], tStyle
                 ))
-                if self._keepMarkdown:
+                if self._keepMD:
                     tmpMarkdown.append(f"{aLine}\n")
 
             elif aLine.startswith(("### ", "###! ")):
@@ -673,10 +720,10 @@ class Tokenizer(ABC):
                             tStyle = self.A_NONE if self._noSep else self.A_CENTRE
                     self._noSep = False
 
-                self._tokens.append((
+                tokens.append((
                     tType, nHead, tText, [], tStyle
                 ))
-                if self._keepMarkdown:
+                if self._keepMD:
                     tmpMarkdown.append(f"{aLine}\n")
 
             elif aLine.startswith("#### "):
@@ -703,10 +750,10 @@ class Tokenizer(ABC):
                             tType = self.T_SEP
                             tStyle = self.A_CENTRE
 
-                self._tokens.append((
+                tokens.append((
                     tType, nHead, tText, [], tStyle
                 ))
-                if self._keepMarkdown:
+                if self._keepMD:
                     tmpMarkdown.append(f"{aLine}\n")
 
             else:
@@ -750,54 +797,91 @@ class Tokenizer(ABC):
                     sAlign |= self.A_IND_R
 
                 # Process formats
-                tLine, fmtPos = self._extractFormats(aLine)
-                self._tokens.append((
-                    self.T_TEXT, nHead, tLine, fmtPos, sAlign
+                tLine, tFmt = self._extractFormats(aLine)
+                tokens.append((
+                    self.T_TEXT, nHead, tLine, tFmt, sAlign
                 ))
-                if self._keepMarkdown:
+                if self._keepMD:
                     tmpMarkdown.append(f"{aLine}\n")
 
         # If we have content, turn off the first page flag
-        if self._isFirst and self._tokens:
+        if self._isFirst and tokens:
             self._isFirst = False  # First document has been processed
 
             # Make sure the token array doesn't start with a page break
             # on the very first page, adding a blank first page.
-            if self._tokens[0][4] & self.A_PBB:
-                token = self._tokens[0]
-                self._tokens[0] = (
-                    token[0], token[1], token[2], token[3], token[4] & ~self.A_PBB
+            if tokens[0][4] & self.A_PBB:
+                cToken = tokens[0]
+                tokens[0] = (
+                    cToken[0], cToken[1], cToken[2], cToken[3], cToken[4] & ~self.A_PBB
                 )
 
         # Always add an empty line at the end of the file
-        self._tokens.append((
+        tokens.append((
             self.T_EMPTY, nHead, "", [], self.A_NONE
         ))
-        if self._keepMarkdown:
+        if self._keepMD:
             tmpMarkdown.append("\n")
-            self._allMarkdown.append("".join(tmpMarkdown))
+            self._markdown.append("".join(tmpMarkdown))
 
         # Second Pass
         # ===========
-        # Some items need a second pass
+        # This second pass strips away consecutive blank lines, and
+        # combines consecutive text lines into the same paragraph.
+        # It also ensures that there isn't paragraph spacing between
+        # meta data lines for formats that has spacing.
+
+        self._tokens = []
+        pToken: T_Token = (self.T_EMPTY, 0, "", [], self.A_NONE)
+        nToken: T_Token = (self.T_EMPTY, 0, "", [], self.A_NONE)
+
+        lineSep = "\n" if self._keepBreaks else " "
+        pLines: list[T_Token] = []
 
-        pToken = (self.T_EMPTY, 0, "", [], self.A_NONE)
-        nToken = (self.T_EMPTY, 0, "", [], self.A_NONE)
-        tCount = len(self._tokens)
-        for n, token in enumerate(self._tokens):
+        tCount = len(tokens)
+        for n, cToken in enumerate(tokens):
 
             if n > 0:
-                pToken = self._tokens[n-1]
+                pToken = tokens[n-1]  # Look behind
             if n < tCount - 1:
-                nToken = self._tokens[n+1]
+                nToken = tokens[n+1]  # Look ahead
 
-            if token[0] == self.T_KEYWORD:
-                aStyle = token[4]
+            if cToken[0] == self.T_EMPTY:
+                # We don't need to keep the empty lines after this pass
+                pass
+
+            elif cToken[0] == self.T_KEYWORD:
+                # Adjust margins for lines in a list of keyword lines
+                aStyle = cToken[4]
                 if pToken[0] == self.T_KEYWORD:
                     aStyle |= self.A_Z_TOPMRG
                 if nToken[0] == self.T_KEYWORD:
                     aStyle |= self.A_Z_BTMMRG
-                self._tokens[n] = (token[0], token[1], token[2], token[3], aStyle)
+                self._tokens.append((
+                    cToken[0], cToken[1], cToken[2], cToken[3], aStyle
+                ))
+
+            elif cToken[0] == self.T_TEXT:
+                # Combine lines from the same paragraph
+                pLines.append(cToken)
+                if nToken[0] != self.T_TEXT:
+                    nLines = len(pLines)
+                    if nLines == 1:
+                        self._tokens.append(pLines[0])
+                    elif nLines > 1:
+                        tTxt = ""
+                        tFmt: T_Formats = []
+                        for aToken in pLines:
+                            tLen = len(tTxt)
+                            tTxt += f"{aToken[2]}{lineSep}"
+                            tFmt.extend((p+tLen, fmt, key) for p, fmt, key in aToken[3])
+                        self._tokens.append((
+                            self.T_TEXT, pLines[0][1], tTxt[:-1], tFmt, pLines[0][4]
+                        ))
+                    pLines = []
+
+            else:
+                self._tokens.append(cToken)
 
         return
 
@@ -840,7 +924,6 @@ class Tokenizer(ABC):
         textWordChars = self._counts.get("textWordChars", 0)
         titleWordChars = self._counts.get("titleWordChars", 0)
 
-        para = []
         for tType, _, tText, _, _ in self._tokens:
             tText = tText.replace(nwUnicode.U_ENDASH, " ")
             tText = tText.replace(nwUnicode.U_EMDASH, " ")
@@ -850,22 +933,19 @@ class Tokenizer(ABC):
             nChars = len(tText)
             nWChars = len("".join(tWords))
 
-            if tType == self.T_EMPTY:
-                if len(para) > 0:
-                    tTemp = "\n".join(para)
-                    tPWords = tTemp.split()
-                    nPWords = len(tPWords)
-                    nPChars = len(tTemp)
-                    nPWChars = len("".join(tPWords))
-
-                    paragraphCount += 1
-                    allWords += nPWords
-                    textWords += nPWords
-                    allChars += nPChars
-                    textChars += nPChars
-                    allWordChars += nPWChars
-                    textWordChars += nPWChars
-                para = []
+            if tType == self.T_TEXT:
+                tPWords = tText.split()
+                nPWords = len(tPWords)
+                nPChars = len(tText)
+                nPWChars = len("".join(tPWords))
+
+                paragraphCount += 1
+                allWords += nPWords
+                textWords += nPWords
+                allChars += nPChars
+                textChars += nPChars
+                allWordChars += nPWChars
+                textWordChars += nPWChars
 
             elif tType in self.L_HEADINGS:
                 titleCount += 1
@@ -881,9 +961,6 @@ class Tokenizer(ABC):
                 allChars += nChars
                 allWordChars += nWChars
 
-            elif tType == self.T_TEXT:
-                para.append(tText.rstrip())
-
             elif tType == self.T_SYNOPSIS and self._doSynopsis:
                 text = "{0}: {1}".format(self._localLookup("Synopsis"), tText)
                 words = text.split()
@@ -935,7 +1012,7 @@ class Tokenizer(ABC):
     def saveRawMarkdown(self, path: str | Path) -> None:
         """Save the raw text to a plain text file."""
         with open(path, mode="w", encoding="utf-8") as outFile:
-            for nwdPage in self._allMarkdown:
+            for nwdPage in self._markdown:
                 outFile.write(nwdPage)
         return
 
@@ -950,7 +1027,7 @@ class Tokenizer(ABC):
                 "buildTimeStr": formatTimeStamp(timeStamp),
             },
             "text": {
-                "nwd": [page.rstrip("\n").split("\n") for page in self._allMarkdown],
+                "nwd": [page.rstrip("\n").split("\n") for page in self._markdown],
             }
         }
         with open(path, mode="w", encoding="utf-8") as fObj:
@@ -961,9 +1038,9 @@ class Tokenizer(ABC):
     #  Internal Functions
     ##
 
-    def _extractFormats(self, text: str) -> tuple[str, list[tuple[int, int]]]:
+    def _extractFormats(self, text: str, skip: int = 0) -> tuple[str, T_Formats]:
         """Extract format markers from a text paragraph."""
-        temp = []
+        temp: list[tuple[int, int, int, str]] = []
 
         # Match Markdown
         for regEx, fmts in self._rxMarkdown:
@@ -971,7 +1048,7 @@ class Tokenizer(ABC):
             while rxItt.hasNext():
                 rxMatch = rxItt.next()
                 temp.extend(
-                    [rxMatch.capturedStart(n), rxMatch.capturedLength(n), fmt]
+                    (rxMatch.capturedStart(n), rxMatch.capturedLength(n), fmt, "")
                     for n, fmt in enumerate(fmts) if fmt > 0
                 )
 
@@ -979,25 +1056,37 @@ class Tokenizer(ABC):
         rxItt = self._rxShortCodes.globalMatch(text, 0)
         while rxItt.hasNext():
             rxMatch = rxItt.next()
-            temp.append([
+            temp.append((
                 rxMatch.capturedStart(1),
                 rxMatch.capturedLength(1),
-                self._shortCodeFmt.get(rxMatch.captured(1).lower(), 0)
-            ])
+                self._shortCodeFmt.get(rxMatch.captured(1).lower(), 0),
+                "",
+            ))
 
-        # Post-process text and format markers
+        # Match Shortcode w/Values
+        rxItt = self._rxShortCodeVals.globalMatch(text, 0)
+        tHandle = self._handle or ""
+        while rxItt.hasNext():
+            rxMatch = rxItt.next()
+            kind = self._shortCodeVals.get(rxMatch.captured(1).lower(), 0)
+            temp.append((
+                rxMatch.capturedStart(0),
+                rxMatch.capturedLength(0),
+                self.FMT_STRIP if kind == skip else kind,
+                f"{tHandle}:{rxMatch.captured(2)}",
+            ))
+
+        # Post-process text and format
         result = text
         formats = []
-        for pos, n, fmt in reversed(sorted(temp, key=lambda x: x[0])):
+        for pos, n, fmt, key in reversed(sorted(temp, key=lambda x: x[0])):
             if fmt > 0:
                 result = result[:pos] + result[pos+n:]
-                formats = [(p-n, f) for p, f in formats]
-                formats.insert(0, (pos, fmt))
+                formats = [(p-n, f, k) for p, f, k in formats]
+                formats.insert(0, (pos, fmt, key))
 
         return result, formats
 
-# END Class Tokenizer
-
 
 class HeadingFormatter:
 
@@ -1067,5 +1156,3 @@ class HeadingFormatter:
             hFormat = hFormat.replace(nwHeadFmt.CHAR_FOCUS, fText)
 
         return hFormat
-
-# END Class HeadingFormatter