novelWriter 2.1.1__py3-none-any.whl → 2.2rc1__py3-none-any.whl

novelwriter/core/options.py

@@ -28,7 +28,7 @@ import json
 import logging
 
 from enum import Enum
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, TypeVar
 from pathlib import Path
 
 from novelwriter.error import logException
@@ -40,6 +40,8 @@ if TYPE_CHECKING:  # pragma: no cover
 
 logger = logging.getLogger(__name__)
 
+NWEnum = TypeVar("NWEnum", bound=Enum)
+
 VALID_MAP = {
     "GuiWritingStats": {
         "winWidth", "winHeight", "widthCol0", "widthCol1", "widthCol2",
@@ -67,6 +69,9 @@ VALID_MAP = {
     "GuiManuscriptBuild": {
         "winWidth", "winHeight", "fmtWidth", "sumWidth",
     },
+    "GuiDocViewerPanel": {
+        "colWidths",
+    }
 }
 
 
@@ -201,11 +206,11 @@ class OptionState:
             return checkBool(self._state[group].get(name, default), default)
         return default
 
-    def getEnum(self, group: str, name: str, lookup: type, default: Enum) -> Enum:
+    def getEnum(self, group: str, name: str, lookup: type, default: NWEnum) -> NWEnum:
         """Return the value mapped to an enum. Otherwise return the
         default value.
         """
-        if issubclass(lookup, Enum):
+        if issubclass(lookup, type(default)):
             if group in self._state:
                 if name in self._state[group]:
                     value = self._state[group][name]

novelwriter/core/project.py

@@ -45,7 +45,7 @@ from novelwriter.core.sessions import NWSessionLog
 from novelwriter.core.projectxml import ProjectXMLReader, ProjectXMLWriter, XMLReadState
 from novelwriter.core.projectdata import NWProjectData
 from novelwriter.common import (
-    checkStringNone, formatInt, formatTimeStamp, hexToInt, makeFileNameSafe, minmax
+    checkStringNone, formatInt, formatTimeStamp, getFileSize, hexToInt, makeFileNameSafe, minmax
 )
 
 if TYPE_CHECKING:  # pragma: no cover
@@ -420,7 +420,7 @@ class NWProject:
         timeStamp = formatTimeStamp(time(), fileSafe=True)
         archName = baseDir / f"{cleanName} {timeStamp}.zip"
         if self._storage.zipIt(archName, compression=2):
-            size = formatInt(archName.stat().st_size)
+            size = formatInt(getFileSize(archName))
             if doNotify:
                 SHARED.info(
                     self.tr("Created a backup of your project of size {0}B.").format(size),

novelwriter/core/projectdata.py

@@ -152,14 +152,14 @@ class NWProjectData:
         """Return the initial count of words for novel and note
         documents.
         """
-        return tuple(self._initCounts)
+        return self._initCounts[0], self._initCounts[1]
 
     @property
     def currCounts(self) -> tuple[int, int]:
         """Return the current count of words for novel and note
         documents.
         """
-        return tuple(self._currCounts)
+        return self._currCounts[0], self._currCounts[1]
 
     @property
     def lastHandle(self) -> dict[str, str | None]:
@@ -170,7 +170,7 @@ class NWProjectData:
 
     @property
     def autoReplace(self) -> dict[str, str]:
-        """Return the autoreplace dictionary."""
+        """Return the auto-replace dictionary."""
         return self._autoReplace
 
     @property

novelwriter/core/tohtml.py

@@ -111,7 +111,7 @@ class ToHtml(Tokenizer):
 
     def getFullResultSize(self) -> int:
         """Return the size of the full HTML result."""
-        return sum([len(x) for x in self._fullHTML])
+        return sum(len(x) for x in self._fullHTML)
 
     def doPreProcessing(self) -> None:
         """Extend the auto-replace to also properly encode some unicode
@@ -122,9 +122,7 @@ class ToHtml(Tokenizer):
         return
 
     def doConvert(self) -> None:
-        """Convert the list of text tokens into a HTML document saved
-        to _result.
-        """
+        """Convert the list of text tokens into an HTML document."""
         if self._genMode == self.M_PREVIEW:
             htmlTags = {  # HTML4 + CSS2 (for Qt)
                 self.FMT_B_B: "<b>",
@@ -133,6 +131,8 @@ class ToHtml(Tokenizer):
                 self.FMT_I_E: "</i>",
                 self.FMT_D_B: "<span style='text-decoration: line-through;'>",
                 self.FMT_D_E: "</span>",
+                self.FMT_U_B: "<u>",
+                self.FMT_U_E: "</u>",
             }
         else:
             htmlTags = {  # HTML5 (for export)
@@ -142,8 +142,15 @@ class ToHtml(Tokenizer):
                 self.FMT_I_E: "</em>",
                 self.FMT_D_B: "<del>",
                 self.FMT_D_E: "</del>",
+                self.FMT_U_B: "<span style='text-decoration: underline;'>",
+                self.FMT_U_E: "</span>",
             }
 
+        htmlTags[self.FMT_SUP_B] = "<sup>"
+        htmlTags[self.FMT_SUP_E] = "</sup>"
+        htmlTags[self.FMT_SUB_B] = "<sub>"
+        htmlTags[self.FMT_SUB_E] = "</sub>"
+
         if self._isNovel and self._genMode != self.M_PREVIEW:
             # For story files, we bump the titles one level up
             h1Cl = " class='title'"
@@ -160,9 +167,9 @@ class ToHtml(Tokenizer):
 
         self._result = ""
 
-        thisPar = []
-        parStyle = None
-        tmpResult = []
+        para = []
+        pStyle = None
+        lines = []
 
         for tType, nHead, tText, tFormat, tStyle in self._tokens:
 
@@ -174,18 +181,16 @@ class ToHtml(Tokenizer):
                 for c in tText:
                     if c == "<":
                         cText.append("&lt;")
-                        tFormat = [[a + 3 if a > i else a, b, c] for a, b, c in tFormat]
+                        tFormat = [[p + 3 if p > i else p, f] for p, f in tFormat]
                         i += 4
                     elif c == ">":
                         cText.append("&gt;")
-                        tFormat = [[a + 3 if a > i else a, b, c] for a, b, c in tFormat]
+                        tFormat = [[p + 3 if p > i else p, f] for p, f in tFormat]
                         i += 4
                     else:
                         cText.append(c)
                         i += 1
-
                 tText = "".join(cText)
-
             else:
                 # If we don't have formatting, we can do a plain replace
                 tText = tText.replace("<", "&lt;").replace(">", "&gt;")
@@ -231,69 +236,70 @@ class ToHtml(Tokenizer):
 
             # Process Text Type
             if tType == self.T_EMPTY:
-                if parStyle is None:
-                    parStyle = ""
-                if len(thisPar) > 1 and self._cssStyles:
-                    parClass = " class='break'"
+                if pStyle is None:
+                    pStyle = ""
+                if len(para) > 1 and self._cssStyles:
+                    pClass = " class='break'"
                 else:
-                    parClass = ""
-                if len(thisPar) > 0:
-                    tTemp = "<br/>".join(thisPar)
-                    tmpResult.append(f"<p{parClass+parStyle}>{tTemp.rstrip()}</p>\n")
-                thisPar = []
-                parStyle = None
+                    pClass = ""
+                if len(para) > 0:
+                    tTemp = "<br/>".join(para)
+                    lines.append(f"<p{pClass+pStyle}>{tTemp.rstrip()}</p>\n")
+                para = []
+                pStyle = None
 
             elif tType == self.T_TITLE:
                 tHead = tText.replace(nwHeadFmt.BR, "<br/>")
-                tmpResult.append(f"<h1 class='title'{hStyle}>{aNm}{tHead}</h1>\n")
+                lines.append(f"<h1 class='title'{hStyle}>{aNm}{tHead}</h1>\n")
 
             elif tType == self.T_UNNUM:
                 tHead = tText.replace(nwHeadFmt.BR, "<br/>")
-                tmpResult.append(f"<{h2}{hStyle}>{aNm}{tHead}</{h2}>\n")
+                lines.append(f"<{h2}{hStyle}>{aNm}{tHead}</{h2}>\n")
 
             elif tType == self.T_HEAD1:
                 tHead = tText.replace(nwHeadFmt.BR, "<br/>")
-                tmpResult.append(f"<{h1}{h1Cl}{hStyle}>{aNm}{tHead}</{h1}>\n")
+                lines.append(f"<{h1}{h1Cl}{hStyle}>{aNm}{tHead}</{h1}>\n")
 
             elif tType == self.T_HEAD2:
                 tHead = tText.replace(nwHeadFmt.BR, "<br/>")
-                tmpResult.append(f"<{h2}{hStyle}>{aNm}{tHead}</{h2}>\n")
+                lines.append(f"<{h2}{hStyle}>{aNm}{tHead}</{h2}>\n")
 
             elif tType == self.T_HEAD3:
                 tHead = tText.replace(nwHeadFmt.BR, "<br/>")
-                tmpResult.append(f"<{h3}{hStyle}>{aNm}{tHead}</{h3}>\n")
+                lines.append(f"<{h3}{hStyle}>{aNm}{tHead}</{h3}>\n")
 
             elif tType == self.T_HEAD4:
                 tHead = tText.replace(nwHeadFmt.BR, "<br/>")
-                tmpResult.append(f"<{h4}{hStyle}>{aNm}{tHead}</{h4}>\n")
+                lines.append(f"<{h4}{hStyle}>{aNm}{tHead}</{h4}>\n")
 
             elif tType == self.T_SEP:
-                tmpResult.append(f"<p class='sep'{hStyle}>{tText}</p>\n")
+                lines.append(f"<p class='sep'{hStyle}>{tText}</p>\n")
 
             elif tType == self.T_SKIP:
-                tmpResult.append(f"<p class='skip'{hStyle}>&nbsp;</p>\n")
+                lines.append(f"<p class='skip'{hStyle}>&nbsp;</p>\n")
 
             elif tType == self.T_TEXT:
                 tTemp = tText
-                if parStyle is None:
-                    parStyle = hStyle
-                for xPos, xLen, xFmt in reversed(tFormat):
-                    tTemp = tTemp[:xPos] + htmlTags[xFmt] + tTemp[xPos+xLen:]
-                thisPar.append(stripEscape(tTemp.rstrip()))
+                if pStyle is None:
+                    pStyle = hStyle
+                for pos, fmt in reversed(tFormat):
+                    tTemp = f"{tTemp[:pos]}{htmlTags[fmt]}{tTemp[pos:]}"
+                para.append(stripEscape(tTemp.rstrip()))
 
             elif tType == self.T_SYNOPSIS and self._doSynopsis:
-                tmpResult.append(self._formatSynopsis(tText))
+                lines.append(self._formatSynopsis(tText, True))
+
+            elif tType == self.T_SHORT and self._doSynopsis:
+                lines.append(self._formatSynopsis(tText, False))
 
             elif tType == self.T_COMMENT and self._doComments:
-                tmpResult.append(self._formatComments(tText))
+                lines.append(self._formatComments(tText))
 
             elif tType == self.T_KEYWORD and self._doKeywords:
                 tTemp = f"<p{hStyle}>{self._formatKeywords(tText)}</p>\n"
-                tmpResult.append(tTemp)
-
-        self._result = "".join(tmpResult)
-        tmpResult = []
+                lines.append(tTemp)
 
+        self._result = "".join(lines)
         if self._genMode != self.M_PREVIEW:
             self._fullHTML.append(self._result)
 
@@ -451,13 +457,15 @@ class ToHtml(Tokenizer):
     #  Internal Functions
     ##
 
-    def _formatSynopsis(self, text: str) -> str:
+    def _formatSynopsis(self, text: str, synopsis: bool) -> str:
         """Apply HTML formatting to synopsis."""
+        if synopsis:
+            sSynop = self._localLookup("Synopsis")
+        else:
+            sSynop = self._localLookup("Short Description")
         if self._genMode == self.M_PREVIEW:
-            sSynop = self._trSynopsis
             return f"<p class='comment'><span class='synopsis'>{sSynop}:</span> {text}</p>\n"
         else:
-            sSynop = self._localLookup("Synopsis")
             return f"<p class='synopsis'><strong>{sSynop}:</strong> {text}</p>\n"
 
     def _formatComments(self, text: str) -> str:
@@ -471,25 +479,18 @@ class ToHtml(Tokenizer):
     def _formatKeywords(self, text: str) -> str:
         """Apply HTML formatting to keywords."""
         valid, bits, _ = self._project.index.scanThis("@"+text)
-        if not valid or not bits:
+        if not valid or not bits or bits[0] not in nwLabels.KEY_NAME:
             return ""
 
-        result = ""
-        tags = []
-        if bits[0] in nwLabels.KEY_NAME:
-            result += f"<span class='tags'>{nwLabels.KEY_NAME[bits[0]]}:</span> "
-            if len(bits) > 1:
-                if bits[0] == nwKeyWords.TAG_KEY:
-                    result += f"<a name='tag_{bits[1]}'>{bits[1]}</a>"
+        result = f"<span class='tags'>{self._localLookup(nwLabels.KEY_NAME[bits[0]])}:</span> "
+        if len(bits) > 1:
+            if bits[0] == nwKeyWords.TAG_KEY:
+                result += f"<a name='tag_{bits[1]}'>{bits[1]}</a>"
+            else:
+                if self._genMode == self.M_PREVIEW:
+                    result += ", ".join(f"<a href='#{bits[0][1:]}={t}'>{t}</a>" for t in bits[1:])
                 else:
-                    if self._genMode == self.M_PREVIEW:
-                        for tTag in bits[1:]:
-                            tags.append(f"<a href='#{bits[0][1:]}={tTag}'>{tTag}</a>")
-                        result += ", ".join(tags)
-                    else:
-                        for tTag in bits[1:]:
-                            tags.append(f"<a href='#tag_{tTag}'>{tTag}</a>")
-                        result += ", ".join(tags)
+                    result += ", ".join(f"<a href='#tag_{t}'>{t}</a>" for t in bits[1:])
 
         return result
 

novelwriter/core/tokenizer.py

@@ -31,27 +31,26 @@ import logging
 from abc import ABC, abstractmethod
 from time import time
 from pathlib import Path
-from operator import itemgetter
 from functools import partial
 
 from PyQt5.QtCore import QCoreApplication, QRegularExpression
+from novelwriter.core.index import processComment
 
-from novelwriter.enum import nwItemLayout
+from novelwriter.enum import nwComment, nwItemLayout
 from novelwriter.common import formatTimeStamp, numberToRoman, checkInt
-from novelwriter.constants import nwConst, nwHeadFmt, nwRegEx, nwUnicode
+from novelwriter.constants import nwHeadFmt, nwRegEx, nwShortcode, nwUnicode
 from novelwriter.core.project import NWProject
 
 logger = logging.getLogger(__name__)
 
+ESCAPES = {r"\*": "*", r"\~": "~", r"\_": "_", r"\[": "[", r"\]": "]", r"\ ": ""}
+RX_ESC = re.compile("|".join([re.escape(k) for k in ESCAPES.keys()]), flags=re.DOTALL)
+
 
 def stripEscape(text) -> str:
-    """Helper function to strip escaped Markdown characters from
-    paragraph text.
-    """
+    """Strip escaped Markdown characters from paragraph text."""
     if "\\" in text:
-        # Checking first is slightly slower when there are escaped
-        # characters in the text, but significantly faster when not
-        return text.replace(r"\*", "*").replace(r"\~", "~").replace(r"\_", "_")
+        return RX_ESC.sub(lambda x: ESCAPES[x.group(0)], text)
     return text
 
 
@@ -65,27 +64,34 @@ class Tokenizer(ABC):
     """
 
     # In-Text Format
-    FMT_B_B = 1  # Begin bold
-    FMT_B_E = 2  # End bold
-    FMT_I_B = 3  # Begin italics
-    FMT_I_E = 4  # End italics
-    FMT_D_B = 5  # Begin strikeout
-    FMT_D_E = 6  # End strikeout
+    FMT_B_B   = 1   # Begin bold
+    FMT_B_E   = 2   # End bold
+    FMT_I_B   = 3   # Begin italics
+    FMT_I_E   = 4   # End italics
+    FMT_D_B   = 5   # Begin strikeout
+    FMT_D_E   = 6   # End strikeout
+    FMT_U_B   = 7   # Begin underline
+    FMT_U_E   = 8   # End underline
+    FMT_SUP_B = 9   # Begin superscript
+    FMT_SUP_E = 10  # End superscript
+    FMT_SUB_B = 11  # Begin subscript
+    FMT_SUB_E = 12  # End subscript
 
     # Block Type
     T_EMPTY    = 1   # Empty line (new paragraph)
     T_SYNOPSIS = 2   # Synopsis comment
-    T_COMMENT  = 3   # Comment line
-    T_KEYWORD  = 4   # Command line
-    T_TITLE    = 5   # Title
-    T_UNNUM    = 6   # Unnumbered
-    T_HEAD1    = 7   # Header 1
-    T_HEAD2    = 8   # Header 2
-    T_HEAD3    = 9   # Header 3
-    T_HEAD4    = 10  # Header 4
-    T_TEXT     = 11  # Text line
-    T_SEP      = 12  # Scene separator
-    T_SKIP     = 13  # Paragraph break
+    T_SHORT    = 3   # Short description comment
+    T_COMMENT  = 4   # Comment line
+    T_KEYWORD  = 5   # Command line
+    T_TITLE    = 6   # Title
+    T_UNNUM    = 7   # Unnumbered
+    T_HEAD1    = 8   # Header 1
+    T_HEAD2    = 9   # Header 2
+    T_HEAD3    = 10   # Header 3
+    T_HEAD4    = 11  # Header 4
+    T_TEXT     = 12  # Text line
+    T_SEP      = 13  # Scene separator
+    T_SKIP     = 14  # Paragraph break
 
     # Block Style
     A_NONE     = 0x0000  # No special style
@@ -163,8 +169,23 @@ class Tokenizer(ABC):
         self._localLookup = self._project.localLookup
         self.tr = partial(QCoreApplication.translate, "Tokenizer")
 
-        # Cached Translations
-        self._trSynopsis = self.tr("Synopsis")
+        # Format RegEx
+        self._rxMarkdown = [
+            (QRegularExpression(nwRegEx.FMT_EI), [0, self.FMT_I_B, 0, self.FMT_I_E]),
+            (QRegularExpression(nwRegEx.FMT_EB), [0, self.FMT_B_B, 0, self.FMT_B_E]),
+            (QRegularExpression(nwRegEx.FMT_ST), [0, self.FMT_D_B, 0, self.FMT_D_E]),
+        ]
+        self._rxShortCodes = QRegularExpression(nwRegEx.FMT_SC)
+        self._rxShortCodeVals = QRegularExpression(nwRegEx.FMT_SV)
+
+        self._shortCodeFmt = {
+            nwShortcode.ITALIC_O: self.FMT_I_B,   nwShortcode.ITALIC_C: self.FMT_I_E,
+            nwShortcode.BOLD_O:   self.FMT_B_B,   nwShortcode.BOLD_C:   self.FMT_B_E,
+            nwShortcode.STRIKE_O: self.FMT_D_B,   nwShortcode.STRIKE_C: self.FMT_D_E,
+            nwShortcode.ULINE_O:  self.FMT_U_B,   nwShortcode.ULINE_C:  self.FMT_U_E,
+            nwShortcode.SUP_O:    self.FMT_SUP_B, nwShortcode.SUP_C:    self.FMT_SUP_E,
+            nwShortcode.SUB_O:    self.FMT_SUB_B, nwShortcode.SUB_C:    self.FMT_SUB_E,
+        }
 
         return
 
@@ -197,7 +218,7 @@ class Tokenizer(ABC):
         return
 
     def setChapterFormat(self, hFormat: str) -> None:
-        """Set the chapert format pattern."""
+        """Set the chapter format pattern."""
         self._fmtChapter = hFormat.strip()
         return
 
@@ -349,14 +370,6 @@ class Tokenizer(ABC):
 
         self._text = text
 
-        docSize = len(self._text)
-        if docSize > nwConst.MAX_DOCSIZE:
-            errVal = self.tr("Document '{0}' is too big ({1} MB). Skipping.").format(
-                self._nwItem.itemName, f"{docSize/1.0e6:.2f}"
-            )
-            self._text = "# {0}\n\n{1}\n\n".format(self.tr("ERROR"), errVal)
-            self._errData.append(errVal)
-
         self._isNone  = self._nwItem.itemLayout == nwItemLayout.NO_LAYOUT
         self._isNovel = self._nwItem.itemLayout == nwItemLayout.DOCUMENT
         self._isNote  = self._nwItem.itemLayout == nwItemLayout.NOTE
@@ -395,19 +408,12 @@ class Tokenizer(ABC):
           4: The internal formatting map of the text, self.FMT_*
           5: The style of the block, self.A_*
         """
-        # RegExes for adding formatting tags within text lines
-        rxFormats = [
-            (QRegularExpression(nwRegEx.FMT_EI), [None, self.FMT_I_B, None, self.FMT_I_E]),
-            (QRegularExpression(nwRegEx.FMT_EB), [None, self.FMT_B_B, None, self.FMT_B_E]),
-            (QRegularExpression(nwRegEx.FMT_ST), [None, self.FMT_D_B, None, self.FMT_D_E]),
-        ]
-
         self._tokens = []
         tmpMarkdown = []
         nHead = 0
         breakNext = False
         for aLine in self._text.splitlines():
-            sLine = aLine.strip()
+            sLine = aLine.strip().lower()
 
             # Check for blank lines
             if len(sLine) == 0:
@@ -430,18 +436,21 @@ class Tokenizer(ABC):
 
             if aLine[0] == "[":
                 # Parse special formatting line
+                # This must be a separate if statement, as it may not
+                # reach a continue statement and must therefore proceed
+                # to check other formats.
 
-                if sLine in ("[NEWPAGE]", "[NEW PAGE]"):
+                if sLine in ("[newpage]", "[new page]"):
                     breakNext = True
                     continue
 
-                elif sLine == "[VSPACE]":
+                elif sLine == "[vspace]":
                     self._tokens.append(
                         (self.T_SKIP, nHead, "", None, sAlign)
                     )
                     continue
 
-                elif sLine.startswith("[VSPACE:") and sLine.endswith("]"):
+                elif sLine.startswith("[vspace:") and sLine.endswith("]"):
                     nSkip = checkInt(sLine[8:-1], 0)
                     if nSkip >= 1:
                         self._tokens.append(
@@ -453,18 +462,23 @@ class Tokenizer(ABC):
                         ]
                     continue
 
-            elif aLine[0] == "%":
-                cLine = aLine[1:].lstrip()
-                synTag = cLine[:9].lower()
-                if synTag == "synopsis:":
+            if aLine[0] == "%":
+                cStyle, cText, _ = processComment(aLine)
+                if cStyle == nwComment.SYNOPSIS:
                     self._tokens.append((
-                        self.T_SYNOPSIS, nHead, cLine[9:].strip(), None, sAlign
+                        self.T_SYNOPSIS, nHead, cText, None, sAlign
+                    ))
+                    if self._doSynopsis and self._keepMarkdown:
+                        tmpMarkdown.append("%s\n" % aLine)
+                elif cStyle == nwComment.SHORT:
+                    self._tokens.append((
+                        self.T_SHORT, nHead, cText, None, sAlign
                     ))
                     if self._doSynopsis and self._keepMarkdown:
                         tmpMarkdown.append("%s\n" % aLine)
                 else:
                     self._tokens.append((
-                        self.T_COMMENT, nHead, aLine[1:].strip(), None, sAlign
+                        self.T_COMMENT, nHead, cText, None, sAlign
                     ))
                     if self._doComments and self._keepMarkdown:
                         tmpMarkdown.append("%s\n" % aLine)
@@ -578,23 +592,10 @@ class Tokenizer(ABC):
                 if indRight:
                     sAlign |= self.A_IND_R
 
-                # Otherwise we use RegEx to find formatting tags within a line of text
-                fmtPos = []
-                for theRX, theKeys in rxFormats:
-                    rxThis = theRX.globalMatch(aLine, 0)
-                    while rxThis.hasNext():
-                        rxMatch = rxThis.next()
-                        for n in range(1, len(theKeys)):
-                            if theKeys[n] is not None:
-                                xPos = rxMatch.capturedStart(n)
-                                xLen = rxMatch.capturedLength(n)
-                                fmtPos.append([xPos, xLen, theKeys[n]])
-
-                # Save the line as is, but append the array of formatting locations
-                # sorted by position
-                fmtPos = sorted(fmtPos, key=itemgetter(0))
+                # Process formats
+                tLine, fmtPos = self._extractFormats(aLine)
                 self._tokens.append((
-                    self.T_TEXT, nHead, aLine, fmtPos, sAlign
+                    self.T_TEXT, nHead, tLine, fmtPos, sAlign
                 ))
                 if self._keepMarkdown:
                     tmpMarkdown.append("%s\n" % aLine)
@@ -771,6 +772,45 @@ class Tokenizer(ABC):
             json.dump(data, fObj, indent=2)
         return
 
+    ##
+    #  Internal Functions
+    ##
+
+    def _extractFormats(self, text: str) -> tuple[str, list[tuple[int, int]]]:
+        """Extract format markers from a text paragraph."""
+        temp = []
+
+        # Match Markdown
+        for regEx, fmts in self._rxMarkdown:
+            rxItt = regEx.globalMatch(text, 0)
+            while rxItt.hasNext():
+                rxMatch = rxItt.next()
+                temp.extend(
+                    [rxMatch.capturedStart(n), rxMatch.capturedLength(n), fmt]
+                    for n, fmt in enumerate(fmts) if fmt > 0
+                )
+
+        # Match Shortcodes
+        rxItt = self._rxShortCodes.globalMatch(text, 0)
+        while rxItt.hasNext():
+            rxMatch = rxItt.next()
+            temp.append([
+                rxMatch.capturedStart(1),
+                rxMatch.capturedLength(1),
+                self._shortCodeFmt.get(rxMatch.captured(1).lower(), 0)
+            ])
+
+        # Post-process text and format markers
+        result = text
+        formats = []
+        for pos, n, fmt in reversed(sorted(temp, key=lambda x: x[0])):
+            if fmt > 0:
+                result = result[:pos] + result[pos+n:]
+                formats = [(p-n, f) for p, f in formats]
+                formats.insert(0, (pos, fmt))
+
+        return result, formats
+
 # END Class Tokenizer