novelWriter 2.3.1__py3-none-any.whl → 2.4rc1__py3-none-any.whl

novelwriter/core/coretools.py

@@ -26,19 +26,19 @@ along with this program. If not, see <https://www.gnu.org/licenses/>.
 """
 from __future__ import annotations
 
-import shutil
 import logging
+import shutil
 
-from pathlib import Path
+from collections.abc import Iterable
 from functools import partial
+from pathlib import Path
 from zipfile import ZipFile, is_zipfile
-from collections.abc import Iterable
 
-from PyQt5.QtCore import QCoreApplication
+from PyQt5.QtCore import QCoreApplication, QRegularExpression
 
 from novelwriter import CONFIG, SHARED
 from novelwriter.common import isHandle, minmax, simplified
-from novelwriter.constants import nwFiles, nwItemClass
+from novelwriter.constants import nwConst, nwFiles, nwItemClass
 from novelwriter.core.item import NWItem
 from novelwriter.core.project import NWProject
 from novelwriter.core.storage import NWStorageCreate
@@ -101,9 +101,7 @@ class DocMerger:
         if srcItem is None:
             return False
 
-        inDoc = self._project.storage.getDocument(srcHandle)
-        docText = (inDoc.readDocument() or "").rstrip("\n")
-
+        docText = self._project.storage.getDocumentText(srcHandle).rstrip("\n")
         if addComment:
             docInfo = srcItem.describeMe()
             docSt, _ = srcItem.getImportStatus(incIcon=False)
@@ -122,9 +120,8 @@ class DocMerger:
             return False
 
         outDoc = self._project.storage.getDocument(self._targetDoc)
-        docText = (outDoc.readDocument() or "").rstrip("\n")
-        if docText:
-            self._targetText.insert(0, docText)
+        if text := (outDoc.readDocument() or "").rstrip("\n"):
+            self._targetText.insert(0, text)
 
         status = outDoc.writeDocument("\n\n".join(self._targetText) + "\n\n")
         if not status:
@@ -292,11 +289,10 @@ class DocDuplicator:
                     newItem.setParent(hMap[newItem.itemParent])
                     self._project.tree.updateItemData(newItem.itemHandle)
                 if newItem.isFileType():
-                    oldDoc = self._project.storage.getDocument(tHandle)
                     newDoc = self._project.storage.getDocument(newItem.itemHandle)
                     if newDoc.fileExists():
                         return
-                    newDoc.writeDocument(oldDoc.readDocument() or "")
+                    newDoc.writeDocument(self._project.storage.getDocumentText(tHandle))
                 yield newItem.itemHandle, nHandle
                 nHandle = None
         return
@@ -304,6 +300,94 @@ class DocDuplicator:
 # END Class DocDuplicator
 
 
+class DocSearch:
+
+    def __init__(self) -> None:
+        self._regEx = QRegularExpression()
+        self.setCaseSensitive(False)
+        self._words = False
+        self._escape = True
+        return
+
+    ##
+    #  Methods
+    ##
+
+    def setCaseSensitive(self, state: bool) -> None:
+        """Set the case sensitive search flag."""
+        opts = QRegularExpression.PatternOption.UseUnicodePropertiesOption
+        if not state:
+            opts |= QRegularExpression.PatternOption.CaseInsensitiveOption
+        self._regEx.setPatternOptions(opts)
+        return
+
+    def setWholeWords(self, state: bool) -> None:
+        """Set the whole words search flag."""
+        self._words = state
+        return
+
+    def setUserRegEx(self, state: bool) -> None:
+        """Set the escape flag to the opposite state."""
+        self._escape = not state
+        return
+
+    def iterSearch(
+        self, project: NWProject, search: str
+    ) -> Iterable[tuple[NWItem, list[tuple[int, int, str]], bool]]:
+        """Iteratively search through documents in a project."""
+        self._regEx.setPattern(self._buildPattern(search))
+        logger.debug("Searching with pattern '%s'", self._regEx.pattern())
+        storage = project.storage
+        for item in project.tree:
+            if item.isFileType():
+                results, capped = self.searchText(storage.getDocumentText(item.itemHandle))
+                yield item, results, capped
+        return
+
+    def searchText(self, text: str) -> tuple[list[tuple[int, int, str]], bool]:
+        """Search a piece of text for RegEx matches."""
+        rxItt = self._regEx.globalMatch(text)
+        count = 0
+        capped = False
+        results = []
+        while rxItt.hasNext():
+            rxMatch = rxItt.next()
+            pos = rxMatch.capturedStart()
+            num = rxMatch.capturedLength()
+            context = text[pos:pos+100].partition("\n")[0]
+            if context:
+                results.append((pos, num, context))
+                count += 1
+                if count >= nwConst.MAX_SEARCH_RESULT:
+                    capped = True
+                    break
+        return results, capped
+
+    ##
+    #  Internal Functions
+    ##
+
+    def _buildPattern(self, search: str) -> str:
+        """Build the search pattern string."""
+        if self._escape:
+            if CONFIG.verQtValue >= 0x050f00:
+                search = QRegularExpression.escape(search)
+            else:
+                # For older Qt versions, we escape manually
+                escaped = ""
+                for c in search:
+                    if c.isalnum() or c == "_":
+                        escaped += c
+                    else:
+                        escaped += f"\\{c}"
+                search = escaped
+        if self._words:
+            search = f"(?:^|\\b){search}(?:$|\\b)"
+        return search
+
+# END Class DocSearch
+
+
 class ProjectBuilder:
     """A class to build a new project from a set of user-defined
     parameter provided by the New Project Wizard.

novelwriter/core/docbuild.py

@@ -52,7 +52,10 @@ class NWBuildDocument:
     manuscript, based on a build definition object (BuildSettings).
     """
 
-    __slots__ = ("_project", "_build", "_queue", "_error", "_cache")
+    __slots__ = (
+        "_project", "_build", "_queue", "_error", "_cache", "_count",
+        "_outline", "_preview"
+    )
 
     def __init__(self, project: NWProject, build: BuildSettings) -> None:
         self._project = project
@@ -60,6 +63,9 @@ class NWBuildDocument:
         self._queue = []
         self._error = None
         self._cache = None
+        self._count = False
+        self._outline = False
+        self._preview = False
         return
 
     ##
@@ -79,6 +85,29 @@ class NWBuildDocument:
         """
         return self._cache
 
+    ##
+    #  Setters
+    ##
+
+    def setCountEnabled(self, state: bool) -> None:
+        """Turn on/off stats for builds."""
+        self._count = state
+        return
+
+    def setBuildOutline(self, state: bool) -> None:
+        """Turn on/off outline for builds."""
+        self._outline = state
+        return
+
+    def setPreviewMode(self, state: bool) -> None:
+        """Set the preview mode of the build. This also enables stats
+        count and outline mode.
+        """
+        self._preview = state
+        self._outline = state
+        self._count = state
+        return
+
     ##
     #  Special Methods
     ##
@@ -153,6 +182,8 @@ class NWBuildDocument:
         makeObj = ToHtml(self._project)
         filtered = self._setupBuild(makeObj)
 
+        makeObj.setPreview(self._preview)
+        makeObj.setLinkHeadings(self._preview)
         for i, tHandle in enumerate(self._queue):
             self._error = None
             if filtered.get(tHandle, (False, 0))[0]:
@@ -160,7 +191,7 @@ class NWBuildDocument:
             else:
                 yield i, False
 
-        if self._build.getBool("format.replaceTabs"):
+        if not (self._build.getBool("html.preserveTabs") or self._preview):
             makeObj.replaceTabs()
 
         self._error = None
@@ -191,6 +222,8 @@ class NWBuildDocument:
         if self._build.getBool("format.replaceTabs"):
             makeObj.replaceTabs(nSpaces=4, spaceChar=" ")
 
+        makeObj.setPreserveBreaks(self._build.getBool("md.preserveBreaks"))
+
         for i, tHandle in enumerate(self._queue):
             self._error = None
             if filtered.get(tHandle, (False, 0))[0]:
@@ -256,26 +289,52 @@ class NWBuildDocument:
         fontInfo = QFontInfo(bldFont)
         textFixed = fontInfo.fixedPitch()
 
-        bldObj.setTitleFormat(self._build.getStr("headings.fmtTitle"))
-        bldObj.setChapterFormat(self._build.getStr("headings.fmtChapter"))
-        bldObj.setUnNumberedFormat(self._build.getStr("headings.fmtUnnumbered"))
+        bldObj.setTitleFormat(
+            self._build.getStr("headings.fmtTitle"),
+            self._build.getBool("headings.hideTitle")
+        )
+        bldObj.setChapterFormat(
+            self._build.getStr("headings.fmtChapter"),
+            self._build.getBool("headings.hideChapter")
+        )
+        bldObj.setUnNumberedFormat(
+            self._build.getStr("headings.fmtUnnumbered"),
+            self._build.getBool("headings.hideUnnumbered")
+        )
         bldObj.setSceneFormat(
             self._build.getStr("headings.fmtScene"),
             self._build.getBool("headings.hideScene")
         )
+        bldObj.setHardSceneFormat(
+            self._build.getStr("headings.fmtHardScene"),
+            self._build.getBool("headings.hideHardScene")
+        )
         bldObj.setSectionFormat(
             self._build.getStr("headings.fmtSection"),
             self._build.getBool("headings.hideSection")
         )
+        bldObj.setTitleStyle(
+            self._build.getBool("headings.centerTitle"),
+            self._build.getBool("headings.breakTitle")
+        )
+        bldObj.setChapterStyle(
+            self._build.getBool("headings.centerChapter"),
+            self._build.getBool("headings.breakChapter")
+        )
+        bldObj.setSceneStyle(
+            self._build.getBool("headings.centerScene"),
+            self._build.getBool("headings.breakScene")
+        )
 
         bldObj.setFont(fontFamily, textSize, textFixed)
         bldObj.setJustify(self._build.getBool("format.justifyText"))
         bldObj.setLineHeight(self._build.getFloat("format.lineHeight"))
 
+        bldObj.setBodyText(self._build.getBool("text.includeBodyText"))
         bldObj.setSynopsis(self._build.getBool("text.includeSynopsis"))
         bldObj.setComments(self._build.getBool("text.includeComments"))
         bldObj.setKeywords(self._build.getBool("text.includeKeywords"))
-        bldObj.setBodyText(self._build.getBool("text.includeBodyText"))
+        bldObj.setIgnoredKeywords(self._build.getStr("text.ignoredKeywords"))
 
         if isinstance(bldObj, ToHtml):
             bldObj.setStyles(self._build.getBool("html.addStyles"))
@@ -287,6 +346,7 @@ class NWBuildDocument:
             bldObj.setHeaderFormat(
                 self._build.getStr("odt.pageHeader"), self._build.getInt("odt.pageCountOffset")
             )
+            bldObj.setFirstLineIndent(self._build.getBool("odt.firstLineIndent"))
 
             scale = nwLabels.UNIT_SCALE.get(self._build.getStr("format.pageUnit"), 1.0)
             pW, pH = nwLabels.PAPER_SIZE.get(self._build.getStr("format.pageSize"), (-1.0, -1.0))
@@ -314,11 +374,18 @@ class NWBuildDocument:
                     bldObj.addRootHeading(tHandle)
                     if convert:
                         bldObj.doConvert()
+                    if self._count:
+                        bldObj.countStats()
+                    if self._outline:
+                        bldObj.buildOutline()
                 elif tItem.isFileType():
                     bldObj.setText(tHandle)
                     bldObj.doPreProcessing()
                     bldObj.tokenizeText()
-                    bldObj.doHeaders()
+                    if self._count:
+                        bldObj.countStats()
+                    if self._outline:
+                        bldObj.buildOutline()
                     if convert:
                         bldObj.doConvert()
                 else:

novelwriter/core/document.py

@@ -31,7 +31,7 @@ from typing import TYPE_CHECKING
 from pathlib import Path
 
 from novelwriter.enum import nwItemLayout, nwItemClass
-from novelwriter.error import formatException
+from novelwriter.error import formatException, logException
 from novelwriter.common import formatTimeStamp, isHandle
 from novelwriter.core.item import NWItem
 
@@ -106,7 +106,28 @@ class NWDocument:
         return self._item
 
     ##
-    #  Class Methods
+    #  Static Methods
+    ##
+
+    @staticmethod
+    def quickReadText(content: Path, tHandle: str) -> str:
+        """Return the text of a document in a fast and efficient way."""
+        if (path := content / f"{tHandle}.nwd").is_file():
+            try:
+                with open(path, mode="r", encoding="utf-8") as inFile:
+                    line = ""
+                    for _ in range(10):
+                        if not (line := inFile.readline()).startswith(r"%%~"):
+                            break
+                    return line + inFile.read()
+            except Exception:
+                logger.error("Cannot read document with handle '%s'", tHandle)
+                logException()
+                return ""
+        return ""
+
+    ##
+    #  Methods
     ##
 
     def fileExists(self) -> bool:
@@ -155,7 +176,7 @@ class NWDocument:
             try:
                 with open(docPath, mode="r", encoding="utf-8") as inFile:
                     # Check the first <= 10 lines for metadata
-                    for i in range(10):
+                    for _ in range(10):
                         line = inFile.readline()
                         if line.startswith(r"%%~"):
                             self._parseMeta(line)

novelwriter/core/index.py

@@ -3,7 +3,6 @@ novelWriter – Project Index
 ===========================
 
 File History:
-Created: 2019-04-22 [0.0.1]  countWords
 Created: 2019-05-27 [0.1.4]  NWIndex
 Created: 2022-05-28 [2.0rc1] IndexItem
 Created: 2022-05-28 [2.0rc1] IndexHeading
@@ -34,13 +33,14 @@ import logging
 from time import time
 from typing import TYPE_CHECKING
 from pathlib import Path
-from collections.abc import ItemsView, Iterable, Iterator
+from collections.abc import ItemsView, Iterable
 
 from novelwriter import SHARED
 from novelwriter.enum import nwComment, nwItemClass, nwItemType, nwItemLayout
 from novelwriter.error import logException
 from novelwriter.common import checkInt, isHandle, isItemClass, isTitleTag, jsonEncode
-from novelwriter.constants import nwFiles, nwKeyWords, nwRegEx, nwUnicode, nwHeaders
+from novelwriter.constants import nwFiles, nwKeyWords, nwHeaders
+from novelwriter.text.counting import standardCounter
 
 if TYPE_CHECKING:  # pragma: no cover
     from novelwriter.core.item import NWItem
@@ -122,9 +122,8 @@ class NWIndex:
         self.clearIndex()
         for nwItem in self._project.tree:
             if nwItem.isFileType():
-                tHandle = nwItem.itemHandle
-                doc = self._project.storage.getDocument(tHandle)
-                self.scanText(tHandle, doc.readDocument() or "", blockSignal=True)
+                text = self._project.storage.getDocumentText(nwItem.itemHandle)
+                self.scanText(nwItem.itemHandle, text, blockSignal=True)
         self._indexBroken = False
         SHARED.indexSignalProxy({"event": "buildIndex"})
         return
@@ -142,17 +141,15 @@ class NWIndex:
         })
         return
 
-    def reIndexHandle(self, tHandle: str | None) -> bool:
+    def reIndexHandle(self, tHandle: str | None) -> None:
         """Put a file back into the index. This is used when files are
         moved from the archive or trash folders back into the active
         project.
         """
         if tHandle and self._project.tree.checkType(tHandle, nwItemType.FILE):
             logger.debug("Re-indexing item '%s'", tHandle)
-            doc = self._project.storage.getDocument(tHandle)
-            self.scanText(tHandle, doc.readDocument() or "")
-            return True
-        return False
+            self.scanText(tHandle, self._project.storage.getDocumentText(tHandle))
+        return
 
     def indexChangedSince(self, checkTime: int | float) -> bool:
         """Check if the index has changed since a given time."""
@@ -266,7 +263,7 @@ class NWIndex:
         self._itemIndex.add(tHandle, tItem)
 
         # Run word counter for the whole text
-        cC, wC, pC = countWords(text)
+        cC, wC, pC = standardCounter(text)
         tItem.setCharCount(cC)
         tItem.setWordCount(wC)
         tItem.setParaCount(pC)
@@ -307,7 +304,7 @@ class NWIndex:
         nTitle = 0           # Line Number of the previous title
         cTitle = TT_NONE     # Tag of the current title
         pTitle = TT_NONE     # Tag of the previous title
-        canSetHeader = True  # First header has not yet been set
+        canSetHead = True  # First heading has not yet been set
 
         lines = text.splitlines()
         for n, line in enumerate(lines, start=1):
@@ -320,9 +317,9 @@ class NWIndex:
                 if hDepth == "H0":
                     continue
 
-                if canSetHeader:
+                if canSetHead:
                     nwItem.setMainHeading(hDepth)
-                    canSetHeader = False
+                    canSetHead = False
 
                 cTitle = self._itemIndex.addItemHeading(tHandle, n, hDepth, hText)
                 if cTitle != TT_NONE:
@@ -383,7 +380,7 @@ class NWIndex:
         return
 
     def _splitHeading(self, line: str) -> tuple[str, str]:
-        """Split a heading into its header level and text value."""
+        """Split a heading into its heading level and text value."""
         if line.startswith("# "):
             return "H1", line[2:].strip()
         elif line.startswith("## "):
@@ -396,11 +393,13 @@ class NWIndex:
             return "H1", line[3:].strip()
         elif line.startswith("##! "):
             return "H2", line[4:].strip()
+        elif line.startswith("###! "):
+            return "H3", line[5:].strip()
         return "H0", ""
 
     def _indexWordCounts(self, tHandle: str, text: str, sTitle: str) -> None:
         """Count text stats and save the counts to the index."""
-        cC, wC, pC = countWords(text)
+        cC, wC, pC = standardCounter(text)
         self._itemIndex.setHeadingCounts(tHandle, sTitle, cC, wC, pC)
         return
 
@@ -515,16 +514,21 @@ class NWIndex:
         """Get the index data for a given item."""
         return self._itemIndex[tHandle]
 
-    def getItemHeader(self, tHandle: str, sTitle: str) -> IndexHeading | None:
-        """Get the header entry for a specific item and heading."""
-        tItem = self._itemIndex[tHandle]
-        if isinstance(tItem, IndexItem):
+    def getItemHeading(self, tHandle: str, sTitle: str) -> IndexHeading | None:
+        """Get the heading entry for a specific item and heading."""
+        if tItem := self._itemIndex[tHandle]:
             return tItem[sTitle]
         return None
 
+    def iterItemHeadings(self, tHandle: str) -> Iterable[tuple[str, IndexHeading]]:
+        """Get all headings for a specific item."""
+        if tItem := self._itemIndex[tHandle]:
+            yield from tItem.items()
+        return []
+
     def novelStructure(
         self, rootHandle: str | None = None, activeOnly: bool = True
-    ) -> Iterator[tuple[str, str, str, IndexHeading]]:
+    ) -> Iterable[tuple[str, str, str, IndexHeading]]:
         """Iterate over all titles in the novel, in the correct order as
         they appear in the tree view and in the respective document
         files, but skipping all note files.
@@ -666,7 +670,7 @@ class NWIndex:
 
     def getTagsData(
         self, activeOnly: bool = True
-    ) -> Iterator[tuple[str, str, str, IndexItem | None, IndexHeading | None]]:
+    ) -> Iterable[tuple[str, str, str, IndexItem | None, IndexHeading | None]]:
         """Return all known tags."""
         for tag, data in self._tagsIndex.items():
             iItem = self._itemIndex[data.get("handle")]
@@ -825,7 +829,7 @@ class ItemIndex:
     class around a single storage dictionary with a set of utility
     functions for setting and accessing the index data. Each indexed
     item is stored in an IndexItem object, which again holds an
-    IndexHeading object for each header of the text.
+    IndexHeading object for each heading of the text.
     """
 
     __slots__ = ("_project", "_items")
@@ -898,13 +902,11 @@ class ItemIndex:
 
             if rHandle is None:
                 for sTitle in self._items[tHandle].headings():
-                    hItem = self._items[tHandle][sTitle]
-                    if hItem:
+                    if hItem := self._items[tHandle][sTitle]:
                         yield tHandle, sTitle, hItem
             elif tItem.itemRoot == rHandle:
                 for sTitle in self._items[tHandle].headings():
-                    hItem = self._items[tHandle][sTitle]
-                    if hItem:
+                    if hItem := self._items[tHandle][sTitle]:
                         yield tHandle, sTitle, hItem
 
         return
@@ -1200,7 +1202,7 @@ class IndexHeading:
     ##
 
     def setLevel(self, level: str) -> None:
-        """Set the level of the header if it's a valid value."""
+        """Set the level of the heading if it's a valid value."""
         if level in nwHeaders.H_VALID:
             self._level = level
         return
@@ -1315,86 +1317,3 @@ def processComment(text: str) -> tuple[nwComment, str, int]:
     if content and (clean := classifier.strip().lower()) in CLASSIFIERS:
         return CLASSIFIERS[clean], content.strip(), text.find(":") + 1
     return nwComment.PLAIN, check, 0
-
-
-def countWords(text: str) -> tuple[int, int, int]:
-    """Count words in a piece of text, skipping special syntax and
-    comments.
-    """
-    charCount = 0
-    wordCount = 0
-    paraCount = 0
-    prevEmpty = True
-
-    if not isinstance(text, str):
-        return charCount, wordCount, paraCount
-
-    # We need to treat dashes as word separators for counting words.
-    # The check+replace approach is much faster than direct replace for
-    # large texts, and a bit slower for small texts, but in the latter
-    # case it doesn't really matter.
-    if nwUnicode.U_ENDASH in text:
-        text = text.replace(nwUnicode.U_ENDASH, " ")
-    if nwUnicode.U_EMDASH in text:
-        text = text.replace(nwUnicode.U_EMDASH, " ")
-
-    # Strip shortcodes
-    if "[" in text:
-        text = nwRegEx.RX_SC.sub("", text)
-
-    for line in text.splitlines():
-
-        countPara = True
-
-        if not line:
-            prevEmpty = True
-            continue
-
-        if line[0] == "@" or line[0] == "%":
-            continue
-
-        if line[0] == "[":
-            check = line.lower()
-            if check.startswith(("[newpage]", "[new page]", "[vspace]")):
-                continue
-            elif check.startswith("[vspace:") and line.endswith("]"):
-                continue
-
-        elif line[0] == "#":
-            if line[:5] == "#### ":
-                line = line[5:]
-                countPara = False
-            elif line[:4] == "### ":
-                line = line[4:]
-                countPara = False
-            elif line[:3] == "## ":
-                line = line[3:]
-                countPara = False
-            elif line[:2] == "# ":
-                line = line[2:]
-                countPara = False
-            elif line[:3] == "#! ":
-                line = line[3:]
-                countPara = False
-            elif line[:4] == "##! ":
-                line = line[4:]
-                countPara = False
-
-        elif line[0] == ">" or line[-1] == "<":
-            if line[:2] == ">>":
-                line = line[2:].lstrip(" ")
-            elif line[:1] == ">":
-                line = line[1:].lstrip(" ")
-            if line[-2:] == "<<":
-                line = line[:-2].rstrip(" ")
-            elif line[-1:] == "<":
-                line = line[:-1].rstrip(" ")
-
-        wordCount += len(line.split())
-        charCount += len(line)
-        if countPara and prevEmpty:
-            paraCount += 1
-
-        prevEmpty = not countPara
-
-    return charCount, wordCount, paraCount