docp 0.1.0b1__py3-none-any.whl → 0.2.0__py3-none-any.whl

docp/__init__.py

@@ -7,25 +7,34 @@
 :Developer: J Berendt
 :Email:     development@s3dev.uk
 
-:Comments:  n/a
+:Comments:  Ths loader modules/classes have *not* been imported due to the
+            heavy dependency requirements. Refer to the loaders/__init__.py
+            module instead.
 
 """
 
 import os
 import sys
 sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)))
+from utils4.user_interface import ui
+# locals
+from .libs._version import __version__
+
+# TODO: Change these to use logging.
 
 # Bring entry-points to the surface.
 try:
-    from loaders.chroma import ChromaLoader
+    from .parsers.pdfparser import PDFParser
 except ImportError as err:
-    # The chroma loader requires a lot of backend which is not required for the parser.
-    msg = f'An error occurred while importing the Chroma loader:\n- {err}'
-    raise ImportError(msg) from err
+    msg = ( 'An error occurred while importing the PDF parser:\n'
+           f'- {err}\n'
+            '  - This can be ignored if the parser is not in use.\n')
+    ui.print_warning(f'\n[ImportError]: {msg}')
 
 try:
-    from .parsers.pdfparser import PDFParser
-    from ._version import __version__
-except ImportError:
-    from parsers.pdfparser import PDFParser
-    from _version import __version__
+    from .parsers.pptxparser import PPTXParser
+except ImportError as err:
+    msg = ( 'An error occurred while importing the PPTX parser:\n'
+           f'- {err}\n'
+            '  - This can be ignored if the parser is not in use.\n')
+    ui.print_warning(f'\n[ImportError]: {msg}')

docp/dbs/chroma.py

@@ -10,11 +10,18 @@
 :Developer: J Berendt
 :Email:     development@s3dev.uk
 
-:Comments:  n/a
+:Comments:  This module uses the
+            ``langchain_community.vectorstores.Chroma`` wrapper class,
+            rather than the base ``chromadb`` library  as it provides the
+            ``add_texts`` method which supports GPU processing and
+            parallelisation; which is implemented by this module's
+            :meth:`~ChromaDB.add_documents` method.
 
 """
+# pylint: disable=import-error
 # pylint: disable=wrong-import-order
 
+from __future__ import annotations
 import chromadb
 import os
 import torch
@@ -81,19 +88,25 @@ class ChromaDB(_Chroma):
         """Accessor to the database's path."""
         return self._path
 
-    def add_documents(self, docs: list):
+    def add_documents(self, docs: list[langchain_core.documents.base.Document]):  # noqa  # pylint: disable=undefined-variable
         """Add multiple documents to the collection.
 
-        This method wraps ``Chroma.add_texts`` method which supports GPU
-        processing and parallelisation. The ID is derived locally from
-        the file's basename, page number and page content.
+        This method overrides the base class' ``add_documents`` method
+        to enable local ID derivation. Knowing *how* the IDs are derived
+        gives us greater understanding and querying ability of the
+        documents in the database. Each ID is derived locally by the
+        :meth:`_preproc` method from the file's basename, page number
+        and page content.
+
+        Additionally, this method wraps the
+        :func:`langchain_community.vectorstores.Chroma.add_texts`
+        method which supports GPU processing and parallelisation.
 
         Args:
             docs (list): A list of ``langchain_core.documents.base.Document``
                 document objects.
 
         """
-        # This method overrides the base class' add_documents method.
         # pylint: disable=arguments-differ
         # pylint: disable=arguments-renamed
         if not isinstance(docs, list):

docp/libs/_version.py

@@ -0,0 +1 @@
+__version__ = '0.2.0'

docp/libs/changelog.py

@@ -0,0 +1,7 @@
+# Changed.
+# ENABLE SPHINX TO ACCESS THE GIT LOG
+"""
+.. git_changelog::
+    :revisions: 99
+    :detailed-message-pre: True
+"""

docp/libs/utilities.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+:Purpose:   This module provides utility-based functionality for the
+            project.
+
+:Platform:  Linux/Windows | Python 3.10+
+:Developer: J Berendt
+:Email:     development@s3dev.uk
+
+:Comments:  n/a
+
+"""
+
+import os
+import sys
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../'))
+import re
+from glob import glob
+from utils4 import futils
+
+
+class Utilities:
+    """General (cross-project) utility functions."""
+
+    @staticmethod
+    def collect_files(path: str, ext: str, recursive: bool) -> list:
+        """Collect all files for a given extension from a path.
+
+        Args:
+            path (str): Full path serving as the root for the search.
+            ext (str, optional): If the ``path`` argument refers to a
+                *directory*, a specific file extension can be specified
+                here. For example: ``ext = 'pdf'``.
+
+                If anything other than ``'**'`` is provided, all
+                alpha-characters are parsed from the string, and prefixed
+                with ``*.``. Meaning, if ``'.pdf'`` is passed, the
+                characters ``'pdf'`` are parsed and prefixed with ``*.``
+                to create ``'*.pdf'``. However, if ``'things.foo'`` is
+                passed, the derived extension will be ``'*.thingsfoo'``.
+                Defaults to '**', for a recursive search.
+
+            recursive (bool): Instruct the search to recurse into
+                sub-directories.
+
+        Returns:
+            list: The list of full file paths returned by the ``glob``
+            call. Any directory-only paths are removed.
+
+        """
+        if ext != '**':
+            ext = f'*.{re.findall("[a-zA-Z]+", ext)[0]}'
+        return list(filter(os.path.isfile, glob(os.path.join(path, ext), recursive=recursive)))
+
+    # !!!: Replace this with utils4.futils when available.
+    @staticmethod
+    def ispdf(path: str) -> bool:
+        """Test the file signature. Verify this is a valid PDF file.
+
+        Args:
+            path (str): Path to the file being tested.
+
+        Returns:
+            bool: True if this is a valid PDF file, otherwise False.
+
+        """
+        with open(path, 'rb') as f:
+            sig = f.read(5)
+        return sig == b'\x25\x50\x44\x46\x2d'
+
+    @staticmethod
+    def iszip(path: str) -> bool:
+        """Test the file signature. Verify this is a valid ZIP archive.
+
+        Args:
+            path (str): Path to the file being tested.
+
+        Returns:
+            bool: True if this is a valid ZIP archive, otherwise False.
+
+        """
+        return futils.iszip(path)
+
+    @staticmethod
+    def parse_to_keywords(resp: str) -> list:
+        """Parse the bot's response into a list of keywords.
+
+        Args:
+            resp (str): Text response directly from the bot.
+
+        Returns:
+            list: A list of keywords extracted from the response,
+            separated by asterisks as bullet points.
+
+        """
+        # Capture asterisk bullet points or a numbered list.
+        rexp = re.compile(r'(?:\*|[0-9]+\.)\s*(.*)\n')
+        trans = {45: ' ', 47: ' '}
+        resp_ = resp.translate(trans).lower()
+        kwds = rexp.findall(resp_)
+        if kwds:
+            return ', '.join(kwds)
+        return ''
+
+
+utilities = Utilities()

docp/loaders/__init__.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+:Purpose:   This module provides the project initilisation logic.
+
+:Platform:  Linux/Windows | Python 3.10+
+:Developer: J Berendt
+:Email:     development@s3dev.uk
+
+:Comments:  n/a
+
+"""
+
+import os
+import sys
+sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)))
+from utils4.user_interface import ui
+
+# TODO: Change these to use logging.
+
+# Bring entry-points to the surface.
+try:
+    from .chromapdfloader import ChromaPDFLoader
+except ImportError as err:
+    # The chroma loader requires a lot of backend which is not required for the parser.
+    msg = ( 'An error occurred while importing the Chroma PDF loader:\n'
+           f'- {err}\n'
+            '  - This can be ignored if the loader is not in use.\n')
+    ui.print_warning(f'\n[ImportError]: {msg}')
+
+try:
+    from .chromapptxloader import ChromaPPTXLoader
+except ImportError as err:
+    # The chroma loader requires a lot of backend which is not required for the parser.
+    msg = ( 'An error occurred while importing the Chroma PPTX loader:\n'
+           f'- {err}\n'
+            '  - This can be ignored if the loader is not in use.\n')
+    ui.print_warning(f'\n[ImportError]: {msg}')

docp/loaders/_chromabaseloader.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 """
-:Purpose:   This module provides functionality to parse and store
-            document data into a Chroma vector database.
+:Purpose:   This module provides the base functionality for parsing and
+            storing a document's data into a Chroma vector database.
 
 :Platform:  Linux/Windows | Python 3.10+
 :Developer: J Berendt
@@ -10,12 +10,23 @@
 
 :Comments:  n/a
 
+.. attention::
+
+            This module is *not* designed to be interacted with
+            directly, only via the appropriate interface class(es).
+
+            Rather, please create an instance of a Chroma
+            document-type-specific loader object using one of the
+            following classes:
+
+                - :class:`~docp.loaders.chromapdfloader.ChromaPDFLoader`
+                - :class:`~docp.loaders.chromapptxloader.ChromaPPTXLoader`
+
 """
 # pylint: disable=no-name-in-module  # langchain.chains.RetrievalQA
 
 import contextlib
 import os
-import re
 from chromadb.api.types import errors as chromadberrors
 from langchain.chains import RetrievalQA
 from langchain.docstore.document import Document
@@ -25,69 +36,51 @@ from utils4.user_interface import ui
 # locals
 try:
     from .dbs.chroma import ChromaDB
-    from .parsers.pdfparser import PDFParser
+    from .libs.utilities import utilities
 except ImportError:
     from dbs.chroma import ChromaDB
-    from parsers.pdfparser import PDFParser
-
-_PRE_ERR = '\n[ERROR]:'
-_PRE_WARN = '\n[WARNING]:'
-
-
-class Tools:
-    """General tools used for loading documents."""
-
-    @staticmethod
-    def parse_to_keywords(resp: str) -> list:
-        """Parse the bot's response into a list of keywords.
-
-        Args:
-            resp (str): Text response directly from the bot.
-
-        Returns:
-            list: A list of keywords extracted from the response,
-            separated by asterisks as bullet points.
-
-        """
-        # Capture asterisk bullet points or a numbered list.
-        rexp = re.compile(r'(?:\*|[0-9]+\.)\s*(.*)\n')
-        trans = {45: ' ', 47: ' '}
-        resp_ = resp.translate(trans).lower()
-        kwds = rexp.findall(resp_)
-        if kwds:
-            return ', '.join(kwds)
-        return ''
+    from libs.utilities import utilities
 
 
 class _ChromaBaseLoader:
     """Base class for loading documents into a Chroma vector database.
 
     Args:
-        path (str): Full path to the file to be parsed and loaded.
-        dbpath (str | Chroma): Either the full path to the Chroma database
-            *directory*, or an instance of a :class:`~dbs.chroma.Chroma`
-            database. If the instance is passed, the ``collection``
-            argument is ignored.
+        dbpath (str | ChromaDB): Either the full path to the Chroma
+            database *directory*, or an instance of a
+            :class:`~docp.dbs.chroma.ChromaDB` class. If the instance is
+            passed, the ``collection`` argument is ignored.
         collection (str, optional): Name of the Chroma database
             collection. Only required if the ``db`` parameter is a path.
             Defaults to None.
+        split_text (bool, optional): Split the document into chunks,
+            before loading it into the database. Defaults to True.
+        load_keywords (bool, optional): Derive keywords from the document
+            and load these into the sister keywords collection.
+            Defaults to False.
+        llm (object, optional): If deriving keywords, this is the LLM
+            which will do the derivation. Defaults to None.
         offline (bool, optional): Remain offline and use the locally
             cached embedding function model. Defaults to False.
 
     """
+    # pylint: disable=assignment-from-no-return  # These are stub methods.
 
-    _PARSERS = {'.pdf': PDFParser}
+    _PFX_ERR = '\n[ERROR]:'
+    _PFX_WARN = '\n[WARNING]:'
 
     def __init__(self,
                  dbpath: str | ChromaDB,
                  collection: str=None,
                  *,
+                 split_text: bool=True,
                  load_keywords: bool=False,
                  llm: object=None,
                  offline: bool=False):
         """Chroma database class initialiser."""
         self._dbpath = dbpath
         self._cname = collection
+        self._split_text = split_text
         self._load_keywords = load_keywords
         self._llm = llm
         self._offline = offline
@@ -111,6 +104,28 @@ class _ChromaBaseLoader:
         """Accessor to the document parser object."""
         return self._p
 
+    def _already_loaded(self) -> bool:
+        """Test if the file has already been loaded into the collection.
+
+        :Logic:
+            This test is performed by querying the collection for a
+            metadata 'source' which equals the filename. As this uses
+            a chromadb 'filter' (i.e. ``$eq``), testing for partial
+            matches is not possible at this time.
+
+            If the filename is different (in any way) from the source's
+            filename in the database, the file will be loaded again.
+
+        Returns:
+            bool: True is the *exact* filename was found in the
+            collection's metadata, otherwise False.
+
+        """
+        if self._dbo.collection.get(where={'source': {'$eq': self._fbase}})['ids']:
+            print(f'-- File already loaded: {self._fbase} - skipping')
+            return True
+        return False
+
     def _check_parameters(self) -> None:
         """Verify the class parameters are viable.
 
@@ -125,22 +140,7 @@ class _ChromaBaseLoader:
                              'must be True and a model instance must be provided.')
 
     def _create_documents(self) -> bool:
-        """Convert each extracted page into a ``Document`` object.
-
-        Returns:
-            bool: True of the pages are loaded as ``Document`` objects
-            successfully. Otherwise False.
-
-        """
-        self._docs = [Document(page_content=page.content,
-                               metadata={'source': self._p.doc.basename,
-                                         'pageno': page.pageno})
-                      for page in self._p.doc.pages if page.hastext]
-        if not self._docs:
-            msg = f'{_PRE_WARN} Text could not be parsed from {self._p.doc.basename}.'
-            ui.print_warning(msg)
-            return False
-        return True
+        """Stub method; overridden by the child class."""
 
     def _get_keywords(self) -> str:
         """Query the document (using the LLM) to extract the keywords."""
@@ -161,24 +161,27 @@ class _ChromaBaseLoader:
                                              return_source_documents=True,
                                              verbose=True)
             resp = qa.invoke(qry)
-        kwds = Tools.parse_to_keywords(resp=resp['result'])
+        kwds = utilities.parse_to_keywords(resp=resp['result'])
         return kwds
 
     def _load(self, path: str, **kwargs):
-        """Load the selected files into the vector store.
+        """Load the provided file into the vector store.
 
         Args:
             path (str): Full path to the file to be loaded.
 
         :Keyword Arguments:
-            Those passed from the loader-specific ``load`` method.
+            Those passed from the document-type-specific loader's
+            :func:`load` method.
 
         """
         # pylint: disable=multiple-statements
         self._fpath = path
         self._fbase = os.path.basename(path)
-        s = self._set_parser()
-        if s: s = self._set_text_splitter()
+        if self._already_loaded():
+            return
+        self._set_parser()
+        s = self._set_text_splitter()
         if s: s = self._parse_text(**kwargs)
         if s: s = self._create_documents()
         if s: s = self._split_texts()
@@ -198,6 +201,7 @@ class _ChromaBaseLoader:
             exceptions being raised.
 
         """
+        # pylint: disable=line-too-long
         try:
             print('- Loading the document into the database ...')
             nrecs_b = self._dbo.collection.count()  # Count records before.
@@ -205,29 +209,14 @@ class _ChromaBaseLoader:
             nrecs_a = self._dbo.collection.count()  # Count records after.
             return self._test_load(nrecs_b=nrecs_b, nrecs_a=nrecs_a)
         except chromadberrors.DuplicateIDError:
-            print('-- Document already loaded; duplicate detected.')
+            print('  -- Document *chunk* already loaded, duplication detected. File may be corrupt.')
             return False  # Prevent from loading keywords.
         except Exception as err:
             reporterror(err)
         return False
 
     def _parse_text(self, **kwargs) -> bool:
-        """Parse text from the document.
-
-        :Keyword Arguments:
-            Those to be passed into the text extraction method.
-
-        Returns:
-            bool: True if the parser's 'text' object is populated,
-            otherwise False.
-
-        """
-        print('- Extracting text ...')
-        self._p.extract_text(**kwargs)
-        if len(self._p.doc.pages) < 2:
-            ui.print_warning(f'No text extracted from {self._p.doc.basename}')
-            return False
-        return True
+        """Stub method, overridden by the child class."""
 
     @staticmethod
     def _print_summary(success: bool):
@@ -266,30 +255,8 @@ class _ChromaBaseLoader:
             return False
         return True
 
-    def _set_parser(self) -> bool:
-        """Set the appropriate document parser.
-
-        :Rationale:
-            The parser is set by the file extension. For example, a file
-            extension ``.pdf`` will set the
-            :class:`parsers.pdfparser.PDFParser` class.
-
-        Returns:
-            bool: True if a file extension appropriate parser was found.
-            Otherwise, False.
-
-        """
-        # pylint: disable=invalid-name  # OK as the variable (Parser) is a class.
-        # TODO: Updated this to use the (not-yet-available) ispdf utility
-        #       function, rather than relying on the file extension.
-        ext = os.path.splitext(self._fpath)[1]
-        Parser = self._PARSERS.get(ext)
-        if not Parser:
-            msg = f'{_PRE_WARN} Document parser not set for {os.path.basename(self._fpath)}.'
-            ui.print_warning(msg)
-            return False
-        self._p = Parser(path=self._fpath)
-        return True
+    def _set_parser(self):
+        """Stub method, overridden by the child class."""
 
     # TODO: Add these to a config file.
     def _set_text_splitter(self) -> bool:
@@ -307,15 +274,24 @@ class _ChromaBaseLoader:
     def _split_texts(self) -> bool:
         """Split the document text using a recursive text splitter.
 
+        Note:
+            If the ``split_text`` parameter was passed as ``False`` on
+            instantiation, the texts will not be split. Rather, the
+            :attr:`_docs` list is simply *copied* to the :attr:`_docss`
+            attribute.
+
         Returns:
-            bool: True if the text was split successfully, otherwise
-            False.
+            bool: True if the text was split (or copied) successfully,
+            otherwise False.
 
         """
-        self._docss = self._splitter.split_documents(self._docs)
+        if self._split_text:
+            self._docss = self._splitter.split_documents(self._docs)
+        else:
+            self._docss = self._docs[:]
         if not self._docss:
-            msg = (f'{_PRE_ERR} An error occurred while splitting the documents for '
-                   f'{self._p.doc.basename}.')
+            msg = (f'{self._PFX_ERR} An error occurred while splitting the documents for '
+                   f'{self._fbase}.')
             ui.print_warning(msg)
             return False
         return True
@@ -358,5 +334,5 @@ class _ChromaBaseLoader:
 
         """
         if nrecs_a == nrecs_b:
-            ui.print_warning(f'{_PRE_WARN} No new documents added. Possibly already loaded?')
+            ui.print_warning(f'{self._PFX_WARN} No new documents added. Possibly already loaded?')
         return nrecs_a == nrecs_b + len(self._docss)