docp 0.0.0.dev1__py3-none-any.whl → 0.2.0__py3-none-any.whl

docp/__init__.py

@@ -1,11 +1,40 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+:Purpose:   This module provides the project initilisation logic.
+
+:Platform:  Linux/Windows | Python 3.10+
+:Developer: J Berendt
+:Email:     development@s3dev.uk
+
+:Comments:  Ths loader modules/classes have *not* been imported due to the
+            heavy dependency requirements. Refer to the loaders/__init__.py
+            module instead.
+
+"""
+
 import os
 import sys
 sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)))
+from utils4.user_interface import ui
+# locals
+from .libs._version import __version__
+
+# TODO: Change these to use logging.
+
 # Bring entry-points to the surface.
 try:
-    from loaders.chroma import LoadChroma
-except ImportError:
-    # The chroma loader requires a lot of backend which is not required for the parser.
-    pass  
-from parsers.pdf import ParsePDF
-from ._version import __version__
+    from .parsers.pdfparser import PDFParser
+except ImportError as err:
+    msg = ( 'An error occurred while importing the PDF parser:\n'
+           f'- {err}\n'
+            '  - This can be ignored if the parser is not in use.\n')
+    ui.print_warning(f'\n[ImportError]: {msg}')
+
+try:
+    from .parsers.pptxparser import PPTXParser
+except ImportError as err:
+    msg = ( 'An error occurred while importing the PPTX parser:\n'
+           f'- {err}\n'
+            '  - This can be ignored if the parser is not in use.\n')
+    ui.print_warning(f'\n[ImportError]: {msg}')

docp/dbs/chroma.py

@@ -0,0 +1,197 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+:Purpose:   This module provides a localised wrapper and specialised
+            functionality around the
+            ``langchain_community.vectorstores.Chroma`` class, for
+            interacting with a Chroma database.
+
+:Platform:  Linux/Windows | Python 3.10+
+:Developer: J Berendt
+:Email:     development@s3dev.uk
+
+:Comments:  This module uses the
+            ``langchain_community.vectorstores.Chroma`` wrapper class,
+            rather than the base ``chromadb`` library  as it provides the
+            ``add_texts`` method which supports GPU processing and
+            parallelisation; which is implemented by this module's
+            :meth:`~ChromaDB.add_documents` method.
+
+"""
+# pylint: disable=import-error
+# pylint: disable=wrong-import-order
+
+from __future__ import annotations
+import chromadb
+import os
+import torch
+from glob import glob
+from hashlib import md5
+from langchain_huggingface import HuggingFaceEmbeddings
+# langchain's Chroma is used rather than the base chromadb as it provides
+# the add_texts method which support GPU processing and parallelisation.
+from langchain_community.vectorstores import Chroma as _Chroma
+
+
+class ChromaDB(_Chroma):
+    """Wrapper class around the ``chromadb`` library.
+
+    Args:
+        path (str): Path to the chroma database's *directory*.
+        collection (str): Collection name.
+        offline (bool, optional): Remain offline, used the cached
+            embedding function model rather than obtaining one online.
+            Defaults to False.
+    """
+    # pylint: disable=line-too-long
+
+    _MODEL_CACHE = os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), '.cache')
+    # Installing torch is a huge overhead, just for this. However, torch
+    # will already be installed as part of the sentence-transformers library,
+    # so we'll use it here.
+    _MODEL_KWARGS = {'device': 'cuda' if torch.cuda.is_available() else 'cpu'}
+    # TODO: Add this to a config file.
+    _MODEL_NAME = 'all-MiniLM-L6-v2'
+
+    def __init__(self, path: str, collection: str, offline: bool=False):
+        """Chroma database class initialiser."""
+        self._path = os.path.realpath(path)
+        self._cname = collection
+        self._offline = offline
+        self._client = None         # Database 'client' object
+        self._dbc = None            # Database 'collection' object.
+        self._set_client()
+        self._set_embedding_fn()
+        super().__init__(client=self._client,
+                          collection_name=self._cname,
+                          embedding_function=self._embfn,
+                          persist_directory=self._path)
+        self._set_collection()
+
+    @property
+    def client(self):
+        """Accessor to the :class:`chromadb.PersistentClient` class."""
+        return self._client
+
+    @property
+    def collection(self):
+        """Accessor to the chromadb client's collection object."""
+        return self._dbc
+
+    @property
+    def embedding_function(self):
+        """Accessor to the embedding function used."""
+        return self._embfn
+
+    @property
+    def path(self) -> str:
+        """Accessor to the database's path."""
+        return self._path
+
+    def add_documents(self, docs: list[langchain_core.documents.base.Document]):  # noqa  # pylint: disable=undefined-variable
+        """Add multiple documents to the collection.
+
+        This method overrides the base class' ``add_documents`` method
+        to enable local ID derivation. Knowing *how* the IDs are derived
+        gives us greater understanding and querying ability of the
+        documents in the database. Each ID is derived locally by the
+        :meth:`_preproc` method from the file's basename, page number
+        and page content.
+
+        Additionally, this method wraps the
+        :func:`langchain_community.vectorstores.Chroma.add_texts`
+        method which supports GPU processing and parallelisation.
+
+        Args:
+            docs (list): A list of ``langchain_core.documents.base.Document``
+                document objects.
+
+        """
+        # pylint: disable=arguments-differ
+        # pylint: disable=arguments-renamed
+        if not isinstance(docs, list):
+            docs = [docs]
+        ids_, docs_, meta_ = self._preproc(docs=docs)
+        self.add_texts(ids=ids_, texts=docs_, metadatas=meta_)
+
+    def show_all(self):
+        """Return the entire contents of the collection.
+
+        This is an alias around ``.collection.get()``.
+
+        """
+        return self._dbc.get()
+
+    def _get_embedding_function_model(self) -> str:
+        """Derive the path to the embedding function model.
+
+        :Note:
+            If ``offline=True`` was passed into the class constructor,
+            the model cache is used, if available - otherwise the user
+            is warned.
+
+            If online usage is allowed, the model is obtained by the
+            means defined by the embedding function constructor.
+
+        Returns:
+            str: The name of the model. Or, if offline, the path to the
+            model's cache to be passed into the embedding function
+            constructor is returned.
+
+        """
+        if self._offline:
+            if not os.path.exists(self._MODEL_CACHE):
+                os.makedirs(self._MODEL_CACHE)
+                msg = ('Offline mode has been chosen, yet the embedding function model cache does not exist. '
+                       'Therefore, a model must be downloaded. Please enable online usage for the first run '
+                       'so a model can be downloaded and stored into the cache for future (offline) use.')
+                raise FileNotFoundError(msg)
+            # Find the cache directory containing the named model, this enables offline use.
+            model_loc = os.path.commonpath(filter(lambda x: 'config.json' in x,
+                                                  glob(os.path.join(self._MODEL_CACHE,
+                                                                    f'*{self._MODEL_NAME}*',
+                                                                    '**'),
+                                                        recursive=True)))
+            return model_loc
+        return self._MODEL_NAME
+
+    @staticmethod
+    def _preproc(docs: list):
+        """Pre-process the document objects to create the IDs.
+
+        Parse the ``Document`` object into its parts for storage.
+        Additionally, create the ID as a hash of the source document's
+        basename, page number and content.
+
+        """
+        ids = []
+        txts = []
+        metas = []
+        for doc in docs:
+            pc = doc.page_content
+            m = doc.metadata
+            pc_, src_ = map(str.encode, (pc, m['source']))
+            pg_ = str(m.get('pageno', 0)).zfill(4)
+            id_ = f'id_{md5(src_).hexdigest()}_{pg_}_{md5(pc_).hexdigest()}'
+            ids.append(id_)
+            txts.append(pc)
+            metas.append(m)
+        return ids, txts, metas
+
+    def _set_client(self):
+        """Set the database client object."""
+        settings = chromadb.Settings(anonymized_telemetry=False)
+        self._client = chromadb.PersistentClient(path=self._path,
+                                                 settings=settings)
+
+    def _set_collection(self):
+        """Set the database collection object."""
+        self._dbc = self._client.get_or_create_collection(self._cname,
+                                                          metadata={'hnsw:space': 'cosine'})
+
+    def _set_embedding_fn(self):
+        """Set the embeddings function object."""
+        model_name = self._get_embedding_function_model()
+        self._embfn = HuggingFaceEmbeddings(model_name=model_name,
+                                            model_kwargs=self._MODEL_KWARGS,
+                                            cache_folder=self._MODEL_CACHE)

docp/libs/_version.py

@@ -0,0 +1 @@
+__version__ = '0.2.0'

docp/libs/changelog.py

@@ -0,0 +1,7 @@
+# Changed.
+# ENABLE SPHINX TO ACCESS THE GIT LOG
+"""
+.. git_changelog::
+    :revisions: 99
+    :detailed-message-pre: True
+"""

docp/libs/utilities.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+:Purpose:   This module provides utility-based functionality for the
+            project.
+
+:Platform:  Linux/Windows | Python 3.10+
+:Developer: J Berendt
+:Email:     development@s3dev.uk
+
+:Comments:  n/a
+
+"""
+
+import os
+import sys
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../'))
+import re
+from glob import glob
+from utils4 import futils
+
+
+class Utilities:
+    """General (cross-project) utility functions."""
+
+    @staticmethod
+    def collect_files(path: str, ext: str, recursive: bool) -> list:
+        """Collect all files for a given extension from a path.
+
+        Args:
+            path (str): Full path serving as the root for the search.
+            ext (str, optional): If the ``path`` argument refers to a
+                *directory*, a specific file extension can be specified
+                here. For example: ``ext = 'pdf'``.
+
+                If anything other than ``'**'`` is provided, all
+                alpha-characters are parsed from the string, and prefixed
+                with ``*.``. Meaning, if ``'.pdf'`` is passed, the
+                characters ``'pdf'`` are parsed and prefixed with ``*.``
+                to create ``'*.pdf'``. However, if ``'things.foo'`` is
+                passed, the derived extension will be ``'*.thingsfoo'``.
+                Defaults to '**', for a recursive search.
+
+            recursive (bool): Instruct the search to recurse into
+                sub-directories.
+
+        Returns:
+            list: The list of full file paths returned by the ``glob``
+            call. Any directory-only paths are removed.
+
+        """
+        if ext != '**':
+            ext = f'*.{re.findall("[a-zA-Z]+", ext)[0]}'
+        return list(filter(os.path.isfile, glob(os.path.join(path, ext), recursive=recursive)))
+
+    # !!!: Replace this with utils4.futils when available.
+    @staticmethod
+    def ispdf(path: str) -> bool:
+        """Test the file signature. Verify this is a valid PDF file.
+
+        Args:
+            path (str): Path to the file being tested.
+
+        Returns:
+            bool: True if this is a valid PDF file, otherwise False.
+
+        """
+        with open(path, 'rb') as f:
+            sig = f.read(5)
+        return sig == b'\x25\x50\x44\x46\x2d'
+
+    @staticmethod
+    def iszip(path: str) -> bool:
+        """Test the file signature. Verify this is a valid ZIP archive.
+
+        Args:
+            path (str): Path to the file being tested.
+
+        Returns:
+            bool: True if this is a valid ZIP archive, otherwise False.
+
+        """
+        return futils.iszip(path)
+
+    @staticmethod
+    def parse_to_keywords(resp: str) -> list:
+        """Parse the bot's response into a list of keywords.
+
+        Args:
+            resp (str): Text response directly from the bot.
+
+        Returns:
+            list: A list of keywords extracted from the response,
+            separated by asterisks as bullet points.
+
+        """
+        # Capture asterisk bullet points or a numbered list.
+        rexp = re.compile(r'(?:\*|[0-9]+\.)\s*(.*)\n')
+        trans = {45: ' ', 47: ' '}
+        resp_ = resp.translate(trans).lower()
+        kwds = rexp.findall(resp_)
+        if kwds:
+            return ', '.join(kwds)
+        return ''
+
+
+utilities = Utilities()

docp/loaders/__init__.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+:Purpose:   This module provides the project initilisation logic.
+
+:Platform:  Linux/Windows | Python 3.10+
+:Developer: J Berendt
+:Email:     development@s3dev.uk
+
+:Comments:  n/a
+
+"""
+
+import os
+import sys
+sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)))
+from utils4.user_interface import ui
+
+# TODO: Change these to use logging.
+
+# Bring entry-points to the surface.
+try:
+    from .chromapdfloader import ChromaPDFLoader
+except ImportError as err:
+    # The chroma loader requires a lot of backend which is not required for the parser.
+    msg = ( 'An error occurred while importing the Chroma PDF loader:\n'
+           f'- {err}\n'
+            '  - This can be ignored if the loader is not in use.\n')
+    ui.print_warning(f'\n[ImportError]: {msg}')
+
+try:
+    from .chromapptxloader import ChromaPPTXLoader
+except ImportError as err:
+    # The chroma loader requires a lot of backend which is not required for the parser.
+    msg = ( 'An error occurred while importing the Chroma PPTX loader:\n'
+           f'- {err}\n'
+            '  - This can be ignored if the loader is not in use.\n')
+    ui.print_warning(f'\n[ImportError]: {msg}')