docp 0.0.0.dev1__py3-none-any.whl → 0.2.0__py3-none-any.whl

docp/loaders/chroma.py.bak

@@ -0,0 +1,196 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+:Purpose:   This module provides the entry point for loading a document
+            into a Chroma database.
+
+:Platform:  Linux/Windows | Python 3.10+
+:Developer: J Berendt
+:Email:     development@s3dev.uk
+
+:Comments:  n/a
+
+:Examples:
+
+    Parse and load a *single* document into a Chroma database
+    collection::
+
+        >>> from docp import ChromaLoader
+
+        >>> l = ChromaLoader(path='/path/to/file.pdf',
+                             dbpath='/path/to/chroma',
+                             collection='spam')
+        >>> l.load()
+
+
+    Parse and load a *directory* of PDF documents into a Chroma
+    database collection::
+
+        >>> from docp import ChromaLoader
+
+        >>> l = ChromaLoader(path='/path/to/directory',
+                             dbpath='/path/to/chroma',
+                             collection='spam')
+        >>> l.load(ext='pdf')
+
+
+    For further example code use, please refer to the
+    :class:`ChromaLoader` class docstring.
+
+"""
+
+import os
+import re
+from glob import glob
+# locals
+try:
+    # from .loaders._chromabaseloader import _ChromaBaseLoader
+    from .loaders._chromabaseloader import _ChromaBasePDFLoader
+    from .loaders._chromabaseloader import _ChromaBasePPTXLoader
+except ImportError:
+    # from loaders._chromabaseloader import _ChromaBaseLoader
+    from loaders._chromabaseloader import _ChromaBasePDFLoader
+    from loaders._chromabaseloader import _ChromaBasePPTXLoader
+
+
+# TODO: The document-type-specific loader will be determined by this
+#       class.
+
+# !!!: This won't work. This class will have to create an instance of either.
+class ChromaLoader(_ChromaBasePDFLoader):
+    """Chroma database document loader.
+
+    Args:
+        path (str): Full path to the file (or *directory*) to be parsed
+            and loaded. Note: If this is a directory, a specific file
+            extension can be passed into the :meth:`load` method using
+            the ``ext`` argument.
+        dbpath (str): Full path to the Chroma database *directory*.
+        collection (str): Name of the Chroma database collection into
+            which the data is to be loaded.
+        load_keywords (bool, optional): Use the provided LLM
+            (via the ``llm`` parameter) to read the document and infer
+            keywords to be loaded into the ``<collection>-kwds``
+            database, for keyword-driven document filtering.
+            Note: This *requires* the ``llm`` parameter and is
+            recommended only for GPU-bound processing. Defaults to False.
+        llm (object, optional): An LLM *instance* which can be provided
+            directly into the
+            :func:`langchain.chains.RetrievalQA.from_chain_type` function
+            for keyword inference. This is *required* for keyword
+            loading. Defaults to None.
+        offline (bool, optional): Remain offline and use the locally
+            cached embedding function model. Defaults to False.
+
+    .. important::
+
+        The *deriving and loading of keywords* is only recommended for
+        **GPU-bound processing**, as the LLM is invoked to infer the
+        keywords for each given document.
+
+        If called on a 'standard' PC, this will take a *long* time to
+        complete, if it completes at all.
+
+    :Example:
+
+        Parse and load a *single* document into a Chroma database
+        collection::
+
+            >>> from docp import ChromaLoader
+
+            >>> l = ChromaLoader(path='/path/to/file.pdf',
+                                 dbpath='/path/to/chroma',
+                                 collection='spam')
+            >>> l.load()
+
+
+        Parse and load a *directory* of PDF documents into a Chroma
+        database collection::
+
+            >>> from docp import ChromaLoader
+
+            >>> l = ChromaLoader(path='/path/to/directory',
+                                 dbpath='/path/to/chroma',
+                                 collection='spam')
+            >>> l.load(ext='pdf')
+
+    """
+
+    def __init__(self,
+                 path: str,
+                 dbpath: str,
+                 collection: str,
+                 *,
+                 load_keywords: bool=False,
+                 llm: object=None,
+                 offline: bool=False):
+        """Chroma database loader class initialiser."""
+        super().__init__(dbpath=dbpath,
+                         collection=collection,
+                         load_keywords=load_keywords,
+                         llm=llm,
+                         offline=offline)
+        self._path = path
+
+    def load(self,
+             *,
+             ext: str='**',
+             recursive: bool=True,
+             remove_header: bool=True,
+             remove_footer: bool=True,
+             remove_newlines: bool=True,
+             ignore_tags: set=None,
+             convert_to_ascii: bool=True) -> None:
+        """Load a document (or documents) into a Chroma database.
+
+        Args:
+            ext (str): If the ``path`` argument refers to a *directory*,
+                a specific file extension can be specified here.
+                For example::
+
+                    ext = 'pdf'
+
+                If anything other than ``'**'`` is provided, all
+                alpha-characters are parsed from the string, and prefixed
+                with ``*.``. Meaning, if ``'.pdf'`` is passed, the
+                characters ``'pdf'`` are parsed and prefixed with ``*.``
+                to create ``'*.pdf'``. However, if ``'things.foo'`` is
+                passed, the derived extension will be ``'*.thingsfoo'``.
+                Defaults to '**', for a recursive search.
+
+            recursive (bool, optional): If True, subdirectories are
+                searched. Defaults to True.
+            remove_header (bool, optional): Attempt to remove the header
+                from each page. Defaults to True.
+            remove_footer (bool, optional): Attempt to remove the footer
+                from each page. Defaults to True.
+            remove_newlines (bool, optional): Replace newline characters
+                with a space. Defaults to True, as this helps with
+                document chunk splitting.
+            ignore_tags (set, optional): If provided, these are the
+                PDF 'marked content' tags which will be ignored. Note
+                that the PDF document must contain tags, otherwise the
+                bounding box method is used and this argument is ignored.
+                Defaults to ``{'Artifact'}``, as these generally
+                relate to a header and/or footer. To include all tags,
+                (not skip any) pass this argument as ``'na'``.
+            convert_to_ascii (bool, optional): Convert all characters to
+                ASCII. Defaults to True.
+
+        """
+        if os.path.isdir(self._path):
+            if ext != '**':
+                ext = f'*.{re.findall("[a-zA-Z]+", ext)[0]}'
+            files = glob(os.path.join(self._path, ext), recursive=recursive)
+            count = len(files)
+            for idx, f in enumerate(files, 1):
+                print(f'\nProcessing {idx} of {count}: {os.path.basename(f)}')
+                self._load(path=f)
+        else:
+            print(f'Processing: {os.path.basename(self._path)} ...')
+            self._load(path=self._path,
+                       remove_header=remove_header,
+                       remove_footer=remove_footer,
+                       remove_newlines=remove_newlines,
+                       ignore_tags=ignore_tags,
+                       convert_to_ascii=convert_to_ascii)

docp/loaders/chromapdfloader.py

@@ -0,0 +1,199 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+:Purpose:   This module provides the entry point for loading PDF files
+            into a Chroma database.
+
+:Platform:  Linux/Windows | Python 3.10+
+:Developer: J Berendt
+:Email:     development@s3dev.uk
+
+:Comments:  n/a
+
+:Examples:
+
+    Parse and load a *single* PDF file into a Chroma database
+    collection::
+
+        >>> from docp.loaders import ChromaPDFLoader
+
+        >>> l = ChromaPDFLoader(dbpath='/path/to/chroma',
+                                collection='spam')
+        >>> l.load(path='/path/to/directory/myfile.pdf')
+
+
+    Parse and load a *directory* of PDF files into a Chroma database
+    collection::
+
+        >>> from docp.loaders import ChromaPDFLoader
+
+        >>> l = ChromaPDFLoader(dbpath='/path/to/chroma',
+                                collection='spam')
+        >>> l.load(path='/path/to/directory', ext='pdf')
+
+
+    For further example code use, please refer to the
+    :class:`ChromaPDFLoader` class docstring.
+
+"""
+
+import os
+# locals
+try:
+    from .libs.utilities import utilities
+    from .loaders._chromabasepdfloader import _ChromaBasePDFLoader
+except ImportError:
+    from libs.utilities import utilities
+    from loaders._chromabasepdfloader import _ChromaBasePDFLoader
+
+
+class ChromaPDFLoader(_ChromaBasePDFLoader):
+    """Chroma database PDF-specific document loader.
+
+    Args:
+        dbpath (str | ChromaDB): Either the full path to the Chroma
+            database *directory*, or an instance of a
+            :class:`~docp.dbs.chroma.ChromaDB` class. If the instance is
+            passed, the ``collection`` argument is ignored.
+        collection (str, optional): Name of the Chroma database
+            collection. Only required if the ``dbpath`` parameter is a
+            path. Defaults to None.
+        split_text (bool, optional): Split the document into chunks,
+            before loading it into the database. Defaults to True.
+        load_keywords (bool, optional): Use an LLM to derive keywords
+            from the document and load these keywords into the sister
+            keywords collection. Defaults to False.
+        llm (object, optional): If deriving keywords, this is the LLM
+            which will do the derivation. Defaults to None.
+        offline (bool, optional): Remain offline and use the locally
+            cached embedding function model. Defaults to False.
+
+    .. important::
+
+        The *deriving and loading of keywords* is only recommended for
+        **GPU-bound processing** as the LLM is invoked to infer the
+        keywords for each given document.
+
+        If called on a 'standard' PC, this will take a *long* time to
+        complete, if it completes at all.
+
+    :Examples:
+
+        Parse and load a *single* PDF file into a Chroma database
+        collection::
+
+            >>> from docp.loaders import ChromaPDFLoader
+
+            >>> l = ChromaPDFLoader(dbpath='/path/to/chroma',
+                                    collection='spam')
+            >>> l.load(path='/path/to/directory/myfile.pdf')
+
+
+        Parse and load a *directory* of PDF files into a Chroma
+        database collection::
+
+            >>> from docp.loaders import ChromaPDFLoader
+
+            >>> l = ChromaPDFLoader(dbpath='/path/to/chroma',
+                                    collection='spam')
+            >>> l.load(path='/path/to/directory', ext='pdf')
+
+    """
+
+    #
+    # No __init__ method here to ensure the ultimate base class'
+    # signature is used and to save passing loads of stuff around, if we
+    # don't have to.
+    #
+
+    def load(self,
+             path: str,
+             *,
+             ext: str='**',
+             recursive: bool=True,
+             remove_header: bool=True,
+             remove_footer: bool=True,
+             remove_newlines: bool=True,
+             ignore_tags: set=None,
+             convert_to_ascii: bool=True,
+             **unused) -> None:
+        """Load a PDF file (or files) into a Chroma database.
+
+        Args:
+            path (str): Full path to the file (or *directory*) to be
+                parsed and loaded. Note: If this is a directory, a
+                specific file extension can be passed into the
+                :meth:`load` method using the ``ext`` argument.
+            ext (str, optional): If the ``path`` argument refers to a
+                *directory*, a specific file extension can be specified
+                here. For example: ``ext = 'pdf'``.
+
+                If anything other than ``'**'`` is provided, all
+                alpha-characters are parsed from the string, and prefixed
+                with ``*.``. Meaning, if ``'.pdf'`` is passed, the
+                characters ``'pdf'`` are parsed and prefixed with ``*.``
+                to create ``'*.pdf'``. However, if ``'things.foo'`` is
+                passed, the derived extension will be ``'*.thingsfoo'``.
+                Defaults to '**', for a recursive search.
+
+            recursive (bool, optional): If True, subdirectories are
+                searched. Defaults to True.
+            remove_header (bool, optional): Attempt to remove the header
+                from each page. Defaults to True.
+            remove_footer (bool, optional): Attempt to remove the footer
+                from each page. Defaults to True.
+            remove_newlines (bool, optional): Replace newline characters
+                with a space. Defaults to True, as this helps with
+                document chunk splitting.
+            ignore_tags (set, optional): If provided, these are the
+                PDF 'marked content' tags which will be ignored. Note
+                that the PDF document must contain tags, otherwise the
+                bounding box method is used and this argument is ignored.
+                Defaults to ``{'Artifact'}``, as these generally
+                relate to a header and/or footer. To include all tags,
+                (not skip any) pass this argument as ``'na'``.
+            convert_to_ascii (bool, optional): Convert all characters to
+                ASCII. Defaults to True.
+
+        :Keyword Args:
+            unused (dict): This enables keywords to be passed into a
+                loader-agnostic ``.load()`` function without raising a
+                'unexpected keyword argument` ``TypeError``.
+
+        """
+        # pylint: disable=unused-argument  # They are 'used' via locals().
+        # Prepare the arguments being sent to the doc parser.
+        kwargs = self._set_kwargs(locals_=locals())
+        # Load multi
+        if os.path.isdir(path):
+            files = utilities.collect_files(path=path, ext=ext, recursive=recursive)
+            count = len(files)
+            for idx, f in enumerate(files, 1):
+                print(f'\nProcessing {idx} of {count}: {os.path.basename(f)}')
+                self._load(path=f, **kwargs)
+        # Load single
+        else:
+            print(f'Processing: {os.path.basename(path)} ...')
+            self._load(path=path, **kwargs)
+
+    @staticmethod
+    def _set_kwargs(locals_: dict) -> dict:
+        r"""Prepare the arguments which are sent to the doc parser.
+
+        As :func:`locals()` is used to capture the :meth:`load` method's
+        arguments for passing into the doc parser, some argument must be
+        removed first.
+
+        Args:
+            locals\_ (dict): The return value from a :func:`locals` call.
+
+        Returns:
+            dict: A *copy* of the provided dictionary with specific
+            key/value pairs removed.
+
+        """
+        # ^^^ The backslash in locals\_ is required for documentation to render correctly.
+        kwargs = locals_.copy()
+        for k in ['self', 'path']:
+            kwargs.pop(k)
+        return kwargs

docp/loaders/chromapptxloader.py

@@ -0,0 +1,192 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+:Purpose:   This module provides the entry point for loading PPTX files
+            into a Chroma database.
+
+:Platform:  Linux/Windows | Python 3.10+
+:Developer: J Berendt
+:Email:     development@s3dev.uk
+
+:Comments:  n/a
+
+:Examples:
+
+    Parse and load a *single* PPTX file into a Chroma database
+    collection::
+
+        >>> from docp.loaders import ChromaPPTXLoader
+
+        >>> l = ChromaPPTXLoader(dbpath='/path/to/chroma',
+                                 collection='spam',
+                                 split_text=False)
+        >>> l.load(path='/path/to/directory/myfile.pptx')
+
+
+    Parse and load a *directory* of PPTX files into a Chroma database
+    collection::
+
+        >>> from docp.loaders import ChromaPPTXLoader
+
+        >>> l = ChromaPPTXLoader(dbpath='/path/to/chroma',
+                                 collection='spam',
+                                 split_text=False)
+        >>> l.load(path='/path/to/directory', ext='pptx')
+
+
+    For further example code use, please refer to the
+    :class:`ChromaPPTXLoader` class docstring.
+
+"""
+
+import os
+# locals
+try:
+    from .libs.utilities import utilities
+    from .loaders._chromabasepptxloader import _ChromaBasePPTXLoader
+except ImportError:
+    from libs.utilities import utilities
+    from loaders._chromabasepptxloader import _ChromaBasePPTXLoader
+
+
+class ChromaPPTXLoader(_ChromaBasePPTXLoader):
+    """Chroma database PPTX-specific document loader.
+
+    Args:
+        dbpath (str | ChromaDB): Either the full path to the Chroma
+            database *directory*, or an instance of a
+            :class:`~docp.dbs.chroma.ChromaDB` class. If the instance is
+            passed, the ``collection`` argument is ignored.
+        collection (str, optional): Name of the Chroma database
+            collection. Only required if the ``db`` parameter is a path.
+            Defaults to None.
+        split_text (bool, optional): Split the document into chunks,
+            before loading it into the database. Defaults to True.
+        load_keywords (bool, optional): Derive keywords from the document
+            and load these into the sister keywords collection.
+            Defaults to False.
+        llm (object, optional): If deriving keywords, this is the LLM
+            which will do the derivation. Defaults to None.
+        offline (bool, optional): Remain offline and use the locally
+            cached embedding function model. Defaults to False.
+
+    .. important::
+
+        The *deriving and loading of keywords* is only recommended for
+        **GPU-bound processing**, as the LLM is invoked to infer the
+        keywords for each given document.
+
+        If called on a 'standard' PC, this will take a *long* time to
+        complete, if it completes at all.
+
+    .. tip::
+
+        It is recommended to pass ``split_text=False`` into the
+        :class:`ChromaPPTXLoader` constructor.
+
+        Often, PowerPoint presentations are structured such that related
+        text is found in the same 'shape' (textbox) on a slide.
+        Splitting the text in these shapes may have undesired results.
+
+    :Examples:
+
+        Parse and load a *single* PPTX file into a Chroma database
+        collection::
+
+            >>> from docp.loaders import ChromaPPTXLoader
+
+            >>> l = ChromaPPTXLoader(dbpath='/path/to/chroma',
+                                     collection='spam',
+                                     split_text=False)  # <-- Note this
+            >>> l.load(path='/path/to/directory/myfile.pptx')
+
+
+        Parse and load a *directory* of PPTX files into a Chroma database
+        collection::
+
+            >>> from docp.loaders import ChromaPPTXLoader
+
+            >>> l = ChromaPPTXLoader(dbpath='/path/to/chroma',
+                                     collection='spam',
+                                     split_text=False)  # <-- Note this
+            >>> l.load(path='/path/to/directory', ext='pptx')
+
+    """
+    def load(self,
+             path: str,
+             *,
+             ext: str='**',
+             recursive: bool=True,
+             remove_newlines: bool=True,
+             convert_to_ascii: bool=True,
+             **unused) -> None:
+        """Load a PDF file (or files) into a Chroma database.
+
+        Args:
+            path (str): Full path to the file (or *directory*) to be
+                parsed and loaded. Note: If this is a directory, a
+                specific file extension can be passed into the
+                :meth:`load` method using the ``ext`` argument.
+            ext (str, optional): If the ``path`` argument refers to a
+                *directory*, a specific file extension can be specified
+                here. For example: ``ext = 'pptx'``.
+
+                If anything other than ``'**'`` is provided, all
+                alpha-characters are parsed from the string, and prefixed
+                with ``*.``. Meaning, if ``'.pptx'`` is passed, the
+                characters ``'pptx'`` are parsed and prefixed with ``*.``
+                to create ``'*.pptx'``. However, if ``'things.foo'`` is
+                passed, the derived extension will be ``'*.thingsfoo'``.
+                Defaults to '**', for a recursive search.
+
+            recursive (bool, optional): If True, subdirectories are
+                searched. Defaults to True.
+            remove_newlines (bool, optional): Replace newline characters
+                with a space. Defaults to True, as this helps with
+                document chunk splitting.
+            convert_to_ascii (bool, optional): Convert all characters to
+                ASCII. Defaults to True.
+
+        :Keyword Args:
+            unused (dict): This enables keywords such as ``remove_header``
+                and ``remove_footer`` (for example) to be passed into a
+                loader-agnostic ``.load()`` function without raising a
+                'unexpected keyword argument` ``TypeError``.
+
+        """
+        # pylint: disable=unused-argument  # They are 'used' via locals().
+        # Prepare the arguments being sent to the doc parser.
+        kwargs = self._set_kwargs(locals_=locals())
+        # Load multi
+        if os.path.isdir(path):
+            files = utilities.collect_files(path=path, ext=ext, recursive=recursive)
+            count = len(files)
+            for idx, f in enumerate(files, 1):
+                print(f'\nProcessing {idx} of {count}: {os.path.basename(f)}')
+                self._load(path=f, **kwargs)
+        # Load single
+        else:
+            print(f'Processing: {os.path.basename(path)} ...')
+            self._load(path=path, **kwargs)
+
+    @staticmethod
+    def _set_kwargs(locals_: dict) -> dict:
+        r"""Prepare the arguments which are sent to the doc parser.
+
+        As :func:`locals()` is used to capture the :meth:`load` method's
+        arguments for passing into the doc parser, some argument must be
+        removed first.
+
+        Args:
+            locals\_ (dict): The return value from a :func:`locals` call.
+
+        Returns:
+            dict: A *copy* of the provided dictionary with specific
+            key/value pairs removed.
+
+        """
+        # ^^^ The backslash in locals\_ is required for documentation to render correctly.
+        kwargs = locals_.copy()
+        for k in ['self', 'path']:
+            kwargs.pop(k)
+        return kwargs

docp/loaders/lutilities.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+:Purpose:   This module provides loader-specific utility functions for
+            the project.
+
+:Platform:  Linux/Windows | Python 3.10+
+:Developer: J Berendt
+:Email:     development@s3dev.uk
+
+:Comments:  This module is here (in the ``docp/loaders``) directory
+            rather than merged with the ``docp/parsers/putilities.py``
+            module as the loaders' dependencies are *heavy*. Keeping the
+            loader functionality separate helps to ease the dependency
+            requirements for parser-only projects.
+
+"""
+
+# locals
+try:
+    from .libs.utilities import utilities
+    from .loaders.chromapdfloader import ChromaPDFLoader
+    from .loaders.chromapptxloader import ChromaPPTXLoader
+except ImportError:
+    from libs.utilities import utilities
+    from loaders.chromapdfloader import ChromaPDFLoader
+    from loaders.chromapptxloader import ChromaPPTXLoader
+
+
+class LoaderUtilities:
+    """Loader-based (cross-project) utility functions."""
+
+    def get_loader(self, path: str) -> ChromaPDFLoader | ChromaPPTXLoader:
+        """Return the appropriate loader for the file type.
+
+        Args:
+            path (str): Full path to the file to be tested.
+
+        Returns:
+            ChromaPDFLoader | ChromaPPTXLoader: The appropriate loader
+            for the file, given the *file signature*; this test is not
+            file extension based.
+
+        """
+        if utilities.ispdf(path=path):
+            return ChromaPDFLoader
+        if utilities.iszip(path=path):
+            return ChromaPPTXLoader
+        raise NotImplementedError('A loader is not available for: os.path.basename(path)')
+
+
+lutilities = LoaderUtilities()