flowtask 5.8.4__cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/components/LangchainLoader/loaders/docx.py

@@ -0,0 +1,91 @@
+from typing import List
+from pathlib import PurePath
+import mammoth
+import docx
+from markdownify import markdownify as md
+from langchain.docstore.document import Document
+from .abstract import AbstractLoader
+
+
+class MSWordLoader(AbstractLoader):
+    """
+    Load Microsoft Docx as Langchain Documents.
+    """
+    def extract_text(self, path):
+        """Extract text from a docx file.
+
+        Args:
+            path (Path): The source of the data.
+
+        Returns:
+            str: The extracted text.
+        """
+        doc = docx.Document(str(path))
+        text = []
+        for paragraph in doc.paragraphs:
+            text.append(paragraph.text)
+        return "\n".join(text)
+
+    async def _load_document(self, path: PurePath) -> List[Document]:
+        """Load data from a source and return it as a Langchain Document.
+
+        Args:
+            path (Path): The source of the data.
+
+        Returns:
+            List[Document]: A list of Langchain Documents.
+        """
+        self.logger.info(
+            f"Loading Word file: {path}"
+        )
+        docs = []
+        with open(path, "rb") as docx_file:
+            doc = docx.Document(str(path))
+            properties = doc.core_properties
+            result = mammoth.convert_to_html(docx_file)
+            # text_file = mammoth.extract_raw_text(docx_file) # Use text File for summary
+            html = result.value  # The generated HTML
+            md_text = md(html)  # The generated Markdown
+
+            print(f"Type of HTML result: {type(html)}")
+            print(f"Length of HTML: {len(html)}")
+            print(f"First 100 characters: {html[:100]}")
+            print(f"Messages from conversion: {result.messages}")
+            # TODO: add summarization and translation if requested
+            summary = ''
+            # try:
+            #     summary = self.get_summary_from_text(md_text, use_gpu=True)
+            # except Exception:
+            #     summary = ''
+            document_meta = {
+                "author": properties.author,
+                "version": properties.version,
+                "title": properties.title,
+                # "created": properties.created.strftime("%Y-%m-%d %H:%M:%S"),
+                # "last_modified": properties.modified.strftime("%Y-%m-%d %H:%M:%S")
+            }
+            metadata = self.create_metadata(
+                path=path,
+                doctype=self.doctype,
+                source_type=self._source_type,
+                summary=summary,
+                doc_metadata=document_meta
+            )
+            # Create document-level context
+            document_context = f"File Name: {path.name}\n"
+            document_context += f"Document Type: {self.doctype}\n"
+            document_context += f"Source Type: {self._source_type}\n"
+            document_context += f"Summary: {summary}\n"
+            document_context += "======\n"
+            # splitting the content:
+            for chunk in self.markdown_splitter.split_text(md_text):
+                _idx = {
+                    **metadata
+                }
+                docs.append(
+                    Document(
+                        page_content=document_context + chunk,
+                        metadata=_idx
+                    )
+                )
+        return docs

flowtask/components/LangchainLoader/loaders/html.py

@@ -0,0 +1,119 @@
+from bs4 import BeautifulSoup
+from langchain.docstore.document import Document
+from .abstract import AbstractLoader
+from pathlib import Path, PurePath
+from markdownify import markdownify as md
+from datetime import datetime
+
+
+class HTMLLoader(AbstractLoader):
+    """
+    Loader for HTML files to convert into Langchain Documents.
+
+    Processes HTML files, extracts relevant content, converts to Markdown,
+    and associates metadata with each document.
+    """
+
+    _extension = ['.html', '.htm']
+
+    def __init__(self, **kwargs):
+        """Initialize the HTMLLoader."""
+        self.elements: list = kwargs.pop('elements', [])
+        super().__init__(**kwargs)
+
+    async def _load_document(self, path: PurePath) -> list[Document]:
+        """
+        Load an HTML file and convert its content into Langchain documents.
+
+        Args:
+            path (PurePath): Path to the HTML file.
+
+        Returns:
+            list[Document]: A list of Langchain documents with content and metadata.
+        """
+        documents = []
+
+        # Check if the file exists and is valid
+        if not self._check_path(path):
+            raise ValueError(f"File {path} does not exist or is not a valid HTML file.")
+
+        # Read and parse the HTML file
+        with open(path, 'r', encoding=self.encoding) as file:
+            html_content = file.read()
+
+        soup = BeautifulSoup(html_content, 'html.parser')
+
+        # Extract the entire <body> content or
+        # Determine the top-level element to process
+        top_element = soup.body or soup
+        if not top_element:
+            raise ValueError(
+                "The HTML file does not contain a <body> or Top element tag."
+            )
+
+        extracted_elements = []
+
+        if self.elements:
+            # Extract content from specific elements
+            for element in self.elements:
+                for tag, selector in element.items():
+                    extracted_elements.extend(top_element.find_all(tag, class_=selector.lstrip('.')))
+
+        if not extracted_elements:
+            extracted_elements = [top_element]
+
+        # Process each extracted element
+        for elem in extracted_elements:
+            # Get the plain text content
+            text = elem.get_text(separator="\n", strip=True)
+
+            # Generate a summary for the extracted text
+            try:
+                summary = self.get_summary_from_text(text, use_gpu=True)
+            except Exception as e:
+                if self.logger:
+                    self.logger.error(f"Error generating summary: {e}")
+                summary = "Summary not available."
+
+            # Create document-level context
+            document_context = f"File Name: {path.name}\n"
+            document_context += f"Document Type: {self.doctype}\n"
+            document_context += f"Source Type: {self._source_type}\n"
+            document_context += f"Element: {elem.name}\n"
+            document_context += f"Summary: {summary}\n\n"
+
+            # Convert the entire <body> to Markdown for better structure
+            markdown_content = md(str(elem))
+
+            # Metadata preparation
+            document_meta = self.create_metadata(
+                path=path,
+                doctype=self.doctype,
+                source_type=self._source_type,
+                summary=summary,
+                doc_metadata={
+                    "type": "html",
+                    "category": self.category,
+                }
+            )
+
+            # Create a single Langchain Document with the full body content
+            document = Document(
+                page_content=document_context + markdown_content,
+                metadata=document_meta
+            )
+            documents.append(document)
+
+            # splitting the content:
+            for chunk in self.markdown_splitter.split_text(text):
+                _idx = {
+                    **document_meta
+                }
+                # Create a Langchain Document
+                documents.append(
+                    Document(
+                        page_content=document_context + chunk,
+                        metadata=_idx
+                    )
+                )
+        return documents

flowtask/components/LangchainLoader/loaders/pdfblocks.py

@@ -0,0 +1,146 @@
+from io import StringIO
+from pathlib import Path
+from datetime import datetime
+import fitz
+import pandas as pd
+from langchain.docstore.document import Document
+from .basepdf import BasePDF
+
+
+class PDFBlocks(BasePDF):
+    """
+    Load a PDF Table as Blocks of text.
+    """
+    _extension = ['.pdf']
+
+    def __init__(
+        self,
+        table_settings: dict = {},
+        **kwargs
+    ):
+        self._skiprows = kwargs.pop('skiprows', None)
+        super().__init__(**kwargs)
+        # Table Settings:
+        self.table_settings = {
+            # "vertical_strategy": "text",
+            # "horizontal_strategy": "text",
+            "intersection_x_tolerance": 5,
+            "intersection_y_tolerance": 5
+        }
+        if table_settings:
+            self.table_settings.update(table_settings)
+
+    def unique_columns(self, df: pd.DataFrame) -> pd.DataFrame:
+        """
+        Rename duplicate columns in the DataFrame to ensure they are unique.
+
+        Args:
+            df (pd.DataFrame): The DataFrame with potential duplicate column names.
+
+        Returns:
+            pd.DataFrame: A DataFrame with unique column names.
+        """
+        seen = {}
+        new_columns = []
+        for col in df.columns:
+            new_col = col
+            count = seen.get(col, 0)
+            while new_col in new_columns:
+                count += 1
+                new_col = f"{col}_{count}"
+            new_columns.append(new_col)
+            seen[col] = count
+        df.columns = new_columns
+        return df
+
+    def get_markdown(self, df: pd.DataFrame) -> str:
+        """
+        Convert a DataFrame to a Markdown string.
+
+        Args:
+            df (pd.DataFrame): The DataFrame to convert.
+
+        Returns:
+            str: The JSON string.
+        """
+        buffer = StringIO()
+        df = self.unique_columns(df)
+        df.to_markdown(buffer)
+        buffer.seek(0)
+        return buffer.getvalue()
+
+    def parse_table(self, table_idx, table, page_number, path) -> pd.DataFrame:
+        df = table.to_pandas()  # convert to pandas DataFrame
+        df = df.dropna(axis=1, how='all')
+        df = df.dropna(how='all', axis=0)  # Drop empty rows
+        page = page_number + 1
+        table_meta = {
+            "url": '',
+            "source": f"{path.name} Page.#{page} Table.#{table_idx}",
+            "filename": path.name,
+            "question": '',
+            "answer": '',
+            "type": 'table',
+            "summary": '',
+            "category": self.category,
+            "source_type": self._source_type,
+            "created_at": datetime.now().strftime("%Y-%m-%d, %H:%M:%S"),
+            "document_meta": {
+                "table_index": table_idx,
+                "table_shape": df.shape,
+                "table_columns": df.columns.tolist(),
+                "description": f"Extracted from Page.#{page}."
+            }
+        }
+        return df, table_meta
+
+    def _load_pdf(self, path: Path) -> list:
+        """
+        Load a PDF file using the Fitz library.
+
+        Args:
+            path (Path): The path to the PDF file.
+
+        Returns:
+            list: A list of Langchain Documents.
+        """
+        if self._check_path(path):
+            self.logger.info(f"Loading PDF file: {path}")
+            pdf = fitz.open(str(path))  # Open the PDF file
+            docs = []
+            # Create document-level context
+            document_context = f"File Name: {path.name}\n"
+            document_context += f"Document Type: {self.doctype}\n"
+            document_context += f"Source Type: {self._source_type}\n\n"
+            for page_number in range(pdf.page_count):
+                page = pdf[page_number]
+                try:
+                    tabs = page.find_tables(**self.table_settings)
+                    for tab_idx, tab in enumerate(tabs):
+                        df, _meta = self.parse_table(tab_idx, tab, page_number, path)
+                        try:
+                            markdown_table = self.get_markdown(df)
+                            docs.append(
+                                Document(
+                                    page_content=document_context + markdown_table,
+                                    metadata=_meta
+                                )
+                            )
+                        except Exception as exc:
+                            print(exc)
+                        ## Sample information:
+                        print('::: Printing Table Information === ')
+                        print(df)
+                        print("::: Printing Column Information === ")
+                        for column, t in df.dtypes.items():
+                            print(column, "->", t, "->", df[column].iloc[0])
+                        # convert into markdown:
+                        txt = df.to_markdown()
+                        if txt:
+                            docs.append(
+                                Document(page_content=document_context + txt, metadata=_meta)
+                            )
+                except Exception as exc:
+                    print(exc)
+                    continue
+            return docs

flowtask/components/LangchainLoader/loaders/pdfmark.py

@@ -0,0 +1,79 @@
+from pathlib import Path
+import fitz
+from pdf4llm import to_markdown
+from langchain.docstore.document import Document
+from langchain.text_splitter import MarkdownTextSplitter
+from .basepdf import BasePDF
+
+
+class PDFMarkdown(BasePDF):
+    """
+    Loader for PDF files converted content to markdown.
+    """
+    def __init__(
+        self,
+        **kwargs
+    ):
+        super().__init__(**kwargs)
+        self._splitter = MarkdownTextSplitter(chunk_size=2048, chunk_overlap=10)
+
+    def _load_pdf(self, path: Path) -> list:
+        """
+        Load a PDF file using the PDFMiner library.
+
+        Args:
+            path (Path): The path to the PDF file.
+
+        Returns:
+            list: A list of Langchain Documents.
+        """
+        if self._check_path(path):
+            self.logger.info(f"Loading PDF file: {path}")
+            docs = []
+            pdf = fitz.open(str(path))
+            md_text = to_markdown(pdf)
+            try:
+                summary = self.get_summary_from_text(md_text, use_gpu=True)
+            except Exception:
+                summary = ''
+            document_meta = {
+                "title": pdf.metadata.get("title", ""),
+                "creationDate": pdf.metadata.get("creationDate", ""),
+                "author": pdf.metadata.get("author", ""),
+            }
+            metadata = self.create_metadata(
+                path=path,
+                doctype=self.doctype,
+                source_type=self._source_type,
+                summary=summary,
+                doc_metadata=document_meta
+            )
+
+            # Prepend document-level context
+            document_context = f"Document Title: {document_meta.get('title', '')}\n"
+            # document_context += f"Document Author: {document_meta.get('author', '')}\n"
+            document_context += f"File Path: {str(path)}\n"
+            document_context += f"Summary: {summary}\n\n"
+
+            for _, chunk in enumerate(self._splitter.split_text(md_text)):
+                docs.append(
+                    Document(
+                        page_content=document_context + chunk,
+                        metadata=metadata
+                    )
+                )
+            # also, creating a document for summary:
+            if summary:
+                _info = {
+                    "category": "Summary",
+                    **metadata
+                }
+                docs.append(
+                    Document(
+                        page_content=f"**Summary:** {summary}",
+                        metadata=_info
+                    )
+                )
+            return docs
+        else:
+            return []

flowtask/components/LangchainLoader/loaders/pdftables.py

@@ -0,0 +1,135 @@
+from io import StringIO
+from pathlib import Path
+from datetime import datetime
+import fitz
+import pandas as pd
+from langchain.docstore.document import Document
+from .basepdf import BasePDF
+
+
+class PDFTables(BasePDF):
+    """
+    Loader for Tables present on PDF Files.
+    """
+    _extension = ['.pdf']
+
+    def __init__(
+        self,
+        table_settings: dict = {},
+        **kwargs
+    ):
+        self._skiprows = kwargs.pop('skiprows', None)
+        super().__init__(**kwargs)
+        # Table Settings:
+        self.table_settings = {
+            # "vertical_strategy": "text",
+            # "horizontal_strategy": "text",
+            "intersection_x_tolerance": 5,
+            "intersection_y_tolerance": 5
+        }
+        if table_settings:
+            self.table_settings.update(table_settings)
+
+    def unique_columns(self, df: pd.DataFrame) -> pd.DataFrame:
+        """
+        Rename duplicate columns in the DataFrame to ensure they are unique.
+
+        Args:
+            df (pd.DataFrame): The DataFrame with potential duplicate column names.
+
+        Returns:
+            pd.DataFrame: A DataFrame with unique column names.
+        """
+        seen = {}
+        new_columns = []
+        for col in df.columns:
+            new_col = col
+            count = seen.get(col, 0)
+            while new_col in new_columns:
+                count += 1
+                new_col = f"{col}_{count}"
+            new_columns.append(new_col)
+            seen[col] = count
+        df.columns = new_columns
+        return df
+
+    def get_markdown(self, df: pd.DataFrame) -> str:
+        """
+        Convert a DataFrame to a Markdown string.
+
+        Args:
+            df (pd.DataFrame): The DataFrame to convert.
+
+        Returns:
+            str: The JSON string.
+        """
+        buffer = StringIO()
+        df = self.unique_columns(df)
+        df.to_markdown(buffer)
+        buffer.seek(0)
+        return buffer.getvalue()
+
+    def parse_table(self, table_idx, table, page_number, path) -> pd.DataFrame:
+        df = table.to_pandas()  # convert to pandas DataFrame
+        df = df.dropna(axis=1, how='all')
+        df = df.dropna(how='all', axis=0)  # Drop empty rows
+        page = page_number + 1
+        table_meta = {
+            "url": '',
+            "source": f"{path.name} Page.#{page} Table.#{table_idx}",
+            "filename": path.name,
+            "question": '',
+            "answer": '',
+            "type": 'table',
+            "summary": '',
+            "category": self.category,
+            "source_type": self._source_type,
+            "created_at": datetime.now().strftime("%Y-%m-%d, %H:%M:%S"),
+            "document_meta": {
+                "table_index": table_idx,
+                "table_shape": df.shape,
+                "table_columns": df.columns.tolist(),
+                "description": f"Extracted from Page.#{page}."
+            }
+        }
+        return df, table_meta
+
+    def _load_pdf(self, path: Path) -> list:
+        """
+        Load a PDF file using the Fitz library.
+
+        Args:
+            path (Path): The path to the PDF file.
+
+        Returns:
+            list: A list of Langchain Documents.
+        """
+        if self._check_path(path):
+            self.logger.info(f"Loading PDF file: {path}")
+            pdf = fitz.open(str(path))  # Open the PDF file
+            docs = []
+            document_context = f"File Name: {path.name}\n"
+            document_context += f"Document Type: {self.doctype}\n"
+            document_context += f"Source Type: {self._source_type}\n\n"
+            for page_number in range(pdf.page_count):
+                page = pdf[page_number]
+                try:
+                    tabs = page.find_tables(**self.table_settings)
+                    for tab_idx, tab in enumerate(tabs):
+                        df, _meta = self.parse_table(tab_idx, tab, page_number, path)
+                        ## Sample information:
+                        print('::: Printing Table Information === ')
+                        print(df)
+                        print("::: Printing Column Information === ")
+                        for column, t in df.dtypes.items():
+                            print(column, "->", t, "->", df[column].iloc[0])
+                        # convert into markdown:
+                        txt = df.to_markdown()
+                        if txt:
+                            docs.append(
+                                Document(page_content=document_context + txt, metadata=_meta)
+                            )
+                except Exception as exc:
+                    print(exc)
+                    continue
+            return docs

flowtask/components/LangchainLoader/loaders/qa.py

@@ -0,0 +1,67 @@
+
+from pathlib import PurePath
+from typing import List
+import pandas as pd
+from langchain.docstore.document import Document
+from .abstract import AbstractLoader
+
+
+class QAFileLoader(AbstractLoader):
+    """
+    Question and Answers File based on Excel, coverted to Langchain Documents.
+    """
+    _extension = ['.xlsx']
+    chunk_size = 768
+
+    def __init__(
+        self,
+        columns: list = ['Question', 'Answer'],
+        **kwargs
+    ):
+        super().__init__(**kwargs)
+        self._columns = columns
+
+    def _load_document(self, path: PurePath) -> list:
+        df = pd.read_excel(path)
+        q = self._columns[0]
+        a = self._columns[1]
+        docs = []
+        for idx, row in df.iterrows():
+            # Question Document
+            document_meta = {
+                "question": row[q],
+                "answer": row[a],
+            }
+            metadata = self.create_metadata(
+                path=path,
+                doctype=self.doctype,
+                source_type=self._source_type,
+                summary=f"Question: {row[q]}?: **{row[a]}**",
+                doc_metadata=document_meta,
+                type="QA",
+                question=row[q],
+                answer=row[a],
+            )
+            doc = Document(
+                page_content=f"**Question:** {row[q]}: **Answer:** {row[a]}",
+                metadata=metadata,
+            )
+            docs.append(doc)
+        return docs
+
+    async def load(self, path: PurePath) -> List[Document]:
+        """Load data from a source and return it as a Langchain Document.
+
+        Args:
+            path (Path): The source of the data.
+
+        Returns:
+            List[Document]: A list of Langchain Documents.
+        """
+        self.logger.info(
+            f"Loading Excel FAQ file: {path}"
+        )
+        docs = []
+        if path.exists():
+            docs = self._load_document(path)
+        return docs

flowtask/components/LangchainLoader/loaders/txt.py

@@ -0,0 +1,55 @@
+from typing import List
+from pathlib import PurePath
+from langchain.docstore.document import Document
+from .abstract import AbstractLoader
+
+
+class TXTLoader(AbstractLoader):
+    """
+    Loader for PDF files.
+    """
+    _extension = ['.txt']
+
+    def _load_document(self, path: PurePath) -> List[Document]:
+        """
+        Load a TXT file.
+
+        Args:
+            path (Path): The path to the TXT file.
+
+        Returns:
+            list: A list of Langchain Documents.
+        """
+        docs = []
+        if self._check_path(path):
+            self.logger.info(f"Loading TXT file: {path}")
+            with open(path, 'r') as file:
+                text = file.read()
+            try:
+                summary = self.get_summary_from_text(text, use_gpu=True)
+            except Exception:
+                summary = ''
+            metadata = self.create_metadata(
+                path=path,
+                doctype=self.doctype,
+                source_type=self._source_type,
+                summary=summary,
+                doc_metadata={}
+            )
+            # Create document-level context
+            document_context = f"File Name: {path.name}\n"
+            document_context += f"Document Type: {self.doctype}\n"
+            document_context += f"Source Type: {self._source_type}\n"
+            document_context += f"Summary: {summary}\n\n"
+            # splitting the content:
+            for chunk in self.markdown_splitter.split_text(text):
+                _idx = {
+                    **metadata
+                }
+                docs.append(
+                    Document(
+                        page_content=document_context + chunk,
+                        metadata=_idx
+                    )
+                )
+        return docs