ai-parrot 0.3.3__cp311-cp311-manylinux_2_28_x86_64.whl → 0.3.6__cp311-cp311-manylinux_2_28_x86_64.whl

{ai_parrot-0.3.3.dist-info → ai_parrot-0.3.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ai-parrot
-Version: 0.3.3
+Version: 0.3.6
 Summary: Live Chatbots based on Langchain chatbots and Agents     Integrated into Navigator Framework or used into aiohttp applications.
 Home-page: https://github.com/phenobarbital/ai-parrot
 Author: Jesus Lara
@@ -26,7 +26,7 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Framework :: AsyncIO
-Requires-Python: >=3.10.12
+Requires-Python: >=3.9.20
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: Cython==3.0.11
@@ -58,11 +58,8 @@ Requires-Dist: sentence-transformers==3.0.1
 Requires-Dist: tabulate==0.9.0
 Requires-Dist: tiktoken==0.7.0
 Requires-Dist: tokenizers==0.19.1
-Requires-Dist: unstructured==0.14.3
-Requires-Dist: unstructured-client==0.18.0
-Requires-Dist: youtube-transcript-api==0.6.2
-Requires-Dist: selenium==4.18.1
-Requires-Dist: webdriver-manager==4.0.1
+Requires-Dist: selenium>=4.18.1
+Requires-Dist: webdriver-manager>=4.0.1
 Requires-Dist: transitions==0.9.0
 Requires-Dist: sentencepiece==0.2.0
 Requires-Dist: duckduckgo-search==5.3.0
@@ -79,26 +76,8 @@ Requires-Dist: mediawikiapi==1.2
 Requires-Dist: pyowm==3.3.0
 Requires-Dist: O365==2.0.35
 Requires-Dist: stackapi==0.3.1
-Requires-Dist: timm==1.0.9
 Requires-Dist: torchvision==0.19.1
-Provides-Extra: all
-Requires-Dist: langchain-milvus==0.1.1; extra == "all"
-Requires-Dist: milvus==2.3.5; extra == "all"
-Requires-Dist: pymilvus==2.4.4; extra == "all"
-Requires-Dist: groq==0.11.0; extra == "all"
-Requires-Dist: langchain-groq==0.1.4; extra == "all"
-Requires-Dist: llama-index-llms-huggingface==0.2.7; extra == "all"
-Requires-Dist: langchain-google-vertexai==1.0.8; extra == "all"
-Requires-Dist: langchain-google-genai==1.0.8; extra == "all"
-Requires-Dist: google-generativeai==0.7.2; extra == "all"
-Requires-Dist: vertexai==1.60.0; extra == "all"
-Requires-Dist: google-cloud-aiplatform>=1.60.0; extra == "all"
-Requires-Dist: grpc-google-iam-v1==0.13.0; extra == "all"
-Requires-Dist: langchain-openai==0.1.21; extra == "all"
-Requires-Dist: openai==1.40.8; extra == "all"
-Requires-Dist: llama-index-llms-openai==0.1.11; extra == "all"
-Requires-Dist: langchain-anthropic==0.1.23; extra == "all"
-Requires-Dist: anthropic==0.34.0; extra == "all"
+Requires-Dist: tf-keras==2.17.0
 Provides-Extra: analytics
 Requires-Dist: annoy==1.17.3; extra == "analytics"
 Requires-Dist: gradio-tools==0.0.9; extra == "analytics"
@@ -123,6 +102,9 @@ Requires-Dist: langchain-groq==0.1.9; extra == "groq"
 Provides-Extra: hunggingfaces
 Requires-Dist: llama-index-llms-huggingface==0.2.7; extra == "hunggingfaces"
 Provides-Extra: loaders
+Requires-Dist: unstructured==0.14.3; extra == "loaders"
+Requires-Dist: unstructured-client==0.18.0; extra == "loaders"
+Requires-Dist: youtube-transcript-api==0.6.2; extra == "loaders"
 Requires-Dist: pymupdf==1.24.4; extra == "loaders"
 Requires-Dist: pymupdf4llm==0.0.1; extra == "loaders"
 Requires-Dist: pdf4llm==0.0.6; extra == "loaders"
@@ -148,7 +130,7 @@ Requires-Dist: paddleocr==2.8.1; extra == "loaders"
 Requires-Dist: ftfy==6.2.3; extra == "loaders"
 Requires-Dist: librosa==0.10.1; extra == "loaders"
 Requires-Dist: XlsxWriter==3.2.0; extra == "loaders"
-Requires-Dist: xformers==0.0.27.post2; extra == "loaders"
+Requires-Dist: timm==1.0.9; extra == "loaders"
 Provides-Extra: milvus
 Requires-Dist: langchain-milvus>=0.1.4; extra == "milvus"
 Requires-Dist: milvus==2.3.5; extra == "milvus"

{ai_parrot-0.3.3.dist-info → ai_parrot-0.3.6.dist-info}/RECORD

@@ -4,7 +4,7 @@ parrot/exceptions.cpython-311-x86_64-linux-gnu.so,sha256=VNyBh3uLxGQgB0l1bkWjQDq
 parrot/manager.py,sha256=NhzXoWxSgtoWHpmYP8cV2Ujq_SlvCbQYQBaohAeL2TM,5935
 parrot/models.py,sha256=RsVQCqhSXBKRPcu-BCga9Y1wyvENFXDCuq3_ObIKvAo,13452
 parrot/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-parrot/version.py,sha256=pbGrvnHWVk2vkgFh0ab5xc4-svi5oC7IvapZz06YLpM,373
+parrot/version.py,sha256=x5KFeKUhuf0IyJRSN3fSKlwqSDRNfHYDoFae9slFeKc,373
 parrot/chatbots/__init__.py,sha256=ypskCnME0xUv6psBEGCEyXCrD0J0ULHSllpVmSxqb4A,200
 parrot/chatbots/abstract.py,sha256=CmDn3k4r9uKImOZRN4L9zxLbCdC-1MPUAorDlfZT-kA,26421
 parrot/chatbots/asktroc.py,sha256=gyWzyvpAnmXwXd-3DEKoIJtAxt6NnP5mUZdZbkFky8s,604
@@ -56,7 +56,7 @@ parrot/loaders/excel.py,sha256=Y1agxm-jG4AgsA2wlPP3p8uBH40wYW1KM2ycTTLKUm4,12441
 parrot/loaders/github.py,sha256=CscyUIqoHTytqCbRUUTcV3QSxI8XoDntq5aTU0vdhzQ,2593
 parrot/loaders/image.py,sha256=A9KCXXoGuhDoyeJaascY7Q1ZK12Kf1ggE1drzJjS3AU,3946
 parrot/loaders/json.py,sha256=6B43k591OpvoJLbsJa8CxJue_lAt713SCdldn8bFW3c,1481
-parrot/loaders/pdf.py,sha256=wGwFnsUmeQqtqk3L2vYt2DkW09LUODUJN-xLjuAa-do,17826
+parrot/loaders/pdf.py,sha256=nyeT4emrewxeO2dUQxW3QOcdk1vg1JYtPKNAV8tThm0,17512
 parrot/loaders/pdfchapters.py,sha256=YhA8Cdx3qXBR0vuTVnQ12XgH1DXT_rp1Tawzh4V2U3o,5637
 parrot/loaders/pdffn.py,sha256=gA-vJEWUiIUwbMxP8Nmvlzlcb39DVV69vGKtSzavUoI,4004
 parrot/loaders/pdfimages.py,sha256=4Q_HKiAee_hALBsG2qF7PpMgKP1AivHXhmcsCkUa9eE,7899
@@ -68,7 +68,7 @@ parrot/loaders/repo.py,sha256=vBqBAnwU6p3_DCvI9DVhi1Bs8iCDYHwFGp0P9zvGRyw,3737
 parrot/loaders/rtd.py,sha256=oKOC9Qn3iwulYx5BEvXy4_kusKRsy5RLYNHS-e5p-1k,1988
 parrot/loaders/txt.py,sha256=AeGroWffFT--7TYlTSTr5Av5zAr8YIp1fxt8r5qdi-A,2802
 parrot/loaders/video.py,sha256=pl5Ho69bp5vrWMqg5tLbsnHUus1LByTDoL6NPk57Ays,2929
-parrot/loaders/videolocal.py,sha256=3EASzbettSO2tboTe3GndR4p6Nihwj6HGZoiPXekYo0,4302
+parrot/loaders/videolocal.py,sha256=o1upjpsMHp1NV4ZQVRc2DS8nKyJI1l0x9p3PjJUr3fU,5018
 parrot/loaders/vimeo.py,sha256=zOvOOIiaZr_bRswJFI7uIMKISgALOxcSim9ZRUFY1Fc,4114
 parrot/loaders/web.py,sha256=3x06JNpfTGFtvSBPAEBVoVdZkpVXePcJeMtj61B2xJk,8867
 parrot/loaders/web_base.py,sha256=5SjQddT0Vhne8C9s30iU3Ex_9O1PJ8kyDmy8EdhGBo0,4380
@@ -103,8 +103,8 @@ resources/users/handlers.py,sha256=BGzqBvPY_OaIF_nONWX4b_B5OyyBrdGuSihIsdlFwjk,2
 resources/users/models.py,sha256=glk7Emv7QCi6i32xRFDrGc8UwK23_LPg0XUOJoHnwRU,6799
 settings/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 settings/settings.py,sha256=9ueEvyLNurUX-AaIeRPV8GKX1c4YjDLbksUAeqEq6Ck,1854
-ai_parrot-0.3.3.dist-info/LICENSE,sha256=vRKOoa7onTsLNvSzJtGtMaNhWWh8B3YAT733Tlu6M4o,1070
-ai_parrot-0.3.3.dist-info/METADATA,sha256=LHLvoMsy1VvMlC33Kl2RKhwnFgj40lMpPDVmVWYj1m8,10624
-ai_parrot-0.3.3.dist-info/WHEEL,sha256=UQ-0qXN3LQUffjrV43_e_ZXj2pgORBqTmXipnkj0E8I,113
-ai_parrot-0.3.3.dist-info/top_level.txt,sha256=qHoO4BhYDfeTkyKnciZSQtn5FSLN3Q-P5xCTkyvbuxg,26
-ai_parrot-0.3.3.dist-info/RECORD,,
+ai_parrot-0.3.6.dist-info/LICENSE,sha256=vRKOoa7onTsLNvSzJtGtMaNhWWh8B3YAT733Tlu6M4o,1070
+ai_parrot-0.3.6.dist-info/METADATA,sha256=UukaICPH6ngFNm_DU3TmP2E89_BpgchSP0oPLxypwso,9721
+ai_parrot-0.3.6.dist-info/WHEEL,sha256=UQ-0qXN3LQUffjrV43_e_ZXj2pgORBqTmXipnkj0E8I,113
+ai_parrot-0.3.6.dist-info/top_level.txt,sha256=qHoO4BhYDfeTkyKnciZSQtn5FSLN3Q-P5xCTkyvbuxg,26
+ai_parrot-0.3.6.dist-info/RECORD,,

parrot/loaders/pdf.py

@@ -49,14 +49,6 @@ class PDFLoader(BasePDF):
         self.parse_images = kwargs.get('parse_images', False)
         self.page_as_images = kwargs.get('page_as_images', False)
         if self.page_as_images is True:
-            # # Load the processor and model from Hugging Face
-            # self.image_processor = DonutProcessor.from_pretrained(
-            #     "naver-clova-ix/donut-base-finetuned-docvqa"
-            # )
-            # self.image_model = VisionEncoderDecoderModel.from_pretrained(
-            #     "naver-clova-ix/donut-base-finetuned-docvqa",
-
-            # )
             # Load the processor and model from Hugging Face
             self.image_processor = LayoutLMv3Processor.from_pretrained(
                 "microsoft/layoutlmv3-base",
@@ -388,6 +380,7 @@ class PDFLoader(BasePDF):
                     # TODO passing the image to a AI visual to get explanation
                     # Get the extracted text from the image
                     text = self.extract_page_text(img_path)
+                    print('TEXT EXTRACTED >> ', text)
                     url = f'/static/images/{img_name}'
                     image_meta = {
                         "url": url,

parrot/loaders/videolocal.py

@@ -1,10 +1,26 @@
 from typing import Any
 from collections.abc import Callable
+import math
 from pathlib import PurePath
 from langchain.docstore.document import Document
 from .basevideo import BaseVideoLoader
 
 
+def split_text(text, max_length):
+    """Split text into chunks of a maximum length, ensuring not to break words."""
+    chunks = []
+    while len(text) > max_length:
+        # Find the last space before the max_length
+        split_point = text.rfind(' ', 0, max_length)
+        # If no space found, split at max_length
+        if split_point == -1:
+            split_point = max_length
+        chunks.append(text[:split_point])
+        text = text[split_point:].strip()
+    chunks.append(text)
+    return chunks
+
+
 class VideoLocalLoader(BaseVideoLoader):
     """
     Generating Video transcripts from local Videos.
@@ -26,14 +42,15 @@ class VideoLocalLoader(BaseVideoLoader):
 
     def load_video(self, path: PurePath) -> list:
         metadata = {
-            "source": f"{path}",
             "url": f"{path.name}",
-            "index": path.stem,
+            "source": f"{path}",
             "filename": f"{path}",
+            "index": path.stem,
             "question": '',
             "answer": '',
             'type': 'video_transcript',
             "source_type": self._source_type,
+            "data": {},
             "summary": '',
             "document_meta": {
                 "language": self._language,
@@ -53,14 +70,17 @@ class VideoLocalLoader(BaseVideoLoader):
             transcript = ''
         # Summarize the transcript
         if transcript:
+            # Split transcript into chunks
+            transcript_chunks = split_text(transcript, 32767)
             summary = self.get_summary_from_text(transcript)
             # Create Two Documents, one is for transcript, second is VTT:
             metadata['summary'] = summary
-            doc = Document(
-                page_content=transcript,
-                metadata=metadata
-            )
-            documents.append(doc)
+            for chunk in transcript_chunks:
+                doc = Document(
+                    page_content=chunk,
+                    metadata=metadata
+                )
+                documents.append(doc)
         if transcript_whisper:
             # VTT version:
             transcript = self.transcript_to_vtt(transcript_whisper, transcript_path)

parrot/version.py

@@ -3,7 +3,7 @@
 __title__ = "ai-parrot"
 __description__ = "Live Chatbots based on Langchain chatbots and Agents \
     Integrated into Navigator Framework or used into aiohttp applications."
-__version__ = "0.3.3"
+__version__ = "0.3.6"
 __author__ = "Jesus Lara"
 __author_email__ = "jesuslarag@gmail.com"
 __license__ = "MIT"