ai-parrot 0.3.10__tar.gz → 0.3.11__tar.gz

{ai_parrot-0.3.10 → ai_parrot-0.3.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ai-parrot
-Version: 0.3.10
+Version: 0.3.11
 Summary: Live Chatbots based on Langchain chatbots and Agents     Integrated into Navigator Framework or used into aiohttp applications.
 Home-page: https://github.com/phenobarbital/ai-parrot
 Author: Jesus Lara
@@ -110,6 +110,8 @@ Requires-Dist: ftfy==6.2.3; extra == "loaders"
 Requires-Dist: librosa==0.10.1; extra == "loaders"
 Requires-Dist: XlsxWriter==3.2.0; extra == "loaders"
 Requires-Dist: timm==1.0.9; extra == "loaders"
+Requires-Dist: simsimd==4.3.1; extra == "loaders"
+Requires-Dist: opencv-python==4.10.0.84; extra == "loaders"
 Requires-Dist: easyocr==1.7.1; extra == "loaders"
 Provides-Extra: anthropic
 Requires-Dist: langchain-anthropic==0.1.11; extra == "anthropic"

{ai_parrot-0.3.10 → ai_parrot-0.3.11}/ai_parrot.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ai-parrot
-Version: 0.3.10
+Version: 0.3.11
 Summary: Live Chatbots based on Langchain chatbots and Agents     Integrated into Navigator Framework or used into aiohttp applications.
 Home-page: https://github.com/phenobarbital/ai-parrot
 Author: Jesus Lara
@@ -110,6 +110,8 @@ Requires-Dist: ftfy==6.2.3; extra == "loaders"
 Requires-Dist: librosa==0.10.1; extra == "loaders"
 Requires-Dist: XlsxWriter==3.2.0; extra == "loaders"
 Requires-Dist: timm==1.0.9; extra == "loaders"
+Requires-Dist: simsimd==4.3.1; extra == "loaders"
+Requires-Dist: opencv-python==4.10.0.84; extra == "loaders"
 Requires-Dist: easyocr==1.7.1; extra == "loaders"
 Provides-Extra: anthropic
 Requires-Dist: langchain-anthropic==0.1.11; extra == "anthropic"

{ai_parrot-0.3.10 → ai_parrot-0.3.11}/ai_parrot.egg-info/SOURCES.txt

@@ -44,6 +44,7 @@ examples/load_pdf.py
 examples/test_bot.py
 examples/test_question.py
 examples/askbuddy/create_bot.py
+examples/askbuddy/load_pdf.py
 parrot/__init__.py
 parrot/conf.py
 parrot/exceptions.c

{ai_parrot-0.3.10 → ai_parrot-0.3.11}/ai_parrot.egg-info/requires.txt

@@ -107,6 +107,8 @@ ftfy==6.2.3
 librosa==0.10.1
 XlsxWriter==3.2.0
 timm==1.0.9
+simsimd==4.3.1
+opencv-python==4.10.0.84
 easyocr==1.7.1
 
 [milvus]

ai_parrot-0.3.11/examples/askbuddy/load_pdf.py

@@ -0,0 +1,52 @@
+"""
+Load RFP documents into bot.
+"""
+import asyncio
+from navconfig import BASE_DIR
+from pprint import pprint
+from pathlib import Path
+from parrot.chatbots.basic import Chatbot
+from parrot.llms.vertex import VertexLLM
+from parrot.loaders import (
+    PDFLoader
+)
+
+async def get_agent():
+    llm = VertexLLM(
+        model='gemini-1.5-pro',
+        temperature=0.1,
+        top_k=30,
+        Top_p=0.5,
+    )
+    agent = Chatbot(
+        name='AskBuddy',
+        llm=llm
+    )
+    # Add LLM
+    await agent.configure()
+    # directory = BASE_DIR.joinpath("docs", "askbuddy")
+    directory = Path('/home/jesuslara/proyectos/navigator/navigator-ai').joinpath("docs", "askbuddy")
+    for filename in directory.glob('*.pdf'):
+        # Loading File by File to avoid overheat in database
+        print(':: Processing: ', filename)
+        # PDF Files
+        loader = PDFLoader(
+            filename,
+            source_type=f"MSO {filename.name}",
+            llm=llm.get_llm(),
+            language="en",
+            parse_images=False,
+            page_as_images=True
+        )
+        docs = loader.load()
+        pprint(docs)
+        await agent.load_documents(
+            docs
+        )
+    return agent
+
+
+
+
+if __name__ == "__main__":
+    agent = asyncio.run(get_agent())

{ai_parrot-0.3.10 → ai_parrot-0.3.11}/parrot/loaders/audio.py

@@ -28,7 +28,7 @@ class AudioLoader(BaseVideoLoader):
         metadata = {
             "source": f"{path}",
             "url": f"{path.name}",
-            "index": path.stem,
+            # "index": path.stem,
             "filename": f"{path}",
             "question": '',
             "answer": '',
@@ -71,7 +71,7 @@ class AudioLoader(BaseVideoLoader):
             docs = []
             for chunk in dialogs:
                 _meta = {
-                    "index": f"{path.stem}:{chunk['id']}",
+                    # "index": f"{path.stem}:{chunk['id']}",
                     "document_meta": {
                         "start": f"{chunk['start_time']}",
                         "end": f"{chunk['end_time']}",

{ai_parrot-0.3.10 → ai_parrot-0.3.11}/parrot/loaders/excel.py

@@ -284,7 +284,7 @@ class ExcelLoader(AbstractLoader):
             df = self.unique_columns(df)
             metadata = {
                 "url": '',
-                "index": '',
+                # "index": '',
                 "source": str(path.name),
                 "filename": path.name,
                 "question": '',
@@ -309,7 +309,7 @@ class ExcelLoader(AbstractLoader):
                         idk = row[col]
                         break
                 _data = {
-                    "index": idk,
+                    # "index": idk,
                     "data": rw,
                     "document_meta": {
                         "row_index": idx,

{ai_parrot-0.3.10 → ai_parrot-0.3.11}/parrot/loaders/pdf.py

@@ -336,7 +336,7 @@ class PDFLoader(BasePDF):
                         "url": f"{path}:#{page_num}",
                         "source": f"{path.name} Page.#{page_num}",
                         "filename": path.name,
-                        "index": f"{page_num}",
+                        # "index": f"{page_num}",
                         "type": 'pdf',
                         "question": '',
                         "answer": '',
@@ -400,7 +400,7 @@ class PDFLoader(BasePDF):
                             "url": url,
                             "source": f"{path.name} Page.#{page_num}",
                             "filename": path.name,
-                            "index": f"{path.name}:{page_num}",
+                            # "index": f"{path.name}:{page_num}",
                             "question": '',
                             "answer": '',
                             "type": 'image',
@@ -430,7 +430,7 @@ class PDFLoader(BasePDF):
                             "url": f"{path.name} Page.#{page_num} Table.#{tab_idx}",
                             "source": f"{path.name} Page.#{page_num} Table.#{tab_idx}",
                             "filename": path.name,
-                            "index": f"{path.name}:{page_num}",
+                            # "index": f"{path.name}:{page_num}",
                             "question": '',
                             "answer": '',
                             "type": 'table',
@@ -476,7 +476,7 @@ class PDFLoader(BasePDF):
                         "url": url,
                         "source": f"{path.name} Page.#{page_num}",
                         "filename": path.name,
-                        "index": f"{path.name}:{page_num}",
+                        # "index": f"{path.name}:{page_num}",
                         "question": '',
                         "answer": '',
                         "type": 'page',

{ai_parrot-0.3.10 → ai_parrot-0.3.11}/parrot/loaders/pdfchapters.py

@@ -78,7 +78,7 @@ class PDFChapterLoader(BasePDF):
             page_number = page_num + 1
             metadata = {
                 "url": '',
-                "index": f"{path.name} #{page_number}",
+                # "index": f"{path.name} #{page_number}",
                 "source": f"{path.name} #{page_number}",
                 "filename": path.name,
                 "source_type": self._source_type,

{ai_parrot-0.3.10 → ai_parrot-0.3.11}/parrot/loaders/pdffn.py

@@ -55,7 +55,7 @@ class PDFFnLoader(BasePDF):
         n = page_number + 1
         return {
             "url": '',
-            "index": f"{path.name} #{page_number}",
+            # "index": f"{path.name} #{page_number}",
             "source": f"{path.name} #{page_number}",
             "filename": path.name,
             "source_type": self._source_type,

{ai_parrot-0.3.10 → ai_parrot-0.3.11}/parrot/loaders/pdfimages.py

@@ -124,7 +124,7 @@ class PDFImageLoader(BasePDF):
                 }
                 for idx, chunk in enumerate(self._splitter.split_text(md_text)):
                     _info = {
-                        "index": f"{idx}",
+                        # "index": f"{idx}",
                         **metadata
                     }
                     docs.append(
@@ -178,7 +178,7 @@ class PDFImageLoader(BasePDF):
                     _meta = {
                         "url": f"{url}",
                         "filename": str(path.name),
-                        "index": f"Page {page_num}, part: {idx}",
+                        # "index": f"Page {page_num}, part: {idx}",
                         "source": str(path.name),
                         "type": 'pdf',
                         "question": prompt_text,

{ai_parrot-0.3.10 → ai_parrot-0.3.11}/parrot/loaders/pdfmark.py

@@ -74,7 +74,7 @@ class PDFMarkdownLoader(BasePDF):
             }
             for idx, chunk in enumerate(self._splitter.split_text(md_text)):
                 _info = {
-                    "index": f"{idx}",
+                    # "index": f"{idx}",
                     **metadata
                 }
                 docs.append(

{ai_parrot-0.3.10 → ai_parrot-0.3.11}/parrot/loaders/pdftables.py

@@ -91,7 +91,7 @@ class PDFTablesLoader(BasePDF):
             "url": '',
             "source": f"{path.name} Page.#{page} Table.#{table_idx}",
             "filename": path.name,
-            "index": f"{path.name}:Table:{table_idx}",
+            # "index": f"{path.name}:Table:{table_idx}",
             "question": '',
             "answer": '',
             "type": 'table',

{ai_parrot-0.3.10 → ai_parrot-0.3.11}/parrot/loaders/qa.py

@@ -46,7 +46,7 @@ class QAFileLoader(AbstractLoader):
                     page_content=f"**Question:** {row[q]}: **Answer:** {row[a]}",
                     metadata={
                         "url": '',
-                        "index": f"{path.name} #{idx}",
+                        # "index": f"{path.name} #{idx}",
                         "source": f"{path.name} Row.#{idx}",
                         "filename": f"{path.name}",
                         "question": row[q],

{ai_parrot-0.3.10 → ai_parrot-0.3.11}/parrot/loaders/rtd.py

@@ -47,7 +47,7 @@ class ReadTheDocsLoader(AbstractLoader):
             metadata = {
                 "source": url,
                 "url": url,
-                "index": suffix,
+                # "index": suffix,
                 "filename": src,
                 "question": '',
                 "answer": '',

{ai_parrot-0.3.10 → ai_parrot-0.3.11}/parrot/loaders/txt.py

@@ -44,7 +44,7 @@ class TXTLoader(AbstractLoader):
                     page_content=text,
                     metadata={
                         "url": '',
-                        "index": str(path.name),
+                        # "index": str(path.name),
                         "source": str(path),
                         "filename": str(path.name),
                         "summary": '',

{ai_parrot-0.3.10 → ai_parrot-0.3.11}/parrot/loaders/videolocal.py

@@ -45,7 +45,7 @@ class VideoLocalLoader(BaseVideoLoader):
             "url": f"{path.name}",
             "source": f"{path}",
             "filename": f"{path}",
-            "index": path.stem,
+            # "index": path.stem,
             "question": '',
             "answer": '',
             'type': 'video_transcript',
@@ -96,7 +96,7 @@ class VideoLocalLoader(BaseVideoLoader):
             docs = []
             for chunk in dialogs:
                 _meta = {
-                    "index": f"{path.stem}:{chunk['id']}",
+                    # "index": f"{path.stem}:{chunk['id']}",
                     "document_meta": {
                         "start": f"{chunk['start_time']}",
                         "end": f"{chunk['end_time']}",

{ai_parrot-0.3.10 → ai_parrot-0.3.11}/parrot/loaders/vimeo.py

@@ -13,7 +13,7 @@ class VimeoLoader(YoutubeLoader):
         metadata = {
             "source": url,
             "url": url,
-            "index": url,
+            # "index": url,
             "filename": video_title,
             "question": '',
             "answer": '',
@@ -73,7 +73,7 @@ class VimeoLoader(YoutubeLoader):
                 dialogs = self.transcript_to_blocks(transcript_whisper)
                 for chunk in dialogs:
                     _meta = {
-                        "index": f"{video_title}:{chunk['id']}",
+                        # "index": f"{video_title}:{chunk['id']}",
                         "document_meta": {
                             "start": f"{chunk['start_time']}",
                             "end": f"{chunk['end_time']}",

{ai_parrot-0.3.10 → ai_parrot-0.3.11}/parrot/loaders/web.py

@@ -164,7 +164,7 @@ class WebLoader(AbstractLoader):
             )
             metadata = {
                 "source": url,
-                "index": page_title,
+                # "index": page_title,
                 "url": url,
                 "filename": page_title,
                 "question": '',

{ai_parrot-0.3.10 → ai_parrot-0.3.11}/parrot/loaders/web_base.py

@@ -75,7 +75,7 @@ class WebBaseLoader(AbstractLoader):
             metadata = {
                 "url": url,
                 "source": url,
-                "index": "",
+                # "index": "",
                 "filename": '',
                 "type": 'webpage',
                 "question": '',

{ai_parrot-0.3.10 → ai_parrot-0.3.11}/parrot/loaders/word.py

@@ -58,7 +58,7 @@ class MSWordLoader(AbstractLoader):
                         "url": '',
                         "source": path.name,
                         "filename": path.name,
-                        "index": str(path.name),
+                        # "index": str(path.name),
                         "type": 'document',
                         "question": '',
                         "answer": '',

{ai_parrot-0.3.10 → ai_parrot-0.3.11}/parrot/loaders/youtube.py

@@ -71,7 +71,7 @@ class YoutubeLoader(VideoLoader):
                 metadata = {
                     "url": f"{url}",
                     "source": f"{url}",
-                    "index": video_title,
+                    # "index": video_title,
                     "filename": video_title,
                     "question": '',
                     "answer": '',
@@ -103,7 +103,7 @@ class YoutubeLoader(VideoLoader):
                 dialogs = self.transcript_to_blocks(transcript_whisper)
                 for chunk in dialogs:
                     _meta = {
-                        "index": f"{video_title}:{chunk['id']}",
+                        # "index": f"{video_title}:{chunk['id']}",
                         "document_meta": {
                             "start": f"{chunk['start_time']}",
                             "end": f"{chunk['end_time']}",
@@ -150,7 +150,7 @@ class YoutubeLoader(VideoLoader):
                 for doc in docs:
                     doc.metadata['source_type'] = self._source_type
                     doc.metadata['summary'] = f"{summary!s}"
-                    doc.metadata['index'] = ''
+                    # doc.metadata['index'] = ''
                     doc.metadata['filename'] = ''
                     doc.metadata['question'] = ''
                     doc.metadata['answer'] = ''
@@ -171,7 +171,7 @@ class YoutubeLoader(VideoLoader):
                 metadata = {
                     "source": url,
                     "url": url,
-                    "index": '',
+                    # "index": '',
                     "filename": '',
                     "question": '',
                     "answer": '',

{ai_parrot-0.3.10 → ai_parrot-0.3.11}/parrot/stores/milvus.py

@@ -350,11 +350,11 @@ class MilvusStore(AbstractStore):
                 auto_id=True,
                 max_length=100
             )
-            schema.add_field(
-                field_name="index",
-                datatype=DataType.VARCHAR,
-                max_length=65535
-            )
+            # schema.add_field(
+            #     field_name="index",
+            #     datatype=DataType.VARCHAR,
+            #     max_length=65535
+            # )
             schema.add_field(
                 field_name="url",
                 datatype=DataType.VARCHAR,

{ai_parrot-0.3.10 → ai_parrot-0.3.11}/parrot/version.py

@@ -3,7 +3,7 @@
 __title__ = "ai-parrot"
 __description__ = "Live Chatbots based on Langchain chatbots and Agents \
     Integrated into Navigator Framework or used into aiohttp applications."
-__version__ = "0.3.10"
+__version__ = "0.3.11"
 __author__ = "Jesus Lara"
 __author_email__ = "jesuslarag@gmail.com"
 __license__ = "MIT"

{ai_parrot-0.3.10 → ai_parrot-0.3.11}/setup.py

@@ -226,6 +226,8 @@ setup(
             "XlsxWriter==3.2.0",
             # "xformers==0.0.27.post2",
             "timm==1.0.9",
+            "simsimd==4.3.1",
+            "opencv-python==4.10.0.84",
             "easyocr==1.7.1"
         ],
         "anthropic": [