ai-parrot 0.3.9__tar.gz → 0.3.11__tar.gz

{ai_parrot-0.3.9 → ai_parrot-0.3.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ai-parrot
-Version: 0.3.9
+Version: 0.3.11
 Summary: Live Chatbots based on Langchain chatbots and Agents     Integrated into Navigator Framework or used into aiohttp applications.
 Home-page: https://github.com/phenobarbital/ai-parrot
 Author: Jesus Lara
@@ -78,6 +78,8 @@ Requires-Dist: O365==2.0.35
 Requires-Dist: stackapi==0.3.1
 Requires-Dist: torchvision==0.19.1
 Requires-Dist: tf-keras==2.17.0
+Requires-Dist: simsimd==4.3.1
+Requires-Dist: opencv-python==4.10.0.84
 Provides-Extra: loaders
 Requires-Dist: unstructured==0.14.3; extra == "loaders"
 Requires-Dist: unstructured-client==0.18.0; extra == "loaders"
@@ -108,6 +110,8 @@ Requires-Dist: ftfy==6.2.3; extra == "loaders"
 Requires-Dist: librosa==0.10.1; extra == "loaders"
 Requires-Dist: XlsxWriter==3.2.0; extra == "loaders"
 Requires-Dist: timm==1.0.9; extra == "loaders"
+Requires-Dist: simsimd==4.3.1; extra == "loaders"
+Requires-Dist: opencv-python==4.10.0.84; extra == "loaders"
 Requires-Dist: easyocr==1.7.1; extra == "loaders"
 Provides-Extra: anthropic
 Requires-Dist: langchain-anthropic==0.1.11; extra == "anthropic"
@@ -141,8 +145,6 @@ Requires-Dist: annoy==1.17.3; extra == "analytics"
 Requires-Dist: gradio_tools==0.0.9; extra == "analytics"
 Requires-Dist: gradio-client==0.2.9; extra == "analytics"
 Requires-Dist: streamlit==1.37.1; extra == "analytics"
-Requires-Dist: simsimd==4.3.1; extra == "analytics"
-Requires-Dist: opencv-python==4.10.0.84; extra == "analytics"
 
 # AI Parrot: Python package for creating Chatbots
 This is an open-source Python package for creating Chatbots based on Langchain and Navigator.

{ai_parrot-0.3.9 → ai_parrot-0.3.11}/ai_parrot.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ai-parrot
-Version: 0.3.9
+Version: 0.3.11
 Summary: Live Chatbots based on Langchain chatbots and Agents     Integrated into Navigator Framework or used into aiohttp applications.
 Home-page: https://github.com/phenobarbital/ai-parrot
 Author: Jesus Lara
@@ -78,6 +78,8 @@ Requires-Dist: O365==2.0.35
 Requires-Dist: stackapi==0.3.1
 Requires-Dist: torchvision==0.19.1
 Requires-Dist: tf-keras==2.17.0
+Requires-Dist: simsimd==4.3.1
+Requires-Dist: opencv-python==4.10.0.84
 Provides-Extra: loaders
 Requires-Dist: unstructured==0.14.3; extra == "loaders"
 Requires-Dist: unstructured-client==0.18.0; extra == "loaders"
@@ -108,6 +110,8 @@ Requires-Dist: ftfy==6.2.3; extra == "loaders"
 Requires-Dist: librosa==0.10.1; extra == "loaders"
 Requires-Dist: XlsxWriter==3.2.0; extra == "loaders"
 Requires-Dist: timm==1.0.9; extra == "loaders"
+Requires-Dist: simsimd==4.3.1; extra == "loaders"
+Requires-Dist: opencv-python==4.10.0.84; extra == "loaders"
 Requires-Dist: easyocr==1.7.1; extra == "loaders"
 Provides-Extra: anthropic
 Requires-Dist: langchain-anthropic==0.1.11; extra == "anthropic"
@@ -141,8 +145,6 @@ Requires-Dist: annoy==1.17.3; extra == "analytics"
 Requires-Dist: gradio_tools==0.0.9; extra == "analytics"
 Requires-Dist: gradio-client==0.2.9; extra == "analytics"
 Requires-Dist: streamlit==1.37.1; extra == "analytics"
-Requires-Dist: simsimd==4.3.1; extra == "analytics"
-Requires-Dist: opencv-python==4.10.0.84; extra == "analytics"
 
 # AI Parrot: Python package for creating Chatbots
 This is an open-source Python package for creating Chatbots based on Langchain and Navigator.

{ai_parrot-0.3.9 → ai_parrot-0.3.11}/ai_parrot.egg-info/SOURCES.txt

@@ -44,6 +44,7 @@ examples/load_pdf.py
 examples/test_bot.py
 examples/test_question.py
 examples/askbuddy/create_bot.py
+examples/askbuddy/load_pdf.py
 parrot/__init__.py
 parrot/conf.py
 parrot/exceptions.c

{ai_parrot-0.3.9 → ai_parrot-0.3.11}/ai_parrot.egg-info/requires.txt

@@ -47,14 +47,14 @@ O365==2.0.35
 stackapi==0.3.1
 torchvision==0.19.1
 tf-keras==2.17.0
+simsimd==4.3.1
+opencv-python==4.10.0.84
 
 [analytics]
 annoy==1.17.3
 gradio_tools==0.0.9
 gradio-client==0.2.9
 streamlit==1.37.1
-simsimd==4.3.1
-opencv-python==4.10.0.84
 
 [anthropic]
 langchain-anthropic==0.1.11
@@ -107,6 +107,8 @@ ftfy==6.2.3
 librosa==0.10.1
 XlsxWriter==3.2.0
 timm==1.0.9
+simsimd==4.3.1
+opencv-python==4.10.0.84
 easyocr==1.7.1
 
 [milvus]

ai_parrot-0.3.11/examples/askbuddy/load_pdf.py

@@ -0,0 +1,52 @@
+"""
+Load RFP documents into bot.
+"""
+import asyncio
+from navconfig import BASE_DIR
+from pprint import pprint
+from pathlib import Path
+from parrot.chatbots.basic import Chatbot
+from parrot.llms.vertex import VertexLLM
+from parrot.loaders import (
+    PDFLoader
+)
+
+async def get_agent():
+    llm = VertexLLM(
+        model='gemini-1.5-pro',
+        temperature=0.1,
+        top_k=30,
+        Top_p=0.5,
+    )
+    agent = Chatbot(
+        name='AskBuddy',
+        llm=llm
+    )
+    # Add LLM
+    await agent.configure()
+    # directory = BASE_DIR.joinpath("docs", "askbuddy")
+    directory = Path('/home/jesuslara/proyectos/navigator/navigator-ai').joinpath("docs", "askbuddy")
+    for filename in directory.glob('*.pdf'):
+        # Loading File by File to avoid overheat in database
+        print(':: Processing: ', filename)
+        # PDF Files
+        loader = PDFLoader(
+            filename,
+            source_type=f"MSO {filename.name}",
+            llm=llm.get_llm(),
+            language="en",
+            parse_images=False,
+            page_as_images=True
+        )
+        docs = loader.load()
+        pprint(docs)
+        await agent.load_documents(
+            docs
+        )
+    return agent
+
+
+
+
+if __name__ == "__main__":
+    agent = asyncio.run(get_agent())

{ai_parrot-0.3.9 → ai_parrot-0.3.11}/parrot/loaders/audio.py

@@ -28,7 +28,7 @@ class AudioLoader(BaseVideoLoader):
         metadata = {
             "source": f"{path}",
             "url": f"{path.name}",
-            "index": path.stem,
+            # "index": path.stem,
             "filename": f"{path}",
             "question": '',
             "answer": '',
@@ -71,7 +71,7 @@ class AudioLoader(BaseVideoLoader):
             docs = []
             for chunk in dialogs:
                 _meta = {
-                    "index": f"{path.stem}:{chunk['id']}",
+                    # "index": f"{path.stem}:{chunk['id']}",
                     "document_meta": {
                         "start": f"{chunk['start_time']}",
                         "end": f"{chunk['end_time']}",

{ai_parrot-0.3.9 → ai_parrot-0.3.11}/parrot/loaders/excel.py

@@ -284,7 +284,7 @@ class ExcelLoader(AbstractLoader):
             df = self.unique_columns(df)
             metadata = {
                 "url": '',
-                "index": '',
+                # "index": '',
                 "source": str(path.name),
                 "filename": path.name,
                 "question": '',
@@ -309,7 +309,7 @@ class ExcelLoader(AbstractLoader):
                         idk = row[col]
                         break
                 _data = {
-                    "index": idk,
+                    # "index": idk,
                     "data": rw,
                     "document_meta": {
                         "row_index": idx,

{ai_parrot-0.3.9 → ai_parrot-0.3.11}/parrot/loaders/pdf.py

@@ -299,7 +299,7 @@ class PDFLoader(BasePDF):
                     "answer": '',
                     "source_type": self._source_type,
                     "data": {},
-                    "summary": '',
+                    "summary": '-',
                     "document_meta": {
                         "title": pdf.metadata.get("title", ""),  # pylint: disable=E1101
                         "creationDate": pdf.metadata.get("creationDate", ""),  # pylint: disable=E1101
@@ -331,12 +331,12 @@ class PDFLoader(BasePDF):
                     try:
                         summary = self.get_summary_from_text(text)
                     except Exception:
-                        summary = ''
+                        summary = '-'
                     metadata = {
-                        "url": '',
+                        "url": f"{path}:#{page_num}",
                         "source": f"{path.name} Page.#{page_num}",
                         "filename": path.name,
-                        "index": f"{page_num}",
+                        # "index": f"{page_num}",
                         "type": 'pdf',
                         "question": '',
                         "answer": '',
@@ -400,7 +400,7 @@ class PDFLoader(BasePDF):
                             "url": url,
                             "source": f"{path.name} Page.#{page_num}",
                             "filename": path.name,
-                            "index": f"{path.name}:{page_num}",
+                            # "index": f"{path.name}:{page_num}",
                             "question": '',
                             "answer": '',
                             "type": 'image',
@@ -427,15 +427,15 @@ class PDFLoader(BasePDF):
                         df = df.dropna(axis=1, how='all')
                         df = df.dropna(how='all', axis=0)  # Drop empty rows
                         table_meta = {
-                            "url": '',
+                            "url": f"{path.name} Page.#{page_num} Table.#{tab_idx}",
                             "source": f"{path.name} Page.#{page_num} Table.#{tab_idx}",
                             "filename": path.name,
-                            "index": f"{path.name}:{page_num}",
+                            # "index": f"{path.name}:{page_num}",
                             "question": '',
                             "answer": '',
                             "type": 'table',
                             "data": {},
-                            "summary": '',
+                            "summary": '-',
                             "document_meta": {
                                 "table_index": tab_idx,
                                 "table_shape": df.shape,
@@ -476,12 +476,12 @@ class PDFLoader(BasePDF):
                         "url": url,
                         "source": f"{path.name} Page.#{page_num}",
                         "filename": path.name,
-                        "index": f"{path.name}:{page_num}",
+                        # "index": f"{path.name}:{page_num}",
                         "question": '',
                         "answer": '',
                         "type": 'page',
                         "data": {},
-                        "summary": '',
+                        "summary": '-',
                         "document_meta": {
                             "image_name": img_name,
                             "page_number": f"{page_number}"

{ai_parrot-0.3.9 → ai_parrot-0.3.11}/parrot/loaders/pdfchapters.py

@@ -78,7 +78,7 @@ class PDFChapterLoader(BasePDF):
             page_number = page_num + 1
             metadata = {
                 "url": '',
-                "index": f"{path.name} #{page_number}",
+                # "index": f"{path.name} #{page_number}",
                 "source": f"{path.name} #{page_number}",
                 "filename": path.name,
                 "source_type": self._source_type,

{ai_parrot-0.3.9 → ai_parrot-0.3.11}/parrot/loaders/pdffn.py

@@ -55,7 +55,7 @@ class PDFFnLoader(BasePDF):
         n = page_number + 1
         return {
             "url": '',
-            "index": f"{path.name} #{page_number}",
+            # "index": f"{path.name} #{page_number}",
             "source": f"{path.name} #{page_number}",
             "filename": path.name,
             "source_type": self._source_type,

{ai_parrot-0.3.9 → ai_parrot-0.3.11}/parrot/loaders/pdfimages.py

@@ -124,7 +124,7 @@ class PDFImageLoader(BasePDF):
                 }
                 for idx, chunk in enumerate(self._splitter.split_text(md_text)):
                     _info = {
-                        "index": f"{idx}",
+                        # "index": f"{idx}",
                         **metadata
                     }
                     docs.append(
@@ -178,7 +178,7 @@ class PDFImageLoader(BasePDF):
                     _meta = {
                         "url": f"{url}",
                         "filename": str(path.name),
-                        "index": f"Page {page_num}, part: {idx}",
+                        # "index": f"Page {page_num}, part: {idx}",
                         "source": str(path.name),
                         "type": 'pdf',
                         "question": prompt_text,

{ai_parrot-0.3.9 → ai_parrot-0.3.11}/parrot/loaders/pdfmark.py

@@ -74,7 +74,7 @@ class PDFMarkdownLoader(BasePDF):
             }
             for idx, chunk in enumerate(self._splitter.split_text(md_text)):
                 _info = {
-                    "index": f"{idx}",
+                    # "index": f"{idx}",
                     **metadata
                 }
                 docs.append(

{ai_parrot-0.3.9 → ai_parrot-0.3.11}/parrot/loaders/pdftables.py

@@ -91,7 +91,7 @@ class PDFTablesLoader(BasePDF):
             "url": '',
             "source": f"{path.name} Page.#{page} Table.#{table_idx}",
             "filename": path.name,
-            "index": f"{path.name}:Table:{table_idx}",
+            # "index": f"{path.name}:Table:{table_idx}",
             "question": '',
             "answer": '',
             "type": 'table',

{ai_parrot-0.3.9 → ai_parrot-0.3.11}/parrot/loaders/qa.py

@@ -46,7 +46,7 @@ class QAFileLoader(AbstractLoader):
                     page_content=f"**Question:** {row[q]}: **Answer:** {row[a]}",
                     metadata={
                         "url": '',
-                        "index": f"{path.name} #{idx}",
+                        # "index": f"{path.name} #{idx}",
                         "source": f"{path.name} Row.#{idx}",
                         "filename": f"{path.name}",
                         "question": row[q],

{ai_parrot-0.3.9 → ai_parrot-0.3.11}/parrot/loaders/rtd.py

@@ -47,7 +47,7 @@ class ReadTheDocsLoader(AbstractLoader):
             metadata = {
                 "source": url,
                 "url": url,
-                "index": suffix,
+                # "index": suffix,
                 "filename": src,
                 "question": '',
                 "answer": '',

{ai_parrot-0.3.9 → ai_parrot-0.3.11}/parrot/loaders/txt.py

@@ -44,7 +44,7 @@ class TXTLoader(AbstractLoader):
                     page_content=text,
                     metadata={
                         "url": '',
-                        "index": str(path.name),
+                        # "index": str(path.name),
                         "source": str(path),
                         "filename": str(path.name),
                         "summary": '',

{ai_parrot-0.3.9 → ai_parrot-0.3.11}/parrot/loaders/videolocal.py

@@ -45,7 +45,7 @@ class VideoLocalLoader(BaseVideoLoader):
             "url": f"{path.name}",
             "source": f"{path}",
             "filename": f"{path}",
-            "index": path.stem,
+            # "index": path.stem,
             "question": '',
             "answer": '',
             'type': 'video_transcript',
@@ -96,7 +96,7 @@ class VideoLocalLoader(BaseVideoLoader):
             docs = []
             for chunk in dialogs:
                 _meta = {
-                    "index": f"{path.stem}:{chunk['id']}",
+                    # "index": f"{path.stem}:{chunk['id']}",
                     "document_meta": {
                         "start": f"{chunk['start_time']}",
                         "end": f"{chunk['end_time']}",

{ai_parrot-0.3.9 → ai_parrot-0.3.11}/parrot/loaders/vimeo.py

@@ -13,7 +13,7 @@ class VimeoLoader(YoutubeLoader):
         metadata = {
             "source": url,
             "url": url,
-            "index": url,
+            # "index": url,
             "filename": video_title,
             "question": '',
             "answer": '',
@@ -73,7 +73,7 @@ class VimeoLoader(YoutubeLoader):
                 dialogs = self.transcript_to_blocks(transcript_whisper)
                 for chunk in dialogs:
                     _meta = {
-                        "index": f"{video_title}:{chunk['id']}",
+                        # "index": f"{video_title}:{chunk['id']}",
                         "document_meta": {
                             "start": f"{chunk['start_time']}",
                             "end": f"{chunk['end_time']}",

{ai_parrot-0.3.9 → ai_parrot-0.3.11}/parrot/loaders/web.py

@@ -164,7 +164,7 @@ class WebLoader(AbstractLoader):
             )
             metadata = {
                 "source": url,
-                "index": page_title,
+                # "index": page_title,
                 "url": url,
                 "filename": page_title,
                 "question": '',

{ai_parrot-0.3.9 → ai_parrot-0.3.11}/parrot/loaders/web_base.py

@@ -75,7 +75,7 @@ class WebBaseLoader(AbstractLoader):
             metadata = {
                 "url": url,
                 "source": url,
-                "index": "",
+                # "index": "",
                 "filename": '',
                 "type": 'webpage',
                 "question": '',

{ai_parrot-0.3.9 → ai_parrot-0.3.11}/parrot/loaders/word.py

@@ -58,7 +58,7 @@ class MSWordLoader(AbstractLoader):
                         "url": '',
                         "source": path.name,
                         "filename": path.name,
-                        "index": str(path.name),
+                        # "index": str(path.name),
                         "type": 'document',
                         "question": '',
                         "answer": '',

{ai_parrot-0.3.9 → ai_parrot-0.3.11}/parrot/loaders/youtube.py

@@ -71,7 +71,7 @@ class YoutubeLoader(VideoLoader):
                 metadata = {
                     "url": f"{url}",
                     "source": f"{url}",
-                    "index": video_title,
+                    # "index": video_title,
                     "filename": video_title,
                     "question": '',
                     "answer": '',
@@ -103,7 +103,7 @@ class YoutubeLoader(VideoLoader):
                 dialogs = self.transcript_to_blocks(transcript_whisper)
                 for chunk in dialogs:
                     _meta = {
-                        "index": f"{video_title}:{chunk['id']}",
+                        # "index": f"{video_title}:{chunk['id']}",
                         "document_meta": {
                             "start": f"{chunk['start_time']}",
                             "end": f"{chunk['end_time']}",
@@ -150,7 +150,7 @@ class YoutubeLoader(VideoLoader):
                 for doc in docs:
                     doc.metadata['source_type'] = self._source_type
                     doc.metadata['summary'] = f"{summary!s}"
-                    doc.metadata['index'] = ''
+                    # doc.metadata['index'] = ''
                     doc.metadata['filename'] = ''
                     doc.metadata['question'] = ''
                     doc.metadata['answer'] = ''
@@ -171,7 +171,7 @@ class YoutubeLoader(VideoLoader):
                 metadata = {
                     "source": url,
                     "url": url,
-                    "index": '',
+                    # "index": '',
                     "filename": '',
                     "question": '',
                     "answer": '',

{ai_parrot-0.3.9 → ai_parrot-0.3.11}/parrot/stores/milvus.py

@@ -350,11 +350,11 @@ class MilvusStore(AbstractStore):
                 auto_id=True,
                 max_length=100
             )
-            schema.add_field(
-                field_name="index",
-                datatype=DataType.VARCHAR,
-                max_length=65535
-            )
+            # schema.add_field(
+            #     field_name="index",
+            #     datatype=DataType.VARCHAR,
+            #     max_length=65535
+            # )
             schema.add_field(
                 field_name="url",
                 datatype=DataType.VARCHAR,

{ai_parrot-0.3.9 → ai_parrot-0.3.11}/parrot/version.py

@@ -3,7 +3,7 @@
 __title__ = "ai-parrot"
 __description__ = "Live Chatbots based on Langchain chatbots and Agents \
     Integrated into Navigator Framework or used into aiohttp applications."
-__version__ = "0.3.9"
+__version__ = "0.3.11"
 __author__ = "Jesus Lara"
 __author_email__ = "jesuslarag@gmail.com"
 __license__ = "MIT"

{ai_parrot-0.3.9 → ai_parrot-0.3.11}/setup.py

@@ -190,7 +190,9 @@ setup(
         "O365==2.0.35",
         "stackapi==0.3.1",
         "torchvision==0.19.1",
-        "tf-keras==2.17.0"
+        "tf-keras==2.17.0",
+        "simsimd==4.3.1",
+        "opencv-python==4.10.0.84"
     ],
     extras_require={
         "loaders": [
@@ -224,6 +226,8 @@ setup(
             "XlsxWriter==3.2.0",
             # "xformers==0.0.27.post2",
             "timm==1.0.9",
+            "simsimd==4.3.1",
+            "opencv-python==4.10.0.84",
             "easyocr==1.7.1"
         ],
         "anthropic": [
@@ -266,8 +270,6 @@ setup(
             "gradio_tools==0.0.9",
             "gradio-client==0.2.9",
             "streamlit==1.37.1",
-            "simsimd==4.3.1",
-            "opencv-python==4.10.0.84"
             # "timm==0.9.16", # image-processor
             # "ultralytics==8.2.4", # image-processor
             # "albumentations-1.4.4",