lionagi 0.1.0__py3-none-any.whl → 0.1.1__py3-none-any.whl

lionagi/integrations/bridge/llamaindex_/get_index.py

@@ -0,0 +1,294 @@
+# TODO: Refactor this code to use the new llama_index API
+
+# class BaseIndex:
+
+#     @staticmethod
+#     def _get_index(
+#         input_=None,
+#         # default to OpenAI
+#         llm=None,
+#         llm_provider=None,
+#         llm_kwargs={},
+#         service_context=None,
+#         service_context_kwargs={},
+#         index_type=None,
+#         index_kwargs={},
+#         rerank_=False,
+#         reranker_type=None,
+#         reranker=None,
+#         rerank_kwargs={},
+#         get_engine=False,
+#         engine_kwargs={},
+#         from_storage=False,
+#         storage_context=None,
+#         strorage_context_kwargs={},
+#         index_id=None,
+#         load_index_from_storage_kwargs={},
+#     ):
+#         """
+#         Creates and returns an index or query engine based on the provided parameters.
+
+#         Args:
+#             chunks: The input data to be indexed or queried.
+#             llm: An instance of a language model for indexing or querying.
+#             llm_provider: A function to provide an instance of a language model.
+#             llm_kwargs: Keyword arguments for configuring the language model.
+#             service_context: An instance of a service context.
+#             service_context_kwargs: Keyword arguments for configuring the service context.
+#             index_type: The type of index to create.
+#             index_kwargs: Keyword arguments for configuring the index.
+#             rerank_: Boolean flag indicating whether reranking should be applied.
+#             reranker_type: The type of reranker to use.
+#             reranker: An instance of a reranker.
+#             rerank_kwargs: Keyword arguments for configuring the reranker.
+#             get_engine: Boolean flag indicating whether to return a query engine.
+#             engine_kwargs: Keyword arguments for configuring the query engine.
+
+#         Returns:
+#             Index or Query Engine: Depending on the 'get_engine' flag, returns an index or query engine.
+
+#         Raises:
+#             Various exceptions if there are errors in creating the index or query engine.
+#         """
+
+#         if from_storage:
+#             from llama_index import StorageContext, load_index_from_storage
+
+#             storage_context = StorageContext.from_defaults(**strorage_context_kwargs)
+
+#             if index_id:
+#                 index = load_index_from_storage(
+#                     storage_context=storage_context,
+#                     index_id=index_id,
+#                     **load_index_from_storage_kwargs,
+#                 )
+#             else:
+#                 raise ValueError("Index ID is required for loading from storage.")
+
+#             if rerank_:
+#                 if not reranker:
+#                     if not reranker_type:
+#                         from llama_index.postprocessor import LLMRerank
+
+#                         reranker_type = LLMRerank
+#                     reranker = reranker_type(
+#                         service_context=service_context, **rerank_kwargs
+#                     )
+#                 engine_kwargs.update({"node_postprocessors": [reranker]})
+
+#             if get_engine:
+#                 return (index, index.as_query_engine(**engine_kwargs))
+#             return index
+
+#         if not llm:
+#             if llm_provider:
+#                 llm = llm_provider(**llm_kwargs)
+#             else:
+#                 from llama_index.llms import OpenAI
+
+#                 llm = OpenAI(**llm_kwargs)
+
+#         if not service_context:
+#             from llama_index import ServiceContext
+
+#             service_context = ServiceContext.from_defaults(
+#                 llm=llm, **service_context_kwargs
+#             )
+
+#         if not index_type:
+#             from llama_index import VectorStoreIndex
+
+#             index_type = VectorStoreIndex
+
+#         index = index_type(input_, service_context=service_context, **index_kwargs)
+
+#         if index_id:
+#             index.index_id = index_id
+
+#         if rerank_:
+#             if not reranker:
+#                 if not reranker_type:
+#                     from llama_index.postprocessor import LLMRerank
+
+#                     reranker_type = LLMRerank
+#                 reranker = reranker_type(
+#                     service_context=service_context, **rerank_kwargs
+#                 )
+#             engine_kwargs.update({"node_postprocessors": [reranker]})
+
+#         if get_engine:
+#             return (index, index.as_query_engine(**engine_kwargs))
+#         return index
+
+
+# class LlamaIndex:
+
+#     @staticmethod
+#     def kg_index(
+#         input_=None,
+#         # default to OpenAI
+#         llm=None,
+#         llm_provider=None,
+#         llm_kwargs={"temperature": 0.1, "model": "gpt-4-1106-preview"},
+#         service_context=None,
+#         service_context_kwargs={},
+#         index_kwargs={"include_embeddings": True},
+#         rerank_=False,
+#         reranker_type=None,
+#         reranker=None,
+#         rerank_kwargs={"choice_batch_size": 5, "top_n": 3},
+#         get_engine=False,
+#         engine_kwargs={"similarity_top_k": 3, "response_mode": "tree_summarize"},
+#         kg_triplet_extract_fn=None,
+#         from_storage=False,
+#         storage_context=None,
+#         strorage_context_kwargs={},
+#         index_id=None,
+#         load_index_from_storage_kwargs={},
+#     ):
+#         """
+#         Creates and returns a KnowledgeGraphIndex based on the provided parameters.
+
+#         Args:
+#             chunks: The input data to be indexed.
+#             llm: An instance of a language model for indexing.
+#             llm_provider: A function to provide an instance of a language model.
+#             llm_kwargs: Keyword arguments for configuring the language model.
+#             service_context: An instance of a service context.
+#             service_context_kwargs: Keyword arguments for configuring the service context.
+#             index_kwargs: Keyword arguments for configuring the index.
+#             rerank_: Boolean flag indicating whether reranking should be applied.
+#             reranker_type: The type of reranker to use.
+#             reranker: An instance of a reranker.
+#             rerank_kwargs: Keyword arguments for configuring the reranker.
+#             get_engine: Boolean flag indicating whether to return a query engine.
+#             engine_kwargs: Keyword arguments for configuring the query engine.
+#             kg_triplet_extract_fn: Optional function for extracting KG triplets.
+
+#         Returns:
+#             KnowledgeGraphIndex or Query Engine: Depending on the 'get_engine' flag,
+#             returns a KnowledgeGraphIndex or query engine.
+
+#         Raises:
+#             Various exceptions if there are errors in creating the index or query engine.
+#         """
+#         from llama_index import KnowledgeGraphIndex
+
+#         index_type_ = ""
+#         if not from_storage:
+#             from llama_index.graph_stores import SimpleGraphStore
+#             from llama_index.storage.storage_context import StorageContext
+
+#             graph_store = SimpleGraphStore()
+#             if storage_context is None:
+#                 storage_context = StorageContext.from_defaults(
+#                     graph_store=graph_store, **strorage_context_kwargs
+#                 )
+#             index_kwargs.update({"storage_context": storage_context})
+#             index_type_ = KnowledgeGraphIndex.from_documents
+
+#         elif from_storage:
+#             index_type_ = KnowledgeGraphIndex
+
+#         if kg_triplet_extract_fn:
+#             index_kwargs.update({"kg_triplet_extract_fn": kg_triplet_extract_fn})
+
+#         if storage_context is None:
+#             from llama_index.graph_stores import SimpleGraphStore
+#             from llama_index.storage.storage_context import StorageContext
+
+#             storage_context = StorageContext.from_defaults(
+#                 graph_store=SimpleGraphStore(), **strorage_context_kwargs
+#             )
+
+#         return BaseIndex._get_index(
+#             input_=input_,
+#             llm=llm,
+#             llm_provider=llm_provider,
+#             llm_kwargs=llm_kwargs,
+#             service_context=service_context,
+#             service_context_kwargs=service_context_kwargs,
+#             index_type=index_type_,
+#             index_kwargs=index_kwargs,
+#             rerank_=rerank_,
+#             reranker_type=reranker_type,
+#             reranker=reranker,
+#             rerank_kwargs=rerank_kwargs,
+#             get_engine=get_engine,
+#             engine_kwargs=engine_kwargs,
+#             from_storage=from_storage,
+#             storage_context=storage_context,
+#             strorage_context_kwargs=strorage_context_kwargs,
+#             index_id=index_id,
+#             load_index_from_storage_kwargs=load_index_from_storage_kwargs,
+#         )
+
+#     @staticmethod
+#     def vector_index(
+#         input_=None,
+#         # default to OpenAI
+#         llm=None,
+#         llm_provider=None,
+#         llm_kwargs={"temperature": 0.1, "model": "gpt-4-1106-preview"},
+#         service_context=None,
+#         service_context_kwargs={},
+#         index_kwargs={"include_embeddings": True},
+#         # default to LLMRerank
+#         rerank_=False,
+#         reranker_type=None,
+#         reranker=None,
+#         rerank_kwargs={"choice_batch_size": 5, "top_n": 3},
+#         get_engine=False,
+#         engine_kwargs={"similarity_top_k": 3, "response_mode": "tree_summarize"},
+#         from_storage=False,
+#         storage_context=None,
+#         strorage_context_kwargs={},
+#         index_id=None,
+#         load_index_from_storage_kwargs={},
+#     ):
+#         """
+#         Creates and returns a vector index or query engine based on the provided parameters.
+
+#         Args:
+#             chunks: The input data to be indexed or queried.
+#             llm: An instance of a language model for indexing or querying.
+#             llm_provider: A function to provide an instance of a language model.
+#             llm_kwargs: Keyword arguments for configuring the language model.
+#             service_context: An instance of a service context.
+#             service_context_kwargs: Keyword arguments for configuring the service context.
+#             index_kwargs: Keyword arguments for configuring the index.
+#             rerank_: Boolean flag indicating whether reranking should be applied.
+#             reranker_type: The type of reranker to use.
+#             reranker: An instance of a reranker.
+#             rerank_kwargs: Keyword arguments for configuring the reranker.
+#             get_engine: Boolean flag indicating whether to return a query engine.
+#             engine_kwargs: Keyword arguments for configuring the query engine.
+
+#         Returns:
+#             Vector Index or Query Engine: Depending on the 'get_engine' flag,
+#             returns a vector index or query engine.
+
+#         Raises:
+#             Various exceptions if there are errors in creating the index or query engine.
+#         """
+
+#         return BaseIndex._get_index(
+#             input_=input_,
+#             llm=llm,
+#             llm_provider=llm_provider,
+#             llm_kwargs=llm_kwargs,
+#             service_context=service_context,
+#             service_context_kwargs=service_context_kwargs,
+#             index_kwargs=index_kwargs,
+#             rerank_=rerank_,
+#             reranker_type=reranker_type,
+#             reranker=reranker,
+#             rerank_kwargs=rerank_kwargs,
+#             get_engine=get_engine,
+#             engine_kwargs=engine_kwargs,
+#             from_storage=from_storage,
+#             storage_context=storage_context,
+#             strorage_context_kwargs=strorage_context_kwargs,
+#             index_id=index_id,
+#             load_index_from_storage_kwargs=load_index_from_storage_kwargs,
+#         )

lionagi/integrations/bridge/llamaindex_/llama_pack.py

@@ -0,0 +1,227 @@
+class LlamaPack:
+
+    @staticmethod
+    def download(pack_name, pack_path):
+        try:
+            from llama_index.llama_pack import download_llama_pack
+
+            return download_llama_pack(pack_name, pack_path)
+        except Exception as e:
+            raise ImportError(f"Error in downloading llama pack: {e}")
+
+    @staticmethod
+    def build(pack_name, pack_path, args=[], **kwargs):
+        pack = LlamaPack.download(pack_name, pack_path)
+        return pack(*args, **kwargs)
+
+    @staticmethod
+    def stock_market_pack(pack_path="./stock_market_data_pack", args=[], **kwargs):
+        name_ = "StockMarketDataQueryEnginePack"
+        return LlamaPack.build(name_, pack_path, args, **kwargs)
+
+    @staticmethod
+    def embedded_table_pack(
+        pack_path="./embedded_tables_unstructured_pack", args=[], **kwargs
+    ):
+        name_ = "RecursiveRetrieverSmallToBigPack"
+        return LlamaPack.build(name_, pack_path, args, **kwargs)
+
+    @staticmethod
+    def rag_evaluator_pack(pack_path="./rag_evaluator_pack", args=[], **kwargs):
+        name_ = "RagEvaluatorPack"
+        return LlamaPack.build(name_, pack_path, args, **kwargs)
+
+    @staticmethod
+    def ollma_pack(pack_path="./ollama_pack", args=[], **kwargs):
+        name_ = "OllamaQueryEnginePack"
+        return LlamaPack.build(name_, pack_path, args, **kwargs)
+
+    @staticmethod
+    def llm_compiler_agent_pack(
+        pack_path="./llm_compiler_agent_pack", args=[], **kwargs
+    ):
+        name_ = "LLMCompilerAgentPack"
+        return LlamaPack.build(name_, pack_path, args, **kwargs)
+
+    @staticmethod
+    def resume_screener_pack(pack_path="./resume_screener_pack", args=[], **kwargs):
+        name_ = "ResumeScreenerPack"
+        return LlamaPack.build(name_, pack_path, args, **kwargs)
+
+    @staticmethod
+    def ragatouille_retriever_pack(pack_path="./ragatouille_pack", args=[], **kwargs):
+        name_ = "RAGatouilleRetrieverPack"
+        return LlamaPack.build(name_, pack_path, args, **kwargs)
+
+    @staticmethod
+    def chain_of_table_pack(pack_path="./chain_of_table_pack", args=[], **kwargs):
+        name_ = "ChainOfTablePack"
+        return LlamaPack.build(name_, pack_path, args, **kwargs)
+
+    @staticmethod
+    def hybrid_fusion_retriever_pack(
+        pack_path="./hybrid_fusion_pack", args=[], **kwargs
+    ):
+        name_ = "HybridFusionRetrieverPack"
+        return LlamaPack.build(name_, pack_path, args, **kwargs)
+
+    @staticmethod
+    def neo4j_query_engine_pack(pack_path="./neo4j_pack", args=[], **kwargs):
+        name_ = "Neo4jQueryEnginePack"
+        return LlamaPack.build(name_, pack_path, args, **kwargs)
+
+    @staticmethod
+    def llava_completion_pack(pack_path="./llava_pack", args=[], **kwargs):
+        name_ = "LlavaCompletionPack"
+        return LlamaPack.build(name_, pack_path, args, **kwargs)
+
+    @staticmethod
+    def sentence_window_retriever_pack(
+        pack_path="./sentence_window_retriever_pack", args=[], **kwargs
+    ):
+        name_ = "SentenceWindowRetrieverPack"
+        return LlamaPack.build(name_, pack_path, args, **kwargs)
+
+    @staticmethod
+    def dense_x_retrieval_pack(pack_path="./dense_pack", args=[], **kwargs):
+        name_ = "DenseXRetrievalPack"
+        return LlamaPack.build(name_, pack_path, args, **kwargs)
+
+    @staticmethod
+    def zephyr_query_engine_pack(pack_path="./zephyr_pack", args=[], **kwargs):
+        name_ = "ZephyrQueryEnginePack"
+        return LlamaPack.build(name_, pack_path, args, **kwargs)
+
+    @staticmethod
+    def query_rewriting_retriever_pack(
+        pack_path="./query_rewriting_pack", args=[], **kwargs
+    ):
+        name_ = "QueryRewritingRetrieverPack"
+        return LlamaPack.build(name_, pack_path, args, **kwargs)
+
+    @staticmethod
+    def fuzzy_citation_engine_pack(
+        pack_path="./fuzzy_citation_pack", args=[], **kwargs
+    ):
+        name_ = "FuzzyCitationEnginePack"
+        return LlamaPack.build(name_, pack_path, args, **kwargs)
+
+    @staticmethod
+    def multidoc_auto_retriever_pack(
+        pack_path="./multidoc_autoretrieval_pack", args=[], **kwargs
+    ):
+        name_ = "MultiDocAutoRetrieverPack"
+        return LlamaPack.build(name_, pack_path, args, **kwargs)
+
+    @staticmethod
+    def auto_merging_retriever_pack(
+        pack_path="./auto_merging_retriever_pack", args=[], **kwargs
+    ):
+        name_ = "AutoMergingRetrieverPack"
+        return LlamaPack.build(name_, pack_path, args, **kwargs)
+
+    @staticmethod
+    def voyage_query_engine_pack(pack_path="./voyage_pack", args=[], **kwargs):
+        name_ = "VoyageQueryEnginePack"
+        return LlamaPack.build(name_, pack_path, args, **kwargs)
+
+    @staticmethod
+    def mix_self_consistency_pack(
+        pack_path="./mix_self_consistency_pack", args=[], **kwargs
+    ):
+        name_ = "MixSelfConsistencyPack"
+        return LlamaPack.build(name_, pack_path, args, **kwargs)
+
+    @staticmethod
+    def rag_fusion_pipeline_pack(
+        pack_path="./rag_fusion_pipeline_pack", args=[], **kwargs
+    ):
+        name_ = "RAGFusionPipelinePack"
+        return LlamaPack.build(name_, pack_path, args, **kwargs)
+
+    @staticmethod
+    def multi_document_agents_pack(
+        pack_path="./multi_doc_agents_pack", args=[], **kwargs
+    ):
+        name_ = "MultiDocumentAgentsPack"
+        return LlamaPack.build(name_, pack_path, args, **kwargs)
+
+    @staticmethod
+    def llama_guard_moderator_pack(pack_path="./llamaguard_pack", args=[], **kwargs):
+        name_ = "LlamaGuardModeratorPack"
+        return LlamaPack.build(name_, pack_path, args, **kwargs)
+
+    @staticmethod
+    def evaluator_benchmarker_pack(
+        pack_path="./eval_benchmark_pack", args=[], **kwargs
+    ):
+        name_ = "EvaluatorBenchmarkerPack"
+        return LlamaPack.build(name_, pack_path, args, **kwargs)
+
+    @staticmethod
+    def amazon_product_extraction_pack(
+        pack_path="./amazon_product_extraction_pack", args=[], **kwargs
+    ):
+        name_ = "AmazonProductExtractionPack"
+        return LlamaPack.build(name_, pack_path, args, **kwargs)
+
+    @staticmethod
+    def llama_dataset_metadata_pack(
+        pack_path="./llama_dataset_metadata_pack", args=[], **kwargs
+    ):
+        name_ = "LlamaDatasetMetadataPack"
+        return LlamaPack.build(name_, pack_path, args, **kwargs)
+
+    @staticmethod
+    def multi_tenancy_rag_pack(pack_path="./multitenancy_rag_pack", args=[], **kwargs):
+        name_ = "MultiTenancyRAGPack"
+        return LlamaPack.build(name_, pack_path, args, **kwargs)
+
+    @staticmethod
+    def gmail_openai_agent_pack(pack_path="./gmail_pack", args=[], **kwargs):
+        name_ = "GmailOpenAIAgentPack"
+        return LlamaPack.build(name_, pack_path, args, **kwargs)
+
+    @staticmethod
+    def snowflake_query_engine_pack(pack_path="./snowflake_pack", args=[], **kwargs):
+        name_ = "SnowflakeQueryEnginePack"
+        return LlamaPack.build(name_, pack_path, args, **kwargs)
+
+    @staticmethod
+    def agent_search_retriever_pack(pack_path="./agent_search_pack", args=[], **kwargs):
+        name_ = "AgentSearchRetrieverPack"
+        return LlamaPack.build(name_, pack_path, args, **kwargs)
+
+    @staticmethod
+    def vectara_rag_pack(pack_path="./vectara_rag_pack", args=[], **kwargs):
+        name_ = "VectaraRagPack"
+        return LlamaPack.build(name_, pack_path, args, **kwargs)
+
+    @staticmethod
+    def chroma_autoretrieval_pack(pack_path="./chroma_pack", args=[], **kwargs):
+        name_ = "ChromaAutoretrievalPack"
+        return LlamaPack.build(name_, pack_path, args, **kwargs)
+
+    @staticmethod
+    def arize_phoenix_query_engine_pack(pack_path="./arize_pack", args=[], **kwargs):
+        name_ = "ArizePhoenixQueryEnginePack"
+        return LlamaPack.build(name_, pack_path, args, **kwargs)
+
+    @staticmethod
+    def redis_ingestion_pipeline_pack(
+        pack_path="./redis_ingestion_pack", args=[], **kwargs
+    ):
+        name_ = "RedisIngestionPipelinePack"
+        return LlamaPack.build(name_, pack_path, args, **kwargs)
+
+    @staticmethod
+    def nebula_graph_query_engine_pack(
+        pack_path="./nebulagraph_pack", args=[], **kwargs
+    ):
+        name_ = "NebulaGraphQueryEnginePack"
+        return LlamaPack.build(name_, pack_path, args, **kwargs)
+
+    @staticmethod
+    def weaviate_retry_engine_pack(pack_path="./weaviate_pack", args=[], **kwargs):
+        name_ = "WeaviateRetryEnginePack"
+        return LlamaPack.build(name_, pack_path, args, **kwargs)

lionagi/integrations/bridge/transformers_/install_.py

@@ -0,0 +1,36 @@
+import subprocess
+from lionagi.libs import SysUtil
+
+
+def get_pytorch_install_command():
+    cpu_arch = SysUtil.get_cpu_architecture()
+
+    if cpu_arch == "apple_silicon":
+        return "pip install --pre torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/nightly/cpu"
+    else:
+        # Default CPU installation
+        return "pip install torch torchvision torchaudio"
+
+
+def install_pytorch():
+    command = get_pytorch_install_command()
+    try:
+        subprocess.run(command.split(), check=True)
+        print("PyTorch installed successfully.")
+    except subprocess.CalledProcessError as e:
+        print(f"Failed to install PyTorch: {e}")
+
+
+def install_transformers():
+    if not SysUtil.is_package_installed("torch"):
+        in_ = input(
+            "PyTorch is required for transformers. Would you like to install it now? (y/n): "
+        )
+        if in_ == "y":
+            install_pytorch()
+    if not SysUtil.is_package_installed("transformers"):
+        in_ = input(
+            "transformers is required. Would you like to install it now? (y/n): "
+        )
+        if in_ == "y":
+            SysUtil.install_import(package_name="transformers", import_name="pipeline")

lionagi/integrations/config/oai_configs.py

@@ -2,7 +2,7 @@
 
 # ChatCompletion
 oai_chat_llmconfig = {
-    "model": "gpt-4-turbo-preview",
+    "model": "gpt-4-turbo",
     "frequency_penalty": 0,
     "max_tokens": None,
     "n": 1,

lionagi/integrations/config/ollama_configs.py

@@ -1 +1 @@
-model = "llama2"
+model = "llama3"

lionagi/integrations/config/openrouter_configs.py

@@ -1,5 +1,5 @@
 openrouter_chat_llmconfig = {
-    "model": "gpt-4-turbo-preview",
+    "model": "gpt-4-turbo",
     "frequency_penalty": 0,
     "max_tokens": None,
     "num": 1,

lionagi/integrations/storage/__init__.py

@@ -0,0 +1,3 @@
+from lionagi.integrations.storage.neo4j import Neo4j
+
+__all__ = ["Neo4j"]