flashrag-dev 0.1.3.dev20241112__tar.gz → 0.1.3.dev20241117__tar.gz

{flashrag-dev-0.1.3.dev20241112 → flashrag-dev-0.1.3.dev20241117}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: flashrag-dev
-Version: 0.1.3.dev20241112
+Version: 0.1.3.dev20241117
 Summary: A library for efficient Retrieval-Augmented Generation research
 Home-page: https://github.com/RUC-NLPIR/FlashRAG
 Author: Jiajie Jin, Yutao Zhu, Chenghao Zhang, Xinyu Yang, Zhicheng Dou
@@ -536,6 +536,11 @@ In the academic research, Wikipedia and MS MARCO are the most commonly used retr
 
 For MS MARCO, it is already processed upon release and can be directly downloaded from its [<u>hosting link</u>](https://huggingface.co/datasets/Tevatron/msmarco-passage-corpus) on Hugging Face.
 
+### Index
+
+To facilitate easier replication of the experiments, we now provide a preprocessed index available in the ModelScope dataset page: [FlashRAG_Dataset/retrieval_corpus/wiki18_100w_e5_index.zip](https://www.modelscope.cn/datasets/hhjinjiajie/FlashRAG_Dataset/file/view/master?id=47985&status=2&fileName=retrieval_corpus%252Fwiki18_100w_e5_index.zip). 
+
+The index was created using the e5-base-v2 retriever on our uploaded wiki18_100w dataset, which is consistent with the index used in our experiments.
 
 ## :raised_hands: Additional FAQs
 

flashrag-dev-0.1.3.dev20241112/flashrag_dev.egg-info/PKG-INFO → flashrag-dev-0.1.3.dev20241117/README.md

@@ -1,19 +1,3 @@
-Metadata-Version: 2.1
-Name: flashrag-dev
-Version: 0.1.3.dev20241112
-Summary: A library for efficient Retrieval-Augmented Generation research
-Home-page: https://github.com/RUC-NLPIR/FlashRAG
-Author: Jiajie Jin, Yutao Zhu, Chenghao Zhang, Xinyu Yang, Zhicheng Dou
-Author-email: jinjiajie@ruc.edu.cn
-License: MIT License
-Requires-Python: >=3.9
-Description-Content-Type: text/markdown
-Provides-Extra: core
-Provides-Extra: retriever
-Provides-Extra: generator
-Provides-Extra: full
-License-File: LICENSE
-
 # <div align="center">⚡FlashRAG: A Python Toolkit for Efficient RAG Research<div>
 
 
@@ -536,6 +520,11 @@ In the academic research, Wikipedia and MS MARCO are the most commonly used retr
 
 For MS MARCO, it is already processed upon release and can be directly downloaded from its [<u>hosting link</u>](https://huggingface.co/datasets/Tevatron/msmarco-passage-corpus) on Hugging Face.
 
+### Index
+
+To facilitate easier replication of the experiments, we now provide a preprocessed index available in the ModelScope dataset page: [FlashRAG_Dataset/retrieval_corpus/wiki18_100w_e5_index.zip](https://www.modelscope.cn/datasets/hhjinjiajie/FlashRAG_Dataset/file/view/master?id=47985&status=2&fileName=retrieval_corpus%252Fwiki18_100w_e5_index.zip). 
+
+The index was created using the e5-base-v2 retriever on our uploaded wiki18_100w dataset, which is consistent with the index used in our experiments.
 
 ## :raised_hands: Additional FAQs
 

{flashrag-dev-0.1.3.dev20241112 → flashrag-dev-0.1.3.dev20241117}/flashrag/pipeline/pipeline.py

@@ -216,10 +216,7 @@ class AdaptivePipeline(BasicPipeline):
         )
 
         self.multi_hop_pipeline = IRCOTPipeline(
-            config,
-            prompt_template=multi_hop_prompt_template,
-            retriever=retriever,
-            generator=generator,
+            config, prompt_template=multi_hop_prompt_template, retriever=retriever, generator=generator, max_iter=5
         )
 
     def run(self, dataset, do_eval=True, pred_process_fun=None):

{flashrag-dev-0.1.3.dev20241112 → flashrag-dev-0.1.3.dev20241117}/flashrag/retriever/utils.py

@@ -27,6 +27,7 @@ def pooling(pooler_output, last_hidden_state, attention_mask=None, pooling_metho
     else:
         raise NotImplementedError("Pooling method not implemented!")
 
+
 def set_default_instruction(model_name, is_query=True, is_zh=False):
     instruction = ""
     if "e5" in model_name.lower():
@@ -37,7 +38,7 @@ def set_default_instruction(model_name, is_query=True, is_zh=False):
 
     if "bge" in model_name.lower():
         if is_query:
-            if 'zh' in model_name.lower() or is_zh:
+            if "zh" in model_name.lower() or is_zh:
                 instruction = "为这个句子生成表示以用于检索相关文章："
             else:
                 instruction = "Represent this sentence for searching relevant passages: "
@@ -60,6 +61,8 @@ def parse_query(model_name, query_list, instruction=None):
                     zh_char += 1
             except:
                 continue
+        if len(str) == 0:
+            return False
         if zh_char / len(str) > 0.2:
             return True
         else:
@@ -67,7 +70,7 @@ def parse_query(model_name, query_list, instruction=None):
 
     if isinstance(query_list, str):
         query_list = [query_list]
-        
+
     if instruction is not None:
         instruction = instruction.strip() + " "
     else:
@@ -78,6 +81,7 @@ def parse_query(model_name, query_list, instruction=None):
 
     return query_list
 
+
 def load_corpus(corpus_path: str):
     corpus = datasets.load_dataset("json", data_files=corpus_path, split="train")
     return corpus
@@ -98,3 +102,16 @@ def load_docs(corpus, doc_idxs):
     results = [corpus[int(idx)] for idx in doc_idxs]
 
     return results
+
+
+def parse_image(image):
+    from PIL import Image
+
+    if isinstance(image, str):
+        if image.startswith("http"):
+            import requests
+
+            image = Image.open(requests.get(image, stream=True).raw)
+        else:
+            image = Image.open(image)
+    return image

flashrag-dev-0.1.3.dev20241117/flashrag/version.py

@@ -0,0 +1 @@
+__version__ = "0.1.3dev20241117"

flashrag-dev-0.1.3.dev20241112/README.md → flashrag-dev-0.1.3.dev20241117/flashrag_dev.egg-info/PKG-INFO

@@ -1,3 +1,19 @@
+Metadata-Version: 2.1
+Name: flashrag-dev
+Version: 0.1.3.dev20241117
+Summary: A library for efficient Retrieval-Augmented Generation research
+Home-page: https://github.com/RUC-NLPIR/FlashRAG
+Author: Jiajie Jin, Yutao Zhu, Chenghao Zhang, Xinyu Yang, Zhicheng Dou
+Author-email: jinjiajie@ruc.edu.cn
+License: MIT License
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Provides-Extra: core
+Provides-Extra: retriever
+Provides-Extra: generator
+Provides-Extra: full
+License-File: LICENSE
+
 # <div align="center">⚡FlashRAG: A Python Toolkit for Efficient RAG Research<div>
 
 
@@ -520,6 +536,11 @@ In the academic research, Wikipedia and MS MARCO are the most commonly used retr
 
 For MS MARCO, it is already processed upon release and can be directly downloaded from its [<u>hosting link</u>](https://huggingface.co/datasets/Tevatron/msmarco-passage-corpus) on Hugging Face.
 
+### Index
+
+To facilitate easier replication of the experiments, we now provide a preprocessed index available in the ModelScope dataset page: [FlashRAG_Dataset/retrieval_corpus/wiki18_100w_e5_index.zip](https://www.modelscope.cn/datasets/hhjinjiajie/FlashRAG_Dataset/file/view/master?id=47985&status=2&fileName=retrieval_corpus%252Fwiki18_100w_e5_index.zip). 
+
+The index was created using the e5-base-v2 retriever on our uploaded wiki18_100w dataset, which is consistent with the index used in our experiments.
 
 ## :raised_hands: Additional FAQs
 

flashrag-dev-0.1.3.dev20241112/flashrag/version.py

@@ -1 +0,0 @@
-__version__ = "0.1.3dev20241112"