flashrag-dev 0.1.4.dev20250608__tar.gz → 0.1.4.dev20250617__tar.gz

{flashrag_dev-0.1.4.dev20250608 → flashrag_dev-0.1.4.dev20250617}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: flashrag_dev
-Version: 0.1.4.dev20250608
+Version: 0.1.4.dev20250617
 Summary: A library for efficient Retrieval-Augmented Generation research
 Home-page: https://github.com/RUC-NLPIR/FlashRAG
 Author: Jiajie Jin, Yutao Zhu, Chenghao Zhang, Xinyu Yang, Zhicheng Dou

{flashrag_dev-0.1.4.dev20250608 → flashrag_dev-0.1.4.dev20250617}/flashrag/prompt/base_prompt.py

@@ -79,7 +79,7 @@ class PromptTemplate:
                     else:
                         print(f"The input text length is greater than the maximum length ({total_tokens + len(encoded_message)} > {self.max_input_len}) and has been truncated!")
                         remaining_tokens = self.max_input_len - total_tokens
-                        truncated_message = self.encoding.decode(encoded_message[:remaining_tokens])
+                        truncated_message = self._get_tokenizer().decode(encoded_message[:remaining_tokens])
                         message['content'] = truncated_message
                         truncated_messages.append(message)
                         break
@@ -93,12 +93,14 @@ class PromptTemplate:
 
         else:
             assert isinstance(prompt, str)
-            # tokenized_prompt = self._get_tokenizer().encode(prompt, truncation=False, return_tensors="pt").input_ids[0]
-            tokenized_prompt = self._get_tokenizer().encode(prompt,truncation=False,return_tensors="pt")
+            try:
+                tokenized_prompt = self._get_tokenizer().encode(prompt, truncation=False, return_tensors="pt").input_ids[0]
+            except:
+                tokenized_prompt = self._get_tokenizer().encode(prompt, truncation=False, return_tensors="pt")[0]
 
-            if len(tokenized_prompt) > self.max_input_len:
+            if len(tokenized_prompt) >= self.max_input_len:
                 print(f"The input text length is greater than the maximum length ({len(tokenized_prompt)} > {self.max_input_len}) and has been truncated!")
-                half = int(self.max_input_len / 2)
+                half = int(self.max_input_len / 2) - 20
                 prompt = self._get_tokenizer().decode(tokenized_prompt[:half], skip_special_tokens=True) + \
                         self._get_tokenizer().decode(tokenized_prompt[-half:], skip_special_tokens=True)
             return prompt
@@ -142,7 +144,11 @@ class PromptTemplate:
             if user_prompt != "":
                 input.append({"role": "user", "content": user_prompt})
             if not self.is_openai:
-                input = self._get_tokenizer().apply_chat_template(input, tokenize=False, add_generation_prompt=True)
+                try:
+                    input = self._get_tokenizer().apply_chat_template(input, tokenize=False, add_generation_prompt=True)
+                except:
+                    print("Warning: the generator tokenizer not support `apply_chat_template`")
+                    input = system_prompt + '\n\n' + user_prompt
         else:
             input = "\n\n".join([prompt for prompt in [system_prompt, user_prompt] if prompt != ""])
 
@@ -186,7 +192,7 @@ class PromptTemplate:
                     **params,
                 )
 
-                possible_prompt_tokens = tokenizer.encode(possible_prompt)
+                possible_prompt_tokens = self._get_tokenizer().encode(possible_prompt)
                 if len(possible_prompt_tokens) <= max_length:
                     final_examplars = examplars[:num]
                     break

flashrag_dev-0.1.4.dev20250617/flashrag/version.py

@@ -0,0 +1 @@
+__version__ = "0.1.4dev20250617"

{flashrag_dev-0.1.4.dev20250608 → flashrag_dev-0.1.4.dev20250617}/flashrag_dev.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: flashrag-dev
-Version: 0.1.4.dev20250608
+Version: 0.1.4.dev20250617
 Summary: A library for efficient Retrieval-Augmented Generation research
 Home-page: https://github.com/RUC-NLPIR/FlashRAG
 Author: Jiajie Jin, Yutao Zhu, Chenghao Zhang, Xinyu Yang, Zhicheng Dou

flashrag_dev-0.1.4.dev20250608/flashrag/version.py

@@ -1 +0,0 @@
-__version__ = "0.1.4dev20250608"