auto-coder 0.1.219__py3-none-any.whl → 0.1.220__py3-none-any.whl

{auto_coder-0.1.219.dist-info → auto_coder-0.1.220.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: auto-coder
-Version: 0.1.219
+Version: 0.1.220
 Summary: AutoCoder: AutoCoder
 Author: allwefantasy
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence

{auto_coder-0.1.219.dist-info → auto_coder-0.1.220.dist-info}/RECORD

@@ -8,7 +8,7 @@ autocoder/chat_auto_coder.py,sha256=NTCWQKBQJluEhay5NGoTl5tdv00Zmu8oFioCjCpJCd8,
 autocoder/chat_auto_coder_lang.py,sha256=ReWukXKVvuzVvpbYk5O9kc1ev7XNmAv3DnuQhmpLmnc,8717
 autocoder/command_args.py,sha256=9aYJ-AmPxP1sQh6ciw04FWHjSn31f2W9afXFwo8wgx4,30441
 autocoder/lang.py,sha256=U6AjVV8Rs1uLyjFCZ8sT6WWuNUxMBqkXXIOs4S120uk,14511
-autocoder/version.py,sha256=CiJSNMnvgzsdXFL0TOFxCpTE04WDBfpfA19PCdyUupg,24
+autocoder/version.py,sha256=AkV-rQzPFI1YGmw3_1zhV77j25aK6xZpk7snIDE9-KU,24
 autocoder/agent/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 autocoder/agent/auto_demand_organizer.py,sha256=NWSAEsEk94vT3lGjfo25kKLMwYdPcpy9e-i21txPasQ,6942
 autocoder/agent/auto_filegroup.py,sha256=CW7bqp0FW1GIEMnl-blyAc2UGT7O9Mom0q66ITz1ckM,6635
@@ -75,10 +75,10 @@ autocoder/index/symbols_utils.py,sha256=CjcjUVajmJZB75Ty3a7kMv1BZphrm-tIBAdOJv6u
 autocoder/pyproject/__init__.py,sha256=7ZuIxD2QBYIwhjmpva8eL2knorKo03yNqUhSyecpt7c,14448
 autocoder/rag/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 autocoder/rag/api_server.py,sha256=dRbhAZVRAOlZ64Cnxf4_rKb4iJwHnrWS9Zr67IVORw0,7288
-autocoder/rag/doc_filter.py,sha256=H0g4s-Q8gJDUbYk9b1qkEyl7RgNzLbRF4UWELVKPDwk,9865
+autocoder/rag/doc_filter.py,sha256=B99Qcy3tcNLuSz2kWbpgfBj2_Igme91zWKOJ2Niq2UY,6652
 autocoder/rag/document_retriever.py,sha256=5oThtxukGuRFF96o3pHKsk306a8diXbhgSrbqyU2BvM,8894
 autocoder/rag/llm_wrapper.py,sha256=sbDxCANiZyWb_ocqNgqu2oy3c2t8orPNRGleEs-Uwl8,2649
-autocoder/rag/long_context_rag.py,sha256=x8ZPg9cwTdFmz6CIzjA1ZPX7_5k3G-ZleL6v3B9CO6Y,24921
+autocoder/rag/long_context_rag.py,sha256=jEhil33y2ryAF393zKdUhrk_QEHUlZ30gZcvPH0Z3MY,25074
 autocoder/rag/rag_config.py,sha256=8LwFcTd8OJWWwi1_WY4IzjqgtT6RyE2j4PjxS5cCTDE,802
 autocoder/rag/rag_entry.py,sha256=6TKtErZ0Us9XSV6HgRKXA6yR3SiZGPHpynOKSaR1wgE,2463
 autocoder/rag/raw_rag.py,sha256=yS2Ur6kG0IRjhCj2_VonwxjY_xls_E62jO5Gz5j2nqE,2952
@@ -122,9 +122,9 @@ autocoder/utils/request_event_queue.py,sha256=r3lo5qGsB1dIjzVQ05dnr0z_9Z3zOkBdP1
 autocoder/utils/request_queue.py,sha256=nwp6PMtgTCiuwJI24p8OLNZjUiprC-TsefQrhMI-yPE,3889
 autocoder/utils/rest.py,sha256=HawagAap3wMIDROGhY1730zSZrJR_EycODAA5qOj83c,8807
 autocoder/utils/tests.py,sha256=BqphrwyycGAvs-5mhH8pKtMZdObwhFtJ5MC_ZAOiLq8,1340
-auto_coder-0.1.219.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
-auto_coder-0.1.219.dist-info/METADATA,sha256=qsE3dhB3-5DCw0cflbsa5zCFs5DqDnyXhzSDum8RNgU,2615
-auto_coder-0.1.219.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-auto_coder-0.1.219.dist-info/entry_points.txt,sha256=0nzHtHH4pNcM7xq4EBA2toS28Qelrvcbrr59GqD_0Ak,350
-auto_coder-0.1.219.dist-info/top_level.txt,sha256=Jqc0_uJSw2GwoFQAa9iJxYns-2mWla-9ok_Y3Gcznjk,10
-auto_coder-0.1.219.dist-info/RECORD,,
+auto_coder-0.1.220.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
+auto_coder-0.1.220.dist-info/METADATA,sha256=uQi6g28X2Lzv4rVNmWu0j3yEQ0W2uo_YQOqdXmpSHwc,2615
+auto_coder-0.1.220.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+auto_coder-0.1.220.dist-info/entry_points.txt,sha256=0nzHtHH4pNcM7xq4EBA2toS28Qelrvcbrr59GqD_0Ak,350
+auto_coder-0.1.220.dist-info/top_level.txt,sha256=Jqc0_uJSw2GwoFQAa9iJxYns-2mWla-9ok_Y3Gcznjk,10
+auto_coder-0.1.220.dist-info/RECORD,,

autocoder/rag/doc_filter.py

@@ -1,24 +1,11 @@
 import time
 from typing import List, Dict, Optional
-from pydantic import BaseModel
-import ray
 from loguru import logger
-import os
 from concurrent.futures import ThreadPoolExecutor, as_completed
-from rich.progress import (
-    Progress,
-    SpinnerColumn,
-    TextColumn,
-    BarColumn,
-    TaskProgressColumn,
-    TimeElapsedColumn,
-)
-from rich.console import Console
 
 from autocoder.rag.relevant_utils import (
     parse_relevance,
-    FilterDoc,
-    DocRelevance,
+    FilterDoc,    
     TaskTiming,
 )
 
@@ -38,14 +25,18 @@ def _check_relevance_with_conversation(
     使用以下文档和对话历史来回答问题。如果文档中没有相关信息，请说"我没有足够的信息来回答这个问题"。
 
     文档：
+    <documents>
     {% for doc in documents %}
     {{ doc }}
     {% endfor %}
+    </documents>
 
     对话历史：
+    <conversations>
     {% for msg in conversations %}
     <{{ msg.role }}>: {{ msg.content }}
     {% endfor %}
+    </conversations>
 
     {% if filter_config %}
     一些提示：
@@ -58,32 +49,6 @@ def _check_relevance_with_conversation(
     """
 
 
-@ray.remote
-class DocFilterWorker:
-    def __init__(self, llm: ByzerLLM):
-        self.llm = llm
-        if self.llm.get_sub_client("recall_model"):
-            self.recall_llm = self.llm.get_sub_client("recall_model")
-        else:
-            self.recall_llm = self.llm
-
-    def filter_doc(
-        self, conversations: List[Dict[str, str]], docs: List[str]
-    ) -> Optional[FilterDoc]:
-        submit_time_1 = time.time()
-        try:
-            v = _check_relevance_with_conversation.with_llm(self.llm).run(
-                conversations=conversations, documents=docs
-            )
-        except Exception as e:
-            logger.error(
-                f"Error in _check_relevance_with_conversation: {str(e)}")
-            return (None, submit_time_1, time.time())
-
-        end_time_2 = time.time()
-        return (v, submit_time_1, end_time_2)
-
-
 class DocFilter:
     def __init__(
         self,
@@ -101,144 +66,67 @@ class DocFilter:
         self.args = args
         self.relevant_score = self.args.rag_doc_filter_relevance or 5
         self.on_ray = on_ray
-        self.path = path
-        if self.on_ray:
-            cpu_count = os.cpu_count() or 1
-            self.workers = [
-                DocFilterWorker.options(
-                    max_concurrency=1000, num_cpus=0).remote(llm)
-                for _ in range(cpu_count)
-            ]
+        self.path = path        
 
     def filter_docs(
         self, conversations: List[Dict[str, str]], documents: List[SourceCode]
     ) -> List[FilterDoc]:
-        if self.on_ray:
-            return self.filter_docs_with_ray(conversations, documents)
-        else:
-            return self.filter_docs_with_threads(conversations, documents)
+        return self.filter_docs_with_threads(conversations, documents)            
 
     def filter_docs_with_threads(
         self, conversations: List[Dict[str, str]], documents: List[SourceCode]
     ) -> List[FilterDoc]:
-
-        console = Console()
+        
         rag_manager = RagConfigManager(path=self.path)
         rag_config = rag_manager.load_config()
-        documents = list(documents)
-        with Progress(
-            SpinnerColumn(),
-            TextColumn("[progress.description]{task.description}"),
-            BarColumn(),
-            TimeElapsedColumn(),
-            console=console,
-        ) as progress:
-            task = progress.add_task(
-                "[cyan]Filtering documents...", total=len(documents)
-            )
-
-            with ThreadPoolExecutor(
-                max_workers=self.args.index_filter_workers or 5
-            ) as executor:
-                future_to_doc = {}
-                for doc in documents:
-                    submit_time = time.time()
-
-                    def _run(conversations, docs):
-                        submit_time_1 = time.time()
-                        try:
-                            llm = ByzerLLM()
-                            llm.skip_nontext_check = True
-                            llm.setup_default_model_name(self.recall_llm.default_model_name)
-
-                            v = (
-                                _check_relevance_with_conversation.with_llm(
-                                    llm)
-                                .options({"llm_config": {"max_length": 10}})
-                                .run(
-                                    conversations=conversations,
-                                    documents=docs,
-                                    filter_config=rag_config.filter_config,
-                                )
-                            )
-                        except Exception as e:
-                            logger.error(
-                                f"Error in _check_relevance_with_conversation: {str(e)}"
-                            )
-                            return (None, submit_time_1, time.time())
-
-                        end_time_2 = time.time()
-                        return (v, submit_time_1, end_time_2)
-
-                    m = executor.submit(
-                        _run,
-                        conversations,
-                        [f"##File: {doc.module_name}\n{doc.source_code}"],
-                    )
-                    future_to_doc[m] = (doc, submit_time)
-
-            relevant_docs = []
-            for future in as_completed(list(future_to_doc.keys())):
-                try:
-                    doc, submit_time = future_to_doc[future]
-                    end_time = time.time()
-                    v, submit_time_1, end_time_2 = future.result()
-                    task_timing = TaskTiming(
-                        submit_time=submit_time,
-                        end_time=end_time,
-                        duration=end_time - submit_time,
-                        real_start_time=submit_time_1,
-                        real_end_time=end_time_2,
-                        real_duration=end_time_2 - submit_time_1,
-                    )
-                    progress.update(task, advance=1)
-
-                    relevance = parse_relevance(v)
-                    if (
-                        relevance
-                        and relevance.is_relevant
-                        and relevance.relevant_score >= self.relevant_score
-                    ):
-                        relevant_docs.append(
-                            FilterDoc(
-                                source_code=doc,
-                                relevance=relevance,
-                                task_timing=task_timing,
+        documents = list(documents)   
+        logger.info(f"Filtering {len(documents)} documents....")
+        with ThreadPoolExecutor(
+            max_workers=self.args.index_filter_workers or 5
+        ) as executor:
+            future_to_doc = {}
+            for doc in documents:
+                submit_time = time.time()
+
+                def _run(conversations, docs):
+                    submit_time_1 = time.time()
+                    try:
+                        llm = ByzerLLM()
+                        llm.skip_nontext_check = True
+                        llm.setup_default_model_name(self.recall_llm.default_model_name)
+
+                        v = (
+                            _check_relevance_with_conversation.with_llm(
+                                llm)
+                            .options({"llm_config": {"max_length": 10}})
+                            .run(
+                                conversations=conversations,
+                                documents=docs,
+                                filter_config=rag_config.filter_config,
                             )
                         )
-                except Exception as exc:
-                    logger.error(
-                        f"Document processing generated an exception: {exc}")
+                    except Exception as e:
+                        logger.error(
+                            f"Error in _check_relevance_with_conversation: {str(e)}"
+                        )
+                        return (None, submit_time_1, time.time())
 
-        # Sort relevant_docs by relevance score in descending order
-        relevant_docs.sort(
-            key=lambda x: x.relevance.relevant_score, reverse=True)
-        return relevant_docs
+                    end_time_2 = time.time()
+                    return (v, submit_time_1, end_time_2)
 
-    def filter_docs_with_ray(
-        self, conversations: List[Dict[str, str]], documents: List[SourceCode]
-    ) -> List[FilterDoc]:
-        futures = []
-        count = 0
-        submit_time = time.time()
-        for doc in documents:
-            worker = self.workers[count % len(self.workers)]
-            count += 1
-            future = worker.filter_doc.remote(
-                conversations, [
-                    f"##File: {doc.module_name}\n{doc.source_code}"]
-            )
-            futures.append((future, doc))
+                m = executor.submit(
+                    _run,
+                    conversations,
+                    [f"##File: {doc.module_name}\n{doc.source_code}"],
+                )
+                future_to_doc[m] = (doc, submit_time)
 
         relevant_docs = []
-        for future, doc in futures:
+        for future in as_completed(list(future_to_doc.keys())):
             try:
-                v, submit_time_1, end_time_2 = ray.get(future)
+                doc, submit_time = future_to_doc[future]
                 end_time = time.time()
-
-                if v is None:
-                    continue
-
+                v, submit_time_1, end_time_2 = future.result()
                 task_timing = TaskTiming(
                     submit_time=submit_time,
                     end_time=end_time,
@@ -246,11 +134,20 @@ class DocFilter:
                     real_start_time=submit_time_1,
                     real_end_time=end_time_2,
                     real_duration=end_time_2 - submit_time_1,
-                )
+                )                
+
+                relevance = parse_relevance(v)
                 logger.info(
-                    f"Document: {doc.module_name} Duration: {task_timing.duration:.2f} seconds/{task_timing.real_duration:.2f}/{task_timing.real_duration-task_timing.duration} seconds"
+                    f"Document filtering progress:\n"
+                    f"  - File: {doc.module_name}\n"
+                    f"  - Relevance: {'Relevant' if relevance and relevance.is_relevant else 'Not Relevant'}\n"
+                    f"  - Score: {relevance.relevant_score if relevance else 'N/A'}\n"
+                    f"  - Raw Response: {v}\n"
+                    f"  - Timing:\n"
+                    f"    * Total Duration: {task_timing.duration:.2f}s\n"
+                    f"    * Real Duration: {task_timing.real_duration:.2f}s\n"
+                    f"    * Queue Time: {(task_timing.real_start_time - task_timing.submit_time):.2f}s"
                 )
-                relevance = parse_relevance(v)
                 if (
                     relevance
                     and relevance.is_relevant
@@ -264,10 +161,16 @@ class DocFilter:
                         )
                     )
             except Exception as exc:
-                logger.error(
-                    f"Document processing generated an exception: {exc}")
+                try:
+                    doc, submit_time = future_to_doc[future]
+                    logger.error(
+                        f"Filtering document generated an exception (doc: {doc.module_name}): {exc}")
+                except Exception as e:
+                    logger.error(
+                        f"Filtering document generated an exception: {exc}")
 
         # Sort relevant_docs by relevance score in descending order
         relevant_docs.sort(
             key=lambda x: x.relevance.relevant_score, reverse=True)
         return relevant_docs
+    

autocoder/rag/long_context_rag.py

@@ -428,6 +428,7 @@ class LongContextRAG:
                 if "only_contexts" in v:
                     query = v["query"]
                     only_contexts = v["only_contexts"]
+                    conversations[-1]["content"] = query
             except json.JSONDecodeError:
                 pass
 
@@ -463,9 +464,10 @@ class LongContextRAG:
             )
 
             if only_contexts:
-                return (
-                    doc.source_code.model_dump_json() + "\n" for doc in relevant_docs
-                ), []
+                final_docs = []
+                for doc in relevant_docs:
+                    final_docs.append(doc.model_dump())
+                return [json.dumps(final_docs,ensure_ascii=False)], []                
 
             if not relevant_docs:
                 return ["没有找到相关的文档来回答这个问题。"], []
@@ -612,7 +614,7 @@ class LongContextRAG:
                     ),
                 }
             ]
-
+            
             chunks = target_llm.stream_chat_oai(
                 conversations=new_conversations,
                 model=model,

autocoder/version.py

@@ -1 +1 @@
-__version__ = "0.1.219"
+__version__ = "0.1.220"