auto-coder 0.1.219__tar.gz → 0.1.220__tar.gz

{auto_coder-0.1.219 → auto_coder-0.1.220}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: auto-coder
-Version: 0.1.219
+Version: 0.1.220
 Summary: AutoCoder: AutoCoder
 Author: allwefantasy
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence

{auto_coder-0.1.219 → auto_coder-0.1.220}/src/auto_coder.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: auto-coder
-Version: 0.1.219
+Version: 0.1.220
 Summary: AutoCoder: AutoCoder
 Author: allwefantasy
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence

auto_coder-0.1.220/src/autocoder/rag/doc_filter.py

@@ -0,0 +1,176 @@
+import time
+from typing import List, Dict, Optional
+from loguru import logger
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+from autocoder.rag.relevant_utils import (
+    parse_relevance,
+    FilterDoc,    
+    TaskTiming,
+)
+
+from autocoder.common import SourceCode, AutoCoderArgs
+from autocoder.rag.rag_config import RagConfigManager
+from byzerllm import ByzerLLM
+import byzerllm
+
+
+@byzerllm.prompt()
+def _check_relevance_with_conversation(
+    conversations: List[Dict[str, str]],
+    documents: List[str],
+    filter_config: Optional[str] = None,
+) -> str:
+    """
+    使用以下文档和对话历史来回答问题。如果文档中没有相关信息，请说"我没有足够的信息来回答这个问题"。
+
+    文档：
+    <documents>
+    {% for doc in documents %}
+    {{ doc }}
+    {% endfor %}
+    </documents>
+
+    对话历史：
+    <conversations>
+    {% for msg in conversations %}
+    <{{ msg.role }}>: {{ msg.content }}
+    {% endfor %}
+    </conversations>
+
+    {% if filter_config %}
+    一些提示：
+    {{ filter_config }}
+    {% endif %}
+
+    请结合提供的文档以及用户对话历史，判断提供的文档是不是能和用户的最后一个问题相关。
+    如果该文档提供的知识能够和用户的问题相关，那么请回复"yes/<relevant>" 否则回复"no/<relevant>"。
+    其中， <relevant> 是你认为文档中和问题的相关度，0-10之间的数字，数字越大表示相关度越高。
+    """
+
+
+class DocFilter:
+    def __init__(
+        self,
+        llm: ByzerLLM,
+        args: AutoCoderArgs,
+        on_ray: bool = False,
+        path: Optional[str] = None,
+    ):
+        self.llm = llm
+        if self.llm.get_sub_client("recall_model"):
+            self.recall_llm = self.llm.get_sub_client("recall_model")
+        else:
+            self.recall_llm = self.llm
+        
+        self.args = args
+        self.relevant_score = self.args.rag_doc_filter_relevance or 5
+        self.on_ray = on_ray
+        self.path = path        
+
+    def filter_docs(
+        self, conversations: List[Dict[str, str]], documents: List[SourceCode]
+    ) -> List[FilterDoc]:
+        return self.filter_docs_with_threads(conversations, documents)            
+
+    def filter_docs_with_threads(
+        self, conversations: List[Dict[str, str]], documents: List[SourceCode]
+    ) -> List[FilterDoc]:
+        
+        rag_manager = RagConfigManager(path=self.path)
+        rag_config = rag_manager.load_config()
+        documents = list(documents)   
+        logger.info(f"Filtering {len(documents)} documents....")
+        with ThreadPoolExecutor(
+            max_workers=self.args.index_filter_workers or 5
+        ) as executor:
+            future_to_doc = {}
+            for doc in documents:
+                submit_time = time.time()
+
+                def _run(conversations, docs):
+                    submit_time_1 = time.time()
+                    try:
+                        llm = ByzerLLM()
+                        llm.skip_nontext_check = True
+                        llm.setup_default_model_name(self.recall_llm.default_model_name)
+
+                        v = (
+                            _check_relevance_with_conversation.with_llm(
+                                llm)
+                            .options({"llm_config": {"max_length": 10}})
+                            .run(
+                                conversations=conversations,
+                                documents=docs,
+                                filter_config=rag_config.filter_config,
+                            )
+                        )
+                    except Exception as e:
+                        logger.error(
+                            f"Error in _check_relevance_with_conversation: {str(e)}"
+                        )
+                        return (None, submit_time_1, time.time())
+
+                    end_time_2 = time.time()
+                    return (v, submit_time_1, end_time_2)
+
+                m = executor.submit(
+                    _run,
+                    conversations,
+                    [f"##File: {doc.module_name}\n{doc.source_code}"],
+                )
+                future_to_doc[m] = (doc, submit_time)
+
+        relevant_docs = []
+        for future in as_completed(list(future_to_doc.keys())):
+            try:
+                doc, submit_time = future_to_doc[future]
+                end_time = time.time()
+                v, submit_time_1, end_time_2 = future.result()
+                task_timing = TaskTiming(
+                    submit_time=submit_time,
+                    end_time=end_time,
+                    duration=end_time - submit_time,
+                    real_start_time=submit_time_1,
+                    real_end_time=end_time_2,
+                    real_duration=end_time_2 - submit_time_1,
+                )                
+
+                relevance = parse_relevance(v)
+                logger.info(
+                    f"Document filtering progress:\n"
+                    f"  - File: {doc.module_name}\n"
+                    f"  - Relevance: {'Relevant' if relevance and relevance.is_relevant else 'Not Relevant'}\n"
+                    f"  - Score: {relevance.relevant_score if relevance else 'N/A'}\n"
+                    f"  - Raw Response: {v}\n"
+                    f"  - Timing:\n"
+                    f"    * Total Duration: {task_timing.duration:.2f}s\n"
+                    f"    * Real Duration: {task_timing.real_duration:.2f}s\n"
+                    f"    * Queue Time: {(task_timing.real_start_time - task_timing.submit_time):.2f}s"
+                )
+                if (
+                    relevance
+                    and relevance.is_relevant
+                    and relevance.relevant_score >= self.relevant_score
+                ):
+                    relevant_docs.append(
+                        FilterDoc(
+                            source_code=doc,
+                            relevance=relevance,
+                            task_timing=task_timing,
+                        )
+                    )
+            except Exception as exc:
+                try:
+                    doc, submit_time = future_to_doc[future]
+                    logger.error(
+                        f"Filtering document generated an exception (doc: {doc.module_name}): {exc}")
+                except Exception as e:
+                    logger.error(
+                        f"Filtering document generated an exception: {exc}")
+
+        # Sort relevant_docs by relevance score in descending order
+        relevant_docs.sort(
+            key=lambda x: x.relevance.relevant_score, reverse=True)
+        return relevant_docs
+    

{auto_coder-0.1.219 → auto_coder-0.1.220}/src/autocoder/rag/long_context_rag.py

@@ -428,6 +428,7 @@ class LongContextRAG:
                 if "only_contexts" in v:
                     query = v["query"]
                     only_contexts = v["only_contexts"]
+                    conversations[-1]["content"] = query
             except json.JSONDecodeError:
                 pass
 
@@ -463,9 +464,10 @@ class LongContextRAG:
             )
 
             if only_contexts:
-                return (
-                    doc.source_code.model_dump_json() + "\n" for doc in relevant_docs
-                ), []
+                final_docs = []
+                for doc in relevant_docs:
+                    final_docs.append(doc.model_dump())
+                return [json.dumps(final_docs,ensure_ascii=False)], []                
 
             if not relevant_docs:
                 return ["没有找到相关的文档来回答这个问题。"], []
@@ -612,7 +614,7 @@ class LongContextRAG:
                     ),
                 }
             ]
-
+            
             chunks = target_llm.stream_chat_oai(
                 conversations=new_conversations,
                 model=model,

auto_coder-0.1.220/src/autocoder/version.py

@@ -0,0 +1 @@
+__version__ = "0.1.220"

auto_coder-0.1.219/src/autocoder/rag/doc_filter.py

@@ -1,273 +0,0 @@
-import time
-from typing import List, Dict, Optional
-from pydantic import BaseModel
-import ray
-from loguru import logger
-import os
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from rich.progress import (
-    Progress,
-    SpinnerColumn,
-    TextColumn,
-    BarColumn,
-    TaskProgressColumn,
-    TimeElapsedColumn,
-)
-from rich.console import Console
-
-from autocoder.rag.relevant_utils import (
-    parse_relevance,
-    FilterDoc,
-    DocRelevance,
-    TaskTiming,
-)
-
-from autocoder.common import SourceCode, AutoCoderArgs
-from autocoder.rag.rag_config import RagConfigManager
-from byzerllm import ByzerLLM
-import byzerllm
-
-
-@byzerllm.prompt()
-def _check_relevance_with_conversation(
-    conversations: List[Dict[str, str]],
-    documents: List[str],
-    filter_config: Optional[str] = None,
-) -> str:
-    """
-    使用以下文档和对话历史来回答问题。如果文档中没有相关信息，请说"我没有足够的信息来回答这个问题"。
-
-    文档：
-    {% for doc in documents %}
-    {{ doc }}
-    {% endfor %}
-
-    对话历史：
-    {% for msg in conversations %}
-    <{{ msg.role }}>: {{ msg.content }}
-    {% endfor %}
-
-    {% if filter_config %}
-    一些提示：
-    {{ filter_config }}
-    {% endif %}
-
-    请结合提供的文档以及用户对话历史，判断提供的文档是不是能和用户的最后一个问题相关。
-    如果该文档提供的知识能够和用户的问题相关，那么请回复"yes/<relevant>" 否则回复"no/<relevant>"。
-    其中， <relevant> 是你认为文档中和问题的相关度，0-10之间的数字，数字越大表示相关度越高。
-    """
-
-
-@ray.remote
-class DocFilterWorker:
-    def __init__(self, llm: ByzerLLM):
-        self.llm = llm
-        if self.llm.get_sub_client("recall_model"):
-            self.recall_llm = self.llm.get_sub_client("recall_model")
-        else:
-            self.recall_llm = self.llm
-
-    def filter_doc(
-        self, conversations: List[Dict[str, str]], docs: List[str]
-    ) -> Optional[FilterDoc]:
-        submit_time_1 = time.time()
-        try:
-            v = _check_relevance_with_conversation.with_llm(self.llm).run(
-                conversations=conversations, documents=docs
-            )
-        except Exception as e:
-            logger.error(
-                f"Error in _check_relevance_with_conversation: {str(e)}")
-            return (None, submit_time_1, time.time())
-
-        end_time_2 = time.time()
-        return (v, submit_time_1, end_time_2)
-
-
-class DocFilter:
-    def __init__(
-        self,
-        llm: ByzerLLM,
-        args: AutoCoderArgs,
-        on_ray: bool = False,
-        path: Optional[str] = None,
-    ):
-        self.llm = llm
-        if self.llm.get_sub_client("recall_model"):
-            self.recall_llm = self.llm.get_sub_client("recall_model")
-        else:
-            self.recall_llm = self.llm
-        
-        self.args = args
-        self.relevant_score = self.args.rag_doc_filter_relevance or 5
-        self.on_ray = on_ray
-        self.path = path
-        if self.on_ray:
-            cpu_count = os.cpu_count() or 1
-            self.workers = [
-                DocFilterWorker.options(
-                    max_concurrency=1000, num_cpus=0).remote(llm)
-                for _ in range(cpu_count)
-            ]
-
-    def filter_docs(
-        self, conversations: List[Dict[str, str]], documents: List[SourceCode]
-    ) -> List[FilterDoc]:
-        if self.on_ray:
-            return self.filter_docs_with_ray(conversations, documents)
-        else:
-            return self.filter_docs_with_threads(conversations, documents)
-
-    def filter_docs_with_threads(
-        self, conversations: List[Dict[str, str]], documents: List[SourceCode]
-    ) -> List[FilterDoc]:
-
-        console = Console()
-        rag_manager = RagConfigManager(path=self.path)
-        rag_config = rag_manager.load_config()
-        documents = list(documents)
-        with Progress(
-            SpinnerColumn(),
-            TextColumn("[progress.description]{task.description}"),
-            BarColumn(),
-            TimeElapsedColumn(),
-            console=console,
-        ) as progress:
-            task = progress.add_task(
-                "[cyan]Filtering documents...", total=len(documents)
-            )
-
-            with ThreadPoolExecutor(
-                max_workers=self.args.index_filter_workers or 5
-            ) as executor:
-                future_to_doc = {}
-                for doc in documents:
-                    submit_time = time.time()
-
-                    def _run(conversations, docs):
-                        submit_time_1 = time.time()
-                        try:
-                            llm = ByzerLLM()
-                            llm.skip_nontext_check = True
-                            llm.setup_default_model_name(self.recall_llm.default_model_name)
-
-                            v = (
-                                _check_relevance_with_conversation.with_llm(
-                                    llm)
-                                .options({"llm_config": {"max_length": 10}})
-                                .run(
-                                    conversations=conversations,
-                                    documents=docs,
-                                    filter_config=rag_config.filter_config,
-                                )
-                            )
-                        except Exception as e:
-                            logger.error(
-                                f"Error in _check_relevance_with_conversation: {str(e)}"
-                            )
-                            return (None, submit_time_1, time.time())
-
-                        end_time_2 = time.time()
-                        return (v, submit_time_1, end_time_2)
-
-                    m = executor.submit(
-                        _run,
-                        conversations,
-                        [f"##File: {doc.module_name}\n{doc.source_code}"],
-                    )
-                    future_to_doc[m] = (doc, submit_time)
-
-            relevant_docs = []
-            for future in as_completed(list(future_to_doc.keys())):
-                try:
-                    doc, submit_time = future_to_doc[future]
-                    end_time = time.time()
-                    v, submit_time_1, end_time_2 = future.result()
-                    task_timing = TaskTiming(
-                        submit_time=submit_time,
-                        end_time=end_time,
-                        duration=end_time - submit_time,
-                        real_start_time=submit_time_1,
-                        real_end_time=end_time_2,
-                        real_duration=end_time_2 - submit_time_1,
-                    )
-                    progress.update(task, advance=1)
-
-                    relevance = parse_relevance(v)
-                    if (
-                        relevance
-                        and relevance.is_relevant
-                        and relevance.relevant_score >= self.relevant_score
-                    ):
-                        relevant_docs.append(
-                            FilterDoc(
-                                source_code=doc,
-                                relevance=relevance,
-                                task_timing=task_timing,
-                            )
-                        )
-                except Exception as exc:
-                    logger.error(
-                        f"Document processing generated an exception: {exc}")
-
-        # Sort relevant_docs by relevance score in descending order
-        relevant_docs.sort(
-            key=lambda x: x.relevance.relevant_score, reverse=True)
-        return relevant_docs
-
-    def filter_docs_with_ray(
-        self, conversations: List[Dict[str, str]], documents: List[SourceCode]
-    ) -> List[FilterDoc]:
-        futures = []
-        count = 0
-        submit_time = time.time()
-        for doc in documents:
-            worker = self.workers[count % len(self.workers)]
-            count += 1
-            future = worker.filter_doc.remote(
-                conversations, [
-                    f"##File: {doc.module_name}\n{doc.source_code}"]
-            )
-            futures.append((future, doc))
-
-        relevant_docs = []
-        for future, doc in futures:
-            try:
-                v, submit_time_1, end_time_2 = ray.get(future)
-                end_time = time.time()
-
-                if v is None:
-                    continue
-
-                task_timing = TaskTiming(
-                    submit_time=submit_time,
-                    end_time=end_time,
-                    duration=end_time - submit_time,
-                    real_start_time=submit_time_1,
-                    real_end_time=end_time_2,
-                    real_duration=end_time_2 - submit_time_1,
-                )
-                logger.info(
-                    f"Document: {doc.module_name} Duration: {task_timing.duration:.2f} seconds/{task_timing.real_duration:.2f}/{task_timing.real_duration-task_timing.duration} seconds"
-                )
-                relevance = parse_relevance(v)
-                if (
-                    relevance
-                    and relevance.is_relevant
-                    and relevance.relevant_score >= self.relevant_score
-                ):
-                    relevant_docs.append(
-                        FilterDoc(
-                            source_code=doc,
-                            relevance=relevance,
-                            task_timing=task_timing,
-                        )
-                    )
-            except Exception as exc:
-                logger.error(
-                    f"Document processing generated an exception: {exc}")
-
-        # Sort relevant_docs by relevance score in descending order
-        relevant_docs.sort(
-            key=lambda x: x.relevance.relevant_score, reverse=True)
-        return relevant_docs

auto_coder-0.1.219/src/autocoder/version.py

@@ -1 +0,0 @@
-__version__ = "0.1.219"