auto-coder 0.1.199__tar.gz → 0.1.201__tar.gz

{auto_coder-0.1.199 → auto_coder-0.1.201}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: auto-coder
-Version: 0.1.199
+Version: 0.1.201
 Summary: AutoCoder: AutoCoder
 Author: allwefantasy
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
@@ -49,6 +49,14 @@ Requires-Dist: python-pptx
 Requires-Dist: watchfiles
 Requires-Dist: cairosvg
 Requires-Dist: matplotlib
+Requires-Dist: mammoth
+Requires-Dist: markdownify
+Requires-Dist: pdfminer.six
+Requires-Dist: puremagic
+Requires-Dist: pydub
+Requires-Dist: youtube-transcript-api
+Requires-Dist: SpeechRecognition
+Requires-Dist: pathvalidate
 
 <p align="center">
   <picture>    

{auto_coder-0.1.199 → auto_coder-0.1.201}/src/auto_coder.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: auto-coder
-Version: 0.1.199
+Version: 0.1.201
 Summary: AutoCoder: AutoCoder
 Author: allwefantasy
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
@@ -49,6 +49,14 @@ Requires-Dist: python-pptx
 Requires-Dist: watchfiles
 Requires-Dist: cairosvg
 Requires-Dist: matplotlib
+Requires-Dist: mammoth
+Requires-Dist: markdownify
+Requires-Dist: pdfminer.six
+Requires-Dist: puremagic
+Requires-Dist: pydub
+Requires-Dist: youtube-transcript-api
+Requires-Dist: SpeechRecognition
+Requires-Dist: pathvalidate
 
 <p align="center">
   <picture>    

{auto_coder-0.1.199 → auto_coder-0.1.201}/src/auto_coder.egg-info/SOURCES.txt

@@ -102,6 +102,7 @@ src/autocoder/regexproject/__init__.py
 src/autocoder/suffixproject/__init__.py
 src/autocoder/tsproject/__init__.py
 src/autocoder/utils/__init__.py
+src/autocoder/utils/_markitdown.py
 src/autocoder/utils/conversation_store.py
 src/autocoder/utils/llm_client_interceptors.py
 src/autocoder/utils/log_capture.py

{auto_coder-0.1.199 → auto_coder-0.1.201}/src/auto_coder.egg-info/requires.txt

@@ -39,3 +39,11 @@ python-pptx
 watchfiles
 cairosvg
 matplotlib
+mammoth
+markdownify
+pdfminer.six
+puremagic
+pydub
+youtube-transcript-api
+SpeechRecognition
+pathvalidate

{auto_coder-0.1.199 → auto_coder-0.1.201}/src/autocoder/rag/cache/base_cache.py

@@ -6,7 +6,7 @@ class DeleteEvent(BaseModel):
     file_paths: List[str]
 
 class AddOrUpdateEvent(BaseModel):
-    file_infos: List[Tuple[str, str, float]]
+    file_infos: List[Tuple[str, str, float, str]]
 
 class BaseCacheManager(ABC):
     @abstractmethod

{auto_coder-0.1.199 → auto_coder-0.1.201}/src/autocoder/rag/cache/byzer_storage_cache.py

@@ -23,13 +23,34 @@ from multiprocessing import Pool
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from autocoder.rag.variable_holder import VariableHolder
 import platform
+import hashlib
+from typing import Union
 
 if platform.system() != "Windows":
     import fcntl
 else:
     fcntl = None
 
-
+def generate_file_md5(file_path: str) -> str:
+    md5_hash = hashlib.md5()
+    with open(file_path, "rb") as f:
+        for chunk in iter(lambda: f.read(4096), b""):
+            md5_hash.update(chunk)
+    return md5_hash.hexdigest()
+
+
+def generate_content_md5(content: Union[str, bytes]) -> str:
+    if isinstance(content, str):
+        content = content.encode("utf-8")
+    md5_hash = hashlib.md5()
+    md5_hash.update(content)
+    return md5_hash.hexdigest()
+
+default_ignore_dirs = [
+    "__pycache__",
+    "node_modules",
+    "_images"
+]
 class ByzerStorageCache(BaseCacheManager):
     def __init__(
         self,
@@ -154,10 +175,10 @@ class ByzerStorageCache(BaseCacheManager):
 
         files_to_process = []
         for file_info in self.get_all_files():
-            file_path, _, modify_time = file_info
+            file_path, _, modify_time, file_md5 = file_info
             if (
-                file_path not in self.cache
-                or self.cache[file_path]["modify_time"] < modify_time
+                file_path not in self.cache                
+                or self.cache[file_path]["md5"] != file_md5
             ):
                 files_to_process.append(file_info)
                 
@@ -175,13 +196,14 @@ class ByzerStorageCache(BaseCacheManager):
 
         items = []
         for file_info, result in zip(files_to_process, results):            
-            file_path, relative_path, modify_time = file_info
+            file_path, relative_path, modify_time, file_md5 = file_info
             content: List[SourceCode] = result
             self.cache[file_path] = {
                 "file_path": file_path,
                 "relative_path": relative_path,
                 "content": [c.model_dump() for c in content],
                 "modify_time": modify_time,
+                "md5": file_md5,
             }
 
             for doc in content:
@@ -295,11 +317,11 @@ class ByzerStorageCache(BaseCacheManager):
         files_to_process = []
         current_files = set()
         for file_info in self.get_all_files():
-            file_path, _, modify_time = file_info
+            file_path, _, _, file_md5 = file_info
             current_files.add(file_path)
             if (
                 file_path not in self.cache
-                or self.cache[file_path]["modify_time"] < modify_time
+                or self.cache[file_path]["md5"] != file_md5
             ):
                 files_to_process.append(file_info)
 
@@ -362,10 +384,10 @@ class ByzerStorageCache(BaseCacheManager):
 
                 
 
-    def get_all_files(self) -> List[Tuple[str, str, float]]:
+    def get_all_files(self) -> List[Tuple[str, str, float, str]]:
         all_files = []
         for root, dirs, files in os.walk(self.path,followlinks=True):
-            dirs[:] = [d for d in dirs if not d.startswith(".")]
+            dirs[:] = [d for d in dirs if not d.startswith(".") and d not in default_ignore_dirs]
 
             if self.ignore_spec:
                 relative_root = os.path.relpath(root, self.path)
@@ -389,6 +411,7 @@ class ByzerStorageCache(BaseCacheManager):
                 file_path = os.path.join(root, file)
                 relative_path = os.path.relpath(file_path, self.path)
                 modify_time = os.path.getmtime(file_path)
-                all_files.append((file_path, relative_path, modify_time))
+                file_md5 = generate_file_md5(file_path)
+                all_files.append((file_path, relative_path, modify_time, file_md5))
 
         return all_files

{auto_coder-0.1.199 → auto_coder-0.1.201}/src/autocoder/rag/cache/simple_cache.py

@@ -1,8 +1,8 @@
 
-from multiprocessing import Pool 
-from autocoder.common import SourceCode 
-from autocoder.rag.cache.base_cache import BaseCacheManager,DeleteEvent,AddOrUpdateEvent
-from typing import Dict, List, Tuple,Any,Optional
+from multiprocessing import Pool
+from autocoder.common import SourceCode
+from autocoder.rag.cache.base_cache import BaseCacheManager, DeleteEvent, AddOrUpdateEvent
+from typing import Dict, List, Tuple, Any, Optional, Union
 import os
 import threading
 import json
@@ -13,8 +13,31 @@ else:
     fcntl = None
 import time
 from loguru import logger
-from autocoder.rag.utils import process_file_in_multi_process,process_file_local
+from autocoder.rag.utils import process_file_in_multi_process, process_file_local
 from autocoder.rag.variable_holder import VariableHolder
+import hashlib
+
+default_ignore_dirs = [
+    "__pycache__",
+    "node_modules",
+    "_images"
+]
+
+
+def generate_file_md5(file_path: str) -> str:
+    md5_hash = hashlib.md5()
+    with open(file_path, "rb") as f:
+        for chunk in iter(lambda: f.read(4096), b""):
+            md5_hash.update(chunk)
+    return md5_hash.hexdigest()
+
+
+def generate_content_md5(content: Union[str, bytes]) -> str:
+    if isinstance(content, str):
+        content = content.encode("utf-8")
+    md5_hash = hashlib.md5()
+    md5_hash.update(content)
+    return md5_hash.hexdigest()
 
 
 class AutoCoderRAGAsyncUpdateQueue(BaseCacheManager):
@@ -52,10 +75,10 @@ class AutoCoderRAGAsyncUpdateQueue(BaseCacheManager):
                 return
             files_to_process = []
             for file_info in self.get_all_files():
-                file_path, _, modify_time = file_info
+                file_path, _, modify_time, file_md5 = file_info
                 if (
                     file_path not in self.cache
-                    or self.cache[file_path]["modify_time"] < modify_time
+                    or self.cache[file_path].get("md5","") != file_md5
                 ):
                     files_to_process.append(file_info)
             if not files_to_process:
@@ -71,7 +94,8 @@ class AutoCoderRAGAsyncUpdateQueue(BaseCacheManager):
                 initializer=initialize_tokenizer,
                 initargs=(VariableHolder.TOKENIZER_PATH,),
             ) as pool:
-                results = pool.map(process_file_in_multi_process, files_to_process)
+                results = pool.map(
+                    process_file_in_multi_process, files_to_process)
 
             for file_info, result in zip(files_to_process, results):
                 self.update_cache(file_info, result)
@@ -83,11 +107,11 @@ class AutoCoderRAGAsyncUpdateQueue(BaseCacheManager):
         files_to_process = []
         current_files = set()
         for file_info in self.get_all_files():
-            file_path, _, modify_time = file_info
+            file_path, _, _, file_md5 = file_info
             current_files.add(file_path)
             if (
                 file_path not in self.cache
-                or self.cache[file_path]["modify_time"] < modify_time
+                or self.cache[file_path].get("md5","") != file_md5
             ):
                 files_to_process.append(file_info)
 
@@ -99,7 +123,8 @@ class AutoCoderRAGAsyncUpdateQueue(BaseCacheManager):
                 self.queue.append(DeleteEvent(file_paths=deleted_files))
         if files_to_process:
             with self.lock:
-                self.queue.append(AddOrUpdateEvent(file_infos=files_to_process))
+                self.queue.append(AddOrUpdateEvent(
+                    file_infos=files_to_process))
 
     def process_queue(self):
         while self.queue:
@@ -111,8 +136,12 @@ class AutoCoderRAGAsyncUpdateQueue(BaseCacheManager):
             elif isinstance(file_list, AddOrUpdateEvent):
                 for file_info in file_list.file_infos:
                     logger.info(f"{file_info[0]} is detected to be updated")
-                    result = process_file_local(file_info[0])
-                    self.update_cache(file_info, result)
+                    try:
+                        result = process_file_local(file_info[0])
+                        self.update_cache(file_info, result)
+                    except Exception as e:
+                        logger.error(
+                            f"SimpleCache Error in process_queue: {e}")
 
             self.write_cache()
 
@@ -138,8 +167,12 @@ class AutoCoderRAGAsyncUpdateQueue(BaseCacheManager):
         if not fcntl:
             with open(cache_file, "w") as f:
                 for data in self.cache.values():
-                    json.dump(data, f, ensure_ascii=False)
-                    f.write("\n")
+                    try:
+                        json.dump(data, f, ensure_ascii=False)
+                        f.write("\n")
+                    except Exception as e:
+                        logger.error(
+                            f"Failed to write {data['file_path']} to .cache/cache.jsonl: {e}")
         else:
             lock_file = cache_file + ".lock"
             with open(lock_file, "w") as lockf:
@@ -149,33 +182,39 @@ class AutoCoderRAGAsyncUpdateQueue(BaseCacheManager):
                     # 写入缓存文件
                     with open(cache_file, "w") as f:
                         for data in self.cache.values():
-                            json.dump(data, f, ensure_ascii=False)
-                            f.write("\n")
+                            try:
+                                json.dump(data, f, ensure_ascii=False)
+                                f.write("\n")
+                            except Exception as e:
+                                logger.error(
+                                    f"Failed to write {data['file_path']} to .cache/cache.jsonl: {e}")
 
                 finally:
                     # 释放文件锁
                     fcntl.flock(lockf, fcntl.LOCK_UN)
 
     def update_cache(
-        self, file_info: Tuple[str, str, float], content: List[SourceCode]
+        self, file_info: Tuple[str, str, float, str], content: List[SourceCode]
     ):
-        file_path, relative_path, modify_time = file_info
+        file_path, relative_path, modify_time, file_md5 = file_info
         self.cache[file_path] = {
             "file_path": file_path,
             "relative_path": relative_path,
             "content": [c.model_dump() for c in content],
             "modify_time": modify_time,
+            "md5": file_md5,
         }
 
-    def get_cache(self,options:Optional[Dict[str,Any]]=None):
+    def get_cache(self, options: Optional[Dict[str, Any]] = None):
         self.load_first()
         self.trigger_update()
         return self.cache
 
     def get_all_files(self) -> List[Tuple[str, str, float]]:
         all_files = []
-        for root, dirs, files in os.walk(self.path,followlinks=True):
-            dirs[:] = [d for d in dirs if not d.startswith(".")]
+        for root, dirs, files in os.walk(self.path, followlinks=True):
+            dirs[:] = [d for d in dirs if not d.startswith(
+                ".") and d not in default_ignore_dirs]
 
             if self.ignore_spec:
                 relative_root = os.path.relpath(root, self.path)
@@ -199,6 +238,8 @@ class AutoCoderRAGAsyncUpdateQueue(BaseCacheManager):
                 file_path = os.path.join(root, file)
                 relative_path = os.path.relpath(file_path, self.path)
                 modify_time = os.path.getmtime(file_path)
-                all_files.append((file_path, relative_path, modify_time))
+                file_md5 = generate_file_md5(file_path)
+                all_files.append(
+                    (file_path, relative_path, modify_time, file_md5))
 
-        return all_files
+        return all_files

{auto_coder-0.1.199 → auto_coder-0.1.201}/src/autocoder/rag/document_retriever.py

@@ -13,13 +13,6 @@ from loguru import logger
 from pydantic import BaseModel
 
 from autocoder.common import SourceCode
-from autocoder.rag.loaders import (
-    extract_text_from_docx,
-    extract_text_from_excel,
-    extract_text_from_pdf,
-    extract_text_from_ppt,
-)
-from autocoder.rag.token_counter import count_tokens_worker, count_tokens
 from uuid import uuid4
 from autocoder.rag.variable_holder import VariableHolder
 from abc import ABC, abstractmethod
@@ -119,7 +112,8 @@ class LocalDocumentRetriever(BaseDocumentRetriever):
         logger.info(f"DocumentRetriever initialized with:")
         logger.info(f"  Path: {self.path}")
         logger.info(f"  Diable auto window: {self.disable_auto_window} ")
-        logger.info(f"  Single file token limit: {self.single_file_token_limit}")
+        logger.info(
+            f"  Single file token limit: {self.single_file_token_limit}")
         logger.info(f"  Small file token limit: {self.small_file_token_limit}")
         logger.info(f"  Small file merge limit: {self.small_file_merge_limit}")
         logger.info(f"  Enable hybrid index: {self.enable_hybrid_index}")
@@ -200,9 +194,10 @@ class LocalDocumentRetriever(BaseDocumentRetriever):
     ) -> Generator[SourceCode, None, None]:
         chunk_size = self.single_file_token_limit
         total_chunks = (doc.tokens + chunk_size - 1) // chunk_size
-        logger.info(f"Splitting document {doc.module_name} into {total_chunks} chunks")
+        logger.info(
+            f"Splitting document {doc.module_name} into {total_chunks} chunks")
         for i in range(0, doc.tokens, chunk_size):
-            chunk_content = doc.source_code[i : i + chunk_size]
+            chunk_content = doc.source_code[i: i + chunk_size]
             chunk_tokens = min(chunk_size, doc.tokens - i)
             chunk_name = f"{doc.module_name}#chunk{i//chunk_size+1}"
             # logger.debug(f"  Created chunk: {chunk_name} (tokens: {chunk_tokens})")

auto_coder-0.1.201/src/autocoder/rag/loaders/docx_loader.py

@@ -0,0 +1,20 @@
+from io import BytesIO
+import docx2txt
+from autocoder.utils._markitdown import MarkItDown
+
+
+def extract_text_from_docx_old(docx_path):
+    with open(docx_path, "rb") as f:
+        docx_content = f.read()
+    docx_file = BytesIO(docx_content)
+    text = docx2txt.process(docx_file)
+    return text
+
+
+def extract_text_from_docx(docx_path):
+    try:
+        md_converter = MarkItDown()
+        result = md_converter.convert(docx_path)
+        return result.text_content
+    except Exception as e:
+        return extract_text_from_docx_old(docx_path)

auto_coder-0.1.201/src/autocoder/rag/loaders/pdf_loader.py

@@ -0,0 +1,22 @@
+from io import BytesIO
+from pypdf import PdfReader
+from autocoder.utils._markitdown import MarkItDown
+
+
+def extract_text_from_pdf_old(file_path):
+    with open(file_path, "rb") as f:
+        pdf_content = f.read()
+    pdf_file = BytesIO(pdf_content)
+    pdf_reader = PdfReader(pdf_file)
+    text = ""
+    for page in pdf_reader.pages:
+        text += page.extract_text()
+    return text
+
+def extract_text_from_pdf(file_path):
+    try:        
+        md_converter = MarkItDown()
+        result = md_converter.convert(file_path)
+        return result.text_content
+    except Exception as e:        
+        return extract_text_from_pdf_old(file_path)

{auto_coder-0.1.199 → auto_coder-0.1.201}/src/autocoder/rag/long_context_rag.py

@@ -210,6 +210,9 @@ class LongContextRAG:
 
         问题：{{ query }}
 
+        要求：
+        1. 注意相应的markdown图片如果存在也要输出,尽可能图文并茂
+        
         回答：
         """
 

{auto_coder-0.1.199 → auto_coder-0.1.201}/src/autocoder/rag/token_limiter.py

@@ -113,7 +113,7 @@ class TokenLimiter:
                 num_count += 1
                 reorder_relevant_docs.append(doc)
                 if "original_doc" in doc.metadata and "chunk_index" in doc.metadata:
-                    original_doc_name = doc.metadata["original_doc"].module_name
+                    original_doc_name = doc.metadata["original_doc"]
 
                     temp_docs = []
                     for temp_doc in relevant_docs[num_count:]:
@@ -122,7 +122,7 @@ class TokenLimiter:
                             and "chunk_index" in temp_doc.metadata
                         ):
                             if (
-                                temp_doc.metadata["original_doc"].module_name
+                                temp_doc.metadata["original_doc"]
                                 == original_doc_name
                             ):
                                 if temp_doc not in reorder_relevant_docs:

{auto_coder-0.1.199 → auto_coder-0.1.201}/src/autocoder/rag/utils.py

@@ -13,11 +13,10 @@ def process_file_in_multi_process(
     file_info: Tuple[str, str, float]
 ) -> List[SourceCode]:
     start_time = time.time()
-    file_path, relative_path, _ = file_info
+    file_path, relative_path, _, _ = file_info
     try:
-        if file_path.endswith(".pdf"):
-            with open(file_path, "rb") as f:
-                content = extract_text_from_pdf(f.read())
+        if file_path.endswith(".pdf"):            
+            content = extract_text_from_pdf(file_path)
             v = [
                 SourceCode(
                     module_name=file_path,
@@ -25,9 +24,8 @@ def process_file_in_multi_process(
                     tokens=count_tokens_worker(content),
                 )
             ]
-        elif file_path.endswith(".docx"):
-            with open(file_path, "rb") as f:
-                content = extract_text_from_docx(f.read())
+        elif file_path.endswith(".docx"):            
+            content = extract_text_from_docx(file_path)
             v = [
                 SourceCode(
                     module_name=f"##File: {file_path}",
@@ -75,9 +73,8 @@ def process_file_in_multi_process(
 def process_file_local(file_path: str) -> List[SourceCode]:
     start_time = time.time()
     try:
-        if file_path.endswith(".pdf"):
-            with open(file_path, "rb") as f:
-                content = extract_text_from_pdf(f.read())
+        if file_path.endswith(".pdf"):            
+            content = extract_text_from_pdf(file_path)
             v = [
                 SourceCode(
                     module_name=file_path,
@@ -85,9 +82,8 @@ def process_file_local(file_path: str) -> List[SourceCode]:
                     tokens=count_tokens(content),
                 )
             ]
-        elif file_path.endswith(".docx"):
-            with open(file_path, "rb") as f:
-                content = extract_text_from_docx(f.read())
+        elif file_path.endswith(".docx"):            
+            content = extract_text_from_docx(file_path)
             v = [
                 SourceCode(
                     module_name=f"##File: {file_path}",