auto-coder 0.1.374__py3-none-any.whl → 0.1.376__py3-none-any.whl

autocoder/rag/simple_rag.py

@@ -1,404 +0,0 @@
-from typing import Any, Dict, List, Optional, Tuple
-from autocoder.common import SourceCode, AutoCoderArgs
-from autocoder.common.llm_rerank import LLMRerank
-from autocoder.rag.simple_directory_reader import AutoCoderSimpleDirectoryReader
-import fsspec
-
-from byzerllm.apps.llama_index.simple_retrieval import SimpleRetrieval
-from byzerllm.apps.llama_index.collection_manager import (
-    CollectionManager,
-    CollectionItem,
-)
-from byzerllm.utils.ray_utils import is_ray_in_client_mode
-
-from llama_index.core import QueryBundle, StorageContext
-from llama_index.core.readers.file.base import default_file_metadata_func
-from llama_index.core import VectorStoreIndex
-from llama_index.core.node_parser import HierarchicalNodeParser, get_leaf_nodes
-from llama_index.core.base.llms.types import ChatMessage, MessageRole
-from llama_index.core.retrievers import AutoMergingRetriever
-from llama_index.core.query_engine import RetrieverQueryEngine, RouterQueryEngine
-from llama_index.core.tools import QueryEngineTool
-from llama_index.core.selectors import (
-    LLMSingleSelector,
-)
-
-import byzerllm
-from loguru import logger
-import hashlib
-import json
-from openai import OpenAI
-
-
-def file_metadata_func(
-    file_path: str, fs: Optional[fsspec.AbstractFileSystem] = None
-) -> Dict:
-    """Get some handy metadata from filesystem.
-
-    Args:
-        file_path: str: file path in str
-    """
-
-    def generate_file_md5(file_path: str) -> str:
-        md5_hash = hashlib.md5()
-        with open(file_path, "rb") as f:
-            for chunk in iter(lambda: f.read(4096), b""):
-                md5_hash.update(chunk)
-        return md5_hash.hexdigest()
-
-    return {
-        **default_file_metadata_func(file_path=file_path, fs=fs),
-        "md5": generate_file_md5(file_path=file_path),
-    }
-
-
-class SimpleRAG:
-    def __init__(self, llm: byzerllm.ByzerLLM, args: AutoCoderArgs, path: str) -> None:
-        from byzerllm.apps.llama_index import get_service_context, get_storage_context
-
-        self.llm = llm
-        self.args = args
-        self.path = path
-        self.collection_manager = CollectionManager(args.base_dir)
-
-        self.collections = self.args.collection
-        if self.args.collections:
-            self.collections = self.args.collections
-
-        self.collections = self.collections.split(",") if self.collections else []
-
-        if not self.collections:
-            logger.warning(
-                """No RAG collection is set, we will use the `default` collection.
-You can set the collection by passing the `--collections`argument in command line or set the `collections` attribute in the config file."""
-            )
-            self.collections = ["default"]
-
-        if not self.llm.default_emb_model_name:
-            raise ValueError("emb_model should be set")
-
-        self.retrieval = byzerllm.ByzerRetrieval()
-
-        if not is_ray_in_client_mode():
-            self.retrieval.launch_gateway()
-            self.service_context = get_service_context(self.llm)
-            self.storage_context_dict: Dict[str, StorageContext] = {}
-            for collection in self.collections:
-                self.storage_context_dict[collection] = get_storage_context(
-                    self.llm,
-                    self.retrieval,
-                    chunk_collection=collection,
-                    namespace=collection,
-                )
-
-            self.simple_retrieval_dict: Dict[str, SimpleRetrieval] = {}
-            for collection in self.collections:
-                self.simple_retrieval_dict[collection] = SimpleRetrieval(
-                    llm=llm, retrieval=self.retrieval, chunk_collection=collection
-                )
-        else:
-            if not args.rag_url:
-                raise ValueError(
-                    "You are in client mode, please provide the RAG URL. e.g. rag_url: http://localhost:8000/v1"
-                )
-            
-            if not args.rag_url.startswith("http://"):
-                raise ValueError("The RAG URL should start with http://")
-            
-            self.client = OpenAI(api_key=args.rag_token, base_url=args.rag_url)
-
-    def _get_indices(self) -> List[Tuple[CollectionItem, VectorStoreIndex]]:
-        indices = []
-        for collection in self.collections:
-            storage_context = self.storage_context_dict[collection]
-            index = VectorStoreIndex.from_vector_store(
-                vector_store=storage_context.vector_store,
-                service_context=self.service_context,
-            )
-            collection_item = self.collection_manager.get_collection(collection)
-            indices.append((collection_item, index))
-
-        return indices
-
-    def _get_query_engine(self, streaming: bool = False):
-        indices = self._get_indices()
-        retrievers = []
-
-        for collection_item, index in indices:
-            retriever = AutoMergingRetriever(
-                index.as_retriever(),
-                storage_context=self.storage_context_dict[collection_item.name],
-            )
-            retrievers.append(retriever)
-
-        query_engines = [
-            RetrieverQueryEngine.from_args(
-                retriever, service_context=self.service_context, streaming=streaming
-            )
-            for retriever in retrievers
-        ]
-
-        if len(query_engines) == 1:
-            return query_engines[0]
-
-        tools = []
-        for (collection_item, index), query_engine in zip(indices, query_engines):
-            tool = QueryEngineTool.from_defaults(
-                query_engine=query_engine,
-                description=collection_item.description,
-            )
-            tools.append(tool)
-
-        query_engine = RouterQueryEngine(
-            selector=LLMSingleSelector.from_defaults(
-                service_context=self.service_context
-            ),
-            query_engine_tools=tools,
-            llm=self.service_context.llm,
-            service_context=self.service_context,
-            verbose=True,
-        )
-        return query_engine
-
-    def _get_retriever(self):
-        indices = self._get_indices()
-        retrievers = []
-
-        for collection_item, index in indices:
-            retriever = AutoMergingRetriever(
-                index.as_retriever(),
-                storage_context=self.storage_context_dict[collection_item.name],
-            )
-            retrievers.append(retriever)
-
-        return retrievers
-
-    def stream_search(self, query: str):
-        query_bundle = QueryBundle(query_str=query)
-        query_engine = self._get_query_engine(streaming=True)
-        streaming_response = query_engine.query(query_bundle)
-        contexts = []
-        for node in streaming_response.source_nodes:
-            contexts.append(
-                {
-                    "raw_chunk": node.node.text,
-                    "doc_url": node.node.metadata["file_path"],
-                    "_id": node.node.id_,
-                }
-            )
-        return streaming_response.response_gen, contexts
-
-    def retrieve(self, query: str) -> List[Dict[str, Any]]:
-        query_bundle = QueryBundle(query_str=query)
-        retrievers = self._get_retriever()
-
-        result = []
-
-        for retriever in retrievers:
-            nodes = retriever.retrieve(query_bundle)
-
-            reranker = LLMRerank(llm=self.llm)
-            retrieved_nodes = reranker.postprocess_nodes(
-                nodes, query_bundle, choice_batch_size=5, top_n=1
-            )
-            result.extend(
-                [
-                    {
-                        "raw_chunk": node.node.text,
-                        "doc_url": node.node.metadata["file_path"],
-                        "_id": node.node.id_,
-                    }
-                    for node in retrieved_nodes
-                ]
-            )
-        return result
-
-    def stream_chat_oai(
-        self,
-        conversations,
-        model: Optional[str] = None,
-        role_mapping=None,
-        llm_config: Dict[str, Any] = {},
-    ):
-        if len(self.collections) != 1:
-            raise ValueError("When chat mode, only one collection can be set")
-
-        index = VectorStoreIndex.from_vector_store(
-            vector_store=self.storage_context_dict[self.collections[0]].vector_store,
-            service_context=self.service_context,
-        )
-        chat_engine = index.as_chat_engine(
-            chat_mode="condense_plus_context",
-            verbose=False,
-        )
-        history = []
-        for conv in conversations[:-1]:
-            if conv["role"] == "user":
-                role = MessageRole.USER
-            elif conv["role"] == "assistant":
-                role = MessageRole.ASSISTANT
-            else:
-                role = MessageRole.SYSTEM
-            history.append(ChatMessage(role=role, content=conv["content"]))
-        return (
-            chat_engine.stream_chat(
-                conversations[-1]["content"], chat_history=history
-            ).response_gen,
-            [],
-        )
-
-    def stream_chat_repl(self, query: str):
-        if len(self.collections) != 1:
-            raise ValueError("When chat mode, only one collection can be set")
-        from llama_index.core.memory import ChatMemoryBuffer
-
-        memory = ChatMemoryBuffer.from_defaults(token_limit=8092)
-        index = VectorStoreIndex.from_vector_store(
-            vector_store=self.storage_context_dict[self.collections[0]].vector_store,
-            service_context=self.service_context,
-        )
-        chat_engine = index.as_chat_engine(
-            chat_mode="condense_plus_context",
-            memory=memory,
-            verbose=False,
-        )
-        chat_engine.streaming_chat_repl()
-
-    def search(self, query: str) -> List[SourceCode]:
-        if not is_ray_in_client_mode():
-            return self.inner_search(query)
-
-        target_query = query
-
-        if isinstance(self.args.enable_rag_search, str):
-            target_query = self.args.enable_rag_search
-
-        response = self.client.chat.completions.create(
-            messages=[{"role": "user", "content": target_query}],
-            model="xxxx",
-        )
-        return [
-            SourceCode(
-                module_name=f"RAG:{target_query}",
-                source_code=response.choices[0].message.content,
-            )
-        ]
-
-    def inner_search(self, query: str) -> List[SourceCode]:
-        if self.args.enable_rag_search:
-            target_query = query
-            if isinstance(self.args.enable_rag_search, str):
-                target_query = self.args.enable_rag_search
-            texts, contexts = self.stream_search(target_query)
-            s = "".join([text for text in texts])
-            urls = ",".join(set([context["doc_url"] for context in contexts]))
-            ## append RAG: prefix is used to protect avoid the source code is modified by the code auto execute
-            return [SourceCode(module_name=f"RAG:{urls}", source_code=s)]
-        elif self.args.enable_rag_context:
-            target_query = query
-            if isinstance(self.args.enable_rag_context, str):
-                target_query = self.args.enable_rag_context
-            contexts = self.retrieve(target_query)
-            for context in contexts:
-                context["raw_chunk"]
-                try:
-                    with open(context["doc_url"], "r") as f:
-                        context["raw_chunk"] = f.read()
-                except Exception as e:
-                    logger.warning(f"Error reading file {context['doc_url']}")
-                    pass
-
-            return [
-                SourceCode(
-                    module_name=context["doc_url"], source_code=context["raw_chunk"]
-                )
-                for context in contexts
-            ]
-        return []
-
-    def build(self):
-
-        if len(self.collections) != 1:
-            raise ValueError("When build, only one collection should be set")
-
-        if is_ray_in_client_mode():
-            raise ValueError(
-                "You are in client mode, please run the build in the server."
-            )
-
-        collection = self.collections[0]
-
-        collection_exists = self.collection_manager.check_collection_exists(collection)
-        if not collection_exists:
-            logger.warning(
-                f"Collection {collection} not found, creating it automatically"
-            )
-            if not self.args.description:
-                logger.error("Please provide a description for the collection")
-                return
-            item = CollectionItem(
-                name=collection, description=self.args.description or ""
-            )
-            self.collection_manager.add_collection(item)
-
-        retrieval_client = self.simple_retrieval_dict[collection]
-        # retrieval_client.delete_from_doc_collection(collection)
-        # retrieval_client.delete_from_chunk_collection(collection)
-
-        required_exts = self.args.required_exts or None
-
-        if required_exts:
-            required_exts = required_exts.split(",")
-
-        documents = AutoCoderSimpleDirectoryReader(
-            self.path,
-            recursive=True,
-            filename_as_id=True,
-            required_exts=required_exts,
-            file_metadata=file_metadata_func,
-        ).load_data()
-        docs_keep = []
-        for document in documents:
-            doc = retrieval_client.get_doc(
-                f"ref_doc_info/{document.doc_id}", collection
-            )
-            if doc:
-                md5 = json.loads(doc["json_data"])["metadata"].get("md5", "")
-                file_path = document.metadata["file_path"]
-                new__md5 = document.metadata["md5"]
-                if md5 != new__md5:
-                    retrieval_client.delete_doc_and_chunks_by_filename(
-                        collection, file_path
-                    )
-                    docs_keep.append(document)
-            else:
-                docs_keep.append(document)
-
-        retrieval_client.commit_doc()
-        retrieval_client.commit_chunk()
-
-        for document in docs_keep:
-            logger.info(f"\nUpsert {document.doc_id}")
-
-        if docs_keep:
-            hirerachical = HierarchicalNodeParser.from_defaults(
-                chunk_sizes=[6000, 3000, 1024]
-            )
-            # sp = SentenceSplitter(chunk_size=1024, chunk_overlap=0)
-
-            nodes = hirerachical.get_nodes_from_documents(docs_keep, show_progress=True)
-
-            leaf_nodes = get_leaf_nodes(nodes)
-            self.storage_context_dict[collection].docstore.add_documents(nodes)
-
-            _ = VectorStoreIndex(
-                nodes=leaf_nodes,
-                store_nodes_override=True,
-                storage_context=self.storage_context_dict[collection],
-                service_context=self.service_context,
-            )
-
-            retrieval_client.commit_doc()
-            retrieval_client.commit_chunk()
-        else:
-            logger.info("There is no new doc to build")

autocoder/regex_project/__init__.py

@@ -1,162 +0,0 @@
-import re
-from autocoder.common import SourceCode,AutoCoderArgs
-from autocoder import common as FileUtils  
-from autocoder.utils.rest import HttpDoc
-import os
-from typing import Optional, Generator, List, Dict, Any, Callable
-from git import Repo
-import byzerllm
-from autocoder.common.search import Search,SearchEngine
-from autocoder.rag.simple_rag import SimpleRAG
-from loguru import logger
-from pydantic import BaseModel,Field
-
-class RegPattern(BaseModel):
-    pattern: str = Field(..., title="Pattern", description="The regex pattern can be used by `re.search` in python.")
-
-class RegexProject():
-    
-    def __init__(self, args: AutoCoderArgs, llm: Optional[byzerllm.ByzerLLM] = None,file_filter=None):
-        self.args = args
-        self.directory = args.source_dir        
-        self.git_url = args.git_url        
-        self.target_file = args.target_file  
-        self.project_type = args.project_type        
-        self.file_filter = file_filter
-        self.sources = []
-        self.llm = llm   
-        self.regex_pattern = self.extract_regex_pattern(self.project_type)
-
-    @byzerllm.prompt()
-    def generate_regex_pattern(self,desc:str)->RegPattern:
-        '''
-        Generate a regex pattern based on the following description:
-
-        {{ desc }}              
-        '''
-        
-
-    def extract_regex_pattern(self, project_type):
-        project_type = project_type.strip()
-        if project_type.startswith("regex://"):
-            return project_type[8:]
-        if project_type.startswith("human://"):
-            desc = project_type[8:]
-            v = self.generate_regex_pattern.with_llm(self.llm).run(desc=desc)
-            if not v:
-                raise ValueError("Fail to generate regex pattern, try again.")
-            logger.info(f"Generated regex pattern: {v.pattern}")
-            return v.pattern
-        else:
-            raise ValueError("Invalid project_type format. Expected 'regex//<pattern>'")
-
-    def output(self):
-        return open(self.target_file, "r").read()                
-
-    def is_regex_match(self, file_path):
-        return re.search(self.regex_pattern, file_path) is not None
-
-    def read_file_content(self, file_path):
-        with open(file_path, "r") as file:
-            return file.read()
-
-    def convert_to_source_code(self, file_path):                               
-        module_name = file_path
-        source_code = self.read_file_content(file_path)            
-        return SourceCode(module_name=module_name, source_code=source_code)
-    
-    def get_source_codes(self) -> Generator[SourceCode, None, None]:
-        for root, dirs, files in os.walk(self.directory):
-            for file in files:
-                file_path = os.path.join(root, file)                            
-                if self.is_regex_match(file_path):                
-                    if self.file_filter is None or self.file_filter(file_path, [self.regex_pattern]):
-                        logger.info(f"collect file: {file_path}")
-                        source_code = self.convert_to_source_code(file_path)
-                        if source_code is not None:
-                            yield source_code
-
-    def get_rest_source_codes(self) -> Generator[SourceCode, None, None]:
-        if self.args.urls:
-            urls = self.args.urls
-            if isinstance(self.args.urls, str):
-                urls = self.args.urls.split(",") 
-            http_doc = HttpDoc(args =self.args, llm=self.llm,urls=urls)
-            sources = http_doc.crawl_urls()    
-            for source in sources:
-                source.tag = "REST"     
-            return sources
-        return []  
-    
-    def get_rag_source_codes(self):        
-        if not self.args.enable_rag_search and not self.args.enable_rag_context:
-            return []
-        rag = SimpleRAG(self.llm,self.args,self.args.source_dir)
-        docs = rag.search(self.args.query)
-        for doc in docs:
-            doc.tag = "RAG"
-        return docs
-
-    def get_search_source_codes(self):
-        temp = self.get_rag_source_codes()
-        if self.args.search_engine and self.args.search_engine_token:
-            if self.args.search_engine == "bing":
-                search_engine = SearchEngine.BING
-            else:
-                search_engine = SearchEngine.GOOGLE
-
-            searcher=Search(args=self.args,llm=self.llm,search_engine=search_engine,subscription_key=self.args.search_engine_token)
-            search_query = self.args.search or self.args.query
-            search_context = searcher.answer_with_the_most_related_context(search_query)  
-            return temp + [SourceCode(module_name="SEARCH_ENGINE", source_code=search_context,tag="SEARCH")]
-        return temp + []                             
-
-    def run(self):
-        if self.git_url is not None:
-            self.clone_repository()        
-
-        if self.target_file is None:   
-            for code in self.get_source_codes():
-                self.sources.append(code)
-                print(f"##File: {code.module_name}")
-                print(code.source_code)   
-
-            for code in self.get_rest_source_codes():
-                self.sources.append(code)
-                print(f"##File: {code.module_name}")
-                print(code.source_code)
-
-            for code in self.get_search_source_codes():
-                self.sources.append(code)
-                print(f"##File: {code.module_name}")
-                print(code.source_code)    
-
-                         
-        else:            
-            with open(self.target_file, "w") as file:
-                for code in self.get_source_codes():                    
-                    self.sources.append(code)
-                    file.write(f"##File: {code.module_name}\n")
-                    file.write(f"{code.source_code}\n\n")
-
-                for code in self.get_rest_source_codes():
-                    self.sources.append(code)
-                    file.write(f"##File: {code.module_name}\n")
-                    file.write(f"{code.source_code}\n\n")
-
-                for code in self.get_search_source_codes():
-                    self.sources.append(code)
-                    file.write(f"##File: {code.module_name}\n")
-                    file.write(f"{code.source_code}\n\n")    
-                                    
-                    
-    
-    def clone_repository(self):   
-        if self.git_url is None:
-            raise ValueError("git_url is required to clone the repository")
-             
-        if os.path.exists(self.directory):
-            print(f"Directory {self.directory} already exists. Skipping cloning.")
-        else:
-            print(f"Cloning repository {self.git_url} into {self.directory}")
-            Repo.clone_from(self.git_url, self.directory)

autocoder/utils/coder.py

@@ -1,125 +0,0 @@
-
-from autocoder.common import AutoCoderArgs
-import byzerllm
-import pydantic
-from enum import Enum
-
-class TaskTypeDef(pydantic.BaseModel):
-    name: str
-    desc: str = ""
-    guidance: str = ""
-
-
-class TaskType(Enum):
-    """By identifying specific types of tasks, we can inject human priors (guidance) to help task solving"""
-    
-    @byzerllm.prompt(render="jinja2")
-    def data_preprocess_prompt():
-        '''
-        The current task is about data preprocessing, please note the following:
-        - Monitor data types per column, applying appropriate methods.
-        - Ensure operations are on existing dataset columns.
-        - Avoid writing processed data to files.
-        - Avoid any change to label column, such as standardization, etc.
-        - Prefer alternatives to one-hot encoding for categorical data.
-        - Only encode or scale necessary columns to allow for potential feature-specific engineering tasks (like time_extract, binning, extraction, etc.) later.
-        - Each step do data preprocessing to train, must do same for test separately at the same time.
-        - Always copy the DataFrame before processing it and use the copy to process.
-        '''
-
-    @byzerllm.prompt(render="jinja2")
-    def image2webpage_prompt():
-        '''
-        The current task is about converting image into webpage code. please note the following:
-        - Single-Step Code Generation: Execute the entire code generation process in a single step, encompassing HTML, CSS, and JavaScript. Avoid fragmenting the code generation into multiple separate steps to maintain consistency and simplify the development workflow.
-        - Save webpages: Be sure to use the save method provided.
-        '''    
-    
-    DATA_PREPROCESS = TaskTypeDef(
-        name="data preprocessing",
-        desc="For preprocessing dataset in a data analysis or machine learning task ONLY,"
-        "general data operation doesn't fall into this type",
-        guidance=data_preprocess_prompt(),
-    )
-    
-    IMAGE2WEBPAGE = TaskTypeDef(
-        name="image2webpage",
-        desc="For converting image into webpage code.",
-        guidance=image2webpage_prompt(),
-    )
-    OTHER = TaskTypeDef(name="other", desc="Any tasks not in the defined categories")
-
-    @property
-    def type_name(self):
-        return self.value.name
-
-    @classmethod
-    def get_type(cls, type_name):
-        for member in cls:
-            if member.type_name == type_name:
-                return member.value
-        return None
-
-class Thought(pydantic.BaseModel):
-    thoughts:str = pydantic.Field(...,description="Thoughts on current situation, reflect on how you should proceed to fulfill the user requirement")
-    state:bool = pydantic.Field(...,description="Decide whether you need to take more actions to complete the user requirement. Return true if you think so. Return false if you think the requirement has been completely fulfilled.")
-
-class Plan(pydantic.BaseModel):
-    task_id:str = pydantic.Field(...,description="unique identifier for a task in plan, can be an ordinal")
-    dependent_task_ids:list[str] = pydantic.Field(...,description="ids of tasks prerequisite to this task")
-    instruction:str = pydantic.Field(...,description="what you should do in this task, one short phrase or sentence")
-    task_type:str = pydantic.Field(...,description="type of this task, should be one of Available Task Types")
-
-class Coder:
-    def __init__(self,llm:byzerllm.ByzerLLM,args:AutoCoderArgs) -> None:
-        self.llm = llm
-        self.args = args
-        self.working_memory = []
-
-    def get_task_type_desc(self):
-        task_type_desc = "\n".join([f"- **{tt.type_name}**: {tt.value.desc}" for tt in TaskType])    
-        return task_type_desc
-    
-    @byzerllm.prompt(llm=lambda self:self.llm,render="jinja2")
-    def write_plan(self,context:str,task_type_desc:str,max_tasks:int=3):
-        '''
-        # Context:
-        {{ context }}
-        # Available Task Types:
-        {{ task_type_desc}}
-        # Task:
-        Based on the context, write a plan or modify an existing plan of what you should do to achieve the goal. A plan consists of one to {{ max_tasks }} tasks.
-        If you are modifying an existing plan, carefully follow the instruction, don't make unnecessary changes. Give the whole plan unless instructed to modify only one task of the plan.
-        If you encounter errors on the current task, revise and output the current single task only.
-        Output a list of jsons following the format:
-        ```json
-        [
-            {
-                "task_id": str = "unique identifier for a task in plan, can be an ordinal",
-                "dependent_task_ids": list[str] = "ids of tasks prerequisite to this task",
-                "instruction": "what you should do in this task, one short phrase or sentence",
-                "task_type": "type of this task, should be one of Available Task Types",
-            },
-            ...
-        ]
-        ```
-        '''
-        pass
-     
-    @byzerllm.prompt(llm=lambda self:self.llm,render="jinja2")
-    def react_think(self,user_requirement:str,context:str)->str:
-        '''
-        # User Requirement
-        {{ user_requirement }}
-        # Context
-        {{ context }}
-
-        Output a json following the format:
-        ```json
-        {
-            "thoughts": str = "Thoughts on current situation, reflect on how you should proceed to fulfill the user requirement",
-            "state": bool = "Decide whether you need to take more actions to complete the user requirement. Return true if you think so. Return false if you think the requirement has been completely fulfilled."
-        }
-        ```
-        '''
-        pass