bioguider 0.2.6__py3-none-any.whl → 0.2.7__py3-none-any.whl

bioguider/agents/agent_tools.py

@@ -1,9 +1,11 @@
 import os
 from typing import Callable
+from markdownify import markdownify as md
 from langchain_openai.chat_models.base import BaseChatOpenAI
 from bioguider.database.summarized_file_db import SummarizedFilesDb
 from bioguider.utils.file_utils import get_file_type
 from bioguider.agents.agent_utils import read_directory, read_file, summarize_file
+from bioguider.rag.data_pipeline import count_tokens
 
 class agent_tool:
     def __init__(
@@ -39,7 +41,14 @@ Returns:
             file_path = os.path.join(self.repo_path, file_path)
         if not os.path.isfile(file_path):
             return None
-        return read_file(file_path)
+        content = read_file(file_path)
+        if file_path.endswith(".html") or file_path.endswith(".htm"):
+            content = md(content, escape_underscores=False)
+        tokens = count_tokens(content)
+        MAX_TOKENS = os.environ.get('OPENAI_MAX_INPUT_TOKENS', 102400)
+        if tokens > int(MAX_TOKENS):
+            content = content[:100000]
+        return content
 
 class summarize_file_tool(agent_tool):
     """ read and summarize the file

bioguider/agents/evaluation_installation_task.py

@@ -6,9 +6,11 @@ from abc import ABC, abstractmethod
 from langchain.prompts import ChatPromptTemplate
 from langchain_openai.chat_models.base import BaseChatOpenAI
 from pydantic import BaseModel, Field
+from markdownify import markdownify as md
 
 from bioguider.agents.agent_utils import read_file
 from bioguider.utils.constants import DEFAULT_TOKEN_USAGE, ProjectMetadata
+from bioguider.rag.data_pipeline import count_tokens
 from .common_agent_2step import CommonAgentTwoSteps, CommonAgentTwoChainSteps
 from .common_agent import CommonConversation
 from ..utils.pyphen_utils import PyphenReadability
@@ -122,8 +124,15 @@ class EvaluationInstallationTask(EvaluationTask):
             return None
         
         files_content = ""
+        MAX_TOKENS = os.environ.get("OPENAI_MAX_INPUT_TOKENS", 102400)
         for f in files:
-            content = read_file(os.path.join(self.repo_path, f))
+            if f.endswith(".html") or f.endswith(".htm"):
+                html = read_file(os.path.join(self.repo_path, f))
+                content = md(html, escape_underscores=False)
+            else:
+                content = read_file(os.path.join(self.repo_path, f))
+            if count_tokens(content) > int(MAX_TOKENS):
+                content = content[:100000]
             files_content += f"""
 {f} content:
 {content}

{bioguider-0.2.6.dist-info → bioguider-0.2.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: bioguider
-Version: 0.2.6
+Version: 0.2.7
 Summary: An AI-Powered package to help biomedical developers to generate clear documentation
 License: MIT
 Author: Cankun Wang
@@ -28,6 +28,7 @@ Requires-Dist: langchain-experimental (>=0.3.4,<0.4.0)
 Requires-Dist: langchain-google-genai (>=2.1.4,<3.0.0)
 Requires-Dist: langchain-openai (>=0.3.8,<0.4.0)
 Requires-Dist: langgraph (>=0.3.11,<0.4.0)
+Requires-Dist: markdownify (>=1.1.0,<2.0.0)
 Requires-Dist: nanoid (>=2.0.0,<3.0.0)
 Requires-Dist: pydantic (>=2.10.6,<3.0.0)
 Requires-Dist: pydantic-settings (>=2.8.1,<3.0.0)

{bioguider-0.2.6.dist-info → bioguider-0.2.7.dist-info}/RECORD

@@ -1,7 +1,7 @@
 bioguider/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 bioguider/agents/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 bioguider/agents/agent_task.py,sha256=SX4iLdGqQttT39qvr-RtXiSpQEzm7Z3ECVw8IGQzpDc,2828
-bioguider/agents/agent_tools.py,sha256=Q7yciRjMwW4J6V4TGZx_smaBYiV_ZtmKkjBb1ofVgM0,5898
+bioguider/agents/agent_tools.py,sha256=B3vwkly5AC5tFp31HfZ07yt18KK2eIrOVk8NJNzukKg,6335
 bioguider/agents/agent_utils.py,sha256=qngmkCiTz46PJebzWmdZLo5XWtyUDZddCQv06PDKBrw,12450
 bioguider/agents/collection_execute_step.py,sha256=6hCCP9LG5goC-EbS6Ye3wKS4vPy7p9NndMYo48Qt3RA,5515
 bioguider/agents/collection_observe_step.py,sha256=iNeV6f16Emk1LMStSR4FXBPZ6Sc0eTjwxEfmoeegV-U,4554
@@ -16,7 +16,7 @@ bioguider/agents/dockergeneration_observe_step.py,sha256=93PO_Y4YyUShVTKRt0nErcj
 bioguider/agents/dockergeneration_plan_step.py,sha256=SB8tQM9PkIKsD2o1DFD7bedcxz6r6hSy8n_EVK60Fz0,7235
 bioguider/agents/dockergeneration_task.py,sha256=ezsweVHJsFpOyOI6rYMt1DZ3PE19dcq4J3Lm-d0IA8M,6220
 bioguider/agents/dockergeneration_task_utils.py,sha256=v7emqrJlVW-A5ZdLmPSdiaMSKCR8uzy9UYzx_1cgzyo,9041
-bioguider/agents/evaluation_installation_task.py,sha256=ULxlw5VvW_w-tH7mVWbvOFjKYzjpUz2dVTCHKqzU_8w,5922
+bioguider/agents/evaluation_installation_task.py,sha256=9fKV7UW3xBHmwY_Q9YVRwoj3xv_ejzbFxojQjwBGIZw,6394
 bioguider/agents/evaluation_task.py,sha256=8tjRPGsRQ9qDxVgYYDo-GhhEiwW71zSS-iyLZUPRbBA,17178
 bioguider/agents/identification_execute_step.py,sha256=_9x_KITVtNJjQlgjfq-LazoUl46Tctaj_W_AVxz5q-w,5488
 bioguider/agents/identification_observe_step.py,sha256=OENwf9XyOSIHvJMp7eoyQOYGjjtPnPT2S29xf1rCATk,3667
@@ -42,7 +42,7 @@ bioguider/utils/file_utils.py,sha256=9VfAHsz1UkFPtzAmvWZvPl1TMaKIYNjNlLgsfB8tNjg
 bioguider/utils/gitignore_checker.py,sha256=pOYUwsS9D5014LxcZb0cj3s2CAYaD2uF_pYJpaNKcho,6532
 bioguider/utils/pyphen_utils.py,sha256=cdZc3qphkvMDeL5NiZ8Xou13M_uVNP7ifJ-FwxO-0BE,2680
 bioguider/utils/utils.py,sha256=YP3HXgU_rvYDWkEcTzWGiYZw-mlfVrqGhUGSc0_4Pms,900
-bioguider-0.2.6.dist-info/LICENSE,sha256=qzkvZcKwwA5DuSuhXMOm2LcO6BdEr4V7jwFZVL2-jL4,1065
-bioguider-0.2.6.dist-info/METADATA,sha256=MBcrwDU4pJKKEkd1jBvtBYd0WDUrbIZjKJuwOk_TQ3Y,1823
-bioguider-0.2.6.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-bioguider-0.2.6.dist-info/RECORD,,
+bioguider-0.2.7.dist-info/LICENSE,sha256=qzkvZcKwwA5DuSuhXMOm2LcO6BdEr4V7jwFZVL2-jL4,1065
+bioguider-0.2.7.dist-info/METADATA,sha256=3SOK0oKDIdeV58nePd69cpwYcxTAcB_juAN034nmjLk,1867
+bioguider-0.2.7.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+bioguider-0.2.7.dist-info/RECORD,,