bioguider 0.2.6__tar.gz → 0.2.7__tar.gz

{bioguider-0.2.6 → bioguider-0.2.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: bioguider
-Version: 0.2.6
+Version: 0.2.7
 Summary: An AI-Powered package to help biomedical developers to generate clear documentation
 License: MIT
 Author: Cankun Wang
@@ -28,6 +28,7 @@ Requires-Dist: langchain-experimental (>=0.3.4,<0.4.0)
 Requires-Dist: langchain-google-genai (>=2.1.4,<3.0.0)
 Requires-Dist: langchain-openai (>=0.3.8,<0.4.0)
 Requires-Dist: langgraph (>=0.3.11,<0.4.0)
+Requires-Dist: markdownify (>=1.1.0,<2.0.0)
 Requires-Dist: nanoid (>=2.0.0,<3.0.0)
 Requires-Dist: pydantic (>=2.10.6,<3.0.0)
 Requires-Dist: pydantic-settings (>=2.8.1,<3.0.0)

{bioguider-0.2.6 → bioguider-0.2.7}/bioguider/agents/agent_tools.py

@@ -1,9 +1,11 @@
 import os
 from typing import Callable
+from markdownify import markdownify as md
 from langchain_openai.chat_models.base import BaseChatOpenAI
 from bioguider.database.summarized_file_db import SummarizedFilesDb
 from bioguider.utils.file_utils import get_file_type
 from bioguider.agents.agent_utils import read_directory, read_file, summarize_file
+from bioguider.rag.data_pipeline import count_tokens
 
 class agent_tool:
     def __init__(
@@ -39,7 +41,14 @@ Returns:
             file_path = os.path.join(self.repo_path, file_path)
         if not os.path.isfile(file_path):
             return None
-        return read_file(file_path)
+        content = read_file(file_path)
+        if file_path.endswith(".html") or file_path.endswith(".htm"):
+            content = md(content, escape_underscores=False)
+        tokens = count_tokens(content)
+        MAX_TOKENS = os.environ.get('OPENAI_MAX_INPUT_TOKENS', 102400)
+        if tokens > int(MAX_TOKENS):
+            content = content[:100000]
+        return content
 
 class summarize_file_tool(agent_tool):
     """ read and summarize the file

{bioguider-0.2.6 → bioguider-0.2.7}/bioguider/agents/evaluation_installation_task.py

@@ -6,9 +6,11 @@ from abc import ABC, abstractmethod
 from langchain.prompts import ChatPromptTemplate
 from langchain_openai.chat_models.base import BaseChatOpenAI
 from pydantic import BaseModel, Field
+from markdownify import markdownify as md
 
 from bioguider.agents.agent_utils import read_file
 from bioguider.utils.constants import DEFAULT_TOKEN_USAGE, ProjectMetadata
+from bioguider.rag.data_pipeline import count_tokens
 from .common_agent_2step import CommonAgentTwoSteps, CommonAgentTwoChainSteps
 from .common_agent import CommonConversation
 from ..utils.pyphen_utils import PyphenReadability
@@ -122,8 +124,15 @@ class EvaluationInstallationTask(EvaluationTask):
             return None
         
         files_content = ""
+        MAX_TOKENS = os.environ.get("OPENAI_MAX_INPUT_TOKENS", 102400)
         for f in files:
-            content = read_file(os.path.join(self.repo_path, f))
+            if f.endswith(".html") or f.endswith(".htm"):
+                html = read_file(os.path.join(self.repo_path, f))
+                content = md(html, escape_underscores=False)
+            else:
+                content = read_file(os.path.join(self.repo_path, f))
+            if count_tokens(content) > int(MAX_TOKENS):
+                content = content[:100000]
             files_content += f"""
 {f} content:
 {content}

{bioguider-0.2.6 → bioguider-0.2.7}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "bioguider"
-version = "0.2.6"
+version = "0.2.7"
 description = "An AI-Powered package to help biomedical developers to generate clear documentation"
 authors = [
     "Cankun Wang <Cankun.Wang@osumc.edu>",
@@ -45,6 +45,7 @@ faiss-cpu = "^1.11.0"
 binaryornot = "^0.4.4"
 textstat = "^0.7.6"
 pyphen = "^0.17.2"
+markdownify = "^1.1.0"
 
 
 [tool.poetry.group.dev.dependencies]