llama-github 0.1.4__tar.gz → 0.1.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {llama_github-0.1.4 → llama_github-0.1.6}/CHANGELOG.md +17 -0
- {llama_github-0.1.4/llama_github.egg-info → llama_github-0.1.6}/PKG-INFO +26 -2
- {llama_github-0.1.4 → llama_github-0.1.6}/llama_github/data_retrieval/github_entities.py +90 -13
- {llama_github-0.1.4 → llama_github-0.1.6}/llama_github/github_rag.py +131 -28
- {llama_github-0.1.4 → llama_github-0.1.6}/llama_github/llm_integration/initial_load.py +78 -15
- llama_github-0.1.6/llama_github/logger.py +15 -0
- llama_github-0.1.6/llama_github/version.py +1 -0
- {llama_github-0.1.4 → llama_github-0.1.6/llama_github.egg-info}/PKG-INFO +26 -2
- {llama_github-0.1.4 → llama_github-0.1.6}/llama_github.egg-info/requires.txt +8 -1
- {llama_github-0.1.4 → llama_github-0.1.6}/setup.cfg +1 -1
- llama_github-0.1.4/llama_github/logger.py +0 -28
- llama_github-0.1.4/llama_github/version.py +0 -1
- {llama_github-0.1.4 → llama_github-0.1.6}/LICENSE +0 -0
- {llama_github-0.1.4 → llama_github-0.1.6}/MANIFEST.in +0 -0
- {llama_github-0.1.4 → llama_github-0.1.6}/README.md +0 -0
- {llama_github-0.1.4 → llama_github-0.1.6}/llama_github/__init__.py +0 -0
- {llama_github-0.1.4 → llama_github-0.1.6}/llama_github/config/__init__.py +0 -0
- {llama_github-0.1.4 → llama_github-0.1.6}/llama_github/config/config.json +0 -0
- {llama_github-0.1.4 → llama_github-0.1.6}/llama_github/config/config.py +0 -0
- {llama_github-0.1.4 → llama_github-0.1.6}/llama_github/data_retrieval/__init__.py +0 -0
- {llama_github-0.1.4 → llama_github-0.1.6}/llama_github/data_retrieval/github_api.py +0 -0
- {llama_github-0.1.4 → llama_github-0.1.6}/llama_github/features/__init__.py +0 -0
- {llama_github-0.1.4 → llama_github-0.1.6}/llama_github/features/feature_flags.py +0 -0
- {llama_github-0.1.4 → llama_github-0.1.6}/llama_github/features/insider_features.py +0 -0
- {llama_github-0.1.4 → llama_github-0.1.6}/llama_github/github_integration/__init__.py +0 -0
- {llama_github-0.1.4 → llama_github-0.1.6}/llama_github/github_integration/github_auth_manager.py +0 -0
- {llama_github-0.1.4 → llama_github-0.1.6}/llama_github/llm_integration/__init__.py +0 -0
- {llama_github-0.1.4 → llama_github-0.1.6}/llama_github/llm_integration/llm_handler.py +0 -0
- {llama_github-0.1.4 → llama_github-0.1.6}/llama_github/rag_processing/__init__.py +0 -0
- {llama_github-0.1.4 → llama_github-0.1.6}/llama_github/rag_processing/rag_processor.py +0 -0
- {llama_github-0.1.4 → llama_github-0.1.6}/llama_github/utils.py +0 -0
- {llama_github-0.1.4 → llama_github-0.1.6}/llama_github.egg-info/SOURCES.txt +0 -0
- {llama_github-0.1.4 → llama_github-0.1.6}/llama_github.egg-info/dependency_links.txt +0 -0
- {llama_github-0.1.4 → llama_github-0.1.6}/llama_github.egg-info/top_level.txt +0 -0
- {llama_github-0.1.4 → llama_github-0.1.6}/setup.py +0 -0
- {llama_github-0.1.4 → llama_github-0.1.6}/tests/test_data_retrieval.py +0 -0
- {llama_github-0.1.4 → llama_github-0.1.6}/tests/test_github_auth_manager.py +0 -0
- {llama_github-0.1.4 → llama_github-0.1.6}/tests/test_initial_load.py +0 -0
- {llama_github-0.1.4 → llama_github-0.1.6}/tests/test_llm_handler.py +0 -0
- {llama_github-0.1.4 → llama_github-0.1.6}/tests/test_logger.py +0 -0
- {llama_github-0.1.4 → llama_github-0.1.6}/tests/test_rag_processor.py +0 -0
|
@@ -5,6 +5,23 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [0.1.6] - 2024-10-30
|
|
9
|
+
|
|
10
|
+
### New Features
|
|
11
|
+
- Enhanced PR content analysis with detailed commit information extraction
|
|
12
|
+
- Improved issue linking detection with support for multiple reference formats
|
|
13
|
+
- Full GitHub URLs, #references, and keyword-based references
|
|
14
|
+
- Added validation for issue numbers
|
|
15
|
+
|
|
16
|
+
### Improvements
|
|
17
|
+
- Added detailed commit metadata extraction including stats and file changes
|
|
18
|
+
- Enhanced error handling for commit fetching
|
|
19
|
+
|
|
20
|
+
## [0.1.5] - 2024-10-14
|
|
21
|
+
|
|
22
|
+
### Optimized
|
|
23
|
+
- requirements.txt updated to more precise list
|
|
24
|
+
|
|
8
25
|
## [0.1.4] - 2024-10-14
|
|
9
26
|
|
|
10
27
|
### Improved
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: llama-github
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.6
|
|
4
4
|
Summary: Llama-github is an open-source Python library that empowers LLM Chatbots, AI Agents, and Auto-dev Agents to conduct Retrieval from actively selected GitHub public projects. It Augments through LLMs and Generates context for any coding question, in order to streamline the development of sophisticated AI-driven applications.
|
|
5
5
|
Home-page: https://github.com/JetXu-LLM/llama-github
|
|
6
6
|
Author: Jet Xu
|
|
@@ -19,9 +19,16 @@ Requires-Dist: langchain
|
|
|
19
19
|
Requires-Dist: numpy
|
|
20
20
|
Requires-Dist: pydantic
|
|
21
21
|
Requires-Dist: requests
|
|
22
|
-
Requires-Dist: torch
|
|
23
22
|
Requires-Dist: transformers
|
|
24
23
|
Requires-Dist: urllib3
|
|
24
|
+
Requires-Dist: sys
|
|
25
|
+
Requires-Dist: platform
|
|
26
|
+
Requires-Dist: subprocess
|
|
27
|
+
Requires-Dist: pydantic_core
|
|
28
|
+
Requires-Dist: langchain_openai
|
|
29
|
+
Requires-Dist: langchain_mistralai
|
|
30
|
+
Requires-Dist: httpx_sse
|
|
31
|
+
Requires-Dist: tokenizers
|
|
25
32
|
Provides-Extra: dev
|
|
26
33
|
Requires-Dist: pytest; extra == "dev"
|
|
27
34
|
Requires-Dist: black; extra == "dev"
|
|
@@ -133,6 +140,23 @@ All notable changes to this project will be documented in this file.
|
|
|
133
140
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
134
141
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
135
142
|
|
|
143
|
+
## [0.1.6] - 2024-10-30
|
|
144
|
+
|
|
145
|
+
### New Features
|
|
146
|
+
- Enhanced PR content analysis with detailed commit information extraction
|
|
147
|
+
- Improved issue linking detection with support for multiple reference formats
|
|
148
|
+
- Full GitHub URLs, #references, and keyword-based references
|
|
149
|
+
- Added validation for issue numbers
|
|
150
|
+
|
|
151
|
+
### Improvements
|
|
152
|
+
- Added detailed commit metadata extraction including stats and file changes
|
|
153
|
+
- Enhanced error handling for commit fetching
|
|
154
|
+
|
|
155
|
+
## [0.1.5] - 2024-10-14
|
|
156
|
+
|
|
157
|
+
### Optimized
|
|
158
|
+
- requirements.txt updated to more precise list
|
|
159
|
+
|
|
136
160
|
## [0.1.4] - 2024-10-14
|
|
137
161
|
|
|
138
162
|
### Improved
|
|
@@ -241,25 +241,79 @@ class Repository:
|
|
|
241
241
|
|
|
242
242
|
def extract_related_issues(self, pr_data: Dict[str, Any]) -> List[int]:
|
|
243
243
|
"""
|
|
244
|
-
Extracts related issue numbers from
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
244
|
+
Extracts related issue numbers from all PR data following GitHub's reference syntax.
|
|
245
|
+
|
|
246
|
+
This function implements GitHub's official autolink reference formats to find:
|
|
247
|
+
1. Full GitHub issue/PR URLs
|
|
248
|
+
2. Numeric references (#123)
|
|
249
|
+
3. Keywords + issue references (fixes #123)
|
|
250
|
+
4. Repository cross-references (owner/repo#123)
|
|
251
|
+
|
|
252
|
+
See: https://docs.github.com/en/get-started/writing-on-github/working-with-advanced-formatting/autolinked-references-and-urls
|
|
253
|
+
|
|
254
|
+
Args:
|
|
255
|
+
pr_data: Dict[str, Any] - The complete pull request data dictionary
|
|
256
|
+
|
|
257
|
+
Returns:
|
|
258
|
+
List[int] - A sorted list of unique issue numbers found in the PR data
|
|
248
259
|
"""
|
|
260
|
+
# GitHub's official closing keywords
|
|
261
|
+
closing_keywords = (
|
|
262
|
+
'close', 'closes', 'closed',
|
|
263
|
+
'fix', 'fixes', 'fixed',
|
|
264
|
+
'resolve', 'resolves', 'resolved'
|
|
265
|
+
)
|
|
266
|
+
|
|
267
|
+
# Regex patterns for GitHub issue references
|
|
249
268
|
patterns = [
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
269
|
+
# Full GitHub issue/PR URL pattern
|
|
270
|
+
rf'(?:https?://)?github\.com/{re.escape(self.full_name)}/(?:issues|pull)/(\d+)',
|
|
271
|
+
|
|
272
|
+
# Standard #123 reference with proper boundaries
|
|
273
|
+
r'(?:^|[^\w/])#(\d+)(?=[^\w/]|$)',
|
|
274
|
+
|
|
275
|
+
# Closing keywords (fixes #123)
|
|
276
|
+
fr'(?:^|[^\w/])(?:{"|".join(closing_keywords)}):?\s+#(\d+)(?=[^\w/]|$)',
|
|
277
|
+
|
|
278
|
+
# Cross-repo reference (owner/repo#123)
|
|
279
|
+
rf'{re.escape(self.full_name)}#(\d+)',
|
|
280
|
+
|
|
281
|
+
# Issue keyword reference (issue #123 or issue: #123)
|
|
282
|
+
r'(?:^|[^\w/])(?:issue|bug|ticket|todo|task)s?:?\s+#?(\d+)(?=[^\w/]|$)'
|
|
253
283
|
]
|
|
284
|
+
|
|
254
285
|
issues = set()
|
|
255
|
-
# Convert PR data to JSON string for pattern matching
|
|
256
|
-
pr_description = json.dumps(pr_data, default=str)
|
|
257
286
|
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
287
|
+
def extract_from_text(text: str) -> None:
|
|
288
|
+
"""Helper function to extract issue numbers from text"""
|
|
289
|
+
if not isinstance(text, str):
|
|
290
|
+
return
|
|
291
|
+
|
|
292
|
+
for pattern in patterns:
|
|
293
|
+
matches = re.findall(pattern, text, re.IGNORECASE | re.MULTILINE)
|
|
294
|
+
# Validate issue numbers (reasonable length and positive values)
|
|
295
|
+
valid_matches = [
|
|
296
|
+
int(match) for match in matches
|
|
297
|
+
if match.isdigit() and len(match) <= 7 and int(match) > 0
|
|
298
|
+
]
|
|
299
|
+
issues.update(valid_matches)
|
|
300
|
+
|
|
301
|
+
def process_value(value: Any) -> None:
|
|
302
|
+
"""Recursively process dictionary values and extract issue numbers"""
|
|
303
|
+
if isinstance(value, dict):
|
|
304
|
+
for v in value.values():
|
|
305
|
+
process_value(v)
|
|
306
|
+
elif isinstance(value, (list, tuple)):
|
|
307
|
+
for item in value:
|
|
308
|
+
process_value(item)
|
|
309
|
+
elif isinstance(value, str):
|
|
310
|
+
extract_from_text(value)
|
|
311
|
+
|
|
312
|
+
# Process all data in pr_data recursively
|
|
313
|
+
process_value(pr_data)
|
|
261
314
|
|
|
262
|
-
return list(issues)
|
|
315
|
+
return sorted(list(issues))
|
|
316
|
+
|
|
263
317
|
|
|
264
318
|
def get_issue_contents(self, issue_numbers: List[int], pr_number: int) -> List[Dict[str, Any]]:
|
|
265
319
|
"""
|
|
@@ -324,11 +378,34 @@ class Repository:
|
|
|
324
378
|
"head_branch": pr.head.ref,
|
|
325
379
|
},
|
|
326
380
|
"related_issues": [],
|
|
381
|
+
"commits": [],
|
|
327
382
|
"file_changes": [],
|
|
328
383
|
"ci_cd_results": [],
|
|
329
384
|
"interactions": []
|
|
330
385
|
}
|
|
331
386
|
|
|
387
|
+
# Fetch and process commits
|
|
388
|
+
try:
|
|
389
|
+
commits = pr.get_commits()
|
|
390
|
+
for commit in commits:
|
|
391
|
+
commit_data = {
|
|
392
|
+
"sha": commit.sha,
|
|
393
|
+
"message": commit.commit.message,
|
|
394
|
+
"author": commit.commit.author.name,
|
|
395
|
+
"date": self.to_isoformat(commit.commit.author.date),
|
|
396
|
+
"stats": {
|
|
397
|
+
"additions": commit.stats.additions,
|
|
398
|
+
"deletions": commit.stats.deletions,
|
|
399
|
+
"total": commit.stats.total
|
|
400
|
+
},
|
|
401
|
+
"files": [f.filename for f in commit.files] # Just keep changed file names
|
|
402
|
+
}
|
|
403
|
+
pr_data["commits"].append(commit_data)
|
|
404
|
+
except GithubException as e:
|
|
405
|
+
logger.exception(f"Error fetching commits for PR #{number}")
|
|
406
|
+
pr_data["commits"] = []
|
|
407
|
+
pr_data["commit_stats"] = {}
|
|
408
|
+
|
|
332
409
|
# Fetch CI/CD results
|
|
333
410
|
try:
|
|
334
411
|
last_commit = pr.get_commits().reversed[0]
|
|
@@ -46,24 +46,26 @@ class GithubRAG:
|
|
|
46
46
|
"""
|
|
47
47
|
Initialize the GithubRAG with the provided credentials and configuration.
|
|
48
48
|
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
- openai_api_key (Optional[str]): API key for OpenAI services -- recommend to use, GPT-4-turbo will be used.
|
|
53
|
-
- mistral_api_key (Optional[str]): API key for Mistral AI services.
|
|
54
|
-
- huggingface_token (Optional[str]): Token for Hugging Face services -- recommend to fill.
|
|
55
|
-
- jina_api_key (Optional[str]): API key for Jina AI services -- s.jina.ai API will be used
|
|
56
|
-
- open_source_models_hg_dir (Optional[str]): Name of open-source models from Hugging Face to replace OpenAI.
|
|
57
|
-
- embedding_model (Optional[str]): Name of Embedding model from Hugging Face, if you have preferred embedding model to be used.
|
|
58
|
-
- rerank_model (Optional[str]): Name of Rerank model from Hugging Face, if you have preferred rerank model to be used.
|
|
59
|
-
- llm (Any): Any kind of LangChain llm chat object - to replace OpenAI or open-source models from Hugging Face.
|
|
60
|
-
- simple_mode (bool): If True, skip embedding and rerank model initialization in LLMManager.
|
|
61
|
-
- **kwargs:
|
|
62
|
-
:param repo_cleanup_interval (Optional[int]): How often to run repo cleanup in seconds within RepositoryPool.
|
|
63
|
-
:param repo_max_idle_time (Optional[int]): Keep a repo in cache until max idle time if not used.
|
|
49
|
+
This constructor sets up the necessary components for GitHub integration,
|
|
50
|
+
RAG processing, and LLM capabilities. It handles authentication, initializes
|
|
51
|
+
the repository pool, and sets up the LLM manager.
|
|
64
52
|
|
|
65
|
-
|
|
66
|
-
|
|
53
|
+
Args:
|
|
54
|
+
github_access_token (Optional[str]): GitHub access token for authentication.
|
|
55
|
+
github_app_credentials (Optional[GitHubAppCredentials]): Credentials for GitHub App authentication.
|
|
56
|
+
openai_api_key (Optional[str]): API key for OpenAI services (GPT-4-turbo will be used).
|
|
57
|
+
mistral_api_key (Optional[str]): API key for Mistral AI services.
|
|
58
|
+
huggingface_token (Optional[str]): Token for Hugging Face services (recommended).
|
|
59
|
+
jina_api_key (Optional[str]): API key for Jina AI services (s.jina.ai API will be used).
|
|
60
|
+
open_source_models_hg_dir (Optional[str]): Directory for open-source models from Hugging Face.
|
|
61
|
+
embedding_model (Optional[str]): Name of the preferred embedding model from Hugging Face.
|
|
62
|
+
rerank_model (Optional[str]): Name of the preferred rerank model from Hugging Face.
|
|
63
|
+
llm (Any): Any LangChain LLM chat object to replace OpenAI or open-source models.
|
|
64
|
+
simple_mode (bool): If True, skip embedding and rerank model initialization in LLMManager.
|
|
65
|
+
**kwargs: Additional keyword arguments for repository pool configuration.
|
|
66
|
+
|
|
67
|
+
Raises:
|
|
68
|
+
Exception: If there's an error during initialization.
|
|
67
69
|
"""
|
|
68
70
|
try:
|
|
69
71
|
logger.info("Initializing GithubRAG...")
|
|
@@ -111,10 +113,23 @@ class GithubRAG:
|
|
|
111
113
|
raise
|
|
112
114
|
|
|
113
115
|
async def async_retrieve_context(self, query, simple_mode: Optional[bool] = None) -> List[str]:
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
116
|
+
"""
|
|
117
|
+
Asynchronously retrieve context based on the given query.
|
|
118
|
+
|
|
119
|
+
This method orchestrates the context retrieval process, including Google search,
|
|
120
|
+
code search, issue search, and repository search. It uses the RAG processor to
|
|
121
|
+
analyze the query and retrieve the most relevant contexts.
|
|
122
|
+
|
|
123
|
+
Args:
|
|
124
|
+
query (str): The query to retrieve context for.
|
|
125
|
+
simple_mode (Optional[bool]): If provided, overrides the instance's simple_mode setting.
|
|
126
|
+
|
|
127
|
+
Returns:
|
|
128
|
+
List[str]: A list of the most relevant context strings.
|
|
129
|
+
|
|
130
|
+
Raises:
|
|
131
|
+
Exception: If there's an error during context retrieval.
|
|
132
|
+
"""
|
|
118
133
|
|
|
119
134
|
if simple_mode is None:
|
|
120
135
|
simple_mode = self.simple_mode
|
|
@@ -191,6 +206,10 @@ class GithubRAG:
|
|
|
191
206
|
"""
|
|
192
207
|
Retrieve context from GitHub code, issue and repo search based on the input query.
|
|
193
208
|
|
|
209
|
+
This method serves as a wrapper for the async_retrieve_context method,
|
|
210
|
+
handling the asynchronous call in different runtime environments (e.g., Jupyter notebook,
|
|
211
|
+
asyncio event loop).
|
|
212
|
+
|
|
194
213
|
Args:
|
|
195
214
|
query (str): The query or question to retrieve context for.
|
|
196
215
|
simple_mode (Optional[bool]): If provided, overrides the instance's simple_mode setting.
|
|
@@ -212,6 +231,24 @@ class GithubRAG:
|
|
|
212
231
|
return self.loop.run_until_complete(self.async_retrieve_context(query, simple_mode=effective_simple_mode))
|
|
213
232
|
|
|
214
233
|
async def code_search_retrieval(self, query, draft_answer: Optional[str] = None):
|
|
234
|
+
"""
|
|
235
|
+
Perform a code search on GitHub based on the given query and draft answer.
|
|
236
|
+
|
|
237
|
+
This method uses the RAG processor to generate search criteria, then performs
|
|
238
|
+
a code search using the GitHub API. It filters and deduplicates the results
|
|
239
|
+
based on star count and relevance.
|
|
240
|
+
|
|
241
|
+
Args:
|
|
242
|
+
query (str): The main query for the code search.
|
|
243
|
+
draft_answer (Optional[str]): A draft answer to refine the search criteria.
|
|
244
|
+
|
|
245
|
+
Returns:
|
|
246
|
+
list: A list of unique, relevant code search results.
|
|
247
|
+
|
|
248
|
+
Raises:
|
|
249
|
+
Exception: If there's an error during the code search retrieval.
|
|
250
|
+
"""
|
|
251
|
+
|
|
215
252
|
result = []
|
|
216
253
|
try:
|
|
217
254
|
logger.info("Retrieving code search...")
|
|
@@ -238,6 +275,24 @@ class GithubRAG:
|
|
|
238
275
|
return result
|
|
239
276
|
|
|
240
277
|
async def issue_search_retrieval(self, query, draft_answer: Optional[str] = None):
|
|
278
|
+
"""
|
|
279
|
+
Perform an issue search on GitHub based on the given query and draft answer.
|
|
280
|
+
|
|
281
|
+
This method uses the RAG processor to generate search criteria, then performs
|
|
282
|
+
an issue search using the GitHub API. It deduplicates the results and transforms
|
|
283
|
+
the API URLs to official GitHub issue webpage URLs.
|
|
284
|
+
|
|
285
|
+
Args:
|
|
286
|
+
query (str): The main query for the issue search.
|
|
287
|
+
draft_answer (Optional[str]): A draft answer to refine the search criteria.
|
|
288
|
+
|
|
289
|
+
Returns:
|
|
290
|
+
list: A list of unique, relevant issue search results with transformed URLs.
|
|
291
|
+
|
|
292
|
+
Raises:
|
|
293
|
+
Exception: If there's an error during the issue search retrieval.
|
|
294
|
+
"""
|
|
295
|
+
|
|
241
296
|
result = []
|
|
242
297
|
try:
|
|
243
298
|
logger.info("Retrieving issue search...")
|
|
@@ -268,6 +323,23 @@ class GithubRAG:
|
|
|
268
323
|
return result
|
|
269
324
|
|
|
270
325
|
async def google_search_retrieval(self, query):
|
|
326
|
+
"""
|
|
327
|
+
Perform a Google search for GitHub-related content based on the given query.
|
|
328
|
+
|
|
329
|
+
This method uses the Jina AI search API to perform a Google search limited to
|
|
330
|
+
GitHub.com. It then retrieves the content of the resulting GitHub URLs using
|
|
331
|
+
the GitHub API.
|
|
332
|
+
|
|
333
|
+
Args:
|
|
334
|
+
query (str): The query to search for on Google.
|
|
335
|
+
|
|
336
|
+
Returns:
|
|
337
|
+
list: A list of dictionaries containing the GitHub URL and its content.
|
|
338
|
+
|
|
339
|
+
Raises:
|
|
340
|
+
Exception: If there's an error during the Google search retrieval.
|
|
341
|
+
"""
|
|
342
|
+
|
|
271
343
|
result = []
|
|
272
344
|
try:
|
|
273
345
|
logger.info("Retrieving google search...")
|
|
@@ -304,15 +376,39 @@ class GithubRAG:
|
|
|
304
376
|
|
|
305
377
|
def _get_repository_rag_info(self, repository: Repository):
|
|
306
378
|
"""
|
|
307
|
-
|
|
379
|
+
Retrieve RAG-related information for a given repository.
|
|
380
|
+
|
|
381
|
+
This helper method fetches the README content and a simple structure
|
|
382
|
+
of the repository using the Repository object.
|
|
383
|
+
|
|
384
|
+
Args:
|
|
385
|
+
repository (Repository): The Repository object to get information from.
|
|
308
386
|
|
|
309
|
-
:
|
|
310
|
-
|
|
387
|
+
Returns:
|
|
388
|
+
tuple: A tuple containing the repository's README content and simple structure.
|
|
311
389
|
"""
|
|
312
|
-
|
|
390
|
+
|
|
313
391
|
return repository.get_readme(), self.rag_processor.get_repo_simple_structure(repository)
|
|
314
392
|
|
|
315
393
|
async def repo_search_retrieval(self, query, draft_answer: Optional[str] = None):
|
|
394
|
+
"""
|
|
395
|
+
Perform a repository search on GitHub based on the given query and draft answer.
|
|
396
|
+
|
|
397
|
+
This method uses the RAG processor to generate search criteria, then performs
|
|
398
|
+
a repository search using the GitHub API. It retrieves README content and
|
|
399
|
+
simple structure for each relevant repository concurrently.
|
|
400
|
+
|
|
401
|
+
Args:
|
|
402
|
+
query (str): The main query for the repository search.
|
|
403
|
+
draft_answer (Optional[str]): A draft answer to refine the search criteria.
|
|
404
|
+
|
|
405
|
+
Returns:
|
|
406
|
+
list: A list of dictionaries containing repository information and content.
|
|
407
|
+
|
|
408
|
+
Raises:
|
|
409
|
+
Exception: If there's an error during the repository search retrieval.
|
|
410
|
+
"""
|
|
411
|
+
|
|
316
412
|
result = []
|
|
317
413
|
results_with_index = []
|
|
318
414
|
try:
|
|
@@ -370,18 +466,20 @@ class GithubRAG:
|
|
|
370
466
|
"""
|
|
371
467
|
Generate an answer based on the given query and optional contexts.
|
|
372
468
|
|
|
373
|
-
This method
|
|
374
|
-
|
|
375
|
-
|
|
469
|
+
This method serves as a wrapper for the async_answer_with_context method,
|
|
470
|
+
handling the asynchronous call in different runtime environments (e.g., Jupyter notebook,
|
|
471
|
+
asyncio event loop).
|
|
376
472
|
|
|
377
473
|
Args:
|
|
378
474
|
query (str): The user's query.
|
|
379
475
|
contexts (Optional[List[Dict[str, Any]]]): Optional list of context dictionaries.
|
|
380
476
|
Each dictionary should contain 'content' and 'url' keys.
|
|
477
|
+
simple_mode (bool): Whether to use simple mode for context retrieval.
|
|
381
478
|
|
|
382
479
|
Returns:
|
|
383
480
|
str: The generated answer.
|
|
384
481
|
"""
|
|
482
|
+
|
|
385
483
|
self.loop = asyncio.get_event_loop()
|
|
386
484
|
ipython = get_ipython()
|
|
387
485
|
if ipython and ipython.has_trait('kernel'):
|
|
@@ -397,14 +495,19 @@ class GithubRAG:
|
|
|
397
495
|
"""
|
|
398
496
|
Asynchronously generate an answer based on the given query and optional contexts.
|
|
399
497
|
|
|
498
|
+
This method retrieves contexts if not provided, extracts relevant information,
|
|
499
|
+
and uses the RAG processor's LLM handler to generate an answer.
|
|
500
|
+
|
|
400
501
|
Args:
|
|
401
502
|
query (str): The user's query.
|
|
402
503
|
contexts (Optional[List[Dict[str, Any]]]): Optional list of context dictionaries.
|
|
403
504
|
Each dictionary should contain 'content' and 'url' keys.
|
|
505
|
+
simple_mode (bool): Whether to use simple mode for context retrieval.
|
|
404
506
|
|
|
405
507
|
Returns:
|
|
406
508
|
str: The generated answer.
|
|
407
509
|
"""
|
|
510
|
+
|
|
408
511
|
if contexts is None:
|
|
409
512
|
contexts = await self.async_retrieve_context(query, simple_mode)
|
|
410
513
|
logger.debug(f"Retrieved contexts: {contexts}")
|
|
@@ -2,14 +2,17 @@
|
|
|
2
2
|
from typing import Optional, Any
|
|
3
3
|
from threading import Lock
|
|
4
4
|
from langchain_openai import ChatOpenAI
|
|
5
|
-
from langchain_community.embeddings import HuggingFaceEmbeddings
|
|
6
|
-
from langchain_community.cross_encoders import HuggingFaceCrossEncoder
|
|
7
5
|
from langchain_mistralai.chat_models import ChatMistralAI
|
|
8
6
|
|
|
9
7
|
from llama_github.config.config import config
|
|
10
8
|
from llama_github.logger import logger
|
|
11
9
|
|
|
12
10
|
class LLMManager:
|
|
11
|
+
"""
|
|
12
|
+
Singleton class for managing Language Models and related components.
|
|
13
|
+
This class handles initialization and access to various models including LLMs,
|
|
14
|
+
embedding models, and reranking models.
|
|
15
|
+
"""
|
|
13
16
|
_instance_lock = Lock()
|
|
14
17
|
_instance = None
|
|
15
18
|
llm = None
|
|
@@ -20,6 +23,9 @@ class LLMManager:
|
|
|
20
23
|
embedding_model = None
|
|
21
24
|
|
|
22
25
|
def __new__(cls, *args, **kwargs):
|
|
26
|
+
"""
|
|
27
|
+
Ensure only one instance of LLMManager is created (Singleton pattern).
|
|
28
|
+
"""
|
|
23
29
|
if cls._instance is None: # First check (unlocked)
|
|
24
30
|
with cls._instance_lock: # Acquire lock
|
|
25
31
|
if cls._instance is None: # Second check (locked)
|
|
@@ -36,6 +42,19 @@ class LLMManager:
|
|
|
36
42
|
rerank_model: Optional[str] = config.get("default_reranker"),
|
|
37
43
|
llm: Any = None,
|
|
38
44
|
simple_mode: bool = False):
|
|
45
|
+
"""
|
|
46
|
+
Initialize the LLMManager with specified models and API keys.
|
|
47
|
+
|
|
48
|
+
Args:
|
|
49
|
+
openai_api_key (Optional[str]): API key for OpenAI.
|
|
50
|
+
mistral_api_key (Optional[str]): API key for Mistral AI.
|
|
51
|
+
huggingface_token (Optional[str]): Token for Hugging Face.
|
|
52
|
+
open_source_models_hg_dir (Optional[str]): Directory for open-source models.
|
|
53
|
+
embedding_model (Optional[str]): Name or path of the embedding model.
|
|
54
|
+
rerank_model (Optional[str]): Name or path of the reranking model.
|
|
55
|
+
llm (Any): Custom LLM instance if provided.
|
|
56
|
+
simple_mode (bool): If True, skip initialization of embedding and reranking models.
|
|
57
|
+
"""
|
|
39
58
|
with self._instance_lock: # Prevent re-initialization
|
|
40
59
|
if self._initialized:
|
|
41
60
|
return
|
|
@@ -43,7 +62,7 @@ class LLMManager:
|
|
|
43
62
|
|
|
44
63
|
self.simple_mode = simple_mode
|
|
45
64
|
|
|
46
|
-
# Initialize
|
|
65
|
+
# Initialize LLM based on provided API keys or custom LLM
|
|
47
66
|
if llm is not None:
|
|
48
67
|
self.llm = llm
|
|
49
68
|
self.model_type = "Custom_langchain_llm"
|
|
@@ -61,33 +80,47 @@ class LLMManager:
|
|
|
61
80
|
# Initialize for Open Source Models
|
|
62
81
|
elif open_source_models_hg_dir is not None and open_source_models_hg_dir != "" and self.llm is None:
|
|
63
82
|
logger.info(f"Initializing {open_source_models_hg_dir}...")
|
|
64
|
-
# load
|
|
83
|
+
# load huggingface models
|
|
65
84
|
self.model_type = "Hubgingface"
|
|
66
85
|
elif self.llm is None:
|
|
67
86
|
# default model is phi3_mini_128k
|
|
68
87
|
self.model_type = "Hubgingface"
|
|
69
88
|
|
|
70
89
|
if not self.simple_mode:
|
|
71
|
-
import
|
|
90
|
+
import sys
|
|
91
|
+
import platform
|
|
92
|
+
import subprocess
|
|
93
|
+
|
|
94
|
+
def get_device():
|
|
95
|
+
if sys.platform.startswith('darwin'): # macOS
|
|
96
|
+
# Check for Apple Silicon (M1/M2)
|
|
97
|
+
if platform.machine() == 'arm64':
|
|
98
|
+
return 'mps'
|
|
99
|
+
elif sys.platform.startswith('linux') or sys.platform.startswith('win'):
|
|
100
|
+
# Check for NVIDIA GPU
|
|
101
|
+
try:
|
|
102
|
+
subprocess.run(['nvidia-smi'], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
103
|
+
return 'cuda'
|
|
104
|
+
except (subprocess.CalledProcessError, FileNotFoundError):
|
|
105
|
+
pass
|
|
106
|
+
|
|
107
|
+
# Default to CPU
|
|
108
|
+
return 'cpu'
|
|
109
|
+
|
|
110
|
+
# Usage
|
|
111
|
+
self.device = get_device()
|
|
112
|
+
|
|
72
113
|
from transformers import AutoModel
|
|
73
114
|
from transformers import AutoModelForSequenceClassification
|
|
74
115
|
from transformers import AutoTokenizer
|
|
75
|
-
#
|
|
76
|
-
if torch.cuda.is_available():
|
|
77
|
-
self.device = torch.device('cuda')
|
|
78
|
-
elif torch.backends.mps.is_available():
|
|
79
|
-
self.device = torch.device('mps')
|
|
80
|
-
else:
|
|
81
|
-
self.device = torch.device('cpu')
|
|
82
|
-
|
|
83
|
-
# initial embedding_model
|
|
116
|
+
# Initialize embedding model
|
|
84
117
|
if self.tokenizer is None:
|
|
85
118
|
logger.info(f"Initializing {embedding_model}...")
|
|
86
119
|
self.tokenizer = AutoTokenizer.from_pretrained(embedding_model)
|
|
87
120
|
self.embedding_model = AutoModel.from_pretrained(
|
|
88
121
|
embedding_model, trust_remote_code=True).to(self.device)
|
|
89
122
|
|
|
90
|
-
#
|
|
123
|
+
# Initialize reranking model
|
|
91
124
|
if self.rerank_model is None:
|
|
92
125
|
logger.info(f"Initializing {rerank_model}...")
|
|
93
126
|
self.rerank_model = AutoModelForSequenceClassification.from_pretrained(
|
|
@@ -97,16 +130,46 @@ class LLMManager:
|
|
|
97
130
|
logger.info("Simple mode enabled. Skipping embedding and rerank model initialization.")
|
|
98
131
|
|
|
99
132
|
def get_llm(self):
|
|
133
|
+
"""
|
|
134
|
+
Get the main Language Model.
|
|
135
|
+
|
|
136
|
+
Returns:
|
|
137
|
+
The initialized Language Model.
|
|
138
|
+
"""
|
|
100
139
|
return self.llm
|
|
101
140
|
|
|
102
141
|
def get_llm_simple(self):
|
|
142
|
+
"""
|
|
143
|
+
Get the simplified Language Model.
|
|
144
|
+
|
|
145
|
+
Returns:
|
|
146
|
+
The initialized simplified Language Model.
|
|
147
|
+
"""
|
|
103
148
|
return self.llm_simple
|
|
104
149
|
|
|
105
150
|
def get_tokenizer(self):
|
|
151
|
+
"""
|
|
152
|
+
Get the tokenizer for the embedding model.
|
|
153
|
+
|
|
154
|
+
Returns:
|
|
155
|
+
The initialized tokenizer.
|
|
156
|
+
"""
|
|
106
157
|
return self.tokenizer
|
|
107
158
|
|
|
108
159
|
def get_rerank_model(self):
|
|
160
|
+
"""
|
|
161
|
+
Get the reranking model.
|
|
162
|
+
|
|
163
|
+
Returns:
|
|
164
|
+
The initialized reranking model.
|
|
165
|
+
"""
|
|
109
166
|
return self.rerank_model
|
|
110
167
|
|
|
111
168
|
def get_embedding_model(self):
|
|
169
|
+
"""
|
|
170
|
+
Get the embedding model.
|
|
171
|
+
|
|
172
|
+
Returns:
|
|
173
|
+
The initialized embedding model.
|
|
174
|
+
"""
|
|
112
175
|
return self.embedding_model
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
|
|
3
|
+
logger = logging.getLogger('llama_github')
|
|
4
|
+
|
|
5
|
+
def configure_logging(level=logging.INFO, handler=None):
|
|
6
|
+
logger.setLevel(level)
|
|
7
|
+
if handler:
|
|
8
|
+
logger.addHandler(handler)
|
|
9
|
+
else:
|
|
10
|
+
# default handler output to console
|
|
11
|
+
ch = logging.StreamHandler()
|
|
12
|
+
ch.setLevel(level)
|
|
13
|
+
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
|
14
|
+
ch.setFormatter(formatter)
|
|
15
|
+
logger.addHandler(ch)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = '0.1.6'
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: llama-github
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.6
|
|
4
4
|
Summary: Llama-github is an open-source Python library that empowers LLM Chatbots, AI Agents, and Auto-dev Agents to conduct Retrieval from actively selected GitHub public projects. It Augments through LLMs and Generates context for any coding question, in order to streamline the development of sophisticated AI-driven applications.
|
|
5
5
|
Home-page: https://github.com/JetXu-LLM/llama-github
|
|
6
6
|
Author: Jet Xu
|
|
@@ -19,9 +19,16 @@ Requires-Dist: langchain
|
|
|
19
19
|
Requires-Dist: numpy
|
|
20
20
|
Requires-Dist: pydantic
|
|
21
21
|
Requires-Dist: requests
|
|
22
|
-
Requires-Dist: torch
|
|
23
22
|
Requires-Dist: transformers
|
|
24
23
|
Requires-Dist: urllib3
|
|
24
|
+
Requires-Dist: sys
|
|
25
|
+
Requires-Dist: platform
|
|
26
|
+
Requires-Dist: subprocess
|
|
27
|
+
Requires-Dist: pydantic_core
|
|
28
|
+
Requires-Dist: langchain_openai
|
|
29
|
+
Requires-Dist: langchain_mistralai
|
|
30
|
+
Requires-Dist: httpx_sse
|
|
31
|
+
Requires-Dist: tokenizers
|
|
25
32
|
Provides-Extra: dev
|
|
26
33
|
Requires-Dist: pytest; extra == "dev"
|
|
27
34
|
Requires-Dist: black; extra == "dev"
|
|
@@ -133,6 +140,23 @@ All notable changes to this project will be documented in this file.
|
|
|
133
140
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
134
141
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
135
142
|
|
|
143
|
+
## [0.1.6] - 2024-10-30
|
|
144
|
+
|
|
145
|
+
### New Features
|
|
146
|
+
- Enhanced PR content analysis with detailed commit information extraction
|
|
147
|
+
- Improved issue linking detection with support for multiple reference formats
|
|
148
|
+
- Full GitHub URLs, #references, and keyword-based references
|
|
149
|
+
- Added validation for issue numbers
|
|
150
|
+
|
|
151
|
+
### Improvements
|
|
152
|
+
- Added detailed commit metadata extraction including stats and file changes
|
|
153
|
+
- Enhanced error handling for commit fetching
|
|
154
|
+
|
|
155
|
+
## [0.1.5] - 2024-10-14
|
|
156
|
+
|
|
157
|
+
### Optimized
|
|
158
|
+
- requirements.txt updated to more precise list
|
|
159
|
+
|
|
136
160
|
## [0.1.4] - 2024-10-14
|
|
137
161
|
|
|
138
162
|
### Improved
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[metadata]
|
|
2
2
|
name = llama-github
|
|
3
|
-
version = 0.1.
|
|
3
|
+
version = 0.1.6
|
|
4
4
|
author = Jet Xu
|
|
5
5
|
author_email = Voldemort.xu@foxmail.com
|
|
6
6
|
description = Llama-github is an open-source Python library that empowers LLM Chatbots, AI Agents, and Auto-dev Agents to conduct Retrieval from actively selected GitHub public projects. It Augments through LLMs and Generates context for any coding question, in order to streamline the development of sophisticated AI-driven applications.
|
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
import logging
|
|
2
|
-
|
|
3
|
-
logger = logging.getLogger('llama_github')
|
|
4
|
-
|
|
5
|
-
def configure_logging(level=logging.INFO, handler=None):
|
|
6
|
-
logger.setLevel(level)
|
|
7
|
-
if handler:
|
|
8
|
-
logger.addHandler(handler)
|
|
9
|
-
else:
|
|
10
|
-
# default handler output to console
|
|
11
|
-
ch = logging.StreamHandler()
|
|
12
|
-
ch.setLevel(level)
|
|
13
|
-
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
|
14
|
-
ch.setFormatter(formatter)
|
|
15
|
-
logger.addHandler(ch)
|
|
16
|
-
|
|
17
|
-
## Logging Configuration
|
|
18
|
-
# By default, `llama_github` does not configure logging to avoid interfering with your application's logging settings. If you want to see log outputs from `llama_github`, you can configure logging as follows:
|
|
19
|
-
|
|
20
|
-
# ```python
|
|
21
|
-
# import logging
|
|
22
|
-
# import llama_github
|
|
23
|
-
|
|
24
|
-
# # Configure the library's logger
|
|
25
|
-
# llama_github.configure_logging(level=logging.INFO)
|
|
26
|
-
|
|
27
|
-
# # Now you can see log outputs
|
|
28
|
-
# logger = logging.getLogger('llama_github')
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = '0.1.4'
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{llama_github-0.1.4 → llama_github-0.1.6}/llama_github/github_integration/github_auth_manager.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|