auto-coder 0.1.255__py3-none-any.whl → 0.1.257__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of auto-coder might be problematic. Click here for more details.
- {auto_coder-0.1.255.dist-info → auto_coder-0.1.257.dist-info}/METADATA +2 -2
- {auto_coder-0.1.255.dist-info → auto_coder-0.1.257.dist-info}/RECORD +30 -27
- autocoder/auto_coder.py +44 -50
- autocoder/chat_auto_coder.py +16 -17
- autocoder/chat_auto_coder_lang.py +1 -1
- autocoder/common/__init__.py +7 -0
- autocoder/common/auto_coder_lang.py +46 -16
- autocoder/common/code_auto_generate.py +45 -5
- autocoder/common/code_auto_generate_diff.py +45 -7
- autocoder/common/code_auto_generate_editblock.py +48 -4
- autocoder/common/code_auto_generate_strict_diff.py +46 -7
- autocoder/common/code_modification_ranker.py +39 -3
- autocoder/dispacher/actions/action.py +60 -40
- autocoder/dispacher/actions/plugins/action_regex_project.py +12 -6
- autocoder/index/entry.py +6 -4
- autocoder/index/filter/quick_filter.py +175 -65
- autocoder/index/index.py +94 -4
- autocoder/models.py +44 -6
- autocoder/privacy/__init__.py +3 -0
- autocoder/privacy/model_filter.py +100 -0
- autocoder/pyproject/__init__.py +1 -0
- autocoder/suffixproject/__init__.py +1 -0
- autocoder/tsproject/__init__.py +1 -0
- autocoder/utils/llms.py +27 -0
- autocoder/utils/model_provider_selector.py +192 -0
- autocoder/version.py +1 -1
- {auto_coder-0.1.255.dist-info → auto_coder-0.1.257.dist-info}/LICENSE +0 -0
- {auto_coder-0.1.255.dist-info → auto_coder-0.1.257.dist-info}/WHEEL +0 -0
- {auto_coder-0.1.255.dist-info → auto_coder-0.1.257.dist-info}/entry_points.txt +0 -0
- {auto_coder-0.1.255.dist-info → auto_coder-0.1.257.dist-info}/top_level.txt +0 -0
|
@@ -2,6 +2,7 @@ from typing import List, Dict, Tuple
|
|
|
2
2
|
from autocoder.common.types import Mode, CodeGenerateResult
|
|
3
3
|
from autocoder.common import AutoCoderArgs
|
|
4
4
|
import byzerllm
|
|
5
|
+
from autocoder.privacy.model_filter import ModelPathFilter
|
|
5
6
|
from autocoder.utils.queue_communicate import queue_communicate, CommunicateEvent, CommunicateEventType
|
|
6
7
|
from autocoder.common import sys_prompt
|
|
7
8
|
from concurrent.futures import ThreadPoolExecutor
|
|
@@ -9,7 +10,8 @@ import json
|
|
|
9
10
|
from autocoder.common.utils_code_auto_generate import chat_with_continue
|
|
10
11
|
from autocoder.common.printer import Printer
|
|
11
12
|
from autocoder.rag.token_counter import count_tokens
|
|
12
|
-
|
|
13
|
+
from autocoder.utils import llms as llm_utils
|
|
14
|
+
from autocoder.common import SourceCodeList
|
|
13
15
|
|
|
14
16
|
class CodeAutoGenerateDiff:
|
|
15
17
|
def __init__(
|
|
@@ -301,9 +303,10 @@ class CodeAutoGenerateDiff:
|
|
|
301
303
|
}
|
|
302
304
|
|
|
303
305
|
def single_round_run(
|
|
304
|
-
self, query: str,
|
|
306
|
+
self, query: str, source_code_list: SourceCodeList
|
|
305
307
|
) -> CodeGenerateResult:
|
|
306
308
|
llm_config = {"human_as_model": self.args.human_as_model}
|
|
309
|
+
source_content = source_code_list.to_str()
|
|
307
310
|
|
|
308
311
|
if self.args.template == "common":
|
|
309
312
|
init_prompt = self.single_round_instruction.prompt(
|
|
@@ -341,6 +344,9 @@ class CodeAutoGenerateDiff:
|
|
|
341
344
|
results = []
|
|
342
345
|
input_tokens_count = 0
|
|
343
346
|
generated_tokens_count = 0
|
|
347
|
+
input_tokens_cost = 0
|
|
348
|
+
generated_tokens_cost = 0
|
|
349
|
+
model_names = []
|
|
344
350
|
|
|
345
351
|
printer = Printer()
|
|
346
352
|
estimated_input_tokens = count_tokens(json.dumps(conversations, ensure_ascii=False))
|
|
@@ -354,13 +360,26 @@ class CodeAutoGenerateDiff:
|
|
|
354
360
|
futures = []
|
|
355
361
|
for llm in self.llms:
|
|
356
362
|
for _ in range(self.generate_times_same_model):
|
|
357
|
-
|
|
358
|
-
|
|
363
|
+
model_names_list = llm_utils.get_llm_names(llm)
|
|
364
|
+
model_name = None
|
|
365
|
+
if model_names_list:
|
|
366
|
+
model_name = model_names_list[0]
|
|
367
|
+
|
|
368
|
+
for _ in range(self.generate_times_same_model):
|
|
369
|
+
model_names.append(model_name)
|
|
370
|
+
futures.append(executor.submit(
|
|
371
|
+
chat_with_continue, llm=llm, conversations=conversations, llm_config=llm_config))
|
|
372
|
+
|
|
359
373
|
temp_results = [future.result() for future in futures]
|
|
360
374
|
for result in temp_results:
|
|
361
375
|
results.append(result.content)
|
|
362
376
|
input_tokens_count += result.input_tokens_count
|
|
363
377
|
generated_tokens_count += result.generated_tokens_count
|
|
378
|
+
model_info = llm_utils.get_model_info(model_name, self.args.product_mode)
|
|
379
|
+
input_cost = model_info.get("input_price",0) if model_info else 0
|
|
380
|
+
output_cost = model_info.get("output_price",0) if model_info else 0
|
|
381
|
+
input_tokens_cost += input_cost * result.input_tokens_count / 1000000
|
|
382
|
+
generated_tokens_cost += output_cost * result.generated_tokens_count / 1000000
|
|
364
383
|
|
|
365
384
|
for result in results:
|
|
366
385
|
conversations_list.append(
|
|
@@ -376,7 +395,9 @@ class CodeAutoGenerateDiff:
|
|
|
376
395
|
|
|
377
396
|
statistics = {
|
|
378
397
|
"input_tokens_count": input_tokens_count,
|
|
379
|
-
"generated_tokens_count": generated_tokens_count
|
|
398
|
+
"generated_tokens_count": generated_tokens_count,
|
|
399
|
+
"input_tokens_cost": input_tokens_cost,
|
|
400
|
+
"generated_tokens_cost": generated_tokens_cost
|
|
380
401
|
}
|
|
381
402
|
|
|
382
403
|
if self.args.request_id and not self.args.skip_events:
|
|
@@ -391,11 +412,28 @@ class CodeAutoGenerateDiff:
|
|
|
391
412
|
return CodeGenerateResult(contents=results, conversations=conversations_list, metadata=statistics)
|
|
392
413
|
|
|
393
414
|
def multi_round_run(
|
|
394
|
-
self, query: str,
|
|
415
|
+
self, query: str, source_code_list: SourceCodeList, max_steps: int = 10
|
|
395
416
|
) -> CodeGenerateResult:
|
|
417
|
+
|
|
418
|
+
# Apply model filter for code_llm
|
|
419
|
+
printer = Printer()
|
|
420
|
+
for llm in self.llms:
|
|
421
|
+
model_filter = ModelPathFilter.from_model_object(llm, self.args)
|
|
422
|
+
filtered_sources = []
|
|
423
|
+
for source in source_code_list.sources:
|
|
424
|
+
if model_filter.is_accessible(source.module_name):
|
|
425
|
+
filtered_sources.append(source)
|
|
426
|
+
else:
|
|
427
|
+
printer.print_in_terminal("index_file_filtered",
|
|
428
|
+
style="yellow",
|
|
429
|
+
file_path=source.path,
|
|
430
|
+
model_name=",".join(llm_utils.get_llm_names(llm)))
|
|
431
|
+
|
|
432
|
+
source_code_list = SourceCodeList(filtered_sources)
|
|
433
|
+
|
|
396
434
|
llm_config = {"human_as_model": self.args.human_as_model}
|
|
397
435
|
result = []
|
|
398
|
-
|
|
436
|
+
source_content = source_code_list.to_str()
|
|
399
437
|
if self.args.template == "common":
|
|
400
438
|
init_prompt = self.multi_round_instruction.prompt(
|
|
401
439
|
instruction=query, content=source_content, context=self.args.context
|
|
@@ -3,6 +3,7 @@ from autocoder.common.types import Mode, CodeGenerateResult
|
|
|
3
3
|
from autocoder.common import AutoCoderArgs
|
|
4
4
|
import byzerllm
|
|
5
5
|
from autocoder.common import sys_prompt
|
|
6
|
+
from autocoder.privacy.model_filter import ModelPathFilter
|
|
6
7
|
from autocoder.utils.queue_communicate import (
|
|
7
8
|
queue_communicate,
|
|
8
9
|
CommunicateEvent,
|
|
@@ -13,6 +14,8 @@ from concurrent.futures import ThreadPoolExecutor
|
|
|
13
14
|
from autocoder.common.utils_code_auto_generate import chat_with_continue
|
|
14
15
|
from autocoder.common.printer import Printer
|
|
15
16
|
from autocoder.rag.token_counter import count_tokens
|
|
17
|
+
from autocoder.utils import llms as llm_utils
|
|
18
|
+
from autocoder.common import SourceCodeList
|
|
16
19
|
|
|
17
20
|
|
|
18
21
|
class CodeAutoGenerateEditBlock:
|
|
@@ -383,10 +386,29 @@ class CodeAutoGenerateEditBlock:
|
|
|
383
386
|
}
|
|
384
387
|
|
|
385
388
|
def single_round_run(
|
|
386
|
-
self, query: str,
|
|
389
|
+
self, query: str, source_code_list: SourceCodeList
|
|
387
390
|
) -> CodeGenerateResult:
|
|
391
|
+
|
|
392
|
+
# Apply model filter for code_llm
|
|
393
|
+
printer = Printer()
|
|
394
|
+
for llm in self.llms:
|
|
395
|
+
model_filter = ModelPathFilter.from_model_object(llm, self.args)
|
|
396
|
+
filtered_sources = []
|
|
397
|
+
for source in source_code_list.sources:
|
|
398
|
+
if model_filter.is_accessible(source.module_name):
|
|
399
|
+
filtered_sources.append(source)
|
|
400
|
+
else:
|
|
401
|
+
printer.print_in_terminal("index_file_filtered",
|
|
402
|
+
style="yellow",
|
|
403
|
+
file_path=source.module_name,
|
|
404
|
+
model_name=",".join(llm_utils.get_llm_names(llm)))
|
|
405
|
+
|
|
406
|
+
source_code_list = SourceCodeList(filtered_sources)
|
|
407
|
+
|
|
388
408
|
llm_config = {"human_as_model": self.args.human_as_model}
|
|
389
409
|
|
|
410
|
+
source_content = source_code_list.to_str()
|
|
411
|
+
|
|
390
412
|
if self.args.template == "common":
|
|
391
413
|
init_prompt = self.single_round_instruction.prompt(
|
|
392
414
|
instruction=query, content=source_content, context=self.args.context
|
|
@@ -424,6 +446,11 @@ class CodeAutoGenerateEditBlock:
|
|
|
424
446
|
input_tokens_count = 0
|
|
425
447
|
generated_tokens_count = 0
|
|
426
448
|
|
|
449
|
+
input_tokens_cost = 0
|
|
450
|
+
generated_tokens_cost = 0
|
|
451
|
+
|
|
452
|
+
model_names = []
|
|
453
|
+
|
|
427
454
|
printer = Printer()
|
|
428
455
|
estimated_input_tokens = count_tokens(
|
|
429
456
|
json.dumps(conversations, ensure_ascii=False))
|
|
@@ -437,14 +464,28 @@ class CodeAutoGenerateEditBlock:
|
|
|
437
464
|
with ThreadPoolExecutor(max_workers=len(self.llms) * self.generate_times_same_model) as executor:
|
|
438
465
|
futures = []
|
|
439
466
|
for llm in self.llms:
|
|
467
|
+
|
|
468
|
+
model_names_list = llm_utils.get_llm_names(llm)
|
|
469
|
+
model_name = None
|
|
470
|
+
if model_names_list:
|
|
471
|
+
model_name = model_names_list[0]
|
|
472
|
+
|
|
440
473
|
for _ in range(self.generate_times_same_model):
|
|
474
|
+
model_names.append(model_name)
|
|
441
475
|
futures.append(executor.submit(
|
|
442
476
|
chat_with_continue, llm=llm, conversations=conversations, llm_config=llm_config))
|
|
477
|
+
|
|
443
478
|
temp_results = [future.result() for future in futures]
|
|
444
|
-
|
|
479
|
+
|
|
480
|
+
for result,model_name in zip(temp_results,model_names):
|
|
445
481
|
results.append(result.content)
|
|
446
482
|
input_tokens_count += result.input_tokens_count
|
|
447
483
|
generated_tokens_count += result.generated_tokens_count
|
|
484
|
+
model_info = llm_utils.get_model_info(model_name,self.args.product_mode)
|
|
485
|
+
input_cost = model_info.get("input_price", 0) if model_info else 0
|
|
486
|
+
output_cost = model_info.get("output_price", 0) if model_info else 0
|
|
487
|
+
input_tokens_cost += input_cost * result.input_tokens_count / 1000000
|
|
488
|
+
generated_tokens_cost += output_cost * result.generated_tokens_count / 1000000
|
|
448
489
|
|
|
449
490
|
for result in results:
|
|
450
491
|
conversations_list.append(
|
|
@@ -461,7 +502,9 @@ class CodeAutoGenerateEditBlock:
|
|
|
461
502
|
|
|
462
503
|
statistics = {
|
|
463
504
|
"input_tokens_count": input_tokens_count,
|
|
464
|
-
"generated_tokens_count": generated_tokens_count
|
|
505
|
+
"generated_tokens_count": generated_tokens_count,
|
|
506
|
+
"input_tokens_cost": input_tokens_cost,
|
|
507
|
+
"generated_tokens_cost": generated_tokens_cost
|
|
465
508
|
}
|
|
466
509
|
|
|
467
510
|
if self.args.request_id and not self.args.skip_events:
|
|
@@ -476,10 +519,11 @@ class CodeAutoGenerateEditBlock:
|
|
|
476
519
|
return CodeGenerateResult(contents=results, conversations=conversations_list, metadata=statistics)
|
|
477
520
|
|
|
478
521
|
def multi_round_run(
|
|
479
|
-
self, query: str,
|
|
522
|
+
self, query: str, source_code_list: SourceCodeList, max_steps: int = 10
|
|
480
523
|
) -> CodeGenerateResult:
|
|
481
524
|
llm_config = {"human_as_model": self.args.human_as_model}
|
|
482
525
|
result = []
|
|
526
|
+
source_content = source_code_list.to_str()
|
|
483
527
|
|
|
484
528
|
if self.args.template == "common":
|
|
485
529
|
init_prompt = self.multi_round_instruction.prompt(
|
|
@@ -9,7 +9,9 @@ import json
|
|
|
9
9
|
from autocoder.common.utils_code_auto_generate import chat_with_continue
|
|
10
10
|
from autocoder.common.printer import Printer
|
|
11
11
|
from autocoder.rag.token_counter import count_tokens
|
|
12
|
-
|
|
12
|
+
from autocoder.utils import llms as llm_utils
|
|
13
|
+
from autocoder.common import SourceCodeList
|
|
14
|
+
from autocoder.privacy.model_filter import ModelPathFilter
|
|
13
15
|
class CodeAutoGenerateStrictDiff:
|
|
14
16
|
def __init__(
|
|
15
17
|
self, llm: byzerllm.ByzerLLM, args: AutoCoderArgs, action=None
|
|
@@ -271,9 +273,10 @@ class CodeAutoGenerateStrictDiff:
|
|
|
271
273
|
}
|
|
272
274
|
|
|
273
275
|
def single_round_run(
|
|
274
|
-
self, query: str,
|
|
276
|
+
self, query: str, source_code_list: SourceCodeList
|
|
275
277
|
) -> CodeGenerateResult:
|
|
276
278
|
llm_config = {"human_as_model": self.args.human_as_model}
|
|
279
|
+
source_content = source_code_list.to_str()
|
|
277
280
|
|
|
278
281
|
if self.args.template == "common":
|
|
279
282
|
init_prompt = self.single_round_instruction.prompt(
|
|
@@ -311,6 +314,9 @@ class CodeAutoGenerateStrictDiff:
|
|
|
311
314
|
results = []
|
|
312
315
|
input_tokens_count = 0
|
|
313
316
|
generated_tokens_count = 0
|
|
317
|
+
input_tokens_cost = 0
|
|
318
|
+
generated_tokens_cost = 0
|
|
319
|
+
model_names = []
|
|
314
320
|
|
|
315
321
|
printer = Printer()
|
|
316
322
|
estimated_input_tokens = count_tokens(json.dumps(conversations, ensure_ascii=False))
|
|
@@ -324,14 +330,27 @@ class CodeAutoGenerateStrictDiff:
|
|
|
324
330
|
futures = []
|
|
325
331
|
for llm in self.llms:
|
|
326
332
|
for _ in range(self.generate_times_same_model):
|
|
327
|
-
|
|
328
|
-
|
|
333
|
+
|
|
334
|
+
model_names_list = llm_utils.get_llm_names(llm)
|
|
335
|
+
model_name = None
|
|
336
|
+
if model_names_list:
|
|
337
|
+
model_name = model_names_list[0]
|
|
338
|
+
|
|
339
|
+
for _ in range(self.generate_times_same_model):
|
|
340
|
+
model_names.append(model_name)
|
|
341
|
+
futures.append(executor.submit(
|
|
342
|
+
chat_with_continue, llm=llm, conversations=conversations, llm_config=llm_config))
|
|
343
|
+
|
|
329
344
|
temp_results = [future.result() for future in futures]
|
|
330
345
|
for result in temp_results:
|
|
331
346
|
results.append(result.content)
|
|
332
347
|
input_tokens_count += result.input_tokens_count
|
|
333
348
|
generated_tokens_count += result.generated_tokens_count
|
|
334
|
-
|
|
349
|
+
model_info = llm_utils.get_model_info(model_name, self.args.product_mode)
|
|
350
|
+
input_cost = model_info.get("input_price", 0) if model_info else 0
|
|
351
|
+
output_cost = model_info.get("output_price", 0) if model_info else 0
|
|
352
|
+
input_tokens_cost += input_cost * result.input_tokens_count / 1000000
|
|
353
|
+
generated_tokens_cost += output_cost * result.generated_tokens_count / 1000000
|
|
335
354
|
for result in results:
|
|
336
355
|
conversations_list.append(
|
|
337
356
|
conversations + [{"role": "assistant", "content": result}])
|
|
@@ -345,7 +364,9 @@ class CodeAutoGenerateStrictDiff:
|
|
|
345
364
|
|
|
346
365
|
statistics = {
|
|
347
366
|
"input_tokens_count": input_tokens_count,
|
|
348
|
-
"generated_tokens_count": generated_tokens_count
|
|
367
|
+
"generated_tokens_count": generated_tokens_count,
|
|
368
|
+
"input_tokens_cost": input_tokens_cost,
|
|
369
|
+
"generated_tokens_cost": generated_tokens_cost
|
|
349
370
|
}
|
|
350
371
|
|
|
351
372
|
if self.args.request_id and not self.args.skip_events:
|
|
@@ -360,10 +381,28 @@ class CodeAutoGenerateStrictDiff:
|
|
|
360
381
|
return CodeGenerateResult(contents=results, conversations=conversations_list, metadata=statistics)
|
|
361
382
|
|
|
362
383
|
def multi_round_run(
|
|
363
|
-
self, query: str,
|
|
384
|
+
self, query: str, source_code_list: SourceCodeList, max_steps: int = 10
|
|
364
385
|
) -> CodeGenerateResult:
|
|
386
|
+
|
|
387
|
+
# Apply model filter for code_llm
|
|
388
|
+
printer = Printer()
|
|
389
|
+
for llm in self.llms:
|
|
390
|
+
model_filter = ModelPathFilter.from_model_object(llm, self.args)
|
|
391
|
+
filtered_sources = []
|
|
392
|
+
for source in source_code_list.sources:
|
|
393
|
+
if model_filter.is_accessible(source.module_name):
|
|
394
|
+
filtered_sources.append(source)
|
|
395
|
+
else:
|
|
396
|
+
printer.print_in_terminal("index_file_filtered",
|
|
397
|
+
style="yellow",
|
|
398
|
+
file_path=source.module_name,
|
|
399
|
+
model_name=",".join(llm_utils.get_llm_names(llm)))
|
|
400
|
+
|
|
401
|
+
source_code_list = SourceCodeList(filtered_sources)
|
|
402
|
+
|
|
365
403
|
llm_config = {"human_as_model": self.args.human_as_model}
|
|
366
404
|
result = []
|
|
405
|
+
source_content = source_code_list.to_str()
|
|
367
406
|
|
|
368
407
|
if self.args.template == "common":
|
|
369
408
|
init_prompt = self.multi_round_instruction.prompt(
|
|
@@ -8,8 +8,8 @@ from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
|
8
8
|
import traceback
|
|
9
9
|
from autocoder.common.utils_code_auto_generate import chat_with_continue
|
|
10
10
|
from byzerllm.utils.str2model import to_model
|
|
11
|
+
from autocoder.utils.llms import get_llm_names, get_model_info
|
|
11
12
|
|
|
12
|
-
from autocoder.utils.llms import get_llm_names
|
|
13
13
|
class RankResult(BaseModel):
|
|
14
14
|
rank_result: List[int]
|
|
15
15
|
|
|
@@ -97,13 +97,42 @@ class CodeModificationRanker:
|
|
|
97
97
|
|
|
98
98
|
# Collect all results
|
|
99
99
|
results = []
|
|
100
|
-
|
|
100
|
+
# 获取模型名称列表
|
|
101
|
+
model_names = []
|
|
102
|
+
for llm in self.llms:
|
|
103
|
+
# 获取当前llm实例对应的模型名称
|
|
104
|
+
names = get_llm_names(llm)
|
|
105
|
+
model_names.extend(names)
|
|
106
|
+
|
|
107
|
+
# 获取模型价格信息
|
|
108
|
+
model_info_map = {}
|
|
109
|
+
for name in model_names:
|
|
110
|
+
# 第二个参数是产品模式,从args中获取
|
|
111
|
+
info = get_model_info(name, self.args.product_mode)
|
|
112
|
+
if info:
|
|
113
|
+
model_info_map[name] = {
|
|
114
|
+
"input_cost": info.get("input_price", 0.0), # 每百万tokens成本
|
|
115
|
+
"output_cost": info.get("output_price", 0.0) # 每百万tokens成本
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
# 计算总成本
|
|
119
|
+
total_input_cost = 0.0
|
|
120
|
+
total_output_cost = 0.0
|
|
121
|
+
|
|
122
|
+
for future, model_name in zip(futures, model_names):
|
|
101
123
|
try:
|
|
102
124
|
result = future.result()
|
|
103
125
|
input_tokens_count += result.input_tokens_count
|
|
104
126
|
generated_tokens_count += result.generated_tokens_count
|
|
105
127
|
v = to_model(result.content,RankResult)
|
|
106
128
|
results.append(v.rank_result)
|
|
129
|
+
|
|
130
|
+
# 计算成本
|
|
131
|
+
info = model_info_map.get(model_name, {})
|
|
132
|
+
# 计算公式:token数 * 单价 / 1000000
|
|
133
|
+
total_input_cost += (result.input_tokens_count * info.get("input_cost", 0.0)) / 1000000
|
|
134
|
+
total_output_cost += (result.generated_tokens_count * info.get("output_cost", 0.0)) / 1000000
|
|
135
|
+
|
|
107
136
|
except Exception as e:
|
|
108
137
|
self.printer.print_in_terminal(
|
|
109
138
|
"ranking_failed_request", style="yellow", error=str(e))
|
|
@@ -113,6 +142,10 @@ class CodeModificationRanker:
|
|
|
113
142
|
raise Exception(
|
|
114
143
|
self.printer.get_message_from_key("ranking_all_failed"))
|
|
115
144
|
|
|
145
|
+
# 四舍五入到4位小数
|
|
146
|
+
total_input_cost = round(total_input_cost, 4)
|
|
147
|
+
total_output_cost = round(total_output_cost, 4)
|
|
148
|
+
|
|
116
149
|
# Calculate scores for each candidate
|
|
117
150
|
candidate_scores = defaultdict(float)
|
|
118
151
|
for rank_result in results:
|
|
@@ -137,7 +170,10 @@ class CodeModificationRanker:
|
|
|
137
170
|
best_candidate=sorted_candidates[0],
|
|
138
171
|
scores=score_details,
|
|
139
172
|
input_tokens=input_tokens_count,
|
|
140
|
-
output_tokens=generated_tokens_count
|
|
173
|
+
output_tokens=generated_tokens_count,
|
|
174
|
+
input_cost=total_input_cost,
|
|
175
|
+
output_cost=total_output_cost,
|
|
176
|
+
model_names=", ".join(model_names)
|
|
141
177
|
)
|
|
142
178
|
|
|
143
179
|
rerank_contents = [generate_result.contents[i]
|
|
@@ -27,6 +27,8 @@ from loguru import logger
|
|
|
27
27
|
import time
|
|
28
28
|
from autocoder.common.printer import Printer
|
|
29
29
|
from autocoder.utils.llms import get_llm_names
|
|
30
|
+
from autocoder.privacy.model_filter import ModelPathFilter
|
|
31
|
+
from autocoder.common import SourceCodeList
|
|
30
32
|
|
|
31
33
|
|
|
32
34
|
class BaseAction:
|
|
@@ -55,14 +57,15 @@ class ActionTSProject(BaseAction):
|
|
|
55
57
|
self.pp = pp
|
|
56
58
|
pp.run()
|
|
57
59
|
|
|
58
|
-
source_code = pp.output()
|
|
60
|
+
# source_code = pp.output()
|
|
61
|
+
source_code_list = SourceCodeList(pp.sources)
|
|
59
62
|
if self.llm:
|
|
60
63
|
if args.in_code_apply:
|
|
61
64
|
old_query = args.query
|
|
62
65
|
args.query = (args.context or "") + "\n\n" + args.query
|
|
63
|
-
|
|
66
|
+
source_code_list = build_index_and_filter_files(
|
|
64
67
|
llm=self.llm, args=args, sources=pp.sources
|
|
65
|
-
)
|
|
68
|
+
)
|
|
66
69
|
if args.in_code_apply:
|
|
67
70
|
args.query = old_query
|
|
68
71
|
|
|
@@ -81,17 +84,21 @@ class ActionTSProject(BaseAction):
|
|
|
81
84
|
html_path=html_path,
|
|
82
85
|
max_iter=self.args.image_max_iter,
|
|
83
86
|
)
|
|
84
|
-
|
|
87
|
+
html_code = ""
|
|
85
88
|
with open(html_path, "r") as f:
|
|
86
89
|
html_code = f.read()
|
|
87
|
-
|
|
90
|
+
|
|
91
|
+
source_code_list.sources.append(SourceCode(
|
|
92
|
+
module_name=html_path,
|
|
93
|
+
source_code=html_code,
|
|
94
|
+
tag="IMAGE"))
|
|
88
95
|
|
|
89
|
-
self.process_content(
|
|
96
|
+
self.process_content(source_code_list)
|
|
90
97
|
return True
|
|
91
98
|
|
|
92
|
-
def process_content(self,
|
|
99
|
+
def process_content(self, source_code_list: SourceCodeList):
|
|
93
100
|
args = self.args
|
|
94
|
-
|
|
101
|
+
content = source_code_list.to_str()
|
|
95
102
|
if args.execute and self.llm and not args.human_as_model:
|
|
96
103
|
content_length = self._get_content_length(content)
|
|
97
104
|
if content_length > self.args.model_max_input_length:
|
|
@@ -116,22 +123,27 @@ class ActionTSProject(BaseAction):
|
|
|
116
123
|
)
|
|
117
124
|
else:
|
|
118
125
|
generate = CodeAutoGenerate(llm=self.llm, args=self.args, action=self)
|
|
126
|
+
|
|
119
127
|
if self.args.enable_multi_round_generate:
|
|
120
128
|
generate_result = generate.multi_round_run(
|
|
121
|
-
query=args.query,
|
|
129
|
+
query=args.query, source_code_list=source_code_list
|
|
122
130
|
)
|
|
123
131
|
else:
|
|
124
132
|
generate_result = generate.single_round_run(
|
|
125
|
-
query=args.query,
|
|
133
|
+
query=args.query, source_code_list=source_code_list
|
|
126
134
|
)
|
|
127
135
|
elapsed_time = time.time() - start_time
|
|
128
|
-
speed = generate_result.metadata.get('generated_tokens_count', 0) / elapsed_time if elapsed_time > 0 else 0
|
|
129
|
-
|
|
136
|
+
speed = generate_result.metadata.get('generated_tokens_count', 0) / elapsed_time if elapsed_time > 0 else 0
|
|
137
|
+
input_tokens_cost = generate_result.metadata.get('input_tokens_cost', 0)
|
|
138
|
+
generated_tokens_cost = generate_result.metadata.get('generated_tokens_cost', 0)
|
|
139
|
+
model_names = ",".join(get_llm_names(generate.llms))
|
|
130
140
|
self.printer.print_in_terminal(
|
|
131
141
|
"code_generation_complete",
|
|
132
142
|
duration=elapsed_time,
|
|
133
143
|
input_tokens=generate_result.metadata.get('input_tokens_count', 0),
|
|
134
144
|
output_tokens=generate_result.metadata.get('generated_tokens_count', 0),
|
|
145
|
+
input_cost=input_tokens_cost,
|
|
146
|
+
output_cost=generated_tokens_cost,
|
|
135
147
|
speed=round(speed, 2),
|
|
136
148
|
model_names=model_names
|
|
137
149
|
)
|
|
@@ -187,11 +199,12 @@ class ActionPyScriptProject(BaseAction):
|
|
|
187
199
|
pp = Level1PyProject(
|
|
188
200
|
script_path=args.script_path, package_name=args.package_name
|
|
189
201
|
)
|
|
190
|
-
|
|
191
|
-
|
|
202
|
+
pp.run()
|
|
203
|
+
source_code_list = SourceCodeList(pp.sources)
|
|
204
|
+
self.process_content(source_code_list)
|
|
192
205
|
return True
|
|
193
206
|
|
|
194
|
-
def process_content(self,
|
|
207
|
+
def process_content(self, source_code_list: SourceCodeList):
|
|
195
208
|
args = self.args
|
|
196
209
|
if args.execute:
|
|
197
210
|
self.printer.print_in_terminal("code_generation_start")
|
|
@@ -212,21 +225,25 @@ class ActionPyScriptProject(BaseAction):
|
|
|
212
225
|
generate = CodeAutoGenerate(llm=self.llm, args=self.args, action=self)
|
|
213
226
|
if self.args.enable_multi_round_generate:
|
|
214
227
|
generate_result = generate.multi_round_run(
|
|
215
|
-
query=args.query,
|
|
228
|
+
query=args.query, source_code_list=source_code_list
|
|
216
229
|
)
|
|
217
230
|
else:
|
|
218
231
|
generate_result = generate.single_round_run(
|
|
219
|
-
query=args.query,
|
|
232
|
+
query=args.query, source_code_list=source_code_list
|
|
220
233
|
)
|
|
221
234
|
|
|
222
235
|
elapsed_time = time.time() - start_time
|
|
223
236
|
speed = generate_result.metadata.get('generated_tokens_count', 0) / elapsed_time if elapsed_time > 0 else 0
|
|
224
|
-
model_names = ",".join(get_llm_names(
|
|
237
|
+
model_names = ",".join(get_llm_names(generate.llms))
|
|
238
|
+
input_tokens_cost = generate_result.metadata.get('input_tokens_cost', 0)
|
|
239
|
+
generated_tokens_cost = generate_result.metadata.get('generated_tokens_cost', 0)
|
|
225
240
|
self.printer.print_in_terminal(
|
|
226
241
|
"code_generation_complete",
|
|
227
242
|
duration=elapsed_time,
|
|
228
243
|
input_tokens=generate_result.metadata.get('input_tokens_count', 0),
|
|
229
244
|
output_tokens=generate_result.metadata.get('generated_tokens_count', 0),
|
|
245
|
+
input_cost=input_tokens_cost,
|
|
246
|
+
output_cost=generated_tokens_cost,
|
|
230
247
|
speed=round(speed, 2),
|
|
231
248
|
model_names=model_names
|
|
232
249
|
)
|
|
@@ -264,13 +281,7 @@ class ActionPyScriptProject(BaseAction):
|
|
|
264
281
|
model=self.llm.default_model_name,
|
|
265
282
|
)
|
|
266
283
|
|
|
267
|
-
end_time = time.time()
|
|
268
|
-
self.printer.print_in_terminal(
|
|
269
|
-
"code_generation_complete",
|
|
270
|
-
duration=end_time - start_time,
|
|
271
|
-
input_tokens=generate_result.metadata.get('input_tokens_count', 0),
|
|
272
|
-
output_tokens=generate_result.metadata.get('generated_tokens_count', 0)
|
|
273
|
-
)
|
|
284
|
+
end_time = time.time()
|
|
274
285
|
with open(self.args.target_file, "w") as file:
|
|
275
286
|
file.write(content)
|
|
276
287
|
|
|
@@ -291,24 +302,24 @@ class ActionPyProject(BaseAction):
|
|
|
291
302
|
pp = PyProject(args=self.args, llm=self.llm)
|
|
292
303
|
self.pp = pp
|
|
293
304
|
pp.run(packages=args.py_packages.split(",") if args.py_packages else [])
|
|
294
|
-
|
|
305
|
+
source_code_list = SourceCodeList(pp.sources)
|
|
295
306
|
|
|
296
307
|
if self.llm:
|
|
297
308
|
old_query = args.query
|
|
298
309
|
if args.in_code_apply:
|
|
299
310
|
args.query = (args.context or "") + "\n\n" + args.query
|
|
300
|
-
|
|
311
|
+
source_code_list = build_index_and_filter_files(
|
|
301
312
|
llm=self.llm, args=args, sources=pp.sources
|
|
302
313
|
)
|
|
303
314
|
if args.in_code_apply:
|
|
304
315
|
args.query = old_query
|
|
305
316
|
|
|
306
|
-
self.process_content(
|
|
317
|
+
self.process_content(source_code_list)
|
|
307
318
|
return True
|
|
308
319
|
|
|
309
|
-
def process_content(self,
|
|
320
|
+
def process_content(self, source_code_list: SourceCodeList):
|
|
310
321
|
args = self.args
|
|
311
|
-
|
|
322
|
+
content = source_code_list.to_str()
|
|
312
323
|
if args.execute and self.llm and not args.human_as_model:
|
|
313
324
|
content_length = self._get_content_length(content)
|
|
314
325
|
if content_length > self.args.model_max_input_length:
|
|
@@ -340,20 +351,24 @@ class ActionPyProject(BaseAction):
|
|
|
340
351
|
|
|
341
352
|
if self.args.enable_multi_round_generate:
|
|
342
353
|
generate_result = generate.multi_round_run(
|
|
343
|
-
query=args.query,
|
|
354
|
+
query=args.query, source_code_list=source_code_list
|
|
344
355
|
)
|
|
345
356
|
else:
|
|
346
357
|
generate_result = generate.single_round_run(
|
|
347
|
-
query=args.query,
|
|
358
|
+
query=args.query, source_code_list=source_code_list
|
|
348
359
|
)
|
|
349
360
|
elapsed_time = time.time() - start_time
|
|
350
361
|
speed = generate_result.metadata.get('generated_tokens_count', 0) / elapsed_time if elapsed_time > 0 else 0
|
|
351
|
-
model_names = ",".join(get_llm_names(
|
|
362
|
+
model_names = ",".join(get_llm_names(generate.llms))
|
|
363
|
+
input_tokens_cost = generate_result.metadata.get('input_tokens_cost', 0)
|
|
364
|
+
generated_tokens_cost = generate_result.metadata.get('generated_tokens_cost', 0)
|
|
352
365
|
self.printer.print_in_terminal(
|
|
353
366
|
"code_generation_complete",
|
|
354
367
|
duration=elapsed_time,
|
|
355
368
|
input_tokens=generate_result.metadata.get('input_tokens_count', 0),
|
|
356
369
|
output_tokens=generate_result.metadata.get('generated_tokens_count', 0),
|
|
370
|
+
input_cost=input_tokens_cost,
|
|
371
|
+
output_cost=generated_tokens_cost,
|
|
357
372
|
speed=round(speed, 2),
|
|
358
373
|
model_names=model_names
|
|
359
374
|
)
|
|
@@ -408,20 +423,21 @@ class ActionSuffixProject(BaseAction):
|
|
|
408
423
|
pp = SuffixProject(args=args, llm=self.llm)
|
|
409
424
|
self.pp = pp
|
|
410
425
|
pp.run()
|
|
411
|
-
|
|
426
|
+
source_code_list = SourceCodeList(pp.sources)
|
|
412
427
|
if self.llm:
|
|
413
428
|
if args.in_code_apply:
|
|
414
429
|
old_query = args.query
|
|
415
430
|
args.query = (args.context or "") + "\n\n" + args.query
|
|
416
|
-
|
|
431
|
+
source_code_list = build_index_and_filter_files(
|
|
417
432
|
llm=self.llm, args=args, sources=pp.sources
|
|
418
433
|
)
|
|
419
434
|
if args.in_code_apply:
|
|
420
435
|
args.query = old_query
|
|
421
|
-
self.process_content(
|
|
436
|
+
self.process_content(source_code_list)
|
|
422
437
|
|
|
423
|
-
def process_content(self,
|
|
438
|
+
def process_content(self, source_code_list: SourceCodeList):
|
|
424
439
|
args = self.args
|
|
440
|
+
content = source_code_list.to_str()
|
|
425
441
|
|
|
426
442
|
if args.execute and self.llm and not args.human_as_model:
|
|
427
443
|
content_length = self._get_content_length(content)
|
|
@@ -449,21 +465,25 @@ class ActionSuffixProject(BaseAction):
|
|
|
449
465
|
generate = CodeAutoGenerate(llm=self.llm, args=self.args, action=self)
|
|
450
466
|
if self.args.enable_multi_round_generate:
|
|
451
467
|
generate_result = generate.multi_round_run(
|
|
452
|
-
query=args.query,
|
|
468
|
+
query=args.query, source_code_list=source_code_list
|
|
453
469
|
)
|
|
454
470
|
else:
|
|
455
471
|
generate_result = generate.single_round_run(
|
|
456
|
-
query=args.query,
|
|
472
|
+
query=args.query, source_code_list=source_code_list
|
|
457
473
|
)
|
|
458
474
|
|
|
459
475
|
elapsed_time = time.time() - start_time
|
|
460
476
|
speed = generate_result.metadata.get('generated_tokens_count', 0) / elapsed_time if elapsed_time > 0 else 0
|
|
461
|
-
model_names = ",".join(get_llm_names(
|
|
477
|
+
model_names = ",".join(get_llm_names(generate.llms))
|
|
478
|
+
input_tokens_cost = generate_result.metadata.get('input_tokens_cost', 0)
|
|
479
|
+
generated_tokens_cost = generate_result.metadata.get('generated_tokens_cost', 0)
|
|
462
480
|
self.printer.print_in_terminal(
|
|
463
481
|
"code_generation_complete",
|
|
464
482
|
duration=elapsed_time,
|
|
465
483
|
input_tokens=generate_result.metadata.get('input_tokens_count', 0),
|
|
466
484
|
output_tokens=generate_result.metadata.get('generated_tokens_count', 0),
|
|
485
|
+
input_cost=input_tokens_cost,
|
|
486
|
+
output_cost=generated_tokens_cost,
|
|
467
487
|
speed=round(speed, 2),
|
|
468
488
|
model_names=model_names
|
|
469
489
|
)
|