spiderforce4ai 2.4.9__py3-none-any.whl → 2.5.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- spiderforce4ai/__init__.py +19 -2
- {spiderforce4ai-2.4.9.dist-info → spiderforce4ai-2.5.1.dist-info}/METADATA +1 -1
- spiderforce4ai-2.5.1.dist-info/RECORD +7 -0
- spiderforce4ai-2.4.9.dist-info/RECORD +0 -7
- {spiderforce4ai-2.4.9.dist-info → spiderforce4ai-2.5.1.dist-info}/WHEEL +0 -0
- {spiderforce4ai-2.4.9.dist-info → spiderforce4ai-2.5.1.dist-info}/entry_points.txt +0 -0
- {spiderforce4ai-2.4.9.dist-info → spiderforce4ai-2.5.1.dist-info}/top_level.txt +0 -0
spiderforce4ai/__init__.py
CHANGED
@@ -5,6 +5,9 @@ import asyncio
|
|
5
5
|
import aiohttp
|
6
6
|
import json
|
7
7
|
import logging
|
8
|
+
|
9
|
+
logger = logging.getLogger(__name__)
|
10
|
+
import logging
|
8
11
|
from typing import List, Dict, Union, Optional, Tuple, Callable, Any
|
9
12
|
from dataclasses import dataclass, asdict
|
10
13
|
from urllib.parse import urljoin, urlparse
|
@@ -746,7 +749,8 @@ class SpiderForce4AI:
|
|
746
749
|
|
747
750
|
for result in successful_results:
|
748
751
|
try:
|
749
|
-
|
752
|
+
# Process content synchronously since it's not an async method
|
753
|
+
result.extraction_result = agent.process_content(result.url, result.markdown)
|
750
754
|
progress.update(llm_task, advance=1)
|
751
755
|
except Exception as e:
|
752
756
|
console.print(f"[red]Error in post-extraction processing for {result.url}: {str(e)}[/red]")
|
@@ -877,7 +881,20 @@ class SpiderForce4AI:
|
|
877
881
|
|
878
882
|
for result in successful_results:
|
879
883
|
try:
|
880
|
-
|
884
|
+
# Get LLM response
|
885
|
+
llm_response = agent.process_content(result.url, result.markdown)
|
886
|
+
if llm_response:
|
887
|
+
# Add URL to the response before transformation
|
888
|
+
llm_response['url'] = result.url
|
889
|
+
# Apply transformation if provided
|
890
|
+
if config.post_agent_transformer_function:
|
891
|
+
try:
|
892
|
+
result.extraction_result = config.post_agent_transformer_function(llm_response)
|
893
|
+
except Exception as e:
|
894
|
+
logger.error(f"Transformer error for {result.url}: {str(e)}")
|
895
|
+
result.extraction_result = llm_response # Use original response if transform fails
|
896
|
+
else:
|
897
|
+
result.extraction_result = llm_response
|
881
898
|
progress.update(llm_task, advance=1)
|
882
899
|
except Exception as e:
|
883
900
|
console.print(f"[red]Error in post-extraction processing for {result.url}: {str(e)}[/red]")
|
@@ -0,0 +1,7 @@
|
|
1
|
+
spiderforce4ai/__init__.py,sha256=ipAArcZXJpB-KT4MSs0NReY5mhpO2X0Zsoh-roibWWI,43401
|
2
|
+
spiderforce4ai/post_extraction_agent.py,sha256=so5Ze7Vz3konpQ0iT7ZxDGE9kIYeTwPTFyzezRc5oys,15392
|
3
|
+
spiderforce4ai-2.5.1.dist-info/METADATA,sha256=VIpmVqwJC6fKufS73tZBYXir8vHaLTScfF3sif28c3o,9012
|
4
|
+
spiderforce4ai-2.5.1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
|
5
|
+
spiderforce4ai-2.5.1.dist-info/entry_points.txt,sha256=ibARQxOlDiL1ho12zbDZt4Uq5RKSIk_qk159ZlZ46hc,59
|
6
|
+
spiderforce4ai-2.5.1.dist-info/top_level.txt,sha256=Kth7A21Js7DCp0j5XBBi-FE45SCLouZkeNZU__Yr9Yk,15
|
7
|
+
spiderforce4ai-2.5.1.dist-info/RECORD,,
|
@@ -1,7 +0,0 @@
|
|
1
|
-
spiderforce4ai/__init__.py,sha256=DUPOKF7-vCVQi7JimsStU1qjk5x3yVUoMnUVOJxOrGk,42360
|
2
|
-
spiderforce4ai/post_extraction_agent.py,sha256=so5Ze7Vz3konpQ0iT7ZxDGE9kIYeTwPTFyzezRc5oys,15392
|
3
|
-
spiderforce4ai-2.4.9.dist-info/METADATA,sha256=kEq3anAkoe_wpPVzpgaJlsSuAzTQHDgXiDFpirXvUQc,9012
|
4
|
-
spiderforce4ai-2.4.9.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
|
5
|
-
spiderforce4ai-2.4.9.dist-info/entry_points.txt,sha256=ibARQxOlDiL1ho12zbDZt4Uq5RKSIk_qk159ZlZ46hc,59
|
6
|
-
spiderforce4ai-2.4.9.dist-info/top_level.txt,sha256=Kth7A21Js7DCp0j5XBBi-FE45SCLouZkeNZU__Yr9Yk,15
|
7
|
-
spiderforce4ai-2.4.9.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|