spiderforce4ai 2.4.9__py3-none-any.whl → 2.5.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -5,6 +5,9 @@ import asyncio
5
5
  import aiohttp
6
6
  import json
7
7
  import logging
8
+
9
+ logger = logging.getLogger(__name__)
10
+ import logging
8
11
  from typing import List, Dict, Union, Optional, Tuple, Callable, Any
9
12
  from dataclasses import dataclass, asdict
10
13
  from urllib.parse import urljoin, urlparse
@@ -746,7 +749,8 @@ class SpiderForce4AI:
746
749
 
747
750
  for result in successful_results:
748
751
  try:
749
- result.extraction_result = asyncio.run(agent.process_content(result.url, result.markdown))
752
+ # Process content synchronously since it's not an async method
753
+ result.extraction_result = agent.process_content(result.url, result.markdown)
750
754
  progress.update(llm_task, advance=1)
751
755
  except Exception as e:
752
756
  console.print(f"[red]Error in post-extraction processing for {result.url}: {str(e)}[/red]")
@@ -877,7 +881,20 @@ class SpiderForce4AI:
877
881
 
878
882
  for result in successful_results:
879
883
  try:
880
- result.extraction_result = asyncio.run(agent.process_content(result.url, result.markdown))
884
+ # Get LLM response
885
+ llm_response = agent.process_content(result.url, result.markdown)
886
+ if llm_response:
887
+ # Add URL to the response before transformation
888
+ llm_response['url'] = result.url
889
+ # Apply transformation if provided
890
+ if config.post_agent_transformer_function:
891
+ try:
892
+ result.extraction_result = config.post_agent_transformer_function(llm_response)
893
+ except Exception as e:
894
+ logger.error(f"Transformer error for {result.url}: {str(e)}")
895
+ result.extraction_result = llm_response # Use original response if transform fails
896
+ else:
897
+ result.extraction_result = llm_response
881
898
  progress.update(llm_task, advance=1)
882
899
  except Exception as e:
883
900
  console.print(f"[red]Error in post-extraction processing for {result.url}: {str(e)}[/red]")
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: spiderforce4ai
3
- Version: 2.4.9
3
+ Version: 2.5.1
4
4
  Summary: Python wrapper for SpiderForce4AI HTML-to-Markdown conversion service with LLM post-processing
5
5
  Home-page: https://petertam.pro
6
6
  Author: Piotr Tamulewicz
@@ -0,0 +1,7 @@
1
+ spiderforce4ai/__init__.py,sha256=ipAArcZXJpB-KT4MSs0NReY5mhpO2X0Zsoh-roibWWI,43401
2
+ spiderforce4ai/post_extraction_agent.py,sha256=so5Ze7Vz3konpQ0iT7ZxDGE9kIYeTwPTFyzezRc5oys,15392
3
+ spiderforce4ai-2.5.1.dist-info/METADATA,sha256=VIpmVqwJC6fKufS73tZBYXir8vHaLTScfF3sif28c3o,9012
4
+ spiderforce4ai-2.5.1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
5
+ spiderforce4ai-2.5.1.dist-info/entry_points.txt,sha256=ibARQxOlDiL1ho12zbDZt4Uq5RKSIk_qk159ZlZ46hc,59
6
+ spiderforce4ai-2.5.1.dist-info/top_level.txt,sha256=Kth7A21Js7DCp0j5XBBi-FE45SCLouZkeNZU__Yr9Yk,15
7
+ spiderforce4ai-2.5.1.dist-info/RECORD,,
@@ -1,7 +0,0 @@
1
- spiderforce4ai/__init__.py,sha256=DUPOKF7-vCVQi7JimsStU1qjk5x3yVUoMnUVOJxOrGk,42360
2
- spiderforce4ai/post_extraction_agent.py,sha256=so5Ze7Vz3konpQ0iT7ZxDGE9kIYeTwPTFyzezRc5oys,15392
3
- spiderforce4ai-2.4.9.dist-info/METADATA,sha256=kEq3anAkoe_wpPVzpgaJlsSuAzTQHDgXiDFpirXvUQc,9012
4
- spiderforce4ai-2.4.9.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
5
- spiderforce4ai-2.4.9.dist-info/entry_points.txt,sha256=ibARQxOlDiL1ho12zbDZt4Uq5RKSIk_qk159ZlZ46hc,59
6
- spiderforce4ai-2.4.9.dist-info/top_level.txt,sha256=Kth7A21Js7DCp0j5XBBi-FE45SCLouZkeNZU__Yr9Yk,15
7
- spiderforce4ai-2.4.9.dist-info/RECORD,,