spiderforce4ai 0.1.9__py3-none-any.whl → 1.1__py3-none-any.whl

spiderforce4ai/__init__.py

@@ -445,7 +445,11 @@ class SpiderForce4AI:
         if not failed_results:
             return []
 
-        console.print("\n[yellow]Retrying failed URLs...[/yellow]")
+        failed_count = len(failed_results)
+        total_count = len([r for r in self.crawl_results])
+        failure_ratio = (failed_count / total_count) * 100
+        
+        console.print(f"\n[yellow]Retrying failed URLs: {failed_count} ({failure_ratio:.1f}% failed)[/yellow]")
         retry_results = []
         
         # Create a new progress bar if one wasn't provided
@@ -519,24 +523,41 @@ class SpiderForce4AI:
             # Identify failed URLs
             failed_results = [r for r in initial_results if r.status == "failed"]
             
-            # Retry failed URLs
+            # Calculate initial failure ratio
+            initial_failed = len(failed_results)
+            total_urls = len(urls)
+            failure_ratio = (initial_failed / total_urls) * 100
+
+            # Retry failed URLs if ratio is acceptable
             if failed_results:
-                retry_results = await self._retry_failed_urls(failed_results, config, progress)
-                
-                # Replace failed results with retry results
-                results = [r for r in initial_results if r.status == "success"] + retry_results
+                if failure_ratio > 20:
+                    console.print(f"\n[red]Failure ratio too high ({failure_ratio:.1f}%) - aborting retry due to possible server overload[/red]")
+                    results = initial_results
+                else:
+                    retry_results = await self._retry_failed_urls(failed_results, config, progress)
+                    # Replace failed results with retry results
+                    results = [r for r in initial_results if r.status == "success"] + retry_results
             else:
                 results = initial_results
             
             # Save final report
             await self._save_report(config)
             
-            # Print final summary
-            successful = len([r for r in results if r.status == "success"])
-            failed = len([r for r in results if r.status == "failed"])
-            console.print(f"\n[green]Final crawling results:[/green]")
-            console.print(f"✓ Successful: {successful}")
-            console.print(f"✗ Failed: {failed}")
+            # Calculate final statistics
+            final_successful = len([r for r in results if r.status == "success"])
+            final_failed = len([r for r in results if r.status == "failed"])
+            
+            # Print detailed summary
+            console.print(f"\n[green]Crawling Summary:[/green]")
+            console.print(f"Total URLs processed: {total_urls}")
+            console.print(f"Initial failures: {initial_failed} ({failure_ratio:.1f}%)")
+            console.print(f"Final results:")
+            console.print(f"  ✓ Successful: {final_successful}")
+            console.print(f"  ✗ Failed: {final_failed}")
+            
+            if initial_failed > 0:
+                retry_successful = initial_failed - final_failed
+                console.print(f"Retry success rate: {retry_successful}/{initial_failed} ({(retry_successful/initial_failed)*100:.1f}%)")
             
             if config.report_file:
                 console.print(f"📊 Report saved to: {config.report_file}")
@@ -624,25 +645,44 @@ class SpiderForce4AI:
             self._save_report_sync(results, config)
             print(f"\nReport saved to: {config.report_file}")
 
-        # Identify failed URLs and retry them
+        # Calculate initial failure statistics
         failed_results = [r for r in results if r.status == "failed"]
-        if failed_results:
-            console.print("\n[yellow]Retrying failed URLs...[/yellow]")
-            for result in failed_results:
-                new_result = _process_url_parallel((result.url, self.base_url, config))
-                if new_result.status == "success":
-                    console.print(f"[green]✓ Retry successful: {result.url}[/green]")
-                    # Replace the failed result with the successful retry
-                    results[results.index(result)] = new_result
-                else:
-                    console.print(f"[red]✗ Retry failed: {result.url} - {new_result.error}[/red]")
+        initial_failed = len(failed_results)
+        total_urls = len(urls)
+        failure_ratio = (initial_failed / total_urls) * 100
 
-        # Print final summary
-        successful = len([r for r in results if r.status == "success"])
-        failed = len([r for r in results if r.status == "failed"])
-        console.print(f"\n[green]Final crawling results:[/green]")
-        console.print(f"✓ Successful: {successful}")
-        console.print(f"✗ Failed: {failed}")
+        # Retry failed URLs if ratio is acceptable
+        if failed_results:
+            if failure_ratio > 20:
+                console.print(f"\n[red]Failure ratio too high ({failure_ratio:.1f}%) - aborting retry due to possible server overload[/red]")
+            else:
+                failed_count = len(failed_results)
+                failure_ratio = (failed_count / total_urls) * 100
+                console.print(f"\n[yellow]Retrying failed URLs: {failed_count} ({failure_ratio:.1f}% failed)[/yellow]")
+                for result in failed_results:
+                    new_result = _process_url_parallel((result.url, self.base_url, config))
+                    if new_result.status == "success":
+                        console.print(f"[green]✓ Retry successful: {result.url}[/green]")
+                        # Replace the failed result with the successful retry
+                        results[results.index(result)] = new_result
+                    else:
+                        console.print(f"[red]✗ Retry failed: {result.url} - {new_result.error}[/red]")
+
+        # Calculate final statistics
+        final_successful = len([r for r in results if r.status == "success"])
+        final_failed = len([r for r in results if r.status == "failed"])
+
+        # Print detailed summary
+        console.print(f"\n[green]Crawling Summary:[/green]")
+        console.print(f"Total URLs processed: {total_urls}")
+        console.print(f"Initial failures: {initial_failed} ({failure_ratio:.1f}%)")
+        console.print(f"Final results:")
+        console.print(f"  ✓ Successful: {final_successful}")
+        console.print(f"  ✗ Failed: {final_failed}")
+
+        if initial_failed > 0:
+            retry_successful = initial_failed - final_failed
+            console.print(f"Retry success rate: {retry_successful}/{initial_failed} ({(retry_successful/initial_failed)*100:.1f}%)")
 
         return results
 

{spiderforce4ai-0.1.9.dist-info → spiderforce4ai-1.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: spiderforce4ai
-Version: 0.1.9
+Version: 1.1
 Summary: Python wrapper for SpiderForce4AI HTML-to-Markdown conversion service
 Home-page: https://petertam.pro
 Author: Piotr Tamulewicz

spiderforce4ai-1.1.dist-info/RECORD

@@ -0,0 +1,5 @@
+spiderforce4ai/__init__.py,sha256=lCviRhfLngSMehFJZwyK4LirPwbWEyZ0RJjCt5FkBcY,28304
+spiderforce4ai-1.1.dist-info/METADATA,sha256=lQfqXn0ifJOmOmLkgr8YTSYUFiu6-HS3YsRD0togylo,7769
+spiderforce4ai-1.1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+spiderforce4ai-1.1.dist-info/top_level.txt,sha256=Kth7A21Js7DCp0j5XBBi-FE45SCLouZkeNZU__Yr9Yk,15
+spiderforce4ai-1.1.dist-info/RECORD,,

spiderforce4ai-0.1.9.dist-info/RECORD

@@ -1,5 +0,0 @@
-spiderforce4ai/__init__.py,sha256=oU_UIdzsQxExaVgD7NCaVm4G-9zMtKGnREfY6xL1uFY,26041
-spiderforce4ai-0.1.9.dist-info/METADATA,sha256=poV1i_-H3AgzFhs9juRDJSfaWO0gVePb5JXN7ynL4Y4,7771
-spiderforce4ai-0.1.9.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-spiderforce4ai-0.1.9.dist-info/top_level.txt,sha256=Kth7A21Js7DCp0j5XBBi-FE45SCLouZkeNZU__Yr9Yk,15
-spiderforce4ai-0.1.9.dist-info/RECORD,,