fraudcrawler 0.5.9__py3-none-any.whl → 0.6.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of fraudcrawler might be problematic. Click here for more details.
- fraudcrawler/__init__.py +2 -2
- fraudcrawler/base/base.py +11 -32
- fraudcrawler/base/client.py +1 -1
- fraudcrawler/base/orchestrator.py +135 -135
- fraudcrawler/base/retry.py +12 -6
- fraudcrawler/launch_demo_pipeline.py +1 -1
- fraudcrawler/processing/processor.py +3 -3
- fraudcrawler/scraping/search.py +352 -125
- fraudcrawler/scraping/url.py +42 -3
- fraudcrawler/scraping/zyte.py +15 -1
- fraudcrawler/settings.py +13 -3
- {fraudcrawler-0.5.9.dist-info → fraudcrawler-0.6.1.dist-info}/METADATA +4 -3
- fraudcrawler-0.6.1.dist-info/RECORD +22 -0
- {fraudcrawler-0.5.9.dist-info → fraudcrawler-0.6.1.dist-info}/WHEEL +1 -1
- fraudcrawler-0.5.9.dist-info/RECORD +0 -22
- {fraudcrawler-0.5.9.dist-info → fraudcrawler-0.6.1.dist-info}/LICENSE +0 -0
- {fraudcrawler-0.5.9.dist-info → fraudcrawler-0.6.1.dist-info}/entry_points.txt +0 -0
|
@@ -72,7 +72,7 @@ class Processor:
|
|
|
72
72
|
"""Context aware logging before the request is made."""
|
|
73
73
|
if retry_state:
|
|
74
74
|
logger.debug(
|
|
75
|
-
f"Classifying product with url={url} using prompt={prompt} (Attempt {retry_state.attempt_number})."
|
|
75
|
+
f"Classifying product with url={url} using prompt={prompt.name} (Attempt {retry_state.attempt_number})."
|
|
76
76
|
)
|
|
77
77
|
else:
|
|
78
78
|
logger.debug(f"retry_state is {retry_state}; not logging before.")
|
|
@@ -84,7 +84,7 @@ class Processor:
|
|
|
84
84
|
"""Context aware logging before sleeping after a failed request."""
|
|
85
85
|
if retry_state and retry_state.outcome:
|
|
86
86
|
logger.warning(
|
|
87
|
-
f"Attempt {retry_state.attempt_number} of classifying product with url={url} using prompt={prompt} "
|
|
87
|
+
f"Attempt {retry_state.attempt_number} of classifying product with url={url} using prompt={prompt.name} "
|
|
88
88
|
f"failed with error: {retry_state.outcome.exception()}. "
|
|
89
89
|
f"Retrying in {retry_state.upcoming_sleep:.0f} seconds."
|
|
90
90
|
)
|
|
@@ -160,7 +160,7 @@ class Processor:
|
|
|
160
160
|
# Call the OpenAI API
|
|
161
161
|
try:
|
|
162
162
|
logger.debug(
|
|
163
|
-
f"Classifying product with url={url} using prompt={prompt.name}
|
|
163
|
+
f"Classifying product with url={url}, using prompt={prompt.name}."
|
|
164
164
|
)
|
|
165
165
|
# Perform the request and retry if necessary. There is some context aware logging
|
|
166
166
|
# - `before`: before the request is made (or before retrying)
|