apcloudy-pipeline 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- apcloudy_pipeline-0.1.0/Apcloudy_Pipeline.egg-info/PKG-INFO +86 -0
- apcloudy_pipeline-0.1.0/Apcloudy_Pipeline.egg-info/SOURCES.txt +23 -0
- apcloudy_pipeline-0.1.0/Apcloudy_Pipeline.egg-info/dependency_links.txt +1 -0
- apcloudy_pipeline-0.1.0/Apcloudy_Pipeline.egg-info/not-zip-safe +1 -0
- apcloudy_pipeline-0.1.0/Apcloudy_Pipeline.egg-info/requires.txt +1 -0
- apcloudy_pipeline-0.1.0/Apcloudy_Pipeline.egg-info/top_level.txt +1 -0
- apcloudy_pipeline-0.1.0/LICENCE +21 -0
- apcloudy_pipeline-0.1.0/PKG-INFO +86 -0
- apcloudy_pipeline-0.1.0/README.md +58 -0
- apcloudy_pipeline-0.1.0/apcloudy_pipeline/__init__.py +18 -0
- apcloudy_pipeline-0.1.0/apcloudy_pipeline/client.py +99 -0
- apcloudy_pipeline-0.1.0/apcloudy_pipeline/exceptions.py +3 -0
- apcloudy_pipeline-0.1.0/apcloudy_pipeline/extensions.py +64 -0
- apcloudy_pipeline-0.1.0/apcloudy_pipeline/log_handler.py +32 -0
- apcloudy_pipeline-0.1.0/apcloudy_pipeline/pipelines.py +14 -0
- apcloudy_pipeline-0.1.0/apcloudy_pipeline/request_logger.py +59 -0
- apcloudy_pipeline-0.1.0/apcloudy_pipeline/utils.py +27 -0
- apcloudy_pipeline-0.1.0/pyproject.toml +3 -0
- apcloudy_pipeline-0.1.0/setup.cfg +4 -0
- apcloudy_pipeline-0.1.0/setup.py +36 -0
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: apcloudy-pipeline
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Scrapy pipeline & extensions for AP Cloudy (logs, stats, requests, items)
|
|
5
|
+
Home-page: https://github.com/fawadss1/apcloudy-pipeline
|
|
6
|
+
Author: Fawad
|
|
7
|
+
Author-email: fawadstar6@email.com
|
|
8
|
+
License: MIT
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Framework :: Scrapy
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Requires-Python: >=3.8
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
License-File: LICENCE
|
|
16
|
+
Requires-Dist: requests>=2.30.0
|
|
17
|
+
Dynamic: author
|
|
18
|
+
Dynamic: author-email
|
|
19
|
+
Dynamic: classifier
|
|
20
|
+
Dynamic: description
|
|
21
|
+
Dynamic: description-content-type
|
|
22
|
+
Dynamic: home-page
|
|
23
|
+
Dynamic: license
|
|
24
|
+
Dynamic: license-file
|
|
25
|
+
Dynamic: requires-dist
|
|
26
|
+
Dynamic: requires-python
|
|
27
|
+
Dynamic: summary
|
|
28
|
+
|
|
29
|
+
# apcloudy-pipeline
|
|
30
|
+
|
|
31
|
+
apcloudy-pipeline is a Scrapy integration that sends **items, requests, logs, and spider statistics**
|
|
32
|
+
to the **Your backend** using **secure HMAC-based authentication**.
|
|
33
|
+
|
|
34
|
+
---
|
|
35
|
+
|
|
36
|
+
## Features
|
|
37
|
+
|
|
38
|
+
- ๐ฆ Item forwarding using Scrapy Item Pipeline
|
|
39
|
+
- ๐ Request and response logging
|
|
40
|
+
- ๐ Spider statistics reporting
|
|
41
|
+
- ๐งพ Spider, user, and Scrapy internal log forwarding
|
|
42
|
+
- ๐ HMAC-secured API communication
|
|
43
|
+
|
|
44
|
+
---
|
|
45
|
+
|
|
46
|
+
## Installation
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
pip install apcloudy-pipeline
|
|
50
|
+
```
|
|
51
|
+
## Configuration
|
|
52
|
+
Add the following settings to your Scrapy project's settings.py file:
|
|
53
|
+
```python
|
|
54
|
+
APCLOUDY_API_URL = "http://localhost:8000/api/v1/"
|
|
55
|
+
APCLOUDY_API_KEY = "api_test_1234567890"
|
|
56
|
+
APCLOUDY_SECRET_KEY = "secret_test_1234567890"
|
|
57
|
+
JOB_ID = 123
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
### Item Pipeline (Required)
|
|
61
|
+
The item pipeline is required to send scraped items to the backend.
|
|
62
|
+
```python
|
|
63
|
+
ITEM_PIPELINES = {
|
|
64
|
+
"apcloudy_pipeline.pipelines.APCloudyItemPipeline": 300,
|
|
65
|
+
}
|
|
66
|
+
```
|
|
67
|
+
### Extensions (Optional)
|
|
68
|
+
Enable the following extensions if you want to send requests, logs, and spider statistics.
|
|
69
|
+
```python
|
|
70
|
+
EXTENSIONS = {
|
|
71
|
+
"apcloudy_pipeline.request_logger.APCloudyRequestLogger": 400,
|
|
72
|
+
"apcloudy_pipeline.extensions.APCloudyLoggingExtension": 510,
|
|
73
|
+
"apcloudy_pipeline.extensions.APCloudyStatsExtension": 520,
|
|
74
|
+
}
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
## Extensions Overview
|
|
78
|
+
|
|
79
|
+
- APCloudyRequestLogger
|
|
80
|
+
Captures request and response metadata such as URL, HTTP method, status code, timing, and fingerprint.
|
|
81
|
+
|
|
82
|
+
- APCloudyLoggingExtension
|
|
83
|
+
Sends spider logs, user logs, Scrapy internal logs, and exception tracebacks to the backend.
|
|
84
|
+
|
|
85
|
+
- APCloudyStatsExtension
|
|
86
|
+
Sends final spider statistics when the crawl finishes.
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
LICENCE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
setup.py
|
|
5
|
+
Apcloudy_Pipeline.egg-info/PKG-INFO
|
|
6
|
+
Apcloudy_Pipeline.egg-info/SOURCES.txt
|
|
7
|
+
Apcloudy_Pipeline.egg-info/dependency_links.txt
|
|
8
|
+
Apcloudy_Pipeline.egg-info/requires.txt
|
|
9
|
+
Apcloudy_Pipeline.egg-info/top_level.txt
|
|
10
|
+
apcloudy_pipeline/__init__.py
|
|
11
|
+
apcloudy_pipeline/client.py
|
|
12
|
+
apcloudy_pipeline/exceptions.py
|
|
13
|
+
apcloudy_pipeline/extensions.py
|
|
14
|
+
apcloudy_pipeline/log_handler.py
|
|
15
|
+
apcloudy_pipeline/pipelines.py
|
|
16
|
+
apcloudy_pipeline/request_logger.py
|
|
17
|
+
apcloudy_pipeline/utils.py
|
|
18
|
+
apcloudy_pipeline.egg-info/PKG-INFO
|
|
19
|
+
apcloudy_pipeline.egg-info/SOURCES.txt
|
|
20
|
+
apcloudy_pipeline.egg-info/dependency_links.txt
|
|
21
|
+
apcloudy_pipeline.egg-info/not-zip-safe
|
|
22
|
+
apcloudy_pipeline.egg-info/requires.txt
|
|
23
|
+
apcloudy_pipeline.egg-info/top_level.txt
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
requests>=2.30.0
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
apcloudy_pipeline
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Fawad
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: apcloudy-pipeline
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Scrapy pipeline & extensions for AP Cloudy (logs, stats, requests, items)
|
|
5
|
+
Home-page: https://github.com/fawadss1/apcloudy-pipeline
|
|
6
|
+
Author: Fawad
|
|
7
|
+
Author-email: fawadstar6@email.com
|
|
8
|
+
License: MIT
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Framework :: Scrapy
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Requires-Python: >=3.8
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
License-File: LICENCE
|
|
16
|
+
Requires-Dist: requests>=2.30.0
|
|
17
|
+
Dynamic: author
|
|
18
|
+
Dynamic: author-email
|
|
19
|
+
Dynamic: classifier
|
|
20
|
+
Dynamic: description
|
|
21
|
+
Dynamic: description-content-type
|
|
22
|
+
Dynamic: home-page
|
|
23
|
+
Dynamic: license
|
|
24
|
+
Dynamic: license-file
|
|
25
|
+
Dynamic: requires-dist
|
|
26
|
+
Dynamic: requires-python
|
|
27
|
+
Dynamic: summary
|
|
28
|
+
|
|
29
|
+
# apcloudy-pipeline
|
|
30
|
+
|
|
31
|
+
apcloudy-pipeline is a Scrapy integration that sends **items, requests, logs, and spider statistics**
|
|
32
|
+
to the **Your backend** using **secure HMAC-based authentication**.
|
|
33
|
+
|
|
34
|
+
---
|
|
35
|
+
|
|
36
|
+
## Features
|
|
37
|
+
|
|
38
|
+
- ๐ฆ Item forwarding using Scrapy Item Pipeline
|
|
39
|
+
- ๐ Request and response logging
|
|
40
|
+
- ๐ Spider statistics reporting
|
|
41
|
+
- ๐งพ Spider, user, and Scrapy internal log forwarding
|
|
42
|
+
- ๐ HMAC-secured API communication
|
|
43
|
+
|
|
44
|
+
---
|
|
45
|
+
|
|
46
|
+
## Installation
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
pip install apcloudy-pipeline
|
|
50
|
+
```
|
|
51
|
+
## Configuration
|
|
52
|
+
Add the following settings to your Scrapy project's settings.py file:
|
|
53
|
+
```python
|
|
54
|
+
APCLOUDY_API_URL = "http://localhost:8000/api/v1/"
|
|
55
|
+
APCLOUDY_API_KEY = "api_test_1234567890"
|
|
56
|
+
APCLOUDY_SECRET_KEY = "secret_test_1234567890"
|
|
57
|
+
JOB_ID = 123
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
### Item Pipeline (Required)
|
|
61
|
+
The item pipeline is required to send scraped items to the backend.
|
|
62
|
+
```python
|
|
63
|
+
ITEM_PIPELINES = {
|
|
64
|
+
"apcloudy_pipeline.pipelines.APCloudyItemPipeline": 300,
|
|
65
|
+
}
|
|
66
|
+
```
|
|
67
|
+
### Extensions (Optional)
|
|
68
|
+
Enable the following extensions if you want to send requests, logs, and spider statistics.
|
|
69
|
+
```python
|
|
70
|
+
EXTENSIONS = {
|
|
71
|
+
"apcloudy_pipeline.request_logger.APCloudyRequestLogger": 400,
|
|
72
|
+
"apcloudy_pipeline.extensions.APCloudyLoggingExtension": 510,
|
|
73
|
+
"apcloudy_pipeline.extensions.APCloudyStatsExtension": 520,
|
|
74
|
+
}
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
## Extensions Overview
|
|
78
|
+
|
|
79
|
+
- APCloudyRequestLogger
|
|
80
|
+
Captures request and response metadata such as URL, HTTP method, status code, timing, and fingerprint.
|
|
81
|
+
|
|
82
|
+
- APCloudyLoggingExtension
|
|
83
|
+
Sends spider logs, user logs, Scrapy internal logs, and exception tracebacks to the backend.
|
|
84
|
+
|
|
85
|
+
- APCloudyStatsExtension
|
|
86
|
+
Sends final spider statistics when the crawl finishes.
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# apcloudy-pipeline
|
|
2
|
+
|
|
3
|
+
apcloudy-pipeline is a Scrapy integration that sends **items, requests, logs, and spider statistics**
|
|
4
|
+
to the **Your backend** using **secure HMAC-based authentication**.
|
|
5
|
+
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
## Features
|
|
9
|
+
|
|
10
|
+
- ๐ฆ Item forwarding using Scrapy Item Pipeline
|
|
11
|
+
- ๐ Request and response logging
|
|
12
|
+
- ๐ Spider statistics reporting
|
|
13
|
+
- ๐งพ Spider, user, and Scrapy internal log forwarding
|
|
14
|
+
- ๐ HMAC-secured API communication
|
|
15
|
+
|
|
16
|
+
---
|
|
17
|
+
|
|
18
|
+
## Installation
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
pip install apcloudy-pipeline
|
|
22
|
+
```
|
|
23
|
+
## Configuration
|
|
24
|
+
Add the following settings to your Scrapy project's settings.py file:
|
|
25
|
+
```python
|
|
26
|
+
APCLOUDY_API_URL = "http://localhost:8000/api/v1/"
|
|
27
|
+
APCLOUDY_API_KEY = "api_test_1234567890"
|
|
28
|
+
APCLOUDY_SECRET_KEY = "secret_test_1234567890"
|
|
29
|
+
JOB_ID = 123
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
### Item Pipeline (Required)
|
|
33
|
+
The item pipeline is required to send scraped items to the backend.
|
|
34
|
+
```python
|
|
35
|
+
ITEM_PIPELINES = {
|
|
36
|
+
"apcloudy_pipeline.pipelines.APCloudyItemPipeline": 300,
|
|
37
|
+
}
|
|
38
|
+
```
|
|
39
|
+
### Extensions (Optional)
|
|
40
|
+
Enable the following extensions if you want to send requests, logs, and spider statistics.
|
|
41
|
+
```python
|
|
42
|
+
EXTENSIONS = {
|
|
43
|
+
"apcloudy_pipeline.request_logger.APCloudyRequestLogger": 400,
|
|
44
|
+
"apcloudy_pipeline.extensions.APCloudyLoggingExtension": 510,
|
|
45
|
+
"apcloudy_pipeline.extensions.APCloudyStatsExtension": 520,
|
|
46
|
+
}
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
## Extensions Overview
|
|
50
|
+
|
|
51
|
+
- APCloudyRequestLogger
|
|
52
|
+
Captures request and response metadata such as URL, HTTP method, status code, timing, and fingerprint.
|
|
53
|
+
|
|
54
|
+
- APCloudyLoggingExtension
|
|
55
|
+
Sends spider logs, user logs, Scrapy internal logs, and exception tracebacks to the backend.
|
|
56
|
+
|
|
57
|
+
- APCloudyStatsExtension
|
|
58
|
+
Sends final spider statistics when the crawl finishes.
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
from .client import APCloudyClient
|
|
2
|
+
from .request_logger import APCloudyRequestLogger
|
|
3
|
+
from .extensions import (
|
|
4
|
+
APCloudyLoggingExtension,
|
|
5
|
+
APCloudyStatsExtension,
|
|
6
|
+
)
|
|
7
|
+
|
|
8
|
+
__title__ = "apcloudy-pipeline"
|
|
9
|
+
__version__ = "0.1.0"
|
|
10
|
+
__author__ = "Fawad Ali"
|
|
11
|
+
__license__ = "MIT"
|
|
12
|
+
|
|
13
|
+
__all__ = [
|
|
14
|
+
"APCloudyClient",
|
|
15
|
+
"APCloudyRequestLogger",
|
|
16
|
+
"APCloudyLoggingExtension",
|
|
17
|
+
"APCloudyStatsExtension",
|
|
18
|
+
]
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import requests
|
|
3
|
+
from scrapy.utils.project import get_project_settings
|
|
4
|
+
from .utils import get_timestamp, hmac_signature, _json_serialize
|
|
5
|
+
from .exceptions import APCloudyPipelineError
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class APCloudyClient:
|
|
9
|
+
"""
|
|
10
|
+
Manages interaction with the AP Cloudy API for sending various types of data.
|
|
11
|
+
|
|
12
|
+
This class provides methods for sending requests, items, logs, and statistical data to the
|
|
13
|
+
AP Cloudy backend. It handles API authentication, including signature generation and
|
|
14
|
+
request header management. It ensures that all necessary credential information is
|
|
15
|
+
configured before usage, raising errors otherwise.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
def __init__(self):
|
|
19
|
+
# Load settings from Scrapy project
|
|
20
|
+
settings = get_project_settings()
|
|
21
|
+
self.api_url = settings.get("APCLOUDY_API_URL")
|
|
22
|
+
self.public_key = settings.get("APCLOUDY_API_KEY")
|
|
23
|
+
self.secret_key = settings.get("APCLOUDY_SECRET_KEY")
|
|
24
|
+
self.job_id = settings.get("JOB_ID")
|
|
25
|
+
|
|
26
|
+
if not all([self.api_url, self.public_key, self.secret_key, self.job_id]):
|
|
27
|
+
raise APCloudyPipelineError(
|
|
28
|
+
"APCloudy API credentials not found in Scrapy settings"
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
def _post(self, endpoint: str, payload: dict):
|
|
32
|
+
"""
|
|
33
|
+
Sends a POST request to the specified endpoint with the provided payload.
|
|
34
|
+
|
|
35
|
+
This method constructs the request with required headers, including
|
|
36
|
+
the API key, timestamp, and signature for authentication. It formats
|
|
37
|
+
the payload as JSON and sends it to the API endpoint via an HTTP POST
|
|
38
|
+
request.
|
|
39
|
+
"""
|
|
40
|
+
raw_body = json.dumps(payload)
|
|
41
|
+
timestamp = get_timestamp()
|
|
42
|
+
signature = hmac_signature(self.secret_key, raw_body, timestamp)
|
|
43
|
+
|
|
44
|
+
headers = {
|
|
45
|
+
"X-API-KEY": self.public_key,
|
|
46
|
+
"X-TIMESTAMP": timestamp,
|
|
47
|
+
"X-SIGNATURE": signature,
|
|
48
|
+
"Content-Type": "application/json"
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
url = f"{self.api_url.rstrip('/')}?type={endpoint.lstrip('/')}"
|
|
52
|
+
try:
|
|
53
|
+
r = requests.post(url, headers=headers, data=raw_body, timeout=10)
|
|
54
|
+
r.raise_for_status()
|
|
55
|
+
except Exception as e:
|
|
56
|
+
raise APCloudyPipelineError(f"Failed to send data to AP Cloudy backend: {e}")
|
|
57
|
+
|
|
58
|
+
def send_request_log(self, request_log: dict):
|
|
59
|
+
"""
|
|
60
|
+
Sends a log entry for a request to the specified endpoint.
|
|
61
|
+
|
|
62
|
+
This method collects the provided request log data, combines it with
|
|
63
|
+
the current job ID, and sends it to a pre-defined resource using
|
|
64
|
+
an internal HTTP POST request.
|
|
65
|
+
"""
|
|
66
|
+
payload = {"job_id": self.job_id, "data": request_log}
|
|
67
|
+
return self._post("requests", payload)
|
|
68
|
+
|
|
69
|
+
def send_item(self, item: dict):
|
|
70
|
+
"""
|
|
71
|
+
Sends an item to be processed with the associated job.
|
|
72
|
+
|
|
73
|
+
This method prepares the payload by combining the provided item data with the
|
|
74
|
+
current job's ID. The constructed payload is then sent to the appropriate
|
|
75
|
+
endpoint for processing.
|
|
76
|
+
"""
|
|
77
|
+
payload = {"job_id": self.job_id, "data": item}
|
|
78
|
+
return self._post("items", payload)
|
|
79
|
+
|
|
80
|
+
def send_log(self, data: dict[str, str]):
|
|
81
|
+
"""
|
|
82
|
+
Sends a log message with a specified logging level to the appropriate logging endpoint. The method constructs
|
|
83
|
+
a log payload containing the job ID, log level, and message, and sends it to the endpoint using an internal
|
|
84
|
+
HTTP POST request.
|
|
85
|
+
"""
|
|
86
|
+
payload = {"job_id": self.job_id, "data": data}
|
|
87
|
+
return self._post("logs", payload)
|
|
88
|
+
|
|
89
|
+
def send_stats(self, stats: dict):
|
|
90
|
+
"""
|
|
91
|
+
Send statistical data to the server for a specific job.
|
|
92
|
+
|
|
93
|
+
This method compiles statistical data into a payload and posts it to a
|
|
94
|
+
predefined endpoint on the server. The functionality is primarily intended
|
|
95
|
+
to facilitate reporting and monitoring of job-related metrics.
|
|
96
|
+
"""
|
|
97
|
+
safe_stats = json.loads(json.dumps(stats, default=_json_serialize))
|
|
98
|
+
payload = {"job_id": self.job_id, "data": safe_stats}
|
|
99
|
+
return self._post("stats", payload)
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import logging
|
|
3
|
+
from datetime import datetime
|
|
4
|
+
|
|
5
|
+
from scrapy import signals
|
|
6
|
+
|
|
7
|
+
from .client import APCloudyClient
|
|
8
|
+
from .log_handler import APCloudyLogHandler
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class APCloudyStatsExtension:
|
|
12
|
+
@classmethod
|
|
13
|
+
def from_crawler(cls, crawler):
|
|
14
|
+
ext = cls()
|
|
15
|
+
ext.client = APCloudyClient()
|
|
16
|
+
crawler.signals.connect(ext.spider_closed, signal=signals.spider_closed)
|
|
17
|
+
return ext
|
|
18
|
+
|
|
19
|
+
def spider_closed(self, spider, reason):
|
|
20
|
+
try:
|
|
21
|
+
stats = spider.crawler.stats.get_stats()
|
|
22
|
+
# Convert non-serializable objects
|
|
23
|
+
safe_stats = json.loads(
|
|
24
|
+
json.dumps(stats, default=lambda o: o.isoformat() if isinstance(o, datetime) else str(o))
|
|
25
|
+
)
|
|
26
|
+
self.client.send_stats(safe_stats)
|
|
27
|
+
except Exception as e:
|
|
28
|
+
logging.error("Failed to send stats:", e)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class APCloudyLoggingExtension:
|
|
32
|
+
"""
|
|
33
|
+
Scrapy extension to send all logs (spider/user/Scrapy internal) to AP Cloudy backend.
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
def __init__(self, log_level=logging.INFO):
|
|
37
|
+
self.log_level = log_level
|
|
38
|
+
self.handler = APCloudyLogHandler(level=log_level)
|
|
39
|
+
|
|
40
|
+
@classmethod
|
|
41
|
+
def from_crawler(cls, crawler):
|
|
42
|
+
level_name = crawler.settings.get("LOG_LEVEL", "INFO")
|
|
43
|
+
log_level = logging._nameToLevel.get(level_name.upper(), logging.INFO)
|
|
44
|
+
ext = cls(log_level)
|
|
45
|
+
|
|
46
|
+
crawler.signals.connect(ext.spider_opened, signal=signals.spider_opened)
|
|
47
|
+
crawler.signals.connect(ext.spider_closed, signal=signals.spider_closed)
|
|
48
|
+
|
|
49
|
+
return ext
|
|
50
|
+
|
|
51
|
+
def spider_opened(self, spider):
|
|
52
|
+
root_logger = logging.getLogger()
|
|
53
|
+
|
|
54
|
+
if self.handler not in root_logger.handlers:
|
|
55
|
+
root_logger.addHandler(self.handler)
|
|
56
|
+
|
|
57
|
+
root_logger.setLevel(self.log_level)
|
|
58
|
+
|
|
59
|
+
logging.getLogger("scrapy").propagate = True
|
|
60
|
+
|
|
61
|
+
def spider_closed(self, spider, reason):
|
|
62
|
+
root_logger = logging.getLogger()
|
|
63
|
+
if self.handler in root_logger.handlers:
|
|
64
|
+
root_logger.removeHandler(self.handler)
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from datetime import datetime
|
|
3
|
+
import json
|
|
4
|
+
|
|
5
|
+
from .client import APCloudyClient
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def _safe_serialize(payload):
|
|
9
|
+
return json.dumps(payload, default=lambda o: o.isoformat() if isinstance(o, datetime) else str(o))
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class APCloudyLogHandler(logging.Handler):
|
|
13
|
+
def __init__(self, level=logging.NOTSET):
|
|
14
|
+
super().__init__(level)
|
|
15
|
+
self.client = APCloudyClient()
|
|
16
|
+
self.formatter = logging.Formatter()
|
|
17
|
+
|
|
18
|
+
def emit(self, record):
|
|
19
|
+
if record.levelno < self.level:
|
|
20
|
+
return
|
|
21
|
+
|
|
22
|
+
log_entry = {
|
|
23
|
+
"level": record.levelname,
|
|
24
|
+
"message": record.getMessage(),
|
|
25
|
+
"exception": None,
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
if record.exc_info:
|
|
29
|
+
log_entry["exception"] = self.formatter.formatException(record.exc_info)
|
|
30
|
+
|
|
31
|
+
payload_json = _safe_serialize(log_entry)
|
|
32
|
+
return self.client.send_log(json.loads(payload_json))
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
from scrapy import Item
|
|
2
|
+
|
|
3
|
+
from .client import APCloudyClient
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class APCloudyItemPipeline:
|
|
7
|
+
"""Scrapy item pipeline to send items to AP Cloudy backend"""
|
|
8
|
+
|
|
9
|
+
def open_spider(self, spider):
|
|
10
|
+
self.client = APCloudyClient()
|
|
11
|
+
|
|
12
|
+
def process_item(self, item: Item, spider):
|
|
13
|
+
self.client.send_item(dict(item))
|
|
14
|
+
return item
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import hashlib
|
|
2
|
+
import time
|
|
3
|
+
|
|
4
|
+
from scrapy import signals
|
|
5
|
+
from scrapy.utils.python import to_bytes
|
|
6
|
+
from scrapy.utils.url import canonicalize_url
|
|
7
|
+
|
|
8
|
+
from apcloudy_pipeline.client import APCloudyClient
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class APCloudyRequestLogger:
|
|
12
|
+
"""
|
|
13
|
+
Scrapy extension to log requests with detailed attributes.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
@classmethod
|
|
17
|
+
def from_crawler(cls, crawler):
|
|
18
|
+
ext = cls()
|
|
19
|
+
ext.client = APCloudyClient()
|
|
20
|
+
crawler.signals.connect(ext.request_scheduled, signal=signals.request_scheduled)
|
|
21
|
+
crawler.signals.connect(ext.response_received, signal=signals.response_received)
|
|
22
|
+
crawler.signals.connect(ext.request_dropped, signal=signals.request_dropped)
|
|
23
|
+
return ext
|
|
24
|
+
|
|
25
|
+
def request_scheduled(self, request, spider):
|
|
26
|
+
# Store start time in request meta
|
|
27
|
+
request.meta['start_time'] = time.time()
|
|
28
|
+
|
|
29
|
+
def response_received(self, response, request, spider):
|
|
30
|
+
start_time = request.meta.get('start_time', time.time())
|
|
31
|
+
response_time = time.time() - start_time
|
|
32
|
+
|
|
33
|
+
request_log = {
|
|
34
|
+
"url": request.url,
|
|
35
|
+
"method": request.method,
|
|
36
|
+
"status_code": response.status,
|
|
37
|
+
"response_time": round(response_time, 2),
|
|
38
|
+
"fingerprint": self.request_fingerprint(request),
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
self.client.send_request_log(request_log)
|
|
42
|
+
|
|
43
|
+
def request_dropped(self, request, spider):
|
|
44
|
+
# handle dropped requests
|
|
45
|
+
request_log = {
|
|
46
|
+
"url": request.url,
|
|
47
|
+
"method": request.method,
|
|
48
|
+
"status_code": 0,
|
|
49
|
+
"response_time": 0,
|
|
50
|
+
"fingerprint": self.request_fingerprint(request),
|
|
51
|
+
}
|
|
52
|
+
self.client.send_request_log(request_log)
|
|
53
|
+
|
|
54
|
+
def request_fingerprint(self, request):
|
|
55
|
+
"""Compute request fingerprint manually if import fails"""
|
|
56
|
+
fp = hashlib.sha1()
|
|
57
|
+
fp.update(to_bytes(request.method))
|
|
58
|
+
fp.update(to_bytes(canonicalize_url(request.url)))
|
|
59
|
+
return fp.hexdigest()
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
from datetime import datetime
|
|
2
|
+
import time
|
|
3
|
+
import hmac
|
|
4
|
+
import hashlib
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def hmac_signature(secret: str, raw_body: str, timestamp: str) -> str:
|
|
8
|
+
"""
|
|
9
|
+
Compute HMAC-SHA256 signature:
|
|
10
|
+
signature = HMAC_SHA256(secret, timestamp + "." + raw_body)
|
|
11
|
+
"""
|
|
12
|
+
message = f"{timestamp}.{raw_body}".encode()
|
|
13
|
+
key = secret.encode()
|
|
14
|
+
signature = hmac.new(key, message, hashlib.sha256).hexdigest()
|
|
15
|
+
return signature
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def get_timestamp() -> str:
|
|
19
|
+
"""Return current timestamp in seconds"""
|
|
20
|
+
return str(int(time.time()))
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _json_serialize(obj):
|
|
24
|
+
"""JSON serializer for objects not serializable by default json code"""
|
|
25
|
+
if isinstance(obj, datetime):
|
|
26
|
+
return obj.isoformat()
|
|
27
|
+
return str(obj)
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
from setuptools import setup, find_packages
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
BASE_DIR = Path(__file__).parent
|
|
5
|
+
|
|
6
|
+
README = (BASE_DIR / "README.md").read_text(encoding="utf-8")
|
|
7
|
+
|
|
8
|
+
setup(
|
|
9
|
+
name="apcloudy-pipeline",
|
|
10
|
+
version="0.1.0",
|
|
11
|
+
author="Fawad",
|
|
12
|
+
author_email="fawadstar6@email.com",
|
|
13
|
+
description="Scrapy pipeline & extensions for AP Cloudy (logs, stats, requests, items)",
|
|
14
|
+
long_description=README,
|
|
15
|
+
long_description_content_type="text/markdown",
|
|
16
|
+
url="https://github.com/fawadss1/apcloudy-pipeline",
|
|
17
|
+
license="MIT",
|
|
18
|
+
|
|
19
|
+
packages=find_packages(exclude=("tests*",)),
|
|
20
|
+
|
|
21
|
+
python_requires=">=3.8",
|
|
22
|
+
|
|
23
|
+
install_requires=[
|
|
24
|
+
"requests>=2.30.0"
|
|
25
|
+
],
|
|
26
|
+
|
|
27
|
+
classifiers=[
|
|
28
|
+
"Programming Language :: Python :: 3",
|
|
29
|
+
"Framework :: Scrapy",
|
|
30
|
+
"License :: OSI Approved :: MIT License",
|
|
31
|
+
"Operating System :: OS Independent",
|
|
32
|
+
],
|
|
33
|
+
|
|
34
|
+
include_package_data=True,
|
|
35
|
+
zip_safe=False,
|
|
36
|
+
)
|