apcloudy-pipeline 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,86 @@
1
+ Metadata-Version: 2.4
2
+ Name: apcloudy-pipeline
3
+ Version: 0.1.0
4
+ Summary: Scrapy pipeline & extensions for AP Cloudy (logs, stats, requests, items)
5
+ Home-page: https://github.com/fawadss1/apcloudy-pipeline
6
+ Author: Fawad
7
+ Author-email: fawadstar6@email.com
8
+ License: MIT
9
+ Classifier: Programming Language :: Python :: 3
10
+ Classifier: Framework :: Scrapy
11
+ Classifier: License :: OSI Approved :: MIT License
12
+ Classifier: Operating System :: OS Independent
13
+ Requires-Python: >=3.8
14
+ Description-Content-Type: text/markdown
15
+ License-File: LICENCE
16
+ Requires-Dist: requests>=2.30.0
17
+ Dynamic: author
18
+ Dynamic: author-email
19
+ Dynamic: classifier
20
+ Dynamic: description
21
+ Dynamic: description-content-type
22
+ Dynamic: home-page
23
+ Dynamic: license
24
+ Dynamic: license-file
25
+ Dynamic: requires-dist
26
+ Dynamic: requires-python
27
+ Dynamic: summary
28
+
29
+ # apcloudy-pipeline
30
+
31
+ apcloudy-pipeline is a Scrapy integration that sends **items, requests, logs, and spider statistics**
32
+ to the **Your backend** using **secure HMAC-based authentication**.
33
+
34
+ ---
35
+
36
+ ## Features
37
+
38
+ - ๐Ÿ“ฆ Item forwarding using Scrapy Item Pipeline
39
+ - ๐ŸŒ Request and response logging
40
+ - ๐Ÿ“Š Spider statistics reporting
41
+ - ๐Ÿงพ Spider, user, and Scrapy internal log forwarding
42
+ - ๐Ÿ” HMAC-secured API communication
43
+
44
+ ---
45
+
46
+ ## Installation
47
+
48
+ ```bash
49
+ pip install apcloudy-pipeline
50
+ ```
51
+ ## Configuration
52
+ Add the following settings to your Scrapy project's settings.py file:
53
+ ```python
54
+ APCLOUDY_API_URL = "http://localhost:8000/api/v1/"
55
+ APCLOUDY_API_KEY = "api_test_1234567890"
56
+ APCLOUDY_SECRET_KEY = "secret_test_1234567890"
57
+ JOB_ID = 123
58
+ ```
59
+
60
+ ### Item Pipeline (Required)
61
+ The item pipeline is required to send scraped items to the backend.
62
+ ```python
63
+ ITEM_PIPELINES = {
64
+ "apcloudy_pipeline.pipelines.APCloudyItemPipeline": 300,
65
+ }
66
+ ```
67
+ ### Extensions (Optional)
68
+ Enable the following extensions if you want to send requests, logs, and spider statistics.
69
+ ```python
70
+ EXTENSIONS = {
71
+ "apcloudy_pipeline.request_logger.APCloudyRequestLogger": 400,
72
+ "apcloudy_pipeline.extensions.APCloudyLoggingExtension": 510,
73
+ "apcloudy_pipeline.extensions.APCloudyStatsExtension": 520,
74
+ }
75
+ ```
76
+
77
+ ## Extensions Overview
78
+
79
+ - APCloudyRequestLogger
80
+ Captures request and response metadata such as URL, HTTP method, status code, timing, and fingerprint.
81
+
82
+ - APCloudyLoggingExtension
83
+ Sends spider logs, user logs, Scrapy internal logs, and exception tracebacks to the backend.
84
+
85
+ - APCloudyStatsExtension
86
+ Sends final spider statistics when the crawl finishes.
@@ -0,0 +1,23 @@
1
+ LICENCE
2
+ README.md
3
+ pyproject.toml
4
+ setup.py
5
+ Apcloudy_Pipeline.egg-info/PKG-INFO
6
+ Apcloudy_Pipeline.egg-info/SOURCES.txt
7
+ Apcloudy_Pipeline.egg-info/dependency_links.txt
8
+ Apcloudy_Pipeline.egg-info/requires.txt
9
+ Apcloudy_Pipeline.egg-info/top_level.txt
10
+ apcloudy_pipeline/__init__.py
11
+ apcloudy_pipeline/client.py
12
+ apcloudy_pipeline/exceptions.py
13
+ apcloudy_pipeline/extensions.py
14
+ apcloudy_pipeline/log_handler.py
15
+ apcloudy_pipeline/pipelines.py
16
+ apcloudy_pipeline/request_logger.py
17
+ apcloudy_pipeline/utils.py
18
+ apcloudy_pipeline.egg-info/PKG-INFO
19
+ apcloudy_pipeline.egg-info/SOURCES.txt
20
+ apcloudy_pipeline.egg-info/dependency_links.txt
21
+ apcloudy_pipeline.egg-info/not-zip-safe
22
+ apcloudy_pipeline.egg-info/requires.txt
23
+ apcloudy_pipeline.egg-info/top_level.txt
@@ -0,0 +1 @@
1
+ requests>=2.30.0
@@ -0,0 +1 @@
1
+ apcloudy_pipeline
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2025 Fawad
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,86 @@
1
+ Metadata-Version: 2.4
2
+ Name: apcloudy-pipeline
3
+ Version: 0.1.0
4
+ Summary: Scrapy pipeline & extensions for AP Cloudy (logs, stats, requests, items)
5
+ Home-page: https://github.com/fawadss1/apcloudy-pipeline
6
+ Author: Fawad
7
+ Author-email: fawadstar6@email.com
8
+ License: MIT
9
+ Classifier: Programming Language :: Python :: 3
10
+ Classifier: Framework :: Scrapy
11
+ Classifier: License :: OSI Approved :: MIT License
12
+ Classifier: Operating System :: OS Independent
13
+ Requires-Python: >=3.8
14
+ Description-Content-Type: text/markdown
15
+ License-File: LICENCE
16
+ Requires-Dist: requests>=2.30.0
17
+ Dynamic: author
18
+ Dynamic: author-email
19
+ Dynamic: classifier
20
+ Dynamic: description
21
+ Dynamic: description-content-type
22
+ Dynamic: home-page
23
+ Dynamic: license
24
+ Dynamic: license-file
25
+ Dynamic: requires-dist
26
+ Dynamic: requires-python
27
+ Dynamic: summary
28
+
29
+ # apcloudy-pipeline
30
+
31
+ apcloudy-pipeline is a Scrapy integration that sends **items, requests, logs, and spider statistics**
32
+ to the **Your backend** using **secure HMAC-based authentication**.
33
+
34
+ ---
35
+
36
+ ## Features
37
+
38
+ - ๐Ÿ“ฆ Item forwarding using Scrapy Item Pipeline
39
+ - ๐ŸŒ Request and response logging
40
+ - ๐Ÿ“Š Spider statistics reporting
41
+ - ๐Ÿงพ Spider, user, and Scrapy internal log forwarding
42
+ - ๐Ÿ” HMAC-secured API communication
43
+
44
+ ---
45
+
46
+ ## Installation
47
+
48
+ ```bash
49
+ pip install apcloudy-pipeline
50
+ ```
51
+ ## Configuration
52
+ Add the following settings to your Scrapy project's settings.py file:
53
+ ```python
54
+ APCLOUDY_API_URL = "http://localhost:8000/api/v1/"
55
+ APCLOUDY_API_KEY = "api_test_1234567890"
56
+ APCLOUDY_SECRET_KEY = "secret_test_1234567890"
57
+ JOB_ID = 123
58
+ ```
59
+
60
+ ### Item Pipeline (Required)
61
+ The item pipeline is required to send scraped items to the backend.
62
+ ```python
63
+ ITEM_PIPELINES = {
64
+ "apcloudy_pipeline.pipelines.APCloudyItemPipeline": 300,
65
+ }
66
+ ```
67
+ ### Extensions (Optional)
68
+ Enable the following extensions if you want to send requests, logs, and spider statistics.
69
+ ```python
70
+ EXTENSIONS = {
71
+ "apcloudy_pipeline.request_logger.APCloudyRequestLogger": 400,
72
+ "apcloudy_pipeline.extensions.APCloudyLoggingExtension": 510,
73
+ "apcloudy_pipeline.extensions.APCloudyStatsExtension": 520,
74
+ }
75
+ ```
76
+
77
+ ## Extensions Overview
78
+
79
+ - APCloudyRequestLogger
80
+ Captures request and response metadata such as URL, HTTP method, status code, timing, and fingerprint.
81
+
82
+ - APCloudyLoggingExtension
83
+ Sends spider logs, user logs, Scrapy internal logs, and exception tracebacks to the backend.
84
+
85
+ - APCloudyStatsExtension
86
+ Sends final spider statistics when the crawl finishes.
@@ -0,0 +1,58 @@
1
+ # apcloudy-pipeline
2
+
3
+ apcloudy-pipeline is a Scrapy integration that sends **items, requests, logs, and spider statistics**
4
+ to the **Your backend** using **secure HMAC-based authentication**.
5
+
6
+ ---
7
+
8
+ ## Features
9
+
10
+ - ๐Ÿ“ฆ Item forwarding using Scrapy Item Pipeline
11
+ - ๐ŸŒ Request and response logging
12
+ - ๐Ÿ“Š Spider statistics reporting
13
+ - ๐Ÿงพ Spider, user, and Scrapy internal log forwarding
14
+ - ๐Ÿ” HMAC-secured API communication
15
+
16
+ ---
17
+
18
+ ## Installation
19
+
20
+ ```bash
21
+ pip install apcloudy-pipeline
22
+ ```
23
+ ## Configuration
24
+ Add the following settings to your Scrapy project's settings.py file:
25
+ ```python
26
+ APCLOUDY_API_URL = "http://localhost:8000/api/v1/"
27
+ APCLOUDY_API_KEY = "api_test_1234567890"
28
+ APCLOUDY_SECRET_KEY = "secret_test_1234567890"
29
+ JOB_ID = 123
30
+ ```
31
+
32
+ ### Item Pipeline (Required)
33
+ The item pipeline is required to send scraped items to the backend.
34
+ ```python
35
+ ITEM_PIPELINES = {
36
+ "apcloudy_pipeline.pipelines.APCloudyItemPipeline": 300,
37
+ }
38
+ ```
39
+ ### Extensions (Optional)
40
+ Enable the following extensions if you want to send requests, logs, and spider statistics.
41
+ ```python
42
+ EXTENSIONS = {
43
+ "apcloudy_pipeline.request_logger.APCloudyRequestLogger": 400,
44
+ "apcloudy_pipeline.extensions.APCloudyLoggingExtension": 510,
45
+ "apcloudy_pipeline.extensions.APCloudyStatsExtension": 520,
46
+ }
47
+ ```
48
+
49
+ ## Extensions Overview
50
+
51
+ - APCloudyRequestLogger
52
+ Captures request and response metadata such as URL, HTTP method, status code, timing, and fingerprint.
53
+
54
+ - APCloudyLoggingExtension
55
+ Sends spider logs, user logs, Scrapy internal logs, and exception tracebacks to the backend.
56
+
57
+ - APCloudyStatsExtension
58
+ Sends final spider statistics when the crawl finishes.
@@ -0,0 +1,18 @@
1
+ from .client import APCloudyClient
2
+ from .request_logger import APCloudyRequestLogger
3
+ from .extensions import (
4
+ APCloudyLoggingExtension,
5
+ APCloudyStatsExtension,
6
+ )
7
+
8
+ __title__ = "apcloudy-pipeline"
9
+ __version__ = "0.1.0"
10
+ __author__ = "Fawad Ali"
11
+ __license__ = "MIT"
12
+
13
+ __all__ = [
14
+ "APCloudyClient",
15
+ "APCloudyRequestLogger",
16
+ "APCloudyLoggingExtension",
17
+ "APCloudyStatsExtension",
18
+ ]
@@ -0,0 +1,99 @@
1
+ import json
2
+ import requests
3
+ from scrapy.utils.project import get_project_settings
4
+ from .utils import get_timestamp, hmac_signature, _json_serialize
5
+ from .exceptions import APCloudyPipelineError
6
+
7
+
8
+ class APCloudyClient:
9
+ """
10
+ Manages interaction with the AP Cloudy API for sending various types of data.
11
+
12
+ This class provides methods for sending requests, items, logs, and statistical data to the
13
+ AP Cloudy backend. It handles API authentication, including signature generation and
14
+ request header management. It ensures that all necessary credential information is
15
+ configured before usage, raising errors otherwise.
16
+ """
17
+
18
+ def __init__(self):
19
+ # Load settings from Scrapy project
20
+ settings = get_project_settings()
21
+ self.api_url = settings.get("APCLOUDY_API_URL")
22
+ self.public_key = settings.get("APCLOUDY_API_KEY")
23
+ self.secret_key = settings.get("APCLOUDY_SECRET_KEY")
24
+ self.job_id = settings.get("JOB_ID")
25
+
26
+ if not all([self.api_url, self.public_key, self.secret_key, self.job_id]):
27
+ raise APCloudyPipelineError(
28
+ "APCloudy API credentials not found in Scrapy settings"
29
+ )
30
+
31
+ def _post(self, endpoint: str, payload: dict):
32
+ """
33
+ Sends a POST request to the specified endpoint with the provided payload.
34
+
35
+ This method constructs the request with required headers, including
36
+ the API key, timestamp, and signature for authentication. It formats
37
+ the payload as JSON and sends it to the API endpoint via an HTTP POST
38
+ request.
39
+ """
40
+ raw_body = json.dumps(payload)
41
+ timestamp = get_timestamp()
42
+ signature = hmac_signature(self.secret_key, raw_body, timestamp)
43
+
44
+ headers = {
45
+ "X-API-KEY": self.public_key,
46
+ "X-TIMESTAMP": timestamp,
47
+ "X-SIGNATURE": signature,
48
+ "Content-Type": "application/json"
49
+ }
50
+
51
+ url = f"{self.api_url.rstrip('/')}?type={endpoint.lstrip('/')}"
52
+ try:
53
+ r = requests.post(url, headers=headers, data=raw_body, timeout=10)
54
+ r.raise_for_status()
55
+ except Exception as e:
56
+ raise APCloudyPipelineError(f"Failed to send data to AP Cloudy backend: {e}")
57
+
58
+ def send_request_log(self, request_log: dict):
59
+ """
60
+ Sends a log entry for a request to the specified endpoint.
61
+
62
+ This method collects the provided request log data, combines it with
63
+ the current job ID, and sends it to a pre-defined resource using
64
+ an internal HTTP POST request.
65
+ """
66
+ payload = {"job_id": self.job_id, "data": request_log}
67
+ return self._post("requests", payload)
68
+
69
+ def send_item(self, item: dict):
70
+ """
71
+ Sends an item to be processed with the associated job.
72
+
73
+ This method prepares the payload by combining the provided item data with the
74
+ current job's ID. The constructed payload is then sent to the appropriate
75
+ endpoint for processing.
76
+ """
77
+ payload = {"job_id": self.job_id, "data": item}
78
+ return self._post("items", payload)
79
+
80
+ def send_log(self, data: dict[str, str]):
81
+ """
82
+ Sends a log message with a specified logging level to the appropriate logging endpoint. The method constructs
83
+ a log payload containing the job ID, log level, and message, and sends it to the endpoint using an internal
84
+ HTTP POST request.
85
+ """
86
+ payload = {"job_id": self.job_id, "data": data}
87
+ return self._post("logs", payload)
88
+
89
+ def send_stats(self, stats: dict):
90
+ """
91
+ Send statistical data to the server for a specific job.
92
+
93
+ This method compiles statistical data into a payload and posts it to a
94
+ predefined endpoint on the server. The functionality is primarily intended
95
+ to facilitate reporting and monitoring of job-related metrics.
96
+ """
97
+ safe_stats = json.loads(json.dumps(stats, default=_json_serialize))
98
+ payload = {"job_id": self.job_id, "data": safe_stats}
99
+ return self._post("stats", payload)
@@ -0,0 +1,3 @@
1
+ class APCloudyPipelineError(Exception):
2
+ """Custom exception for AP Cloudy Pipeline errors"""
3
+ pass
@@ -0,0 +1,64 @@
1
+ import json
2
+ import logging
3
+ from datetime import datetime
4
+
5
+ from scrapy import signals
6
+
7
+ from .client import APCloudyClient
8
+ from .log_handler import APCloudyLogHandler
9
+
10
+
11
+ class APCloudyStatsExtension:
12
+ @classmethod
13
+ def from_crawler(cls, crawler):
14
+ ext = cls()
15
+ ext.client = APCloudyClient()
16
+ crawler.signals.connect(ext.spider_closed, signal=signals.spider_closed)
17
+ return ext
18
+
19
+ def spider_closed(self, spider, reason):
20
+ try:
21
+ stats = spider.crawler.stats.get_stats()
22
+ # Convert non-serializable objects
23
+ safe_stats = json.loads(
24
+ json.dumps(stats, default=lambda o: o.isoformat() if isinstance(o, datetime) else str(o))
25
+ )
26
+ self.client.send_stats(safe_stats)
27
+ except Exception as e:
28
+ logging.error("Failed to send stats:", e)
29
+
30
+
31
+ class APCloudyLoggingExtension:
32
+ """
33
+ Scrapy extension to send all logs (spider/user/Scrapy internal) to AP Cloudy backend.
34
+ """
35
+
36
+ def __init__(self, log_level=logging.INFO):
37
+ self.log_level = log_level
38
+ self.handler = APCloudyLogHandler(level=log_level)
39
+
40
+ @classmethod
41
+ def from_crawler(cls, crawler):
42
+ level_name = crawler.settings.get("LOG_LEVEL", "INFO")
43
+ log_level = logging._nameToLevel.get(level_name.upper(), logging.INFO)
44
+ ext = cls(log_level)
45
+
46
+ crawler.signals.connect(ext.spider_opened, signal=signals.spider_opened)
47
+ crawler.signals.connect(ext.spider_closed, signal=signals.spider_closed)
48
+
49
+ return ext
50
+
51
+ def spider_opened(self, spider):
52
+ root_logger = logging.getLogger()
53
+
54
+ if self.handler not in root_logger.handlers:
55
+ root_logger.addHandler(self.handler)
56
+
57
+ root_logger.setLevel(self.log_level)
58
+
59
+ logging.getLogger("scrapy").propagate = True
60
+
61
+ def spider_closed(self, spider, reason):
62
+ root_logger = logging.getLogger()
63
+ if self.handler in root_logger.handlers:
64
+ root_logger.removeHandler(self.handler)
@@ -0,0 +1,32 @@
1
+ import logging
2
+ from datetime import datetime
3
+ import json
4
+
5
+ from .client import APCloudyClient
6
+
7
+
8
+ def _safe_serialize(payload):
9
+ return json.dumps(payload, default=lambda o: o.isoformat() if isinstance(o, datetime) else str(o))
10
+
11
+
12
+ class APCloudyLogHandler(logging.Handler):
13
+ def __init__(self, level=logging.NOTSET):
14
+ super().__init__(level)
15
+ self.client = APCloudyClient()
16
+ self.formatter = logging.Formatter()
17
+
18
+ def emit(self, record):
19
+ if record.levelno < self.level:
20
+ return
21
+
22
+ log_entry = {
23
+ "level": record.levelname,
24
+ "message": record.getMessage(),
25
+ "exception": None,
26
+ }
27
+
28
+ if record.exc_info:
29
+ log_entry["exception"] = self.formatter.formatException(record.exc_info)
30
+
31
+ payload_json = _safe_serialize(log_entry)
32
+ return self.client.send_log(json.loads(payload_json))
@@ -0,0 +1,14 @@
1
+ from scrapy import Item
2
+
3
+ from .client import APCloudyClient
4
+
5
+
6
+ class APCloudyItemPipeline:
7
+ """Scrapy item pipeline to send items to AP Cloudy backend"""
8
+
9
+ def open_spider(self, spider):
10
+ self.client = APCloudyClient()
11
+
12
+ def process_item(self, item: Item, spider):
13
+ self.client.send_item(dict(item))
14
+ return item
@@ -0,0 +1,59 @@
1
+ import hashlib
2
+ import time
3
+
4
+ from scrapy import signals
5
+ from scrapy.utils.python import to_bytes
6
+ from scrapy.utils.url import canonicalize_url
7
+
8
+ from apcloudy_pipeline.client import APCloudyClient
9
+
10
+
11
+ class APCloudyRequestLogger:
12
+ """
13
+ Scrapy extension to log requests with detailed attributes.
14
+ """
15
+
16
+ @classmethod
17
+ def from_crawler(cls, crawler):
18
+ ext = cls()
19
+ ext.client = APCloudyClient()
20
+ crawler.signals.connect(ext.request_scheduled, signal=signals.request_scheduled)
21
+ crawler.signals.connect(ext.response_received, signal=signals.response_received)
22
+ crawler.signals.connect(ext.request_dropped, signal=signals.request_dropped)
23
+ return ext
24
+
25
+ def request_scheduled(self, request, spider):
26
+ # Store start time in request meta
27
+ request.meta['start_time'] = time.time()
28
+
29
+ def response_received(self, response, request, spider):
30
+ start_time = request.meta.get('start_time', time.time())
31
+ response_time = time.time() - start_time
32
+
33
+ request_log = {
34
+ "url": request.url,
35
+ "method": request.method,
36
+ "status_code": response.status,
37
+ "response_time": round(response_time, 2),
38
+ "fingerprint": self.request_fingerprint(request),
39
+ }
40
+
41
+ self.client.send_request_log(request_log)
42
+
43
+ def request_dropped(self, request, spider):
44
+ # handle dropped requests
45
+ request_log = {
46
+ "url": request.url,
47
+ "method": request.method,
48
+ "status_code": 0,
49
+ "response_time": 0,
50
+ "fingerprint": self.request_fingerprint(request),
51
+ }
52
+ self.client.send_request_log(request_log)
53
+
54
+ def request_fingerprint(self, request):
55
+ """Compute request fingerprint manually if import fails"""
56
+ fp = hashlib.sha1()
57
+ fp.update(to_bytes(request.method))
58
+ fp.update(to_bytes(canonicalize_url(request.url)))
59
+ return fp.hexdigest()
@@ -0,0 +1,27 @@
1
+ from datetime import datetime
2
+ import time
3
+ import hmac
4
+ import hashlib
5
+
6
+
7
+ def hmac_signature(secret: str, raw_body: str, timestamp: str) -> str:
8
+ """
9
+ Compute HMAC-SHA256 signature:
10
+ signature = HMAC_SHA256(secret, timestamp + "." + raw_body)
11
+ """
12
+ message = f"{timestamp}.{raw_body}".encode()
13
+ key = secret.encode()
14
+ signature = hmac.new(key, message, hashlib.sha256).hexdigest()
15
+ return signature
16
+
17
+
18
+ def get_timestamp() -> str:
19
+ """Return current timestamp in seconds"""
20
+ return str(int(time.time()))
21
+
22
+
23
+ def _json_serialize(obj):
24
+ """JSON serializer for objects not serializable by default json code"""
25
+ if isinstance(obj, datetime):
26
+ return obj.isoformat()
27
+ return str(obj)
@@ -0,0 +1,3 @@
1
+ [build-system]
2
+ requires = ["setuptools>=61.0", "wheel"]
3
+ build-backend = "setuptools.build_meta"
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,36 @@
1
+ from setuptools import setup, find_packages
2
+ from pathlib import Path
3
+
4
+ BASE_DIR = Path(__file__).parent
5
+
6
+ README = (BASE_DIR / "README.md").read_text(encoding="utf-8")
7
+
8
+ setup(
9
+ name="apcloudy-pipeline",
10
+ version="0.1.0",
11
+ author="Fawad",
12
+ author_email="fawadstar6@email.com",
13
+ description="Scrapy pipeline & extensions for AP Cloudy (logs, stats, requests, items)",
14
+ long_description=README,
15
+ long_description_content_type="text/markdown",
16
+ url="https://github.com/fawadss1/apcloudy-pipeline",
17
+ license="MIT",
18
+
19
+ packages=find_packages(exclude=("tests*",)),
20
+
21
+ python_requires=">=3.8",
22
+
23
+ install_requires=[
24
+ "requests>=2.30.0"
25
+ ],
26
+
27
+ classifiers=[
28
+ "Programming Language :: Python :: 3",
29
+ "Framework :: Scrapy",
30
+ "License :: OSI Approved :: MIT License",
31
+ "Operating System :: OS Independent",
32
+ ],
33
+
34
+ include_package_data=True,
35
+ zip_safe=False,
36
+ )