pocong 0.1.3__py3-none-any.whl → 1.0.1__py3-none-any.whl

pocong/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-08-23T21:57:15+0700",
+ "date": "2025-09-21T22:32:32+0700",
  "dirty": false,
  "error": null,
- "full-revisionid": "49273c80614427be28b9ed5ed8db6214c15148d4",
- "version": "0.1.3"
+ "full-revisionid": "85eac04a9a369e96c0c4819feecca0cc5d23c173",
+ "version": "1.0.1"
 }
 '''  # END VERSION_JSON
 

pocong/proxy_spiders/__init__.py

@@ -0,0 +1,72 @@
+# Init file for spiders module to make it a package
+import random
+
+import requests
+
+
+class GetProxy():
+    '''
+    Class to get proxies using Scrapy spiders and validate them.
+    '''
+    def __init__(self):
+        self._items = []
+
+    def _check_proxy(self, x):
+        proxy = f"http://{x['ip']}:{x['port']}"
+        try:
+            response = requests.get("https://httpbin.org/ip", proxies={'https': proxy}, timeout=10)
+            if response.status_code == 200 and response.json().get('origin') == x['ip']:
+                return response.status_code
+            return 0
+        except requests.RequestException:
+            return 0
+
+    def _get_proxy_from_scrape(self):
+        import subprocess
+        import sys
+        import json
+        # Run the spider in a subprocess to avoid reactor restart error
+        code = (
+            'import pandas as pd;'
+            'from scrapy.crawler import CrawlerProcess;'
+            'from pocong.proxy_spiders.spiders.free_proxy_list_net_spider import ProxySpider;'
+            'from pocong.proxy_spiders.pipelines import collected_items;'
+            'process = CrawlerProcess(settings={"LOG_LEVEL": "ERROR", "ITEM_PIPELINES": {"pocong.proxy_spiders.pipelines.Pipelines": 1}});'  # noqa: E501
+            'process.crawl(ProxySpider);'
+            'process.start();'
+            'process.stop();'
+            'df = pd.DataFrame(collected_items);'
+            'df = df[df["https"] == "yes"];'
+            'df = df.drop_duplicates(subset=["ip", "port"]);'
+            'print(df.to_json(orient="records"))'
+        )
+        result = subprocess.run([sys.executable, '-c', code], capture_output=True, text=True)
+        proxies_json = json.loads(result.stdout.strip()) if result.stdout.strip() else []
+        return proxies_json
+
+    def get_proxy(self):
+        '''
+        Get a working proxy from the list of proxies.
+        parameter: None
+        return: dict or None
+        '''
+        proxies_json = self._get_proxy_from_scrape()
+        for proxy in proxies_json:
+            if self._check_proxy(proxy) == 200:
+                return proxy
+
+    def get_proxy_random(self):
+        '''
+        Get a random working proxy from the list of proxies.
+        parameter: None
+        return: dict or None
+        '''
+        proxies_json = self._get_proxy_from_scrape()
+        retry = 0
+        proxy = None
+        while retry < 20:
+            retry += 1
+            proxy = random.choice(proxies_json)
+            if self._check_proxy(proxy) == 200:
+                break
+        return proxy

pocong/proxy_spiders/pipelines.py

@@ -0,0 +1,7 @@
+collected_items = []
+
+
+class Pipelines:
+    def process_item(self, item, spider):
+        collected_items.append(dict(item))
+        return item

pocong/proxy_spiders/spiders/__init__.py

@@ -0,0 +1 @@
+# Init file for spiders module to make it a package

pocong/proxy_spiders/spiders/free_proxy_list_net_spider.py

@@ -0,0 +1,25 @@
+import scrapy
+
+
+class ProxySpider(scrapy.Spider):
+    name = "example"
+    start_urls = [
+        'https://free-proxy-list.net/en/',
+    ]
+
+    def parse(self, response):
+        # Extract proxy table rows
+        rows = response.css('table tbody tr')
+        for row in rows:
+            columns = row.css('td')
+            if len(columns) >= 8:
+                yield {
+                    'ip': columns[0].css('::text').get(),
+                    'port': columns[1].css('::text').get(),
+                    'code': columns[2].css('::text').get(),
+                    'country': columns[3].css('::text').get(),
+                    'anonymity': columns[4].css('::text').get(),
+                    'google': columns[5].css('::text').get(),
+                    'https': columns[6].css('::text').get(),
+                    'last_checked': columns[7].css('::text').get(),
+                }

{pocong-0.1.3.dist-info → pocong-1.0.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pocong
-Version: 0.1.3
+Version: 1.0.1
 Summary: Python Oriented Crawling Ongoing (POCONG): a simple crawling framework
 Home-page: https://gitlab.com/mohsin3107/pocong
 Author: Singgih
@@ -19,6 +19,9 @@ Classifier: Topic :: Software Development :: Libraries
 Classifier: Topic :: Internet :: WWW/HTTP
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
+Requires-Dist: Scrapy>=2.5.0
+Requires-Dist: pandas>=1.3.0
+Requires-Dist: requests>=2.25.0
 Requires-Dist: Click>=7.0
 Provides-Extra: dev
 Requires-Dist: pytest; extra == "dev"
@@ -57,3 +60,55 @@ POCONG is a lightweight web crawling framework built in Python.
 ## Installation
 ```bash
 pip install pocong
+```
+
+## Usage: Get Proxy from proxy_spiders
+
+You can use the `get_proxy` and `get_proxy_random` methods from `proxy_spiders` to fetch working proxies.
+
+```python
+from pocong.proxy_spiders import GetProxy
+
+gp = GetProxy()
+
+# Get the first working proxy
+proxy = gp.get_proxy()
+print("First working proxy:", proxy)
+```
+```python
+from pocong.proxy_spiders import GetProxy
+
+gp = GetProxy()
+
+# Get a random working proxy
+random_proxy = gp.get_proxy_random()
+print("Random working proxy:", random_proxy)
+```
+
+Sample output:
+```
+First working proxy: {'ip': '123.45.67.89', 'port': '8080', 'https': 'yes', ...}
+Random working proxy: {'ip': '98.76.54.32', 'port': '3128', 'https': 'yes', ...}
+```
+
+You can use the returned proxy dictionary with the `requests` library, for example:
+
+```python
+import requests
+
+proxy = gp.get_proxy()
+if proxy:
+    proxies = {
+        'http': f"http://{proxy['ip']}:{proxy['port']}",
+        'https': f"http://{proxy['ip']}:{proxy['port']}"
+    }
+    response = requests.get('https://httpbin.org/ip', proxies=proxies)
+    print(response.json())
+else:
+    print("No working proxy found.")
+```
+
+- `get_proxy()` will return the first working proxy found.
+- `get_proxy_random()` will return a random working proxy (with up to 20 retries).
+
+Both methods return a dictionary with proxy details (e.g., `{ 'ip': '...', 'port': '...', ... }`) or `None` if no working proxy is found.

pocong-1.0.1.dist-info/RECORD

@@ -0,0 +1,14 @@
+pocong/__init__.py,sha256=Hl0PkSkg6LV6IRLzXnGc0K2GY-drxkZEpt5qTAVDUkY,109
+pocong/_version.py,sha256=t_ZIUHG9ovHL3x_o5mb9F_Ih_UhgDKE9j9yI8c92xyk,497
+pocong/cli.py,sha256=_f_aU4pckbQ_baF9oHwbqwmBFiQFn5Irvi-v5rDZ70o,529
+pocong/pocong.py,sha256=h0hwdogXGFqerm-5ZPeT-irPn91pCcQRjiHThXsRzEk,19
+pocong/utils.py,sha256=MAbbL9PXRWnBpJKgI869ZfY42Eph73zcbJyK0jH2Nak,35
+pocong/proxy_spiders/__init__.py,sha256=RBlvqba1wIhCJPn4n5LNB3SLiCQHDcGOhtrFydWY7T4,2556
+pocong/proxy_spiders/pipelines.py,sha256=k8DRupjvN7qnIk0uFNJ_3JEFlDadtO0PCBH0iOsPKp4,145
+pocong/proxy_spiders/spiders/__init__.py,sha256=4-oTTycftRXl_6z92SjSi_XmDfP-1xAaVj39HMggLWc,52
+pocong/proxy_spiders/spiders/free_proxy_list_net_spider.py,sha256=AV-8_KF7UMRkdcuCaqdhGtbsMawpJt9G3NF6S7aVQO4,886
+pocong-1.0.1.dist-info/METADATA,sha256=-t70ukwfphmnul9khbMmbYyIOIowevrC9eriXZrv5so,3411
+pocong-1.0.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+pocong-1.0.1.dist-info/entry_points.txt,sha256=Q3F4OQIZJzlnS2tnEuTzcn2tN4S5Btd08o_9Otdb4bM,43
+pocong-1.0.1.dist-info/top_level.txt,sha256=ZMo2AlCPGpM4N7hHVSNoIjbM1D90yjFhRra0YmCfTO4,7
+pocong-1.0.1.dist-info/RECORD,,

pocong-0.1.3.dist-info/RECORD

@@ -1,10 +0,0 @@
-pocong/__init__.py,sha256=Hl0PkSkg6LV6IRLzXnGc0K2GY-drxkZEpt5qTAVDUkY,109
-pocong/_version.py,sha256=6DzWhBmSmCgH-UnOBP9FE_v5R2nrgAZOwvaIfjiu-CE,497
-pocong/cli.py,sha256=_f_aU4pckbQ_baF9oHwbqwmBFiQFn5Irvi-v5rDZ70o,529
-pocong/pocong.py,sha256=h0hwdogXGFqerm-5ZPeT-irPn91pCcQRjiHThXsRzEk,19
-pocong/utils.py,sha256=MAbbL9PXRWnBpJKgI869ZfY42Eph73zcbJyK0jH2Nak,35
-pocong-0.1.3.dist-info/METADATA,sha256=BsjoWSXB7UYifsd7F5thMiWK3rTyfjX5n3Kfl47gG_0,1925
-pocong-0.1.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-pocong-0.1.3.dist-info/entry_points.txt,sha256=Q3F4OQIZJzlnS2tnEuTzcn2tN4S5Btd08o_9Otdb4bM,43
-pocong-0.1.3.dist-info/top_level.txt,sha256=ZMo2AlCPGpM4N7hHVSNoIjbM1D90yjFhRra0YmCfTO4,7
-pocong-0.1.3.dist-info/RECORD,,