pocong 1.0.0__py3-none-any.whl → 1.0.1__py3-none-any.whl

pocong/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-09-21T17:59:23+0700",
+ "date": "2025-09-21T22:32:32+0700",
  "dirty": false,
  "error": null,
- "full-revisionid": "9a2d0f58b89a546044fc52948ce274767aa450d4",
- "version": "1.0.0"
+ "full-revisionid": "85eac04a9a369e96c0c4819feecca0cc5d23c173",
+ "version": "1.0.1"
 }
 '''  # END VERSION_JSON
 

pocong/proxy_spiders/__init__.py

@@ -2,10 +2,6 @@
 import random
 
 import requests
-import pandas as pd
-from scrapy.crawler import CrawlerProcess
-
-from pocong.proxy_spiders.spiders.free_proxy_list_net_spider import ProxySpider
 
 
 class GetProxy():
@@ -13,37 +9,39 @@ class GetProxy():
     Class to get proxies using Scrapy spiders and validate them.
     '''
     def __init__(self):
-        pass
+        self._items = []
 
     def _check_proxy(self, x):
         proxy = f"http://{x['ip']}:{x['port']}"
         try:
             response = requests.get("https://httpbin.org/ip", proxies={'https': proxy}, timeout=10)
             if response.status_code == 200 and response.json().get('origin') == x['ip']:
-                print(f"checking proxy: {proxy} success")  # noqa
                 return response.status_code
-            print(f"checking proxy: {proxy} failed")  # noqa
             return 0
         except requests.RequestException:
-            print(f"checking proxy: {proxy} failed")  # noqa
             return 0
 
-    def _run_example_spider(self):
-        process = CrawlerProcess(settings={
-            "LOG_LEVEL": "ERROR",
-            "ITEM_PIPELINES": {'pocong.proxy_spiders.pipelines.Pipelines': 1},
-        })
-        process.crawl(ProxySpider)
-        process.start()
-        from pocong.proxy_spiders.pipelines import collected_items
-        return collected_items
-
     def _get_proxy_from_scrape(self):
-        items = self._run_example_spider()
-        df = pd.DataFrame(items)
-        df = df[df['https'] == 'yes']
-        df = df.drop_duplicates(subset=['ip', 'port'])
-        proxies_json = df.to_dict(orient='records')
+        import subprocess
+        import sys
+        import json
+        # Run the spider in a subprocess to avoid reactor restart error
+        code = (
+            'import pandas as pd;'
+            'from scrapy.crawler import CrawlerProcess;'
+            'from pocong.proxy_spiders.spiders.free_proxy_list_net_spider import ProxySpider;'
+            'from pocong.proxy_spiders.pipelines import collected_items;'
+            'process = CrawlerProcess(settings={"LOG_LEVEL": "ERROR", "ITEM_PIPELINES": {"pocong.proxy_spiders.pipelines.Pipelines": 1}});'  # noqa: E501
+            'process.crawl(ProxySpider);'
+            'process.start();'
+            'process.stop();'
+            'df = pd.DataFrame(collected_items);'
+            'df = df[df["https"] == "yes"];'
+            'df = df.drop_duplicates(subset=["ip", "port"]);'
+            'print(df.to_json(orient="records"))'
+        )
+        result = subprocess.run([sys.executable, '-c', code], capture_output=True, text=True)
+        proxies_json = json.loads(result.stdout.strip()) if result.stdout.strip() else []
         return proxies_json
 
     def get_proxy(self):

{pocong-1.0.0.dist-info → pocong-1.0.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pocong
-Version: 1.0.0
+Version: 1.0.1
 Summary: Python Oriented Crawling Ongoing (POCONG): a simple crawling framework
 Home-page: https://gitlab.com/mohsin3107/pocong
 Author: Singgih

{pocong-1.0.0.dist-info → pocong-1.0.1.dist-info}/RECORD

@@ -1,14 +1,14 @@
 pocong/__init__.py,sha256=Hl0PkSkg6LV6IRLzXnGc0K2GY-drxkZEpt5qTAVDUkY,109
-pocong/_version.py,sha256=hFgXEoBMIBnyj4LP_PTO6GNR5nrvefsXEARVrbSCj7o,497
+pocong/_version.py,sha256=t_ZIUHG9ovHL3x_o5mb9F_Ih_UhgDKE9j9yI8c92xyk,497
 pocong/cli.py,sha256=_f_aU4pckbQ_baF9oHwbqwmBFiQFn5Irvi-v5rDZ70o,529
 pocong/pocong.py,sha256=h0hwdogXGFqerm-5ZPeT-irPn91pCcQRjiHThXsRzEk,19
 pocong/utils.py,sha256=MAbbL9PXRWnBpJKgI869ZfY42Eph73zcbJyK0jH2Nak,35
-pocong/proxy_spiders/__init__.py,sha256=q3ifQZd4_TipTmVYklCdvgVZEkuLR91Qo0LwM4CBMnA,2384
+pocong/proxy_spiders/__init__.py,sha256=RBlvqba1wIhCJPn4n5LNB3SLiCQHDcGOhtrFydWY7T4,2556
 pocong/proxy_spiders/pipelines.py,sha256=k8DRupjvN7qnIk0uFNJ_3JEFlDadtO0PCBH0iOsPKp4,145
 pocong/proxy_spiders/spiders/__init__.py,sha256=4-oTTycftRXl_6z92SjSi_XmDfP-1xAaVj39HMggLWc,52
 pocong/proxy_spiders/spiders/free_proxy_list_net_spider.py,sha256=AV-8_KF7UMRkdcuCaqdhGtbsMawpJt9G3NF6S7aVQO4,886
-pocong-1.0.0.dist-info/METADATA,sha256=2A3QXiD-vF7rgHM6sN4Yt9-O8KbRm_5DGaKGfCaKMjw,3411
-pocong-1.0.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-pocong-1.0.0.dist-info/entry_points.txt,sha256=Q3F4OQIZJzlnS2tnEuTzcn2tN4S5Btd08o_9Otdb4bM,43
-pocong-1.0.0.dist-info/top_level.txt,sha256=ZMo2AlCPGpM4N7hHVSNoIjbM1D90yjFhRra0YmCfTO4,7
-pocong-1.0.0.dist-info/RECORD,,
+pocong-1.0.1.dist-info/METADATA,sha256=-t70ukwfphmnul9khbMmbYyIOIowevrC9eriXZrv5so,3411
+pocong-1.0.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+pocong-1.0.1.dist-info/entry_points.txt,sha256=Q3F4OQIZJzlnS2tnEuTzcn2tN4S5Btd08o_9Otdb4bM,43
+pocong-1.0.1.dist-info/top_level.txt,sha256=ZMo2AlCPGpM4N7hHVSNoIjbM1D90yjFhRra0YmCfTO4,7
+pocong-1.0.1.dist-info/RECORD,,