pocong 1.0.0__tar.gz → 1.0.1__tar.gz

{pocong-1.0.0 → pocong-1.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pocong
-Version: 1.0.0
+Version: 1.0.1
 Summary: Python Oriented Crawling Ongoing (POCONG): a simple crawling framework
 Home-page: https://gitlab.com/mohsin3107/pocong
 Author: Singgih

{pocong-1.0.0 → pocong-1.0.1}/src/pocong/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-09-21T17:59:23+0700",
+ "date": "2025-09-21T22:32:32+0700",
  "dirty": false,
  "error": null,
- "full-revisionid": "9a2d0f58b89a546044fc52948ce274767aa450d4",
- "version": "1.0.0"
+ "full-revisionid": "85eac04a9a369e96c0c4819feecca0cc5d23c173",
+ "version": "1.0.1"
 }
 '''  # END VERSION_JSON
 

{pocong-1.0.0 → pocong-1.0.1}/src/pocong/proxy_spiders/__init__.py

@@ -2,10 +2,6 @@
 import random
 
 import requests
-import pandas as pd
-from scrapy.crawler import CrawlerProcess
-
-from pocong.proxy_spiders.spiders.free_proxy_list_net_spider import ProxySpider
 
 
 class GetProxy():
@@ -13,37 +9,39 @@ class GetProxy():
     Class to get proxies using Scrapy spiders and validate them.
     '''
     def __init__(self):
-        pass
+        self._items = []
 
     def _check_proxy(self, x):
         proxy = f"http://{x['ip']}:{x['port']}"
         try:
             response = requests.get("https://httpbin.org/ip", proxies={'https': proxy}, timeout=10)
             if response.status_code == 200 and response.json().get('origin') == x['ip']:
-                print(f"checking proxy: {proxy} success")  # noqa
                 return response.status_code
-            print(f"checking proxy: {proxy} failed")  # noqa
             return 0
         except requests.RequestException:
-            print(f"checking proxy: {proxy} failed")  # noqa
             return 0
 
-    def _run_example_spider(self):
-        process = CrawlerProcess(settings={
-            "LOG_LEVEL": "ERROR",
-            "ITEM_PIPELINES": {'pocong.proxy_spiders.pipelines.Pipelines': 1},
-        })
-        process.crawl(ProxySpider)
-        process.start()
-        from pocong.proxy_spiders.pipelines import collected_items
-        return collected_items
-
     def _get_proxy_from_scrape(self):
-        items = self._run_example_spider()
-        df = pd.DataFrame(items)
-        df = df[df['https'] == 'yes']
-        df = df.drop_duplicates(subset=['ip', 'port'])
-        proxies_json = df.to_dict(orient='records')
+        import subprocess
+        import sys
+        import json
+        # Run the spider in a subprocess to avoid reactor restart error
+        code = (
+            'import pandas as pd;'
+            'from scrapy.crawler import CrawlerProcess;'
+            'from pocong.proxy_spiders.spiders.free_proxy_list_net_spider import ProxySpider;'
+            'from pocong.proxy_spiders.pipelines import collected_items;'
+            'process = CrawlerProcess(settings={"LOG_LEVEL": "ERROR", "ITEM_PIPELINES": {"pocong.proxy_spiders.pipelines.Pipelines": 1}});'  # noqa: E501
+            'process.crawl(ProxySpider);'
+            'process.start();'
+            'process.stop();'
+            'df = pd.DataFrame(collected_items);'
+            'df = df[df["https"] == "yes"];'
+            'df = df.drop_duplicates(subset=["ip", "port"]);'
+            'print(df.to_json(orient="records"))'
+        )
+        result = subprocess.run([sys.executable, '-c', code], capture_output=True, text=True)
+        proxies_json = json.loads(result.stdout.strip()) if result.stdout.strip() else []
         return proxies_json
 
     def get_proxy(self):

{pocong-1.0.0 → pocong-1.0.1}/src/pocong.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pocong
-Version: 1.0.0
+Version: 1.0.1
 Summary: Python Oriented Crawling Ongoing (POCONG): a simple crawling framework
 Home-page: https://gitlab.com/mohsin3107/pocong
 Author: Singgih