crawlo 1.0.1__tar.gz → 1.0.3__tar.gz

{crawlo-1.0.1 → crawlo-1.0.3}/LICENSE

@@ -1,23 +1,23 @@
-MIT License
-
-Modifications:
-
-Copyright (c) 2020 crawl-coder <2251018029@qq.com>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+MIT License
+
+Modifications:
+
+Copyright (c) 2020 crawl-coder <2251018029@qq.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.

crawlo-1.0.3/MANIFEST.in

@@ -0,0 +1,17 @@
+include README.md
+include LICENSE
+include requirements.txt  # 如果根目录有全局requirements.txt
+include VERSION           # 如果根目录有全局VERSION文件
+
+# 包内文件包含
+recursive-include crawlo/utils/js *
+recursive-include crawlo/templates *
+
+# 测试文件（如果需要在分发包中包含测试）
+recursive-include tests *
+
+# 排除项
+global-exclude __pycache__ *.py[cod] .DS_Store *.so
+global-exclude *.bak *.swp *.orig *.rej
+prune samples              # 排除示例目录
+prune docs                 # 排除文档目录

{crawlo-1.0.1/crawlo.egg-info → crawlo-1.0.3}/PKG-INFO

@@ -1,36 +1,48 @@
-Metadata-Version: 2.4
-Name: crawlo
-Version: 1.0.1
-Summary: Crawlo是一款支持异步的python爬虫框架
-Home-page: https://github.com/crawl-coder/Crawlo.git
-Author: crawl-coder
-Author-email: crawlo@qq.com
-License: MIT
-Classifier: Programming Language :: Python :: 3
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Operating System :: OS Independent
-Requires-Python: >=3.6
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: aiohttp>=3.12.6
-Requires-Dist: httpx>=0.28.1
-Requires-Dist: DBUtils>=2.0
-Requires-Dist: parsel>=1.10.0
-Requires-Dist: pymysql>=1.1.1
-Requires-Dist: ujson>=5.10.0
-Provides-Extra: render
-Requires-Dist: webdriver-manager>=4.0.0; extra == "render"
-Requires-Dist: playwright; extra == "render"
-Requires-Dist: selenium>=3.141.0; extra == "render"
-Provides-Extra: all
-Requires-Dist: bitarray>=1.5.3; extra == "all"
-Requires-Dist: PyExecJS>=1.5.1; extra == "all"
-Requires-Dist: pymongo>=3.10.1; extra == "all"
-Requires-Dist: redis-py-cluster>=2.1.0; extra == "all"
-Requires-Dist: webdriver-manager>=4.0.0; extra == "all"
-Requires-Dist: playwright; extra == "all"
-Requires-Dist: selenium>=3.141.0; extra == "all"
-Dynamic: license-file
-
-# Crawlo
-异步通用爬虫框架
+Metadata-Version: 2.4
+Name: crawlo
+Version: 1.0.3
+Summary: Crawlo 是一款基于异步IO的高性能Python爬虫框架，支持分布式抓取。
+Home-page: https://github.com/crawl-coder/Crawlo.git
+Author: crawl-coder
+Author-email: crawlo@qq.com
+License: MIT
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown
+Requires-Dist: aiohttp>=3.12.14
+Requires-Dist: aiomysql>=0.2.0
+Requires-Dist: aioredis>=2.0.1
+Requires-Dist: asyncmy>=0.2.10
+Requires-Dist: cssselect>=1.2.0
+Requires-Dist: dateparser>=1.2.2
+Requires-Dist: httpx[http2]>=0.27.0
+Requires-Dist: lxml>=5.2.1
+Requires-Dist: motor>=3.7.0
+Requires-Dist: parsel>=1.9.1
+Requires-Dist: pydantic>=2.11.7
+Requires-Dist: pymongo>=4.11
+Requires-Dist: PyMySQL>=1.1.1
+Requires-Dist: python-dateutil>=2.9.0.post0
+Requires-Dist: redis>=6.2.0
+Requires-Dist: requests>=2.32.4
+Requires-Dist: six>=1.17.0
+Requires-Dist: ujson>=5.9.0
+Requires-Dist: urllib3>=2.5.0
+Requires-Dist: w3lib>=2.1.2
+Provides-Extra: render
+Requires-Dist: webdriver-manager>=4.0.0; extra == "render"
+Requires-Dist: playwright; extra == "render"
+Requires-Dist: selenium>=3.141.0; extra == "render"
+Provides-Extra: all
+Requires-Dist: bitarray>=1.5.3; extra == "all"
+Requires-Dist: PyExecJS>=1.5.1; extra == "all"
+Requires-Dist: pymongo>=3.10.1; extra == "all"
+Requires-Dist: redis-py-cluster>=2.1.0; extra == "all"
+Requires-Dist: webdriver-manager>=4.0.0; extra == "all"
+Requires-Dist: playwright; extra == "all"
+Requires-Dist: selenium>=3.141.0; extra == "all"
+
+# Crawlo
+Crawlo 是一款基于异步IO的高性能Python爬虫框架，支持分布式抓取与数据管道。

crawlo-1.0.3/README.md

@@ -0,0 +1,2 @@
+# Crawlo
+Crawlo 是一款基于异步IO的高性能Python爬虫框架，支持分布式抓取与数据管道。

{crawlo-1.0.1 → crawlo-1.0.3}/crawlo/__init__.py

@@ -1,6 +1,9 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-from crawlo.network.request import Request
-from crawlo.network.response import Response
-from crawlo.items.items import Item
-from .__version__ import __version__
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+# from crawlo.spider import Spider
+from crawlo.items.items import Item
+from crawlo.network.request import Request
+from crawlo.network.response import Response
+from crawlo.downloader import DownloaderBase
+from crawlo.middleware import BaseMiddleware
+from .__version__ import __version__

crawlo-1.0.3/crawlo/__version__.py

@@ -0,0 +1 @@
+__version__ = "1.0.3"

{crawlo-1.0.1 → crawlo-1.0.3}/crawlo/core/__init__.py

@@ -1,2 +1,2 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-

{crawlo-1.0.1 → crawlo-1.0.3}/crawlo/core/engine.py

@@ -1,159 +1,159 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-import asyncio
-from inspect import iscoroutine
-from typing import Optional, Generator, Callable
-
-from crawlo import Request, Item
-from crawlo.spider import Spider
-from crawlo.utils.log import get_logger
-from crawlo.exceptions import OutputError
-from crawlo.core.scheduler import Scheduler
-from crawlo.core.processor import Processor
-from crawlo.task_manager import TaskManager
-from crawlo.utils.project import load_class
-from crawlo.downloader import DownloaderBase
-from crawlo.utils.func_tools import transform
-from crawlo.event import spider_opened, spider_error, request_scheduled
-
-
-class Engine(object):
-
-    def __init__(self, crawler):
-        self.running = False
-        self.normal = True
-        self.crawler = crawler
-        self.settings = crawler.settings
-        self.spider: Optional[Spider] = None
-        self.downloader: Optional[DownloaderBase] = None
-        self.scheduler: Optional[Scheduler] = None
-        self.processor: Optional[Processor] = None
-        self.start_requests: Optional[Generator] = None
-        self.task_manager: Optional[TaskManager] = TaskManager(self.settings.get_int('CONCURRENCY'))
-
-        self.logger = get_logger(name=self.__class__.__name__)
-
-    def _get_downloader_cls(self):
-        downloader_cls = load_class(self.settings.get('DOWNLOADER'))
-        if not issubclass(downloader_cls, DownloaderBase):
-            raise TypeError(f'Downloader {downloader_cls.__name__} is not subclass of DownloaderBase.')
-        return downloader_cls
-
-    def engine_start(self):
-        self.running = True
-        self.logger.info(
-            f"Crawlo (version {self.settings.get_int('VERSION')}) started. "
-            f"(project name : {self.settings.get('PROJECT_NAME')})"
-        )
-
-    async def start_spider(self, spider):
-        self.spider = spider
-
-        self.scheduler = Scheduler.create_instance(self.crawler)
-        if hasattr(self.scheduler, 'open'):
-            self.scheduler.open()
-
-        downloader_cls = self._get_downloader_cls()
-        self.downloader = downloader_cls(self.crawler)
-        if hasattr(self.downloader, 'open'):
-            self.downloader.open()
-
-        self.processor = Processor(self.crawler)
-        if hasattr(self.processor, 'open'):
-            self.processor.open()
-
-        self.start_requests = iter(spider.start_requests())
-        await self._open_spider()
-
-    async def crawl(self):
-        """
-        Crawl the spider
-        """
-        while self.running:
-            if request := await self._get_next_request():
-                await self._crawl(request)
-            try:
-                start_request = next(self.start_requests)
-            except StopIteration:
-                self.start_requests = None
-            except Exception as exp:
-                # 1、发去请求的request全部运行完毕
-                # 2、调度器是否空闲
-                # 3、下载器是否空闲
-                if not await self._exit():
-                    continue
-                self.running = False
-                if self.start_requests is not None:
-                    self.logger.error(f"启动请求时发生错误: {str(exp)}")
-            else:
-                # 请求入队
-                await self.enqueue_request(start_request)
-
-        if not self.running:
-            await self.close_spider()
-
-    async def _open_spider(self):
-        asyncio.create_task(self.crawler.subscriber.notify(spider_opened))
-        crawling = asyncio.create_task(self.crawl())
-        await crawling
-
-    async def _crawl(self, request):
-        # TODO 实现并发
-        async def crawl_task():
-            outputs = await self._fetch(request)
-            # TODO 处理output
-            if outputs:
-                await self._handle_spider_output(outputs)
-
-        # asyncio.create_task(crawl_task())
-        self.task_manager.create_task(crawl_task())
-
-    async def _fetch(self, request):
-        async def _successful(_response):
-            callback: Callable = request.callback or self.spider.parse
-            if _outputs := callback(_response):
-                if iscoroutine(_outputs):
-                    await _outputs
-                else:
-                    return transform(_outputs, _response)
-
-        _response = await self.downloader.fetch(request)
-        if _response is None:
-            return None
-        output = await _successful(_response)
-        return output
-
-    async def enqueue_request(self, start_request):
-        await self._schedule_request(start_request)
-
-    async def _schedule_request(self, request):
-        # TODO 去重
-        if await self.scheduler.enqueue_request(request):
-            asyncio.create_task(self.crawler.subscriber.notify(request_scheduled, request, self.crawler.spider))
-
-    async def _get_next_request(self):
-        return await self.scheduler.next_request()
-
-    async def _handle_spider_output(self, outputs):
-        async for spider_output in outputs:
-            if isinstance(spider_output, (Request, Item)):
-                await self.processor.enqueue(spider_output)
-            elif isinstance(spider_output, Exception):
-                asyncio.create_task(
-                    self.crawler.subscriber.notify(spider_error, spider_output, self.spider)
-                )
-                raise spider_output
-            else:
-                raise OutputError(f'{type(self.spider)} must return `Request` or `Item`.')
-
-    async def _exit(self):
-        if self.scheduler.idle() and self.downloader.idle() and self.task_manager.all_done() and self.processor.idle():
-            return True
-        return False
-
-    async def close_spider(self):
-        await asyncio.gather(*self.task_manager.current_task)
-        await self.scheduler.close()
-        await self.downloader.close()
-        if self.normal:
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import asyncio
+from inspect import iscoroutine
+from typing import Optional, Generator, Callable
+
+from crawlo import Request, Item
+from crawlo.spider import Spider
+from crawlo.utils.log import get_logger
+from crawlo.exceptions import OutputError
+from crawlo.core.scheduler import Scheduler
+from crawlo.core.processor import Processor
+from crawlo.task_manager import TaskManager
+from crawlo.utils.project import load_class
+from crawlo.downloader import DownloaderBase
+from crawlo.utils.func_tools import transform
+from crawlo.event import spider_opened, spider_error, request_scheduled
+
+
+class Engine(object):
+
+    def __init__(self, crawler):
+        self.running = False
+        self.normal = True
+        self.crawler = crawler
+        self.settings = crawler.settings
+        self.spider: Optional[Spider] = None
+        self.downloader: Optional[DownloaderBase] = None
+        self.scheduler: Optional[Scheduler] = None
+        self.processor: Optional[Processor] = None
+        self.start_requests: Optional[Generator] = None
+        self.task_manager: Optional[TaskManager] = TaskManager(self.settings.get_int('CONCURRENCY'))
+
+        self.logger = get_logger(name=self.__class__.__name__)
+
+    def _get_downloader_cls(self):
+        downloader_cls = load_class(self.settings.get('DOWNLOADER'))
+        if not issubclass(downloader_cls, DownloaderBase):
+            raise TypeError(f'Downloader {downloader_cls.__name__} is not subclass of DownloaderBase.')
+        return downloader_cls
+
+    def engine_start(self):
+        self.running = True
+        self.logger.info(
+            f"Crawlo (version {self.settings.get_int('VERSION')}) started. "
+            f"(project name : {self.settings.get('PROJECT_NAME')})"
+        )
+
+    async def start_spider(self, spider):
+        self.spider = spider
+
+        self.scheduler = Scheduler.create_instance(self.crawler)
+        if hasattr(self.scheduler, 'open'):
+            self.scheduler.open()
+
+        downloader_cls = self._get_downloader_cls()
+        self.downloader = downloader_cls(self.crawler)
+        if hasattr(self.downloader, 'open'):
+            self.downloader.open()
+
+        self.processor = Processor(self.crawler)
+        if hasattr(self.processor, 'open'):
+            self.processor.open()
+
+        self.start_requests = iter(spider.start_requests())
+        await self._open_spider()
+
+    async def crawl(self):
+        """
+        Crawl the spider
+        """
+        while self.running:
+            if request := await self._get_next_request():
+                await self._crawl(request)
+            try:
+                start_request = next(self.start_requests)
+            except StopIteration:
+                self.start_requests = None
+            except Exception as exp:
+                # 1、发去请求的request全部运行完毕
+                # 2、调度器是否空闲
+                # 3、下载器是否空闲
+                if not await self._exit():
+                    continue
+                self.running = False
+                if self.start_requests is not None:
+                    self.logger.error(f"启动请求时发生错误: {str(exp)}")
+            else:
+                # 请求入队
+                await self.enqueue_request(start_request)
+
+        if not self.running:
+            await self.close_spider()
+
+    async def _open_spider(self):
+        asyncio.create_task(self.crawler.subscriber.notify(spider_opened))
+        crawling = asyncio.create_task(self.crawl())
+        await crawling
+
+    async def _crawl(self, request):
+        # TODO 实现并发
+        async def crawl_task():
+            outputs = await self._fetch(request)
+            # TODO 处理output
+            if outputs:
+                await self._handle_spider_output(outputs)
+
+        # asyncio.create_task(crawl_task())
+        self.task_manager.create_task(crawl_task())
+
+    async def _fetch(self, request):
+        async def _successful(_response):
+            callback: Callable = request.callback or self.spider.parse
+            if _outputs := callback(_response):
+                if iscoroutine(_outputs):
+                    await _outputs
+                else:
+                    return transform(_outputs, _response)
+
+        _response = await self.downloader.fetch(request)
+        if _response is None:
+            return None
+        output = await _successful(_response)
+        return output
+
+    async def enqueue_request(self, start_request):
+        await self._schedule_request(start_request)
+
+    async def _schedule_request(self, request):
+        # TODO 去重
+        if await self.scheduler.enqueue_request(request):
+            asyncio.create_task(self.crawler.subscriber.notify(request_scheduled, request, self.crawler.spider))
+
+    async def _get_next_request(self):
+        return await self.scheduler.next_request()
+
+    async def _handle_spider_output(self, outputs):
+        async for spider_output in outputs:
+            if isinstance(spider_output, (Request, Item)):
+                await self.processor.enqueue(spider_output)
+            elif isinstance(spider_output, Exception):
+                asyncio.create_task(
+                    self.crawler.subscriber.notify(spider_error, spider_output, self.spider)
+                )
+                raise spider_output
+            else:
+                raise OutputError(f'{type(self.spider)} must return `Request` or `Item`.')
+
+    async def _exit(self):
+        if self.scheduler.idle() and self.downloader.idle() and self.task_manager.all_done() and self.processor.idle():
+            return True
+        return False
+
+    async def close_spider(self):
+        await asyncio.gather(*self.task_manager.current_task)
+        await self.scheduler.close()
+        await self.downloader.close()
+        if self.normal:
             await self.crawler.close()

{crawlo-1.0.1 → crawlo-1.0.3}/crawlo/core/processor.py

@@ -1,40 +1,40 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-from asyncio import Queue
-from typing import Union, Optional
-
-from crawlo import Request, Item
-from crawlo.pipelines.pipeline_manager import PipelineManager
-
-
-class Processor(object):
-
-    def __init__(self, crawler):
-        self.crawler = crawler
-        self.queue: Queue = Queue()
-        self.pipelines: Optional[PipelineManager] = None
-
-    def open(self):
-        self.pipelines = PipelineManager.from_crawler(self.crawler)
-
-    async def process(self):
-        while not self.idle():
-            result = await self.queue.get()
-            if isinstance(result, Request):
-                await self.crawler.engine.enqueue_request(result)
-            else:
-                assert isinstance(result, Item)
-                await self._process_item(result)
-
-    async def _process_item(self, item):
-        await self.pipelines.process_item(item=item)
-
-    async def enqueue(self, output: Union[Request, Item]):
-        await self.queue.put(output)
-        await self.process()
-
-    def idle(self) -> bool:
-        return len(self) == 0
-
-    def __len__(self):
-        return self.queue.qsize()
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from asyncio import Queue
+from typing import Union, Optional
+
+from crawlo import Request, Item
+from crawlo.pipelines.pipeline_manager import PipelineManager
+
+
+class Processor(object):
+
+    def __init__(self, crawler):
+        self.crawler = crawler
+        self.queue: Queue = Queue()
+        self.pipelines: Optional[PipelineManager] = None
+
+    def open(self):
+        self.pipelines = PipelineManager.from_crawler(self.crawler)
+
+    async def process(self):
+        while not self.idle():
+            result = await self.queue.get()
+            if isinstance(result, Request):
+                await self.crawler.engine.enqueue_request(result)
+            else:
+                assert isinstance(result, Item)
+                await self._process_item(result)
+
+    async def _process_item(self, item):
+        await self.pipelines.process_item(item=item)
+
+    async def enqueue(self, output: Union[Request, Item]):
+        await self.queue.put(output)
+        await self.process()
+
+    def idle(self) -> bool:
+        return len(self) == 0
+
+    def __len__(self):
+        return self.queue.qsize()