crawl4weibo 0.1.0__tar.gz

crawl4weibo-0.1.0/PKG-INFO

@@ -0,0 +1,134 @@
+Metadata-Version: 2.4
+Name: crawl4weibo
+Version: 0.1.0
+Summary: A professional Weibo crawler library
+Home-page: https://github.com/yourusername/crawl4weibo
+Author: Your Name
+Author-email: Kritoooo <krito2023@gmail.com>
+License: MIT
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+Requires-Dist: requests>=2.25.0
+Requires-Dist: lxml>=4.6.0
+Requires-Dist: tqdm>=4.60.0
+Requires-Dist: python-dateutil>=2.8.0
+Provides-Extra: dev
+Requires-Dist: pytest>=6.0; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
+Requires-Dist: black; extra == "dev"
+Requires-Dist: isort; extra == "dev"
+Requires-Dist: flake8; extra == "dev"
+Dynamic: author
+Dynamic: home-page
+Dynamic: requires-python
+
+# Crawl4Weibo
+
+一个专业的微博爬虫Python库，基于实际测试成功的方案，无需Cookie即可使用。
+
+## 特性
+
+- 🚀 **开箱即用**: 无需Cookie，一行代码初始化
+- 🛡️ **防反爬**: 自动处理432错误和请求限制  
+- 📱 **真实模拟**: 使用真实手机浏览器UA
+- 🔄 **智能重试**: 自动重试机制
+- 📊 **结构化数据**: 清晰的数据模型
+
+## 安装
+
+```bash
+pip install -e .
+```
+
+## 快速开始
+
+```python
+from crawl4weibo import WeiboClient
+
+# 初始化（无需Cookie）
+client = WeiboClient()
+
+# 获取用户信息
+user = client.get_user_by_uid("1195230310")
+print(f"用户: {user.screen_name}")
+print(f"粉丝: {user.followers_count:,}")
+
+# 获取微博
+posts = client.get_user_posts("1195230310")
+for post in posts:
+    print(f"微博: {post.text[:50]}...")
+    print(f"点赞: {post.attitudes_count}")
+
+# 搜索用户
+users = client.search_users("技术博主")
+for user in users:
+    print(f"用户: {user.screen_name}")
+
+# 搜索微博  
+posts = client.search_posts("人工智能")
+for post in posts:
+    print(f"内容: {post.text[:50]}...")
+```
+
+## API参考
+
+### WeiboClient
+
+#### 初始化
+```python
+WeiboClient(cookies=None, log_level="INFO", log_file=None)
+```
+
+#### 主要方法
+
+- `get_user_by_uid(uid)` - 获取用户信息
+- `get_user_posts(uid, page=1)` - 获取用户微博
+- `search_users(query, page=1, count=10)` - 搜索用户
+- `search_posts(query, page=1)` - 搜索微博
+
+### 数据模型
+
+**User (用户)**:
+- `screen_name` - 用户名
+- `followers_count` - 粉丝数  
+- `posts_count` - 微博数
+- `verified` - 是否认证
+
+**Post (微博)**:
+- `text` - 微博文本
+- `attitudes_count` - 点赞数
+- `comments_count` - 评论数
+- `created_at` - 发布时间
+
+## 运行示例
+
+```bash
+python examples/simple_example.py
+```
+
+## 技术实现
+
+基于你提供的成功代码实现：
+
+```python
+# 核心技术栈
+- Android Chrome UA模拟
+- 移动端API接口
+- 自动session管理  
+- 432错误智能重试
+- 随机请求间隔
+```
+
+## 许可证
+
+MIT License

crawl4weibo-0.1.0/README.md

@@ -0,0 +1,100 @@
+# Crawl4Weibo
+
+一个专业的微博爬虫Python库，基于实际测试成功的方案，无需Cookie即可使用。
+
+## 特性
+
+- 🚀 **开箱即用**: 无需Cookie，一行代码初始化
+- 🛡️ **防反爬**: 自动处理432错误和请求限制  
+- 📱 **真实模拟**: 使用真实手机浏览器UA
+- 🔄 **智能重试**: 自动重试机制
+- 📊 **结构化数据**: 清晰的数据模型
+
+## 安装
+
+```bash
+pip install -e .
+```
+
+## 快速开始
+
+```python
+from crawl4weibo import WeiboClient
+
+# 初始化（无需Cookie）
+client = WeiboClient()
+
+# 获取用户信息
+user = client.get_user_by_uid("1195230310")
+print(f"用户: {user.screen_name}")
+print(f"粉丝: {user.followers_count:,}")
+
+# 获取微博
+posts = client.get_user_posts("1195230310")
+for post in posts:
+    print(f"微博: {post.text[:50]}...")
+    print(f"点赞: {post.attitudes_count}")
+
+# 搜索用户
+users = client.search_users("技术博主")
+for user in users:
+    print(f"用户: {user.screen_name}")
+
+# 搜索微博  
+posts = client.search_posts("人工智能")
+for post in posts:
+    print(f"内容: {post.text[:50]}...")
+```
+
+## API参考
+
+### WeiboClient
+
+#### 初始化
+```python
+WeiboClient(cookies=None, log_level="INFO", log_file=None)
+```
+
+#### 主要方法
+
+- `get_user_by_uid(uid)` - 获取用户信息
+- `get_user_posts(uid, page=1)` - 获取用户微博
+- `search_users(query, page=1, count=10)` - 搜索用户
+- `search_posts(query, page=1)` - 搜索微博
+
+### 数据模型
+
+**User (用户)**:
+- `screen_name` - 用户名
+- `followers_count` - 粉丝数  
+- `posts_count` - 微博数
+- `verified` - 是否认证
+
+**Post (微博)**:
+- `text` - 微博文本
+- `attitudes_count` - 点赞数
+- `comments_count` - 评论数
+- `created_at` - 发布时间
+
+## 运行示例
+
+```bash
+python examples/simple_example.py
+```
+
+## 技术实现
+
+基于你提供的成功代码实现：
+
+```python
+# 核心技术栈
+- Android Chrome UA模拟
+- 移动端API接口
+- 自动session管理  
+- 432错误智能重试
+- 随机请求间隔
+```
+
+## 许可证
+
+MIT License

crawl4weibo-0.1.0/crawl4weibo/__init__.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+crawl4weibo - A professional Weibo crawler library
+"""
+
+__version__ = "0.1.0"
+__author__ = "Your Name"
+__email__ = "your.email@example.com"
+
+from .core.client import WeiboClient
+from .models.user import User
+from .models.post import Post
+from .exceptions.base import CrawlError, AuthenticationError, RateLimitError, UserNotFoundError, NetworkError, ParseError
+
+__all__ = [
+    "WeiboClient",
+    "User", 
+    "Post",
+    "CrawlError",
+    "AuthenticationError", 
+    "RateLimitError",
+    "UserNotFoundError",
+    "NetworkError", 
+    "ParseError",
+]

crawl4weibo-0.1.0/crawl4weibo/core/__init__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Core module for crawl4weibo
+"""

crawl4weibo-0.1.0/crawl4weibo/core/client.py

@@ -0,0 +1,195 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+微博爬虫客户端 - 基于实际测试成功的代码
+"""
+
+import requests
+import time
+import random
+from typing import List, Dict, Any, Optional, Union
+
+from ..utils.parser import WeiboParser
+from ..utils.logger import setup_logger
+from ..models.user import User
+from ..models.post import Post
+from ..exceptions.base import CrawlError, UserNotFoundError, ParseError, NetworkError
+
+
+class WeiboClient:
+    """微博爬虫客户端"""
+    
+    def __init__(self, cookies: Optional[Union[str, Dict[str, str]]] = None,
+                 log_level: str = "INFO", log_file: Optional[str] = None,
+                 user_agent: Optional[str] = None):
+        """
+        初始化微博客户端
+        
+        Args:
+            cookies: 可选的Cookie字符串或字典
+            log_level: 日志级别
+            log_file: 日志文件路径
+            user_agent: 可选的User-Agent字符串
+        """
+        self.logger = setup_logger(
+            level=getattr(__import__('logging'), log_level.upper()),
+            log_file=log_file
+        )
+        
+        # 创建session
+        self.session = requests.Session()
+        
+        # 设置经过验证的headers
+        default_user_agent = (
+            "Mozilla/5.0 (Linux; Android 13; SM-G9980) "
+            "AppleWebKit/537.36 (KHTML, like Gecko) "
+            "Chrome/112.0.5615.135 Mobile Safari/537.36"
+        )
+        self.session.headers.update({
+            "User-Agent": user_agent or default_user_agent,
+            "Referer": "https://m.weibo.cn/",
+            "Accept": "application/json, text/plain, */*",
+            "X-Requested-With": "XMLHttpRequest"
+        })
+        
+        # 添加Cookie（如果提供）
+        if cookies:
+            self._set_cookies(cookies)
+        
+        # 初始化session
+        self._init_session()
+        
+        # 解析器
+        self.parser = WeiboParser()
+        
+        self.logger.info("WeiboClient initialized successfully")
+    
+    def _set_cookies(self, cookies: Union[str, Dict[str, str]]):
+        """设置cookies"""
+        if isinstance(cookies, str):
+            cookie_dict = {}
+            for pair in cookies.split(';'):
+                if '=' in pair:
+                    key, value = pair.split('=', 1)
+                    cookie_dict[key.strip()] = value.strip()
+            self.session.cookies.update(cookie_dict)
+        elif isinstance(cookies, dict):
+            self.session.cookies.update(cookies)
+    
+    def _init_session(self):
+        """初始化session，获取首页cookie"""
+        try:
+            self.logger.debug("初始化session...")
+            self.session.get("https://m.weibo.cn/", timeout=5)
+            time.sleep(random.uniform(2, 4))
+        except Exception as e:
+            self.logger.warning(f"Session初始化失败: {e}")
+    
+    def _request(self, url: str, params: Dict[str, Any], max_retries: int = 3) -> Dict[str, Any]:
+        """发送请求并处理重试"""
+        for attempt in range(1, max_retries + 1):
+            try:
+                response = self.session.get(url, params=params, timeout=5)
+                
+                if response.status_code == 200:
+                    return response.json()
+                elif response.status_code == 432:
+                    if attempt < max_retries:
+                        sleep_time = random.uniform(4, 7)
+                        self.logger.warning(f"遇到432错误，等待 {sleep_time:.1f} 秒后重试...")
+                        time.sleep(sleep_time)
+                        continue
+                    else:
+                        raise NetworkError("遇到432反爬虫拦截")
+                else:
+                    response.raise_for_status()
+                    
+            except requests.exceptions.RequestException as e:
+                if attempt < max_retries:
+                    sleep_time = random.uniform(2, 5)
+                    self.logger.warning(f"请求失败，等待 {sleep_time:.1f} 秒后重试: {e}")
+                    time.sleep(sleep_time)
+                    continue
+                else:
+                    raise NetworkError(f"请求失败: {e}")
+        
+        raise CrawlError("达到最大重试次数")
+    
+    def get_user_by_uid(self, uid: str) -> User:
+        """根据UID获取用户信息"""
+        url = "https://m.weibo.cn/api/container/getIndex"
+        params = {"containerid": f"100505{uid}"}
+        
+        data = self._request(url, params)
+        
+        if not data.get("data") or not data["data"].get("userInfo"):
+            raise UserNotFoundError(f"用户 {uid} 不存在")
+        
+        user_info = self.parser.parse_user_info(data)
+        user = User.from_dict(user_info)
+        
+        self.logger.info(f"获取用户: {user.screen_name}")
+        return user
+    
+    def get_user_posts(self, uid: str, page: int = 1) -> List[Post]:
+        """获取用户微博"""
+        time.sleep(random.uniform(1, 3))  # 请求间隔
+        
+        url = "https://m.weibo.cn/api/container/getIndex"
+        params = {"containerid": f"107603{uid}", "page": page}
+        
+        data = self._request(url, params)
+        
+        if not data.get("data"):
+            return []
+        
+        posts_data = self.parser.parse_posts(data)
+        posts = [Post.from_dict(post_data) for post_data in posts_data]
+        
+        self.logger.info(f"获取到 {len(posts)} 条微博")
+        return posts
+    
+    def search_users(self, query: str, page: int = 1, count: int = 10) -> List[User]:
+        """搜索用户"""
+        time.sleep(random.uniform(1, 3))
+        
+        url = "https://m.weibo.cn/api/container/getIndex"
+        params = {
+            "containerid": f"100103type=3&q={query}",
+            "page": page,
+            "count": count
+        }
+        
+        data = self._request(url, params)
+        users = []
+        cards = data.get("data", {}).get("cards", [])
+        
+        for card in cards:
+            if card.get("card_type") == 11:
+                card_group = card.get("card_group", [])
+                for group_card in card_group:
+                    if group_card.get("card_type") == 10:
+                        user_data = group_card.get("user", {})
+                        if user_data:
+                            users.append(User.from_dict(user_data))
+        
+        self.logger.info(f"搜索到 {len(users)} 个用户")
+        return users
+    
+    def search_posts(self, query: str, page: int = 1) -> List[Post]:
+        """搜索微博"""
+        time.sleep(random.uniform(1, 3))
+        
+        url = "https://m.weibo.cn/api/container/getIndex"
+        params = {
+            "containerid": f"100103type=1&q={query}",
+            "page": page
+        }
+        
+        data = self._request(url, params)
+        posts_data = self.parser.parse_posts(data)
+        posts = [Post.from_dict(post_data) for post_data in posts_data]
+        
+        self.logger.info(f"搜索到 {len(posts)} 条微博")
+        return posts

crawl4weibo-0.1.0/crawl4weibo/exceptions/__init__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Exceptions for crawl4weibo
+"""

crawl4weibo-0.1.0/crawl4weibo/exceptions/base.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Base exceptions for crawl4weibo
+"""
+
+
+class CrawlError(Exception):
+    """Base exception for crawl4weibo"""
+    
+    def __init__(self, message="An error occurred during crawling", code=None):
+        self.message = message
+        self.code = code
+        super().__init__(self.message)
+
+
+class AuthenticationError(CrawlError):
+    """Raised when authentication fails"""
+    
+    def __init__(self, message="Authentication failed", code="AUTH_ERROR"):
+        super().__init__(message, code)
+
+
+class RateLimitError(CrawlError):
+    """Raised when rate limit is exceeded"""
+    
+    def __init__(self, message="Rate limit exceeded", code="RATE_LIMIT", retry_after=None):
+        self.retry_after = retry_after
+        super().__init__(message, code)
+
+
+class NetworkError(CrawlError):
+    """Raised when network request fails"""
+    
+    def __init__(self, message="Network request failed", code="NETWORK_ERROR"):
+        super().__init__(message, code)
+
+
+class ParseError(CrawlError):
+    """Raised when response parsing fails"""
+    
+    def __init__(self, message="Failed to parse response", code="PARSE_ERROR"):
+        super().__init__(message, code)
+
+
+class UserNotFoundError(CrawlError):
+    """Raised when user is not found"""
+    
+    def __init__(self, message="User not found", code="USER_NOT_FOUND"):
+        super().__init__(message, code)
+
+
+class InvalidConfigError(CrawlError):
+    """Raised when configuration is invalid"""
+    
+    def __init__(self, message="Invalid configuration", code="INVALID_CONFIG"):
+        super().__init__(message, code)

crawl4weibo-0.1.0/crawl4weibo/models/__init__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Data models for crawl4weibo
+"""

crawl4weibo-0.1.0/crawl4weibo/models/post.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Post model for crawl4weibo
+"""
+
+from dataclasses import dataclass, field
+from typing import Optional, List, Dict, Any
+from datetime import datetime
+
+
+@dataclass
+class Post:
+    """Weibo post model"""
+    
+    id: str
+    user_id: str
+    text: str = ""
+    created_at: Optional[datetime] = None
+    source: str = ""
+    reposts_count: int = 0
+    comments_count: int = 0
+    attitudes_count: int = 0
+    pic_urls: List[str] = field(default_factory=list)
+    video_url: str = ""
+    is_original: bool = True
+    retweeted_status: Optional["Post"] = None
+    location: str = ""
+    topic_ids: List[str] = field(default_factory=list)
+    at_users: List[str] = field(default_factory=list)
+    raw_data: Dict[str, Any] = field(default_factory=dict)
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "Post":
+        """Create Post instance from dictionary"""
+        retweeted_status = None
+        if data.get("retweeted_status"):
+            retweeted_status = cls.from_dict(data["retweeted_status"])
+        
+        post_data = {
+            "id": str(data.get("id", "")),
+            "user_id": str(data.get("user_id", "")),
+            "text": data.get("text", ""),
+            "created_at": data.get("created_at"),
+            "source": data.get("source", ""),
+            "reposts_count": data.get("reposts_count", 0),
+            "comments_count": data.get("comments_count", 0),
+            "attitudes_count": data.get("attitudes_count", 0),
+            "pic_urls": data.get("pic_urls", []),
+            "video_url": data.get("video_url", ""),
+            "is_original": data.get("is_original", True),
+            "retweeted_status": retweeted_status,
+            "location": data.get("location", ""),
+            "topic_ids": data.get("topic_ids", []),
+            "at_users": data.get("at_users", []),
+            "raw_data": data,
+        }
+        return cls(**post_data)
+    
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert Post instance to dictionary"""
+        result = {
+            "id": self.id,
+            "user_id": self.user_id,
+            "text": self.text,
+            "created_at": self.created_at,
+            "source": self.source,
+            "reposts_count": self.reposts_count,
+            "comments_count": self.comments_count,
+            "attitudes_count": self.attitudes_count,
+            "pic_urls": self.pic_urls,
+            "video_url": self.video_url,
+            "is_original": self.is_original,
+            "location": self.location,
+            "topic_ids": self.topic_ids,
+            "at_users": self.at_users,
+        }
+        
+        if self.retweeted_status:
+            result["retweeted_status"] = self.retweeted_status.to_dict()
+            
+        return result

crawl4weibo-0.1.0/crawl4weibo/models/user.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+User model for crawl4weibo
+"""
+
+from dataclasses import dataclass, field
+from typing import Optional, Dict, Any
+from datetime import datetime
+
+
+@dataclass
+class User:
+    """Weibo user model"""
+    
+    id: str
+    screen_name: str = ""
+    gender: str = ""
+    location: str = ""
+    description: str = ""
+    followers_count: int = 0
+    following_count: int = 0
+    posts_count: int = 0
+    verified: bool = False
+    verified_reason: str = ""
+    avatar_url: str = ""
+    cover_image_url: str = ""
+    birthday: Optional[str] = None
+    education: str = ""
+    company: str = ""
+    registration_time: Optional[datetime] = None
+    sunshine_credit: str = ""
+    raw_data: Dict[str, Any] = field(default_factory=dict)
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "User":
+        """Create User instance from dictionary"""
+        user_data = {
+            "id": str(data.get("id", "")),
+            "screen_name": data.get("screen_name", ""),
+            "gender": data.get("gender", ""),
+            "location": data.get("location", ""),
+            "description": data.get("description", ""),
+            "followers_count": data.get("followers_count", 0),
+            "following_count": data.get("following_count", 0),
+            "posts_count": data.get("posts_count", 0),
+            "verified": data.get("verified", False),
+            "verified_reason": data.get("verified_reason", ""),
+            "avatar_url": data.get("avatar_url", ""),
+            "cover_image_url": data.get("cover_image_url", ""),
+            "birthday": data.get("birthday"),
+            "education": data.get("education", ""),
+            "company": data.get("company", ""),
+            "registration_time": data.get("registration_time"),
+            "sunshine_credit": data.get("sunshine_credit", ""),
+            "raw_data": data,
+        }
+        return cls(**user_data)
+    
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert User instance to dictionary"""
+        return {
+            "id": self.id,
+            "screen_name": self.screen_name,
+            "gender": self.gender,
+            "location": self.location,
+            "description": self.description,
+            "followers_count": self.followers_count,
+            "following_count": self.following_count,
+            "posts_count": self.posts_count,
+            "verified": self.verified,
+            "verified_reason": self.verified_reason,
+            "avatar_url": self.avatar_url,
+            "cover_image_url": self.cover_image_url,
+            "birthday": self.birthday,
+            "education": self.education,
+            "company": self.company,
+            "registration_time": self.registration_time,
+            "sunshine_credit": self.sunshine_credit,
+        }

crawl4weibo-0.1.0/crawl4weibo/utils/__init__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Utility functions for crawl4weibo
+"""

crawl4weibo-0.1.0/crawl4weibo/utils/logger.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Logger utilities for crawl4weibo
+"""
+
+import logging
+import sys
+from pathlib import Path
+
+
+def setup_logger(name="crawl4weibo", level=logging.INFO, log_file=None):
+    """
+    Setup logger with console and optional file output
+    
+    Args:
+        name (str): Logger name
+        level (int): Logging level
+        log_file (str, optional): Log file path
+        
+    Returns:
+        logging.Logger: Configured logger
+    """
+    logger = logging.getLogger(name)
+    
+    if logger.handlers:
+        return logger
+    
+    logger.setLevel(level)
+    
+    formatter = logging.Formatter(
+        "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+    )
+    
+    console_handler = logging.StreamHandler(sys.stdout)
+    console_handler.setLevel(level)
+    console_handler.setFormatter(formatter)
+    logger.addHandler(console_handler)
+    
+    if log_file:
+        log_path = Path(log_file)
+        log_path.parent.mkdir(parents=True, exist_ok=True)
+        
+        file_handler = logging.FileHandler(log_file, encoding='utf-8')
+        file_handler.setLevel(level)
+        file_handler.setFormatter(formatter)
+        logger.addHandler(file_handler)
+    
+    return logger
+
+
+def get_logger(name="crawl4weibo"):
+    """Get existing logger or create new one"""
+    return logging.getLogger(name)

crawl4weibo-0.1.0/crawl4weibo/utils/parser.py

@@ -0,0 +1,168 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+HTML/JSON parsing utilities for crawl4weibo
+"""
+
+import re
+from typing import List, Dict, Any, Optional
+from datetime import datetime
+from lxml import etree
+
+from ..exceptions.base import ParseError
+from .logger import get_logger
+
+
+class WeiboParser:
+    """Parser for Weibo API responses and HTML content"""
+    
+    def __init__(self):
+        self.logger = get_logger()
+    
+    def parse_user_info(self, response_data: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Parse user information from API response
+        
+        Args:
+            response_data: Raw API response data
+            
+        Returns:
+            Dict containing parsed user information
+        """
+        try:
+            if "data" not in response_data or "userInfo" not in response_data["data"]:
+                raise ParseError("Invalid user info response format")
+            
+            user_info = response_data["data"]["userInfo"]
+            
+            return {
+                "id": str(user_info.get("id", "")),
+                "screen_name": user_info.get("screen_name", ""),
+                "gender": user_info.get("gender", ""),
+                "description": user_info.get("description", ""),
+                "followers_count": user_info.get("followers_count", 0),
+                "following_count": user_info.get("follow_count", 0),
+                "posts_count": user_info.get("statuses_count", 0),
+                "verified": user_info.get("verified", False),
+                "verified_reason": user_info.get("verified_reason", ""),
+                "avatar_url": user_info.get("profile_image_url", ""),
+                "cover_image_url": user_info.get("cover_image_phone", ""),
+            }
+        except Exception as e:
+            self.logger.error(f"Failed to parse user info: {e}")
+            raise ParseError(f"Failed to parse user info: {e}")
+    
+    def parse_posts(self, response_data: Dict[str, Any]) -> List[Dict[str, Any]]:
+        """
+        Parse posts from API response
+        
+        Args:
+            response_data: Raw API response data
+            
+        Returns:
+            List of parsed post dictionaries
+        """
+        try:
+            if "data" not in response_data or "cards" not in response_data["data"]:
+                return []
+            
+            posts = []
+            cards = response_data["data"]["cards"]
+            
+            for card in cards:
+                if card.get("card_type") == 9 and "mblog" in card:
+                    post_data = self._parse_single_post(card["mblog"])
+                    if post_data:
+                        posts.append(post_data)
+            
+            return posts
+        except Exception as e:
+            self.logger.error(f"Failed to parse posts: {e}")
+            raise ParseError(f"Failed to parse posts: {e}")
+    
+    def _parse_single_post(self, mblog: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+        """Parse a single post from mblog data"""
+        try:
+            post = {
+                "id": str(mblog.get("id", "")),
+                "user_id": str(mblog.get("user", {}).get("id", "")),
+                "text": self._clean_text(mblog.get("text", "")),
+                "created_at": self._parse_time(mblog.get("created_at", "")),
+                "source": mblog.get("source", ""),
+                "reposts_count": mblog.get("reposts_count", 0),
+                "comments_count": mblog.get("comments_count", 0),
+                "attitudes_count": mblog.get("attitudes_count", 0),
+                "pic_urls": self._extract_pic_urls(mblog),
+                "video_url": self._extract_video_url(mblog),
+                "is_original": not mblog.get("retweeted_status"),
+                "location": mblog.get("geo", {}).get("name", ""),
+                "topic_ids": self._extract_topics(mblog.get("text", "")),
+                "at_users": self._extract_at_users(mblog.get("text", "")),
+            }
+            
+            if mblog.get("retweeted_status"):
+                post["retweeted_status"] = self._parse_single_post(mblog["retweeted_status"])
+            
+            return post
+        except Exception as e:
+            self.logger.error(f"Failed to parse single post: {e}")
+            return None
+    
+    def _clean_text(self, text: str) -> str:
+        """Remove HTML tags and clean text"""
+        if not text:
+            return ""
+        
+        text = re.sub(r'<[^>]+>', '', text)
+        text = re.sub(r'\s+', ' ', text).strip()
+        return text
+    
+    def _parse_time(self, time_str: str) -> Optional[datetime]:
+        """Parse time string to datetime"""
+        if not time_str:
+            return None
+        
+        try:
+            return datetime.strptime(time_str, "%a %b %d %H:%M:%S %z %Y")
+        except ValueError:
+            try:
+                return datetime.strptime(time_str, "%Y-%m-%d %H:%M:%S")
+            except ValueError:
+                self.logger.warning(f"Failed to parse time: {time_str}")
+                return None
+    
+    def _extract_pic_urls(self, mblog: Dict[str, Any]) -> List[str]:
+        """Extract picture URLs from post"""
+        pic_urls = []
+        
+        if "pics" in mblog:
+            for pic in mblog["pics"]:
+                if "large" in pic and "url" in pic["large"]:
+                    pic_urls.append(pic["large"]["url"])
+        
+        return pic_urls
+    
+    def _extract_video_url(self, mblog: Dict[str, Any]) -> str:
+        """Extract video URL from post"""
+        if "page_info" in mblog and mblog["page_info"].get("type") == "video":
+            media_info = mblog["page_info"].get("media_info", {})
+            return media_info.get("stream_url", "")
+        
+        return ""
+    
+    def _extract_topics(self, text: str) -> List[str]:
+        """Extract topic hashtags from text"""
+        if not text:
+            return []
+        
+        topics = re.findall(r'#([^#]+)#', text)
+        return [topic.strip() for topic in topics if topic.strip()]
+    
+    def _extract_at_users(self, text: str) -> List[str]:
+        """Extract @mentioned users from text"""
+        if not text:
+            return []
+        
+        mentions = re.findall(r'@([^\s@]+)', text)
+        return [mention.strip() for mention in mentions if mention.strip()]

crawl4weibo-0.1.0/crawl4weibo.egg-info/PKG-INFO

@@ -0,0 +1,134 @@
+Metadata-Version: 2.4
+Name: crawl4weibo
+Version: 0.1.0
+Summary: A professional Weibo crawler library
+Home-page: https://github.com/yourusername/crawl4weibo
+Author: Your Name
+Author-email: Kritoooo <krito2023@gmail.com>
+License: MIT
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+Requires-Dist: requests>=2.25.0
+Requires-Dist: lxml>=4.6.0
+Requires-Dist: tqdm>=4.60.0
+Requires-Dist: python-dateutil>=2.8.0
+Provides-Extra: dev
+Requires-Dist: pytest>=6.0; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
+Requires-Dist: black; extra == "dev"
+Requires-Dist: isort; extra == "dev"
+Requires-Dist: flake8; extra == "dev"
+Dynamic: author
+Dynamic: home-page
+Dynamic: requires-python
+
+# Crawl4Weibo
+
+一个专业的微博爬虫Python库，基于实际测试成功的方案，无需Cookie即可使用。
+
+## 特性
+
+- 🚀 **开箱即用**: 无需Cookie，一行代码初始化
+- 🛡️ **防反爬**: 自动处理432错误和请求限制  
+- 📱 **真实模拟**: 使用真实手机浏览器UA
+- 🔄 **智能重试**: 自动重试机制
+- 📊 **结构化数据**: 清晰的数据模型
+
+## 安装
+
+```bash
+pip install -e .
+```
+
+## 快速开始
+
+```python
+from crawl4weibo import WeiboClient
+
+# 初始化（无需Cookie）
+client = WeiboClient()
+
+# 获取用户信息
+user = client.get_user_by_uid("1195230310")
+print(f"用户: {user.screen_name}")
+print(f"粉丝: {user.followers_count:,}")
+
+# 获取微博
+posts = client.get_user_posts("1195230310")
+for post in posts:
+    print(f"微博: {post.text[:50]}...")
+    print(f"点赞: {post.attitudes_count}")
+
+# 搜索用户
+users = client.search_users("技术博主")
+for user in users:
+    print(f"用户: {user.screen_name}")
+
+# 搜索微博  
+posts = client.search_posts("人工智能")
+for post in posts:
+    print(f"内容: {post.text[:50]}...")
+```
+
+## API参考
+
+### WeiboClient
+
+#### 初始化
+```python
+WeiboClient(cookies=None, log_level="INFO", log_file=None)
+```
+
+#### 主要方法
+
+- `get_user_by_uid(uid)` - 获取用户信息
+- `get_user_posts(uid, page=1)` - 获取用户微博
+- `search_users(query, page=1, count=10)` - 搜索用户
+- `search_posts(query, page=1)` - 搜索微博
+
+### 数据模型
+
+**User (用户)**:
+- `screen_name` - 用户名
+- `followers_count` - 粉丝数  
+- `posts_count` - 微博数
+- `verified` - 是否认证
+
+**Post (微博)**:
+- `text` - 微博文本
+- `attitudes_count` - 点赞数
+- `comments_count` - 评论数
+- `created_at` - 发布时间
+
+## 运行示例
+
+```bash
+python examples/simple_example.py
+```
+
+## 技术实现
+
+基于你提供的成功代码实现：
+
+```python
+# 核心技术栈
+- Android Chrome UA模拟
+- 移动端API接口
+- 自动session管理  
+- 432错误智能重试
+- 随机请求间隔
+```
+
+## 许可证
+
+MIT License

crawl4weibo-0.1.0/crawl4weibo.egg-info/SOURCES.txt

@@ -0,0 +1,19 @@
+README.md
+pyproject.toml
+setup.py
+crawl4weibo/__init__.py
+crawl4weibo.egg-info/PKG-INFO
+crawl4weibo.egg-info/SOURCES.txt
+crawl4weibo.egg-info/dependency_links.txt
+crawl4weibo.egg-info/requires.txt
+crawl4weibo.egg-info/top_level.txt
+crawl4weibo/core/__init__.py
+crawl4weibo/core/client.py
+crawl4weibo/exceptions/__init__.py
+crawl4weibo/exceptions/base.py
+crawl4weibo/models/__init__.py
+crawl4weibo/models/post.py
+crawl4weibo/models/user.py
+crawl4weibo/utils/__init__.py
+crawl4weibo/utils/logger.py
+crawl4weibo/utils/parser.py

crawl4weibo-0.1.0/crawl4weibo.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

crawl4weibo-0.1.0/crawl4weibo.egg-info/requires.txt

@@ -0,0 +1,11 @@
+requests>=2.25.0
+lxml>=4.6.0
+tqdm>=4.60.0
+python-dateutil>=2.8.0
+
+[dev]
+pytest>=6.0
+pytest-cov
+black
+isort
+flake8

crawl4weibo-0.1.0/crawl4weibo.egg-info/top_level.txt

@@ -0,0 +1 @@
+crawl4weibo

crawl4weibo-0.1.0/pyproject.toml

@@ -0,0 +1,53 @@
+[build-system]
+requires = ["setuptools>=45", "wheel", "setuptools_scm[toml]>=6.2"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "crawl4weibo"
+description = "A professional Weibo crawler library"
+readme = "README.md"
+requires-python = ">=3.7"
+license = {text = "MIT"}
+authors = [
+    {name = "Kritoooo", email = "krito2023@gmail.com"},
+]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.7",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+]
+dependencies = [
+    "requests>=2.25.0",
+    "lxml>=4.6.0",
+    "tqdm>=4.60.0",
+    "python-dateutil>=2.8.0",
+]
+dynamic = ["version"]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=6.0",
+    "pytest-cov",
+    "black",
+    "isort",
+    "flake8",
+]
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["crawl4weibo*"]
+
+[tool.black]
+line-length = 88
+target-version = ['py37']
+
+[tool.isort]
+profile = "black"
+line_length = 88

crawl4weibo-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

crawl4weibo-0.1.0/setup.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from setuptools import setup, find_packages
+
+with open("README.md", "r", encoding="utf-8") as fh:
+    long_description = fh.read()
+
+setup(
+    name="crawl4weibo",
+    version="0.1.0",
+    author="Your Name",
+    author_email="your.email@example.com",
+    description="A professional Weibo crawler library",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    url="https://github.com/yourusername/crawl4weibo",
+    packages=find_packages(),
+    classifiers=[
+        "Development Status :: 3 - Alpha",
+        "Intended Audience :: Developers",
+        "License :: OSI Approved :: MIT License",
+        "Operating System :: OS Independent",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
+    ],
+    python_requires=">=3.7",
+    install_requires=[
+        "requests>=2.25.0",
+        "lxml>=4.6.0",
+        "tqdm>=4.60.0",
+        "python-dateutil>=2.8.0",
+    ],
+    extras_require={
+        "dev": [
+            "pytest>=6.0",
+            "pytest-cov",
+            "black",
+            "isort",
+            "flake8",
+        ],
+    },
+)