travel-agent-cli 0.2.0 → 0.2.1

package/python/collectors/weibo.py

@@ -0,0 +1,235 @@
+"""微博数据采集器
+
+支持：
+1. 微博开放平台 API
+2. 网页爬取（备用方案）
+"""
+import asyncio
+from typing import List, Optional
+from datetime import datetime
+
+from config.models import SocialPost, SourceType
+from config.settings import get_settings
+from utils.storage import Storage
+from utils.http import get_client
+
+
+class WeiboCollector:
+    """微博采集器"""
+
+    def __init__(self):
+        self.storage = Storage()
+        self.settings = get_settings()
+
+        # 微博开放平台 API
+        self.api_base = "https://api.weibo.com"
+        self.access_token: Optional[str] = None
+
+    def _get_auth_token(self) -> Optional[str]:
+        """获取微博 API 访问令牌"""
+        if not self.settings.weibo_app_key or not self.settings.weibo_app_secret:
+            return None
+
+        # TODO: 实现 OAuth2.0 认证流程
+        # 参考：https://open.weibo.com/wiki/Oauth2
+        return self.access_token
+
+    async def search(
+        self, keyword: str = "旅行", count: int = 20, page: int = 1
+    ) -> List[SocialPost]:
+        """搜索微博内容
+
+        优先使用官方 API，失败则降级为网页爬取
+        """
+        # 方案 1: 使用微博开放平台 API
+        token = self._get_auth_token()
+        if token:
+            try:
+                return await self._search_by_api(keyword, count, page)
+            except Exception as e:
+                print(f"微博 API 调用失败：{e}，降级为网页爬取")
+
+        # 方案 2: 网页爬取
+        return await self._search_by_web(keyword, count, page)
+
+    async def _search_by_api(
+        self, keyword: str, count: int, page: int
+    ) -> List[SocialPost]:
+        """通过微博 API 搜索"""
+        # 参考：https://open.weibo.com/wiki/2/search/statuses
+        url = f"{self.api_base}/2/search/statuses.json"
+        params = {
+            "q": keyword,
+            "count": count,
+            "page": page,
+            "access_token": self.access_token,
+        }
+
+        async with get_client() as client:
+            response = await client.get(url, params=params)
+            response.raise_for_status()
+            data = response.json()
+
+            posts = []
+            for status in data.get("statuses", []):
+                post = SocialPost(
+                    id=str(status["id"]),
+                    source=SourceType.WEIBO,
+                    title=status.get("text", "")[:100],
+                    content=status.get("text", ""),
+                    author=status["user"]["screen_name"],
+                    author_id=str(status["user"]["id"]),
+                    likes=status.get("attitudes_count", 0),
+                    comments=status.get("comments_count", 0),
+                    shares=status.get("reposts_count", 0),
+                    url=f"https://weibo.com/{status['id']}",
+                    published_at=datetime.fromisoformat(
+                        status["created_at"].replace("CST", "+08:00")
+                    )
+                    if "created_at" in status
+                    else None,
+                )
+                posts.append(post)
+                await self.storage.save_post(post)
+
+            return posts
+
+    async def _search_by_web(
+        self, keyword: str, count: int, page: int
+    ) -> List[SocialPost]:
+        """通过网页爬取搜索"""
+        collected_posts = []
+
+        try:
+            async with get_client() as client:
+                # 微博搜索结果页
+                url = "https://s.weibo.com/weibo"
+                params = {
+                    "q": keyword,
+                    "typeall": "sub",
+                    "suball": "1",
+                    "page": page,
+                }
+
+                headers = {
+                    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",
+                    "Referer": "https://weibo.com/",
+                }
+
+                response = await client.get(url, params=params, headers=headers)
+
+                if response.status_code == 200:
+                    # 解析 HTML
+                    from bs4 import BeautifulSoup
+
+                    soup = BeautifulSoup(response.text, "html.parser")
+
+                    # 查找微博列表
+                    for item in soup.select('.card-wrap'):
+                        try:
+                            content_elem = item.select_one('.content')
+                            if not content_elem:
+                                continue
+
+                            # 提取信息
+                            title = content_elem.get_text(strip=True)[:100]
+                            content = content_elem.get_text(strip=True)
+                            author_elem = item.select_one('.name')
+                            author = author_elem.get_text(strip=True) if author_elem else "未知"
+
+                            # 互动数据
+                            likes_elem = item.select_one('.card-act li:last-child')
+                            likes = 0
+                            if likes_elem:
+                                likes_text = likes_elem.get_text()
+                                import re
+                                match = re.search(r'\d+', likes_text)
+                                if match:
+                                    likes = int(match.group())
+
+                            # 生成 ID
+                            post_id = f"weibo_{hash(title) % 1000000}"
+
+                            post = SocialPost(
+                                id=post_id,
+                                source=SourceType.WEIBO,
+                                title=title,
+                                content=content,
+                                author=author,
+                                author_id=hash(author),
+                                likes=likes,
+                                url="https://s.weibo.com",
+                                published_at=datetime.now(),
+                            )
+                            collected_posts.append(post)
+                            await self.storage.save_post(post)
+
+                            if len(collected_posts) >= count:
+                                break
+
+                        except Exception as e:
+                            continue
+
+        except Exception as e:
+            print(f"微博爬取失败：{e}")
+
+        # 如果没有爬到数据，返回示例数据
+        if not collected_posts:
+            collected_posts = self._get_sample_posts(keyword)
+
+        return collected_posts
+
+    def _get_sample_posts(self, keyword: str) -> List[SocialPost]:
+        """返回示例微博（用于测试）"""
+        samples = [
+            {
+                "id": "weibo_001",
+                "title": "春天去哪里旅行？这 5 个目的地强烈推荐！",
+                "content": "春暖花开，正是出游好时节！推荐 5 个宝藏目的地：1.云南罗平油菜花 2.西藏林芝桃花 3.江西婺源 4.江苏兴化 5.贵州平坝樱花...",
+                "author": "旅游博主小王",
+                "author_id": "wb_user_001",
+                "likes": 8923,
+                "comments": 445,
+                "shares": 1234,
+                "url": "https://weibo.com/001",
+            },
+            {
+                "id": "weibo_002",
+                "title": "特种兵式旅游到底值不值？",
+                "content": "24 小时打卡 8 个景点，日行 3 万步...这种旅游方式引发热议。我觉得...",
+                "author": "旅行观察家",
+                "author_id": "wb_user_002",
+                "likes": 15234,
+                "comments": 2345,
+                "shares": 890,
+                "url": "https://weibo.com/002",
+            },
+            {
+                "id": "weibo_003",
+                "title": "五一假期出行预测：这些城市最火",
+                "content": "根据各大平台数据，五一假期最热门目的地预测：北京、上海、成都、重庆、西安...",
+                "author": "旅行大数据",
+                "author_id": "wb_user_003",
+                "likes": 5678,
+                "comments": 234,
+                "shares": 567,
+                "url": "https://weibo.com/003",
+            },
+        ]
+
+        return [
+            SocialPost(
+                id=s["id"],
+                source=SourceType.WEIBO,
+                title=s["title"],
+                content=s["content"],
+                author=s["author"],
+                author_id=s["author_id"],
+                likes=s["likes"],
+                comments=s["comments"],
+                shares=s["shares"],
+                url=s["url"],
+                published_at=datetime.now(),
+            )
+            for s in samples
+        ]

package/python/collectors/wenlv.py

@@ -0,0 +1,155 @@
+"""文旅局信息采集器
+
+爬取各地文旅局官网、公众号的推广信息
+"""
+import asyncio
+from typing import List, Dict
+from datetime import datetime
+
+from config.models import WenlvInfo, SourceType
+from utils.storage import Storage
+from utils.http import get_client
+
+
+# 主要文旅局官网列表
+WENLV_WEBSITES = [
+    {"name": "文化和旅游部", "url": "https://www.mct.gov.cn", "region": "全国"},
+    {"name": "北京文旅局", "url": "https://wlj.beijing.gov.cn", "region": "北京"},
+    {"name": "上海文旅局", "url": "https://whlyj.sh.gov.cn", "region": "上海"},
+    {"name": "广东文旅厅", "url": "http://whlt.gd.gov.cn", "region": "广东"},
+    {"name": "云南文旅厅", "url": "http://whlt.yn.gov.cn", "region": "云南"},
+    {"name": "四川文旅厅", "url": "http://whlyt.sc.gov.cn", "region": "四川"},
+    {"name": "浙江文旅厅", "url": "http://ct.zj.gov.cn", "region": "浙江"},
+    {"name": "江苏文旅厅", "url": "http://wlt.jiangsu.gov.cn", "region": "江苏"},
+    {"name": "陕西文旅厅", "url": "http://whlyt.shaanxi.gov.cn", "region": "陕西"},
+    {"name": "海南旅文厅", "url": "http://wlt.hainan.gov.cn", "region": "海南"},
+]
+
+
+class WenlvCollector:
+    """文旅信息采集器"""
+
+    def __init__(self):
+        self.storage = Storage()
+
+    async def collect(self) -> List[WenlvInfo]:
+        """采集各地文旅信息"""
+        # 初始化数据库
+        await self.storage.init_db()
+
+        all_info = []
+
+        for site in WENLV_WEBSITES:
+            try:
+                info_list = await self._collect_site(site)
+                all_info.extend(info_list)
+                print(f"已采集 {site['name']}: {len(info_list)} 条")
+            except Exception as e:
+                print(f"采集 {site['name']} 失败：{e}")
+
+        # 如果没有采集到数据，返回示例数据
+        if not all_info:
+            all_info = self._get_sample_info()
+
+        return all_info
+
+    async def _collect_site(self, site: Dict) -> List[WenlvInfo]:
+        """采集单个网站"""
+        collected = []
+
+        async with get_client() as client:
+            # 尝试访问官网
+            response = await client.get(site["url"])
+
+            if response.status_code == 200:
+                from bs4 import BeautifulSoup
+
+                soup = BeautifulSoup(response.text, "html.parser")
+
+                # 查找新闻列表（不同网站结构不同，需要针对性适配）
+                # 这里是一个通用尝试
+                for link in soup.find_all("a", href=True)[:20]:
+                    text = link.get_text(strip=True)
+                    href = link["href"]
+
+                    # 过滤有效链接
+                    if len(text) < 10 or len(text) > 100:
+                        continue
+
+                    # 判断信息类型
+                    info_type = "其他"
+                    if any(kw in text for kw in ["政策", "通知", "公告"]):
+                        info_type = "政策"
+                    elif any(kw in text for kw in ["活动", "节庆", "优惠"]):
+                        info_type = "活动"
+                    elif any(kw in text for kw in ["路线", "推荐", "攻略"]):
+                        info_type = "推荐路线"
+
+                    # 处理相对 URL
+                    if href.startswith("/"):
+                        base = site["url"].rstrip("/")
+                        href = f"{base}{href}"
+
+                    info = WenlvInfo(
+                        id=f"wenlv_{hash(href) % 1000000}",
+                        source=SourceType.WENLV,
+                        title=text,
+                        content=text,
+                        region=site["region"],
+                        url=href,
+                        info_type=info_type,
+                        published_at=datetime.now(),
+                    )
+                    collected.append(info)
+                    await self.storage.save_wenlv(info)
+
+        return collected
+
+    def _get_sample_info(self) -> List[WenlvInfo]:
+        """返回示例文旅信息"""
+        samples = [
+            {
+                "title": "文化和旅游部关于促进乡村旅游高质量发展的指导意见",
+                "region": "全国",
+                "info_type": "政策",
+                "url": "https://www.mct.gov.cn/example1",
+            },
+            {
+                "title": "云南推出 10 条精品旅游线路，五一假期出发",
+                "region": "云南",
+                "info_type": "推荐路线",
+                "url": "http://whlt.yn.gov.cn/example2",
+            },
+            {
+                "title": "海南离岛免税购物额度提升至 10 万元",
+                "region": "海南",
+                "info_type": "政策",
+                "url": "http://wlt.hainan.gov.cn/example3",
+            },
+            {
+                "title": "四川启动文旅消费季，发放 5000 万消费券",
+                "region": "四川",
+                "info_type": "活动",
+                "url": "http://whlyt.sc.gov.cn/example4",
+            },
+            {
+                "title": "浙江推出'诗画江南'文旅品牌",
+                "region": "浙江",
+                "info_type": "推荐路线",
+                "url": "http://ct.zj.gov.cn/example5",
+            },
+        ]
+
+        return [
+            WenlvInfo(
+                id=f"wenlv_{i}",
+                source=SourceType.WENLV,
+                title=s["title"],
+                content=s["title"],  # 简化处理
+                region=s["region"],
+                url=s["url"],
+                info_type=s["info_type"],
+                published_at=datetime.now(),
+            )
+            for i, s in enumerate(samples)
+        ]

package/python/collectors/xiaohongshu.py

@@ -0,0 +1,170 @@
+"""小红书数据采集器
+
+参考：MediaCrawler, xhs_one_spider
+注意：小红书有严格的反爬机制，需要处理签名和验证码
+"""
+import asyncio
+import hashlib
+import time
+from typing import List, Optional
+from datetime import datetime
+import re
+
+from config.models import SocialPost, SourceType
+from utils.storage import Storage
+from utils.http import get_client
+
+
+class XiaohongshuCollector:
+    """小红书采集器"""
+
+    def __init__(self):
+        self.storage = Storage()
+        self.base_url = "https://www.xiaohongshu.com"
+        self.api_base = "https://edith.xiaohongshu.com"
+
+    def _generate_xs_token(self) -> str:
+        """生成 X-s 签名 token（简化版，实际需要根据最新算法更新）"""
+        timestamp = str(int(time.time() * 1000))
+        return hashlib.md5(timestamp.encode()).hexdigest()
+
+    async def search(self, keyword: str = "旅行", page: int = 1, page_size: int = 20) -> List[SocialPost]:
+        """搜索小红书笔记
+
+        注意：由于小红书反爬严格，这里提供两种方案：
+        1. 使用官方开放平台 API（需要申请）
+        2. 模拟网页爬取（需要处理签名和验证码）
+
+        此处为简化示例，实际使用需要更新签名算法
+        """
+        collected_posts = []
+
+        # 方案 1: 尝试使用开放平台 API
+        # 需要申请 https://open.xiaohongshu.com/
+
+        # 方案 2: 网页爬取（需要处理反爬）
+        # 这里提供一个框架，实际使用需要逆向分析最新接口
+
+        try:
+            async with get_client() as client:
+                # 搜索页面
+                url = f"{self.base_url}/explore/search"
+                params = {
+                    "source": "web_search",
+                    "keyword": keyword,
+                    "page": page,
+                    "page_size": page_size,
+                }
+
+                headers = {
+                    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",
+                    "Referer": self.base_url,
+                }
+
+                response = await client.get(url, params=params, headers=headers)
+
+                if response.status_code == 200:
+                    # 解析搜索结果
+                    # 注意：实际需要根据返回结构调整
+                    pass
+
+        except Exception as e:
+            print(f"小红书采集失败：{e}")
+
+        # 如果自动采集失败，可以使用数据导出工具手动获取
+        # 参考：xhs_one_spider, MediaCrawler 等项目
+        print(f"小红书搜索关键词：{keyword}")
+        print("提示：小红书有严格反爬，建议使用以下方案：")
+        print("  1. 申请官方开放平台 API: https://open.xiaohongshu.com/")
+        print("  2. 使用 MediaCrawler 等开源工具手动采集后导入")
+
+        # 返回示例数据用于测试
+        return self._get_sample_posts(keyword)
+
+    def _get_sample_posts(self, keyword: str) -> List[SocialPost]:
+        """返回示例帖子（用于测试）"""
+        samples = [
+            {
+                "id": "xhs_001",
+                "title": "三亚旅行 | 5 天 4 晚超详细攻略，人均 3000+ 搞定",
+                "content": "三亚真的是国内海岛游的天花板！这次整理了 5 天 4 晚的详细攻略...",
+                "author": "旅行达人小 A",
+                "author_id": "user_001",
+                "likes": 15234,
+                "comments": 892,
+                "shares": 1523,
+                "tags": ["三亚", "旅行攻略", "海岛游", "海南"],
+                "url": "https://www.xiaohongshu.com/discovery/item/001",
+            },
+            {
+                "id": "xhs_002",
+                "title": "云南大理 | 这才是正确的打开方式！",
+                "content": "大理真的太美了！洱海骑行、古城漫步、苍山徒步...",
+                "author": "背包客小李",
+                "author_id": "user_002",
+                "likes": 8921,
+                "comments": 445,
+                "shares": 678,
+                "tags": ["大理", "云南", "洱海", "古城"],
+                "url": "https://www.xiaohongshu.com/discovery/item/002",
+            },
+            {
+                "id": "xhs_003",
+                "title": "川西小环线 | 自驾 3 天看遍雪山草甸",
+                "content": "川西真的是自驾天堂！这条小环线可以看到雪山、草甸、藏寨...",
+                "author": "自驾达人老王",
+                "author_id": "user_003",
+                "likes": 12456,
+                "comments": 623,
+                "shares": 1890,
+                "tags": ["川西", "自驾", "雪山", "四姑娘山"],
+                "url": "https://www.xiaohongshu.com/discovery/item/003",
+            },
+            {
+                "id": "xhs_004",
+                "title": "西安美食攻略 | 100 元吃遍回民街",
+                "content": "西安真的是美食天堂！肉夹馍、羊肉泡馍、凉皮...",
+                "author": "吃货小分队",
+                "author_id": "user_004",
+                "likes": 23456,
+                "comments": 1234,
+                "shares": 2345,
+                "tags": ["西安", "美食", "回民街", "陕西"],
+                "url": "https://www.xiaohongshu.com/discovery/item/004",
+            },
+            {
+                "id": "xhs_005",
+                "title": "桂林阳朔 | 山水画中的慢生活",
+                "content": "桂林山水甲天下，阳朔风光甲桂林！遇龙河漂流、十里画廊...",
+                "author": "慢旅行者",
+                "author_id": "user_005",
+                "likes": 7654,
+                "comments": 321,
+                "shares": 543,
+                "tags": ["桂林", "阳朔", "漓江", "广西"],
+                "url": "https://www.xiaohongshu.com/discovery/item/005",
+            },
+        ]
+
+        return [
+            SocialPost(
+                id=s["id"],
+                source=SourceType.XIAOHONGSHU,
+                title=s["title"],
+                content=s["content"],
+                author=s["author"],
+                author_id=s["author_id"],
+                likes=s["likes"],
+                comments=s["comments"],
+                shares=s["shares"],
+                tags=s["tags"],
+                url=s["url"],
+                published_at=datetime.now(),
+            )
+            for s in samples
+        ]
+
+    async def collect_detail(self, note_id: str) -> Optional[SocialPost]:
+        """采集单篇笔记详情"""
+        # 实现类似逻辑
+        pass

package/python/config/__init__.py

@@ -0,0 +1,30 @@
+"""配置模块"""
+from config.settings import get_settings
+from config.prompts import (
+    DESTINATION_ANALYSIS_PROMPT,
+    ROUTE_PLANNING_PROMPT,
+    REPORT_SUMMARY_PROMPT,
+)
+from config.models import (
+    SocialPost,
+    WenlvInfo,
+    FlightInfo,
+    HotelInfo,
+    DestinationRecommendation,
+    TravelRoute,
+    Report,
+)
+
+__all__ = [
+    "get_settings",
+    "DESTINATION_ANALYSIS_PROMPT",
+    "ROUTE_PLANNING_PROMPT",
+    "REPORT_SUMMARY_PROMPT",
+    "SocialPost",
+    "WenlvInfo",
+    "FlightInfo",
+    "HotelInfo",
+    "DestinationRecommendation",
+    "TravelRoute",
+    "Report",
+]