travel-agent-cli 0.2.0 → 0.2.1

package/python/utils/http.py

@@ -0,0 +1,73 @@
+"""HTTP 工具模块"""
+import httpx
+import asyncio
+from typing import Optional, Dict, Any
+from config.settings import get_settings
+
+
+def get_client() -> httpx.AsyncClient:
+    """获取配置好的 HTTP 客户端"""
+    settings = get_settings()
+
+    headers = {
+        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+        "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+    }
+
+    # 新版 httpx 使用 transport 来配置代理
+    if settings.http_proxy or settings.https_proxy:
+        # httpx 使用 mounts 来配置代理
+        proxy_url = settings.https_proxy or settings.http_proxy
+        if proxy_url:
+            return httpx.AsyncClient(
+                timeout=httpx.Timeout(settings.request_timeout),
+                proxies=proxy_url,
+                headers=headers,
+            )
+
+    return httpx.AsyncClient(
+        timeout=httpx.Timeout(settings.request_timeout),
+        headers=headers,
+    )
+
+
+async def fetch_with_retry(
+    url: str,
+    method: str = "GET",
+    headers: Optional[Dict[str, str]] = None,
+    params: Optional[Dict[str, Any]] = None,
+    data: Optional[Dict[str, Any]] = None,
+    json_data: Optional[Dict[str, Any]] = None,
+) -> Optional[httpx.Response]:
+    """带重试的 HTTP 请求"""
+    settings = get_settings()
+
+    async with get_client() as client:
+        for attempt in range(settings.max_retries):
+            try:
+                if method == "GET":
+                    response = await client.get(url, headers=headers, params=params)
+                elif method == "POST":
+                    response = await client.post(
+                        url, headers=headers, params=params, data=data, json=json_data
+                    )
+                else:
+                    raise ValueError(f"不支持的 HTTP 方法：{method}")
+
+                if response.status_code == 200:
+                    return response
+                elif response.status_code == 429:
+                    #  rate limited, wait and retry
+                    await asyncio.sleep(settings.request_delay * (attempt + 1) * 2)
+                else:
+                    response.raise_for_status()
+
+            except httpx.HTTPError as e:
+                if attempt < settings.max_retries - 1:
+                    await asyncio.sleep(settings.request_delay * (attempt + 1))
+                else:
+                    print(f"请求失败 {url}: {e}")
+                    return None
+
+    return None

package/python/utils/storage.py

@@ -0,0 +1,288 @@
+"""数据存储工具"""
+import aiosqlite
+from pathlib import Path
+from typing import Optional, List, Dict, Any
+from datetime import datetime
+import json
+
+from config.models import (
+    SocialPost,
+    WenlvInfo,
+    FlightInfo,
+    HotelInfo,
+    DestinationRecommendation,
+)
+from config.settings import get_settings
+
+
+class Storage:
+    """SQLite 数据存储"""
+
+    def __init__(self, db_path: Optional[str] = None):
+        settings = get_settings()
+        self.db_path = db_path or settings.database_path
+        self._initialized = False
+
+    async def ensure_init(self):
+        """确保数据库已初始化"""
+        if not self._initialized:
+            await self.init_db()
+            self._initialized = True
+
+    async def init_db(self):
+        """初始化数据库表"""
+        async with aiosqlite.connect(self.db_path) as db:
+            # 社交媒体帖子表
+            await db.execute(
+                """
+                CREATE TABLE IF NOT EXISTS social_posts (
+                    id TEXT PRIMARY KEY,
+                    source TEXT NOT NULL,
+                    title TEXT,
+                    content TEXT,
+                    author TEXT,
+                    author_id TEXT,
+                    images TEXT,
+                    likes INTEGER DEFAULT 0,
+                    comments INTEGER DEFAULT 0,
+                    shares INTEGER DEFAULT 0,
+                    tags TEXT,
+                    url TEXT,
+                    published_at TIMESTAMP,
+                    collected_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+                )
+            """
+            )
+
+            # 文旅信息表
+            await db.execute(
+                """
+                CREATE TABLE IF NOT EXISTS wenlv_info (
+                    id TEXT PRIMARY KEY,
+                    source TEXT NOT NULL,
+                    title TEXT,
+                    content TEXT,
+                    region TEXT,
+                    url TEXT,
+                    info_type TEXT,
+                    published_at TIMESTAMP,
+                    collected_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+                )
+            """
+            )
+
+            # 航班信息表
+            await db.execute(
+                """
+                CREATE TABLE IF NOT EXISTS flight_info (
+                    id TEXT PRIMARY KEY,
+                    departure_city TEXT,
+                    arrival_city TEXT,
+                    price REAL,
+                    currency TEXT DEFAULT 'CNY',
+                    airline TEXT,
+                    departure_time TEXT,
+                    travel_time TEXT,
+                    platform TEXT DEFAULT 'fliggy',
+                    collected_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+                )
+            """
+            )
+
+            # 酒店信息表
+            await db.execute(
+                """
+                CREATE TABLE IF NOT EXISTS hotel_info (
+                    id TEXT PRIMARY KEY,
+                    name TEXT,
+                    destination TEXT,
+                    price_per_night REAL,
+                    currency TEXT DEFAULT 'CNY',
+                    rating REAL,
+                    stars INTEGER,
+                    platform TEXT DEFAULT 'fliggy',
+                    url TEXT,
+                    collected_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+                )
+            """
+            )
+
+            # 推荐结果表
+            await db.execute(
+                """
+                CREATE TABLE IF NOT EXISTS recommendations (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    name TEXT,
+                    rank INTEGER,
+                    score REAL,
+                    reason TEXT,
+                    estimated_cost TEXT,
+                    suggested_days INTEGER,
+                    best_time TEXT,
+                    highlights TEXT,
+                    flight_info_id TEXT,
+                    hotel_info_id TEXT,
+                    generated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+                    FOREIGN KEY (flight_info_id) REFERENCES flight_info(id),
+                    FOREIGN KEY (hotel_info_id) REFERENCES hotel_info(id)
+                )
+            """
+            )
+
+            await db.commit()
+
+    async def save_post(self, post: SocialPost):
+        """保存社交媒体帖子"""
+        async with aiosqlite.connect(self.db_path) as db:
+            await db.execute(
+                """
+                INSERT OR REPLACE INTO social_posts
+                (id, source, title, content, author, author_id, images, likes, comments, shares, tags, url, published_at, collected_at)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+            """,
+                (
+                    post.id,
+                    post.source.value,
+                    post.title,
+                    post.content,
+                    post.author,
+                    post.author_id,
+                    json.dumps(post.images),
+                    post.likes,
+                    post.comments,
+                    post.shares,
+                    json.dumps(post.tags),
+                    post.url,
+                    post.published_at,
+                    post.collected_at,
+                ),
+            )
+            await db.commit()
+
+    async def save_wenlv(self, info: WenlvInfo):
+        """保存文旅信息"""
+        async with aiosqlite.connect(self.db_path) as db:
+            await db.execute(
+                """
+                INSERT OR REPLACE INTO wenlv_info
+                (id, source, title, content, region, url, info_type, published_at, collected_at)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+            """,
+                (
+                    info.id,
+                    info.source.value,
+                    info.title,
+                    info.content,
+                    info.region,
+                    info.url,
+                    info.info_type,
+                    info.published_at,
+                    info.collected_at,
+                ),
+            )
+            await db.commit()
+
+    async def save_flight(self, flight: FlightInfo):
+        """保存航班信息"""
+        async with aiosqlite.connect(self.db_path) as db:
+            await db.execute(
+                """
+                INSERT OR REPLACE INTO flight_info
+                (id, departure_city, arrival_city, price, currency, airline, departure_time, travel_time, platform)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+            """,
+                (
+                    flight.id,
+                    flight.departure_city,
+                    flight.arrival_city,
+                    flight.price,
+                    flight.currency,
+                    flight.airline,
+                    flight.departure_time,
+                    flight.travel_time,
+                    flight.platform,
+                ),
+            )
+            await db.commit()
+
+    async def save_hotel(self, hotel: HotelInfo):
+        """保存酒店信息"""
+        async with aiosqlite.connect(self.db_path) as db:
+            await db.execute(
+                """
+                INSERT OR REPLACE INTO hotel_info
+                (id, name, destination, price_per_night, currency, rating, stars, platform, url)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+            """,
+                (
+                    hotel.id,
+                    hotel.name,
+                    hotel.destination,
+                    hotel.price_per_night,
+                    hotel.currency,
+                    hotel.rating,
+                    hotel.stars,
+                    hotel.platform,
+                    hotel.url,
+                ),
+            )
+            await db.commit()
+
+    async def get_recent_posts(
+        self, source: Optional[str] = None, limit: int = 100
+    ) -> List[SocialPost]:
+        """获取最近的帖子"""
+        await self.ensure_init()
+        async with aiosqlite.connect(self.db_path) as db:
+            db.row_factory = aiosqlite.Row
+            query = "SELECT * FROM social_posts ORDER BY collected_at DESC LIMIT ?"
+            if source:
+                query = (
+                    "SELECT * FROM social_posts WHERE source = ? ORDER BY collected_at DESC LIMIT ?"
+                )
+                cursor = await db.execute(query, (source, limit))
+            else:
+                cursor = await db.execute(query, (limit,))
+
+            rows = await cursor.fetchall()
+            return [
+                SocialPost(
+                    id=row["id"],
+                    source=row["source"],
+                    title=row["title"],
+                    content=row["content"],
+                    author=row["author"],
+                    author_id=row["author_id"],
+                    images=json.loads(row["images"] or "[]"),
+                    likes=row["likes"],
+                    comments=row["comments"],
+                    shares=row["shares"],
+                    tags=json.loads(row["tags"] or "[]"),
+                    url=row["url"],
+                    published_at=row["published_at"],
+                )
+                for row in rows
+            ]
+
+    async def get_recent_wenlv(self, limit: int = 100) -> List[WenlvInfo]:
+        """获取最近的文旅信息"""
+        await self.ensure_init()
+        async with aiosqlite.connect(self.db_path) as db:
+            db.row_factory = aiosqlite.Row
+            cursor = await db.execute(
+                "SELECT * FROM wenlv_info ORDER BY collected_at DESC LIMIT ?", (limit,)
+            )
+            rows = await cursor.fetchall()
+            return [
+                WenlvInfo(
+                    id=row["id"],
+                    source=row["source"],
+                    title=row["title"],
+                    content=row["content"],
+                    region=row["region"],
+                    url=row["url"],
+                    info_type=row["info_type"],
+                    published_at=row["published_at"],
+                )
+                for row in rows
+            ]