feedship 1.0.0__tar.gz

feedship-1.0.0/PKG-INFO

@@ -0,0 +1,44 @@
+Metadata-Version: 2.4
+Name: feedship
+Version: 1.0.0
+Summary: A personal information delivery system - collect, subscribe to, and organize information sources from the internet
+Requires-Python: >=3.10
+Requires-Dist: feedparser>=6.0.0
+Requires-Dist: httpx>=0.28.0
+Requires-Dist: click>=8.1.0
+Requires-Dist: beautifulsoup4>=4.12.0
+Requires-Dist: lxml>=6.0.0
+Requires-Dist: rich>=13.0.0
+Requires-Dist: PyGithub>=2.0.0
+Requires-Dist: dynaconf>=3.2.13
+Requires-Dist: trafilatura>=1.0.0
+Requires-Dist: robotexclusionrulesparser>=1.7.1
+Requires-Dist: platformdirs>=4.9.4
+Requires-Dist: numpy<2,>=1.26.0
+Requires-Dist: scikit-learn>=1.7.2
+Requires-Dist: pyyaml>=6.0.3
+Requires-Dist: uvloop>=0.22.0
+Requires-Dist: nanoid>=2.0.0
+Requires-Dist: scrapling>=0.4.0
+Requires-Dist: msgspec>=0.20.0
+Provides-Extra: test
+Requires-Dist: pytest>=9.0.2; extra == "test"
+Requires-Dist: pytest-asyncio>=1.0.0; extra == "test"
+Requires-Dist: pytest-cov>=7.0.0; extra == "test"
+Requires-Dist: pytest-mock>=3.15.0; extra == "test"
+Requires-Dist: pytest-click>=1.1.0; extra == "test"
+Requires-Dist: pytest-httpx>=0.36.0; extra == "test"
+Requires-Dist: pytest-xdist>=3.8.0; extra == "test"
+Requires-Dist: ruff>=0.6.0; extra == "test"
+Requires-Dist: pre-commit>=3.0.0; extra == "test"
+Provides-Extra: ml
+Requires-Dist: sentence-transformers>=3.0.0; extra == "ml"
+Requires-Dist: torch>=2.0.0; extra == "ml"
+Requires-Dist: safetensors>=0.4.3; extra == "ml"
+Requires-Dist: transformers>=4.40.0; extra == "ml"
+Provides-Extra: cloudflare
+Requires-Dist: scrapling>=0.4.0; extra == "cloudflare"
+Requires-Dist: playwright>=1.49.0; extra == "cloudflare"
+Requires-Dist: curl-cffi>=0.14.0; extra == "cloudflare"
+Requires-Dist: socksio>=1.0.0; extra == "cloudflare"
+Requires-Dist: browserforge>=1.2.0; extra == "cloudflare"

feedship-1.0.0/README.md

@@ -0,0 +1,75 @@
+# feedship
+
+Personal information system for collecting, subscribing to, and organizing information sources from the internet.
+
+## Features
+
+- **Feed subscription** - RSS/Atom feeds and GitHub releases
+- **Web article extraction** - Crawl webpages with Readability
+- **Full-text search** - FTS5-powered search across all content
+- **Semantic search** - Vector embeddings with ChromaDB
+- **CLI tool** - Full-featured command-line interface
+
+## Tech Stack
+
+Python 3.10+ | click | feedparser | httpx | BeautifulSoup4 | sqlite3 | ChromaDB | sentence-transformers
+
+## Installation
+
+```bash
+pip install feedship
+# or
+uv pip install feedship
+```
+
+### Optional Dependencies
+
+```bash
+# ML/AI features for auto-tagging
+pip install feedship[ml]
+
+# All features
+pip install feedship[ml,semantic]
+```
+
+## Quick Start
+
+### Add a Feed
+
+```bash
+feedship feed add <url> [options]
+
+# Examples:
+feedship feed add https://example.com/feed.xml
+feedship feed add https://github.com/python/cpython
+
+# Options:
+--discover [on|off]          Enable feed discovery (default: on)
+--automatic [on|off]          Auto-add all discovered feeds (default: off)
+--discover-depth N           Discovery depth 1-10 (default: 1)
+--weight FLOAT               Feed weight for semantic search (default: 0.3)
+```
+
+### Fetch & List
+
+```bash
+feedship fetch --all        # Fetch all feeds
+feedship feed list          # List all feeds
+feedship article list        # List articles
+feedship article list --limit 50
+```
+
+### Search
+
+```bash
+feedship search "machine learning"
+feedship search "python" --limit 10
+```
+
+## Documentation
+
+- @docs/feed.md - Feed provider architecture, fetch flow, refactoring status
+- @docs/providers.md - Provider/TagParser interfaces, registration
+- @docs/structure.md - Application structure, source files, structural rules
+- @docs/cli.md - CLI command reference
+- @docs/Automatic Discovery Feed.md - Automatic feed discovery system

feedship-1.0.0/feedship.egg-info/PKG-INFO

@@ -0,0 +1,44 @@
+Metadata-Version: 2.4
+Name: feedship
+Version: 1.0.0
+Summary: A personal information delivery system - collect, subscribe to, and organize information sources from the internet
+Requires-Python: >=3.10
+Requires-Dist: feedparser>=6.0.0
+Requires-Dist: httpx>=0.28.0
+Requires-Dist: click>=8.1.0
+Requires-Dist: beautifulsoup4>=4.12.0
+Requires-Dist: lxml>=6.0.0
+Requires-Dist: rich>=13.0.0
+Requires-Dist: PyGithub>=2.0.0
+Requires-Dist: dynaconf>=3.2.13
+Requires-Dist: trafilatura>=1.0.0
+Requires-Dist: robotexclusionrulesparser>=1.7.1
+Requires-Dist: platformdirs>=4.9.4
+Requires-Dist: numpy<2,>=1.26.0
+Requires-Dist: scikit-learn>=1.7.2
+Requires-Dist: pyyaml>=6.0.3
+Requires-Dist: uvloop>=0.22.0
+Requires-Dist: nanoid>=2.0.0
+Requires-Dist: scrapling>=0.4.0
+Requires-Dist: msgspec>=0.20.0
+Provides-Extra: test
+Requires-Dist: pytest>=9.0.2; extra == "test"
+Requires-Dist: pytest-asyncio>=1.0.0; extra == "test"
+Requires-Dist: pytest-cov>=7.0.0; extra == "test"
+Requires-Dist: pytest-mock>=3.15.0; extra == "test"
+Requires-Dist: pytest-click>=1.1.0; extra == "test"
+Requires-Dist: pytest-httpx>=0.36.0; extra == "test"
+Requires-Dist: pytest-xdist>=3.8.0; extra == "test"
+Requires-Dist: ruff>=0.6.0; extra == "test"
+Requires-Dist: pre-commit>=3.0.0; extra == "test"
+Provides-Extra: ml
+Requires-Dist: sentence-transformers>=3.0.0; extra == "ml"
+Requires-Dist: torch>=2.0.0; extra == "ml"
+Requires-Dist: safetensors>=0.4.3; extra == "ml"
+Requires-Dist: transformers>=4.40.0; extra == "ml"
+Provides-Extra: cloudflare
+Requires-Dist: scrapling>=0.4.0; extra == "cloudflare"
+Requires-Dist: playwright>=1.49.0; extra == "cloudflare"
+Requires-Dist: curl-cffi>=0.14.0; extra == "cloudflare"
+Requires-Dist: socksio>=1.0.0; extra == "cloudflare"
+Requires-Dist: browserforge>=1.2.0; extra == "cloudflare"

feedship-1.0.0/feedship.egg-info/SOURCES.txt

@@ -0,0 +1,51 @@
+README.md
+pyproject.toml
+feedship.egg-info/PKG-INFO
+feedship.egg-info/SOURCES.txt
+feedship.egg-info/dependency_links.txt
+feedship.egg-info/entry_points.txt
+feedship.egg-info/requires.txt
+feedship.egg-info/top_level.txt
+src/__init__.py
+src/constants.py
+src/models.py
+src/application/__init__.py
+src/application/articles.py
+src/application/combine.py
+src/application/config.py
+src/application/feed.py
+src/application/fetch.py
+src/application/related.py
+src/application/rerank.py
+src/application/search.py
+src/cli/__init__.py
+src/cli/__main__.py
+src/cli/article.py
+src/cli/discover.py
+src/cli/feed.py
+src/cli/ui.py
+src/discovery/__init__.py
+src/discovery/common_paths.py
+src/discovery/deep_crawl.py
+src/discovery/models.py
+src/discovery/parser.py
+src/providers/__init__.py
+src/providers/base.py
+src/providers/default_provider.py
+src/providers/github_release_provider.py
+src/providers/rss_provider.py
+src/providers/webpage_provider.py
+src/storage/__init__.py
+src/storage/vector.py
+src/storage/sqlite/__init__.py
+src/storage/sqlite/impl.py
+src/storage/sqlite/init.py
+src/utils/__init__.py
+src/utils/asyncio_utils.py
+src/utils/github.py
+src/utils/scraping_utils.py
+tests/test_cli.py
+tests/test_config.py
+tests/test_fetch.py
+tests/test_providers.py
+tests/test_storage.py

feedship-1.0.0/feedship.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

feedship-1.0.0/feedship.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+feedship = src.cli:cli

feedship-1.0.0/feedship.egg-info/requires.txt

@@ -0,0 +1,42 @@
+feedparser>=6.0.0
+httpx>=0.28.0
+click>=8.1.0
+beautifulsoup4>=4.12.0
+lxml>=6.0.0
+rich>=13.0.0
+PyGithub>=2.0.0
+dynaconf>=3.2.13
+trafilatura>=1.0.0
+robotexclusionrulesparser>=1.7.1
+platformdirs>=4.9.4
+numpy<2,>=1.26.0
+scikit-learn>=1.7.2
+pyyaml>=6.0.3
+uvloop>=0.22.0
+nanoid>=2.0.0
+scrapling>=0.4.0
+msgspec>=0.20.0
+
+[cloudflare]
+scrapling>=0.4.0
+playwright>=1.49.0
+curl-cffi>=0.14.0
+socksio>=1.0.0
+browserforge>=1.2.0
+
+[ml]
+sentence-transformers>=3.0.0
+torch>=2.0.0
+safetensors>=0.4.3
+transformers>=4.40.0
+
+[test]
+pytest>=9.0.2
+pytest-asyncio>=1.0.0
+pytest-cov>=7.0.0
+pytest-mock>=3.15.0
+pytest-click>=1.1.0
+pytest-httpx>=0.36.0
+pytest-xdist>=3.8.0
+ruff>=0.6.0
+pre-commit>=3.0.0

feedship-1.0.0/feedship.egg-info/top_level.txt

@@ -0,0 +1 @@
+src

feedship-1.0.0/pyproject.toml

@@ -0,0 +1,106 @@
+[project]
+name = "feedship"
+version = "1.0.0"
+description = "A personal information delivery system - collect, subscribe to, and organize information sources from the internet"
+requires-python = ">=3.10"
+dependencies = [
+    "feedparser>=6.0.0",
+    "httpx>=0.28.0",
+    "click>=8.1.0",
+    "beautifulsoup4>=4.12.0",
+    "lxml>=6.0.0",
+    "rich>=13.0.0",
+    "PyGithub>=2.0.0",
+    "dynaconf>=3.2.13",
+    "trafilatura>=1.0.0",
+    "robotexclusionrulesparser>=1.7.1",
+    "platformdirs>=4.9.4",
+    "numpy>=1.26.0,<2",
+    "scikit-learn>=1.7.2",
+    "pyyaml>=6.0.3",
+    "uvloop>=0.22.0",  # async event loop
+    "nanoid>=2.0.0",  # URL-safe ID generation
+    "scrapling>=0.4.0",  # CSS-selector HTML parsing
+    "msgspec>=0.20.0",  # Required by scrapling browser engine
+]
+
+[project.optional-dependencies]
+test = [
+    "pytest>=9.0.2",
+    "pytest-asyncio>=1.0.0",
+    "pytest-cov>=7.0.0",
+    "pytest-mock>=3.15.0",
+    "pytest-click>=1.1.0",
+    "pytest-httpx>=0.36.0",
+    "pytest-xdist>=3.8.0",
+    "ruff>=0.6.0",
+    "pre-commit>=3.0.0",
+]
+ml = [
+    "sentence-transformers>=3.0.0",
+    "torch>=2.0.0",
+    "safetensors>=0.4.3",
+    "transformers>=4.40.0",
+]
+cloudflare = [
+    "scrapling>=0.4.0",
+    "playwright>=1.49.0",
+    "curl-cffi>=0.14.0",
+    "socksio>=1.0.0",
+    "browserforge>=1.2.0",
+]
+
+[project.scripts]
+feedship = "src.cli:cli"
+
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["src*"]
+
+[tool.pytest.ini_options]
+minversion = "9.0"
+testpaths = ["tests"]
+asyncio_mode = "auto"
+addopts = [
+    "-v",
+    "--tb=short",
+    "--strict-markers",
+]
+markers = [
+    "asyncio: mark test as async",
+    "integration: integration test requiring full app",
+    "slow: tests that take significant time",
+]
+filterwarnings = [
+    "ignore::DeprecationWarning",
+]
+
+[tool.ruff]
+target-version = "py310"
+line-length = 88
+
+[tool.ruff.lint]
+select = [
+    "E", "W",  # pycodestyle errors & warnings
+    "F",       # Pyflakes
+    "I",       # isort import sorting
+    "UP",      # pyupgrade modernization
+    "B",       # flake8-bugbear common traps
+    "C4",      # flake8-comprehensions comprehension improvements
+    "SIM",     # flake8-simplify simplify code
+]
+ignore = ["E501"]  # line too long
+
+[tool.ruff.lint.per-file-ignores]
+"__init__.py" = ["F401", "E402"]
+"src/cli/*.py" = ["E402"]
+"tests/**/*.py" = ["E402", "F401"]
+
+[tool.ruff.format]
+quote-style = "double"
+indent-style = "space"
+docstring-code-format = true

feedship-1.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

feedship-1.0.0/src/application/__init__.py

@@ -0,0 +1 @@
+"""Application layer - use cases and orchestration."""

feedship-1.0.0/src/application/articles.py

@@ -0,0 +1,128 @@
+"""Article operations for RSS reader.
+
+Provides functions for listing and retrieving articles from the database.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from src.storage import (
+    get_article as storage_get_article,
+)
+from src.storage import (
+    get_article_detail as storage_get_article_detail,
+)
+from src.storage import (
+    list_articles as storage_list_articles,
+)
+from src.storage import (
+    search_articles as storage_search_articles,
+)
+
+
+@dataclass
+class ArticleListItem:
+    """Represents an article with feed name for list display.
+
+    Attributes:
+        id: Unique identifier for the article.
+        feed_id: ID of the feed this article belongs to.
+        feed_name: Name of the feed.
+        title: Title of the article.
+        link: URL link to the full article.
+        guid: Global unique identifier from the feed.
+        pub_date: Publication date from the feed.
+        description: Short description or summary.
+    """
+
+    id: str
+    feed_id: str
+    feed_name: str
+    title: str | None
+    link: str | None
+    guid: str
+    pub_date: str | None
+    description: str | None
+    vec_sim: float = 0.0
+    bm25_score: float = 0.0
+    freshness: float = 0.0
+    source_weight: float = 0.3
+    ce_score: float = 0.0
+    final_score: float = 0.0
+    score: float = 1.0
+
+
+def list_articles(
+    limit: int = 20,
+    feed_id: str | None = None,
+    since: str | None = None,
+    until: str | None = None,
+    on: list[str] | None = None,
+) -> list[ArticleListItem]:
+    """List articles ordered by publication date.
+
+    Args:
+        limit: Maximum number of articles to return (default 20).
+        feed_id: Optional feed ID to filter articles by a specific feed.
+        since: Optional start date (inclusive), format YYYY-MM-DD.
+        until: Optional end date (inclusive), format YYYY-MM-DD.
+        on: Optional list of specific dates to match.
+
+    Returns:
+        List of ArticleListItem objects.
+    """
+    return storage_list_articles(
+        limit=limit, feed_id=feed_id, since=since, until=until, on=on
+    )
+
+
+def get_article(article_id: str) -> ArticleListItem | None:
+    """Get a single article by ID.
+
+    Args:
+        article_id: The ID of the article to retrieve.
+
+    Returns:
+        ArticleListItem object if found, None otherwise.
+    """
+    return storage_get_article(article_id)
+
+
+def get_article_detail(article_id: str) -> dict | None:
+    """Get full article details including content.
+
+    Args:
+        article_id: The ID of the article (can be truncated 8-char or full 32-char).
+
+    Returns:
+        Dict with all article fields.
+        Returns None if article not found.
+    """
+    return storage_get_article_detail(article_id)
+
+
+def search_articles(
+    query: str,
+    limit: int = 20,
+    feed_id: str | None = None,
+    since: str | None = None,
+    until: str | None = None,
+    on: list[str] | None = None,
+) -> list[ArticleListItem]:
+    """Search articles using FTS5 full-text search.
+
+    Args:
+        query: FTS5 query string (space-separated = AND, use quotes for phrases)
+        limit: Maximum number of results (default 20)
+        feed_id: Optional feed ID to filter by specific feed
+        since: Optional start date (inclusive), format YYYY-MM-DD.
+        until: Optional end date (inclusive), format YYYY-MM-DD.
+        on: Optional list of specific dates to match.
+
+    Returns:
+        List of ArticleListItem objects.
+    """
+    return storage_search_articles(
+        query=query, limit=limit, feed_id=feed_id, since=since, until=until, on=on
+    )

feedship-1.0.0/src/application/combine.py

@@ -0,0 +1,63 @@
+"""Unified score combination using Newton's cooling law for freshness.
+
+This module provides combine_scores() which merges multiple scoring signals
+into a final ranking score using weighted combination with time-decay freshness.
+"""
+
+from __future__ import annotations
+
+import math
+from datetime import datetime, timezone
+
+from src.application.articles import ArticleListItem
+from src.storage.vector import _pub_date_to_timestamp
+
+
+def combine_scores(
+    candidates: list[ArticleListItem],
+    alpha: float = 0.3,
+    beta: float = 0.3,
+    gamma: float = 0.2,
+    delta: float = 0.2,
+) -> list[ArticleListItem]:
+    """Combine multiple scoring signals into final_score using weighted combination.
+
+    Newton's cooling law: freshness = exp(-days_ago / half_life_days)
+    half_life_days is fixed at 7 (one week).
+
+    Args:
+        candidates: List of ArticleListItem candidates to score.
+        alpha: Weight for Cross-Encoder score (ce_score).
+        beta: Weight for freshness (time decay).
+        gamma: Weight for vector similarity (vec_sim).
+        delta: Weight for BM25 score (bm25_score).
+
+    Returns:
+        List of candidates sorted by final_score descending.
+    """
+    half_life_days = 7
+    now = datetime.now(timezone.utc)
+
+    for c in candidates:
+        # Calculate freshness using Newton's cooling law
+        if c.pub_date:
+            timestamp = _pub_date_to_timestamp(c.pub_date)
+            if timestamp is not None:
+                pub_dt = datetime.fromtimestamp(timestamp, tz=timezone.utc)
+                days_ago = (now - pub_dt).days
+                c.freshness = math.exp(-days_ago / half_life_days)
+            else:
+                c.freshness = 0.0
+        else:
+            c.freshness = 0.0
+
+        # ce_score = 0 means not reranked, treat as no contribution
+        ce = c.ce_score if c.ce_score > 0 else 0.0
+
+        # Final score = weighted combination of 4 signals
+        c.final_score = (
+            alpha * ce + beta * c.freshness + gamma * c.vec_sim + delta * c.bm25_score
+        )
+
+    candidates.sort(key=lambda x: x.final_score, reverse=True)
+    return candidates

feedship-1.0.0/src/application/config.py

@@ -0,0 +1,36 @@
+"""Application configuration loaded from config.yaml via dynaconf."""
+
+from pathlib import Path
+from zoneinfo import ZoneInfo
+
+from dynaconf import Dynaconf
+
+_settings: Dynaconf | None = None
+
+
+def _get_settings() -> Dynaconf:
+    global _settings
+    if _settings is None:
+        _settings = Dynaconf(
+            envvar_prefix="RADAR",
+            settings_files=[
+                Path(__file__).parent.parent / "config.yaml",
+            ],
+        )
+    return _settings
+
+
+def get_timezone() -> ZoneInfo:
+    """Return the configured timezone as a ZoneInfo object."""
+    tz_name = _get_settings().get("timezone", "Asia/Shanghai")
+    return ZoneInfo(tz_name)
+
+
+def get_default_feed_weight() -> float:
+    """Return the default feed weight for semantic search ranking."""
+    return _get_settings().get("feed.default.weight", 0.3)
+
+
+def get_bm25_factor() -> float:
+    """Return the BM25 sigmoid normalization factor (default 0.5)."""
+    return _get_settings().get("bm25_factor", 0.5)