fplkit 1.4.1__tar.gz → 1.4.2__tar.gz

{fplkit-1.4.1 → fplkit-1.4.2}/CHANGELOG.md

@@ -1,6 +1,12 @@
 # Changelog
 
 All notable changes to this project will be documented in this file.
+## [1.4.1] - 2026-05-17
+
+### Refactoring
+
+- refactor FPL scraper to fetch API directly instead of intercepting (#37)
+
 ## [1.4.0] - 2026-05-17
 
 ### Bug Fixes
@@ -108,5 +114,3 @@ All notable changes to this project will be documented in this file.
 ### Features
 
 - CI pipeline and PyPI publish for v1.0.0 (#1)
-
-

{fplkit-1.4.1 → fplkit-1.4.2}/CLAUDE.md

@@ -3,7 +3,7 @@ fpl-cli - CLI tool for Fantasy Premier League analysis (classic + draft).
 
 ## Setup & Dev
 ```bash
-source .venv/bin/activate && pip install -e ".[dev]"
+source .venv/bin/activate && pip install -e ".[dev]"            # local only — web sessions install globally via setup script
 ruff check fpl_cli/            # Lint
 pyright fpl_cli/               # Type check
 pytest tests/                 # Tests

{fplkit-1.4.1 → fplkit-1.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fplkit
-Version: 1.4.1
+Version: 1.4.2
 Summary: CLI tool for Fantasy Premier League analysis - classic and draft formats
 Project-URL: Homepage, https://github.com/rossgroomio/fpl-cli
 Project-URL: Repository, https://github.com/rossgroomio/fpl-cli

{fplkit-1.4.1 → fplkit-1.4.2}/fpl_cli/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.4.1'
-__version_tuple__ = version_tuple = (1, 4, 1)
+__version__ = version = '1.4.2'
+__version_tuple__ = version_tuple = (1, 4, 2)
 
 __commit_id__ = commit_id = None

{fplkit-1.4.1 → fplkit-1.4.2}/fpl_cli/agents/data/scout.py

@@ -49,10 +49,16 @@ class ScoutAgent(Agent):
         Returns:
             Formatted position reference string.
         """
-        # Filter to relevant players: ownership > 1% OR form > 3 OR minutes > 500
+        # Filter to relevant players: ownership > 1% OR form > 3 OR minutes > 500.
+        # Also include any player with active news (e.g. injury return, late-fitness
+        # flags) so post-injury returnees with low minutes still appear in the
+        # reference and don't force the LLM to guess their club.
         relevant = [
             p for p in players
-            if p.selected_by_percent > 1.0 or p.form > 3.0 or p.minutes > 500
+            if p.selected_by_percent > 1.0
+            or p.form > 3.0
+            or p.minutes > 500
+            or bool(p.news)
         ]
 
         # Group by position

{fplkit-1.4.1 → fplkit-1.4.2}/fpl_cli/prompts/scout.py

@@ -34,11 +34,16 @@ ALWAYS:
 - Flag confidence level (High/Medium/Low) based on source agreement and recency
 - Interpret signals for the user - don't just report raw quotes
 - Note when trusted voices disagree with mainstream opinion
+- Source every player's current club from the <player_reference> block - never infer it from prior-season knowledge
+- Treat <player_reference> as a strict allowlist: every player you name anywhere in the output (BUY tables, SELL tables, Differential Whispers, Contrarian Take) MUST appear in it
 
 NEVER:
 - Repeat statistical analysis (xG, form scores, PPG) - the user has this
 - Give generic advice without specific sourcing
 - Present rumour as fact without flagging uncertainty
+- Guess a player's club from memory; player transfers between seasons make prior knowledge unreliable
+- Surface players who are not in <player_reference>, even if community chatter mentions them - they are not in this season's FPL pool (likely transferred abroad, retired, or never were in the PL)
+- Emit placeholder rows like "PlayerX? — not listed in your FPL pool, so skipping". Silently exclude out-of-pool names instead; return fewer rows (or omit the section) rather than padding
 </rules>"""
 
 SCOUT_USER_PROMPT_TEMPLATE = """Surface qualitative FPL intelligence for Gameweek {gameweek}.
@@ -51,12 +56,21 @@ This query runs 24-30 hours before deadline, after most press conferences.
 3. Check r/FantasyPL for community momentum - who's being quietly accumulated, what the RMT thread is converging on
 4. Identify narrative breaks - players being over-sold or under-bought based on reactive sentiment
 5. Note any blank/double GW planning implications from @BenCrellin
+
+When scanning external sources, treat any player name you don't find in <player_reference> as out-of-pool: they're not in this season's FPL game (transferred abroad, retired, or wrong league) and must not appear anywhere in your output, even if the community is discussing them.
 </research_focus>
 
-<player_positions>
-IMPORTANT: Use this official FPL position reference when categorising players. Do NOT guess positions.
+<player_reference>
+Authoritative directory of player → (club, position) for THIS season, sourced from live FPL data. Each entry is formatted as `Name (CLUB)`.
+
+Use this as the ONLY source of truth for both:
+- The "Club" column in your output tables (clubs change between seasons; do NOT use prior-season knowledge)
+- The position grouping (DEF / MID / FWD)
+
+If a player you want to recommend is not listed here, either omit them or explicitly flag "club unknown" rather than guessing.
+
 {position_reference}
-</player_positions>
+</player_reference>
 
 <unavailable_players>
 CRITICAL: The following players are INJURED, SUSPENDED, or otherwise UNAVAILABLE according to current FPL data. Do NOT recommend any of these players under any circumstances, even if web sources suggest them (those sources may be outdated).
@@ -70,24 +84,24 @@ CRITICAL: The following players are INJURED, SUSPENDED, or otherwise UNAVAILABLE
 
 | Player | Club | Signal | Source | Confidence |
 |--------|------|--------|--------|------------|
-[2-3 players with qualitative buy signals - what do the watchers see that stats don't show yet?]
+[Up to 3 players from <player_reference> with qualitative buy signals - what do the watchers see that stats don't show yet? Fewer rows are fine; omit the section entirely if no one in the pool qualifies. Do NOT pad with out-of-pool names.]
 
 **MIDFIELDERS**
 
 | Player | Club | Signal | Source | Confidence |
 |--------|------|--------|--------|------------|
-[2-3 players with qualitative buy signals]
+[Up to 3 players from <player_reference> with qualitative buy signals. Fewer or zero is acceptable.]
 
 **FORWARDS**
 
 | Player | Club | Signal | Source | Confidence |
 |--------|------|--------|--------|------------|
-[1-2 players with qualitative buy signals]
+[Up to 2 players from <player_reference> with qualitative buy signals. Fewer or zero is acceptable.]
 
 ## SELL Signals
 | Player | Club | Signal Type | Detail | Confidence |
 |--------|------|-------------|--------|------------|
-[3-5 players to avoid - categorise as: Rotation Risk, Eye-Test Warning, or Narrative Trap]
+[Up to 5 players from <player_reference> to avoid - categorise as: Rotation Risk, Eye-Test Warning, or Narrative Trap. Fewer rows are fine; omit if no one qualifies.]
 
 ## Strategic Intel
 - **Blank/Double GW Watch:** Any planning implications from Ben Crellin or fixture news
@@ -100,7 +114,8 @@ CRITICAL: The following players are INJURED, SUSPENDED, or otherwise UNAVAILABLE
 - Confidence = High when multiple trusted sources agree + recent (last 48h); Medium when single source or older; Low when speculative/emerging
 - Prioritise actionable, time-sensitive intel over comprehensive coverage
 - If a trusted voice contradicts the crowd, highlight it
-- CRITICAL: Always check <player_positions> reference before assigning a player to DEFENDERS/MIDFIELDERS/FORWARDS
+- CRITICAL: Always check <player_reference> before filling the Club column or assigning a player to DEFENDERS/MIDFIELDERS/FORWARDS. The reference reflects current-season clubs; your prior knowledge does not.
+- CRITICAL: <player_reference> is the complete allowlist of in-pool players. If a name you're considering is not in it, drop the row — never emit "Player? — not in pool" placeholders, and never pad to hit a row count.
 </quality_requirements>"""
 
 

{fplkit-1.4.1 → fplkit-1.4.2}/fpl_cli/scraper/fpl_prices.py

@@ -217,27 +217,40 @@ class FPLPriceScraper:
             finally:
                 await browser.close()
 
+    # `or {}` on each step: FPL briefly returns {"player": null} in the post-login
+    # window before the session is fully hydrated. .get(key, {}) only defaults on
+    # missing keys, not on explicit nulls, so chain through `or {}` to be safe.
+    _ME_RETRY_ATTEMPTS = 5
+    _ME_RETRY_DELAY_MS = 1000
+
     async def _fetch_my_team(self, page) -> dict | None:
         """Fetch /api/me/ and /api/my-team/{id}/ from the authenticated session.
 
         Returns the my-team payload, or None if either call fails (in which case the
         caller falls back to DOM extraction).
         """
-        try:
-            me = await page.evaluate(
-                "async () => (await fetch('/api/me/', {credentials: 'include'})).json()"
-            )
-        except Exception as e:  # noqa: BLE001 — scraper resilience
-            logger.debug("Failed to fetch /api/me/: %s", e)
-            return None
+        me: dict | None = None
+        entry_id = None
+        for attempt in range(self._ME_RETRY_ATTEMPTS):
+            try:
+                me = await page.evaluate(
+                    "async () => (await fetch('/api/me/', {credentials: 'include'})).json()"
+                )
+            except Exception as e:  # noqa: BLE001 — scraper resilience
+                logger.debug("Failed to fetch /api/me/: %s", e)
+                return None
+            entry_id = ((me or {}).get("player") or {}).get("entry")
+            if entry_id:
+                break
+            if attempt < self._ME_RETRY_ATTEMPTS - 1:
+                await page.wait_for_timeout(self._ME_RETRY_DELAY_MS)
 
-        # `or {}` on each step: FPL briefly returns {"player": null} in the post-login
-        # window before the session is fully hydrated. .get(key, {}) only defaults on
-        # missing keys, not on explicit nulls, so chain through `or {}` to be safe.
-        player = (me or {}).get("player") or {}
-        entry_id = player.get("entry")
         if not entry_id:
-            logger.debug("No entry id in /api/me/ response: %s", me)
+            logger.debug(
+                "No entry id in /api/me/ after %d retries: %s",
+                self._ME_RETRY_ATTEMPTS,
+                me,
+            )
             return None
 
         try:
@@ -380,12 +393,22 @@ class FPLPriceScraper:
 
     async def _extract_finances(self, page, my_entry_response: dict | None = None) -> TeamFinances:
         """Extract financial data - tries intercepted API data first, falls back to DOM."""
-        if my_entry_response:
+        fallback_reason: str | None = None
+        if my_entry_response is None:
+            fallback_reason = (
+                "API /api/me/ returned no entry id after retries (post-login hydration race?)"
+            )
+        else:
             finances = await self._extract_from_intercepted(page, my_entry_response)
             if finances and not finances.is_suspect:
                 return finances
+            fallback_reason = (
+                "API /api/my-team/ returned no usable squad data; fell back to DOM extraction"
+            )
 
-        return await self._extract_via_dom(page)
+        finances = await self._extract_via_dom(page)
+        finances.extraction_errors.append(fallback_reason)
+        return finances
 
     async def _extract_from_intercepted(self, page, my_entry_response: dict) -> TeamFinances | None:
         """Build TeamFinances from intercepted /api/my-team/ response data."""

{fplkit-1.4.1 → fplkit-1.4.2}/tests/test_agents_data.py

@@ -1206,6 +1206,46 @@ Sources:
             assert isinstance(result.data["content_clean"], str)
             assert isinstance(result.data["citations"], list)
 
+    def test_build_position_reference_includes_news_flagged_players(self, agent):
+        """Low-minutes players with active news (e.g. injury returnees) must
+        appear in the reference so the LLM doesn't have to guess their club."""
+        from fpl_cli.models.player import PlayerPosition
+        from tests.conftest import make_player
+
+        # A defender just back from injury: low ownership, low form, low
+        # minutes — would be excluded by the old filter, but has news.
+        returnee = make_player(
+            id=99,
+            web_name="Returnee",
+            team_id=7,
+            position=PlayerPosition.DEFENDER,
+            selected_by_percent=0.3,
+            form=1.0,
+            minutes=120,
+            news="Back in training, hopeful for involvement",
+        )
+        ref = agent.build_position_reference([returnee], {7: "MCI"})
+        assert "Returnee (MCI)" in ref
+
+    def test_build_position_reference_excludes_irrelevant_players(self, agent):
+        """Players with no news, no minutes, no form, and tiny ownership
+        should still be excluded to keep the reference compact."""
+        from fpl_cli.models.player import PlayerPosition
+        from tests.conftest import make_player
+
+        irrelevant = make_player(
+            id=100,
+            web_name="Bench",
+            team_id=7,
+            position=PlayerPosition.DEFENDER,
+            selected_by_percent=0.1,
+            form=0.0,
+            minutes=0,
+            news="",
+        )
+        ref = agent.build_position_reference([irrelevant], {7: "MCI"})
+        assert "Bench" not in ref
+
     async def test_run_cleans_citations_correctly(self, agent):
         """Test that citations are properly cleaned from content."""
         response_with_citations = LLMResponse(

{fplkit-1.4.1 → fplkit-1.4.2}/tests/test_scraper.py

@@ -256,26 +256,100 @@ class TestFPLPriceScraper:
         assert page.evaluate.call_args_list[1].args[1] == 5955459
 
     async def test_fetch_my_team_returns_none_on_missing_entry(self):
-        """If /api/me/ returns no player.entry, abort and let DOM fallback take over."""
+        """If /api/me/ never returns a player.entry, abort and let DOM fallback take over."""
         from unittest.mock import AsyncMock, MagicMock
 
         scraper = FPLPriceScraper()
         page = MagicMock()
         page.evaluate = AsyncMock(return_value={})
+        page.wait_for_timeout = AsyncMock()
 
         result = await scraper._fetch_my_team(page)
         assert result is None
+        assert page.evaluate.call_count == scraper._ME_RETRY_ATTEMPTS
+        # No trailing wait after the final failed attempt
+        assert page.wait_for_timeout.call_count == scraper._ME_RETRY_ATTEMPTS - 1
 
     async def test_fetch_my_team_handles_null_player(self):
-        """FPL briefly returns {'player': null} in the post-login window; must not crash."""
+        """FPL briefly returns {'player': null} post-login; we retry then give up cleanly."""
         from unittest.mock import AsyncMock, MagicMock
 
         scraper = FPLPriceScraper()
         page = MagicMock()
         page.evaluate = AsyncMock(return_value={"player": None})
+        page.wait_for_timeout = AsyncMock()
 
         result = await scraper._fetch_my_team(page)
         assert result is None
+        assert page.evaluate.call_count == scraper._ME_RETRY_ATTEMPTS
+        # No trailing wait after the final failed attempt
+        assert page.wait_for_timeout.call_count == scraper._ME_RETRY_ATTEMPTS - 1
+
+    async def test_fetch_my_team_retries_until_hydrated(self):
+        """Null player on early polls, then a real entry id: retry succeeds and fetches my-team."""
+        from unittest.mock import AsyncMock, MagicMock
+
+        scraper = FPLPriceScraper()
+        page = MagicMock()
+        page.evaluate = AsyncMock(side_effect=[
+            {"player": None},
+            {"player": None},
+            {"player": {"entry": 12345}},
+            {"picks": [{"element": 1, "selling_price": 50, "purchase_price": 50}],
+             "transfers": {"bank": 10, "limit": 1}},
+        ])
+        page.wait_for_timeout = AsyncMock()
+
+        result = await scraper._fetch_my_team(page)
+        assert result is not None
+        assert result["transfers"]["bank"] == 10
+        # 3 /api/me/ calls + 1 /api/my-team/ call
+        assert page.evaluate.call_count == 4
+        # And we waited between the failing /api/me/ attempts (twice)
+        assert page.wait_for_timeout.call_count == 2
+
+    async def test_fetch_my_team_evaluate_error_no_retry(self):
+        """A hard page.evaluate exception is a different failure mode - don't retry."""
+        from unittest.mock import AsyncMock, MagicMock
+
+        scraper = FPLPriceScraper()
+        page = MagicMock()
+        page.evaluate = AsyncMock(side_effect=RuntimeError("eval failed"))
+        page.wait_for_timeout = AsyncMock()
+
+        result = await scraper._fetch_my_team(page)
+        assert result is None
+        assert page.evaluate.call_count == 1
+        assert page.wait_for_timeout.call_count == 0
+
+    async def test_extract_finances_records_api_failure_in_errors(self):
+        """When the API path is skipped, the DOM fallback's result gets a hint appended."""
+        from unittest.mock import AsyncMock, MagicMock
+
+        scraper = FPLPriceScraper()
+        dom_result = TeamFinances(bank=0.0, free_transfers=0, squad=[], total_value=0.0)
+        with patch.object(scraper, "_extract_via_dom", AsyncMock(return_value=dom_result)):
+            result = await scraper._extract_finances(MagicMock(), my_entry_response=None)
+
+        assert result is dom_result
+        assert any("api/me" in err.lower() for err in result.extraction_errors)
+
+    async def test_extract_finances_records_intercepted_failure_in_errors(self):
+        """If /api/me/ succeeded but the intercepted payload is unusable, the DOM fallback still gets a hint."""
+        from unittest.mock import AsyncMock, MagicMock
+
+        scraper = FPLPriceScraper()
+        dom_result = TeamFinances(bank=0.0, free_transfers=0, squad=[], total_value=0.0)
+        with (
+            patch.object(scraper, "_extract_from_intercepted", AsyncMock(return_value=None)),
+            patch.object(scraper, "_extract_via_dom", AsyncMock(return_value=dom_result)),
+        ):
+            result = await scraper._extract_finances(
+                MagicMock(), my_entry_response={"picks": [], "transfers": {}}
+            )
+
+        assert result is dom_result
+        assert any("my-team" in err.lower() for err in result.extraction_errors)
 
     def test_cache_file_path(self):
         """Test cache file path is correct."""