matrx-social 0.1.1__tar.gz

matrx_social-0.1.1/.gitignore

@@ -0,0 +1,351 @@
+*.pyc
+secrets/
+ignore/
+temp/
+logs/
+# The broad `logs/` rule above is for RUNTIME log output, but it also matched
+# the dashboard's SOURCE directory and silently swallowed an entire feature's
+# files (only the pre-existing index.tsx stayed tracked), breaking the prod
+# Docker build with "Could not resolve ./structured-tab". Re-include the source.
+!apps/dashboard/src/features/logs/
+!apps/dashboard/src/features/logs/**
+todo
+text_notes/
+aidream/secrets/2.env
+automation_matrix/matrix_processing/temp/*
+cd
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+.venv/
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+# The blanket lib/ rule above is from the standard Python .gitignore template
+# and was silently swallowing TS source under the SPA `src/lib/` folders.
+# Re-allow them explicitly so frontend builds don't ship without their lib layer.
+!apps/dashboard/src/lib/
+!apps/dashboard/src/lib/**
+!apps/dashboard/src/features/crawler/lib/
+!apps/dashboard/src/features/crawler/lib/**
+!apps/workflow-studio/src/lib/
+!apps/workflow-studio/src/lib/**
+# Shared package source: a `lib/` folder under apps/shared/<pkg>/src is TS source, never a Python build dir.
+# 2026-10-16: data's src/access/lib/governedActionError.ts stayed untracked, so the publish gate (a clean
+# checkout) failed TS2307 and blocked data and every dependent.
+!apps/shared/*/src/**/lib/
+!apps/shared/*/src/**/lib/**
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+ai/tests/clean_response.json
+ai/tests/cx_storage_response.json
+ai/tests/execution_test.py
+ai/tests/final_response.json
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.env_remote
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+.env.armanonly
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# random armani files
+/armani_dev/secrets/
+/armani/
+/_armani/
+
+
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+.idea/
+.vscode/
+/node_modules/
+
+# Frontend pnpm workspace (apps/) — node_modules at the workspace root and any
+# member, plus Vite caches and build output. The unified lockfile (apps/pnpm-lock.yaml)
+# IS committed; everything below is regenerated.
+#
+# 🚨 BOTH SPELLINGS. `node_modules/` with the slash matches a DIRECTORY only, so
+# a SYMLINK named `node_modules` slipped straight past it: five of them — each
+# an absolute path into one machine's checkout — were committed on main and
+# broke `pnpm install --frozen-lockfile` on every CI runner, where the symlink
+# is dangling and pnpm's `mkdir` of it answers ENOENT. Every package release was
+# blocked by it. The bare spelling below matches the symlink too.
+node_modules
+node_modules/
+apps/**/.vite/
+apps/**/dist/
+.vite/
+
+dump.rdb
+
+frontend/
+
+# AME Temp Files and directory structure
+# Ignore all files in the temp directory and its subdirectories
+/temp/**/*
+/tmp/**/*
+
+# Allow .gitkeep files to retain directory structure
+!/temp/**/.gitkeep
+!/tmp/**/.gitkeep
+
+# Armani
+.history*
+.history/
+/local_data/
+local_reports_data/
+webscraper/quick_scrapes/temp/
+automation_matrix/ai_apis/fireworks/_dev/*
+automation_matrix/ai_apis/fireworks/_dev/fireworks_sample.py
+*.pdf
+*.flac
+*.mp3
+*.wav
+miniconda.sh
+/database/python_sql/temp_data/
+ .history*
+ .history/
+.history/
+
+_dev/
+/_dev/
+requirements_filtered.txt
+
+# matrx-dev-tools backups
+.env-backups/
+# Matrx Ship config (contains API key)
+.matrx-ship.json
+
+# Matrx config (contains API keys)
+.matrx.json
+.matrx-tools.conf
+
+# Claude Code local worktrees and per-user settings
+.claude/worktrees/
+.worktrees/
+.claude/settings.local.json
+
+# Append-only snapshots from matrx_utils.update_history (unbounded; do not commit)
+common/utils/data_in_code/data_history.json
+packages/matrx-utils/matrx_utils/data_in_code/data_history.json
+
+# Tool-dispatch debug logs — one file per server start, never committed
+.matrx-debug/
+# Per-definition BEFORE snapshots written by a live migration script. They are
+# the recovery record for one machine's run, not repo content — the durable
+# audit trail is the `metadata.migrations` stamp on the row itself.
+.matrx-migrations/
+
+# Trial Bench records written on THIS machine — the trial's JSON record, its §8
+# report and the index that finds them (aidream/services/masterworks/bench/index.py,
+# and the runs a product-started trial writes beside it). They are never repo
+# content: a record carries every arm's full text INCLUDING the expert's ground
+# truth, which is material deliberately withheld from the Rulebook, and a trial
+# that matters is published deliberately under common-docs/projects/<its trial>/.
+# Untracked-but-not-ignored meant one `git add -A` away from committing an
+# expert's withheld work.
+.matrx/bench/
+
+# macOS Finder metadata
+.DS_Store
+**/.DS_Store
+
+# Environment files
+.env
+.env.*
+*.env
+*.env.*
+
+# Keep safe templates trackable
+!.env.example
+!.env.sample
+!.env.template
+
+# Never commit local OAuth client/token artifacts
+credentials.json
+token*.pickle
+token*.json
+tests_trials/rag_tests/kg_export_output/
+
+# Pooler-contamination watch output (machine-local evidence, not source)
+db/evidence/
+
+# Pooler contamination evidence — machine-local forensic capture, not source
+db/evidence/
+db/mirror/.env.mirror
+supabase/.temp/
+**/supabase/.temp/
+# Packed npm artifacts. Kept locally for a one-time bootstrap publish; never
+# committed (they are large binaries regenerable with `pnpm pack`).
+apps/shared/*/*.tgz
+
+# Guard forcing-function scratch packages. These tests plant real files inside
+# the real tree the guard scans (a temp dir would prove nothing) and delete
+# them in teardown — ignoring them keeps the auto-commit tooling from
+# capturing one mid-run.
+_kind_boundary_scratch*/
+_kind_marker_law_scratch*/
+
+# packed npm tarballs are build output, never source (doctrine: typescript-package-standard)
+*.tgz
+
+# Peers' kit scratch witnesses. A `.kit-<name>-witness.XXXXXX/` directory is a
+# full COPY of kit/src planted inside the packages tree — three of them held a
+# duplicate `format.ts` on 2026-09-12 — and it sits in the `check:package-twins`
+# scan ROOT. `git ls-files` only reaches TRACKED files, so the guard never saw
+# them; the risk was a broad `git add` committing three formatter twins in one
+# stroke, which is the exact class that guard exists to stop. Ignored, never
+# deleted: they are another agent's evidence.
+apps/shared/.kit-*-witness.*/
+
+# Agent worktrees. Extra checkouts of this same repo, never product source.
+# They live at .wt/<slug> (disk-groom). Committing the folder would add a
+# second copy of the entire tree and a symlink into common-docs.
+.wt/
+.matrx/fixer-dispatch.json
+.matrx/fixer-ledger.json
+
+# Release streak ledger (scripts/release_repeats.py): per-machine state, never committed
+.matrx/release-repeats.json
+.matrx/release-repeats.json.tmp
+.matrx/fixer-ledger.json
+/.config-manifest.json

matrx_social-0.1.1/FEATURE.md

@@ -0,0 +1,53 @@
+# matrx-social
+
+Provider-agnostic social data for Social Intelligence (`common-docs/systems/marketing/social/`). Depends only on pydantic + httpx; owns no ledger, no DB, no mock mode.
+
+- **Contracts** (`contracts.py`): `Platform`, `SocialProfile`, `SocialPost`, `MetricSet`, `Comment`, `Transcript`, `Ad`, `Page[T]`, `ProviderResult[T]`. A metric the provider does not give is `None`, never `0`. Every result carries `provider`, `fallback_reason`, `cost_credits`, and the `raw` payload.
+- **Interface** (`interface.py`): `SocialProvider` — `get_post`, `get_profile`, `list_posts`, `get_comments`, `get_transcript`, `search`, `trending`, `search_ads`, `capabilities`.
+- **ScrapeCreators** (`providers/scrapecreators/`): primary. Only operations whose real payload was captured and normalized are in `capabilities()`; anything else raises `UnsupportedError`. Snapchat and comments/search on several platforms are NOT wired yet (see `common-docs/systems/marketing/social/STATE.md` for the measured picture). A lookup of a missing account returns HTTP 200 + `account_deactivated` and is billed 1 credit; the adapter raises `NotFoundError(credits_charged=1)`. A lookup the platform refused answers HTTP 200 + `success: true` with `error`/`errorStatus`/`message` (e.g. Instagram `not_found` 404 "Profile is private"); `_get` raises `NotFoundError` for it (other `errorStatus` values: `SocialProviderError`, retryable only on 5xx) — never an empty normalized profile.
+- **Pinterest** (wired 2026-10-09): no profile endpoint exists. `get_profile` reads `/v1/pinterest/user/boards` (1 credit; the owner block gives handle, id, followers; `extras.boards` holds the boards on that page; `post_count` stays `None` unless the boards listing is complete). A board URL resolves to its owner. `list_posts` = pins of the account's boards, ONE board per page (cursor = board index, 1 credit/page, boards with 0 pins skipped; `effective_params` names the board). `get_post` = `/v1/pinterest/pin`; keyword `search` = `/v1/pinterest/search` (items carry NO engagement numbers: all metrics `None`; non-pin cards skipped and counted in `effective_params.skipped_non_pins`). No comments, no transcript. **Outlier metric = saves** (`aggregatedStats.saves`, the platform's own measure, larger than `repinCount` which rides in `extras.repins`); likes = reaction total, comments = `commentCount`, shares = `shareCount`, views always `None`. Pin video shape was not captured (no video pin in any probe): a video pin without a parsable mp4 carries `extras.video_unparsed`.
+- **Reddit** (wired 2026-10-09): no user endpoint, so the trackable unit is a SUBREDDIT (`handle` = subreddit name, `display_name` = `r/name`, id = `t5_…`). `get_profile` = `/v1/reddit/subreddit/details` (`followers` = `subscribers`, `None` when the vendor sends null, as it did for r/Entrepreneur); `list_posts` = `/v1/reddit/subreddit` (sort=new, `after` cursor); `get_post` = `/v1/reddit/post`; `get_comments` = `/v1/reddit/post/comments` (cursor from `more.cursor`); keyword `search` = `/v1/reddit/search` (posts). **Outlier metric = score (net upvotes) stored as `likes`**; comments = `num_comments`; views `None`. No transcript wired (the vendor has one for video posts).
+- **Threads** now also has keyword `search` (`/v1/threads/search`, one page, no cursor).
+- **Outlier metric per platform** lives in `aidream/services/social/stats.py::PRIMARY_METRIC` (views; Pinterest saves; Reddit and Threads likes) and is mirrored in matrx-frontend `features/marketing/social/outlier.ts`. The stored `post_stat.baseline_views` column holds the baseline of that platform's primary metric; `velocity_24h` is only computed for views platforms.
+- **Transcripts** (`normalize.parse_transcript`): captions are cleaned (`clean_caption_text`: zero-width characters stripped, whitespace collapsed) and rolling/overlapping segments de-duplicated (`dedupe_caption_segments`); text is rebuilt from the segments, never taken from the vendor's `transcript_only_text` (YouTube repeats every line there: a 20-minute video counted 54,558 words, really ~11k). Test on the real payload `tests/fixtures/yt_transcript_rolling.json`.
+- **HTTP content refusals**: a structured HTTP 403 verdict naming a restricted/private/missing post or profile is classified by the same `errors.refusal_kind` as a 200 refusal. Unknown 403s and credential/IP refusals remain `AuthError`; zero stated credits stay zero. Regression fixture: `tests/fixtures/ig_post_age_restricted.json`.
+- **Mastodon** (`providers/mastodon_public/`, MKT-109): free, keyless, always configured. An account is read from ITS OWN server (`/api/v1/accounts/lookup`, `/accounts/<id>/statuses`); handles are `user@host` (a bare `user` is `mastodon.social`); profile fields ride in `extras["fields"]`, the first (verified first) link in `extras["external_url"]`. Cost is always 0. A host that does not answer the Mastodon API is `NotFoundError(reason="missing")`.
+- **Failure types**: a vendor payload missing a key the adapter needs raises `UnexpectedShapeError(endpoint, missing)` (the caller records it and may fall back). `AuthError`/`OutOfCreditsError` are the account-level failures a caller treats as critical (`ProviderAccountError` is what a caller raises when every provider fails that way). Defaults the adapter applied (trending region, Meta ads country) are reported on `ProviderResult.effective_params`.
+- **Routing is not in this package.** The provider router (chain order, fallback, chat-vs-screen `read_context`, definitive-refusal stop, credit aggregation) was deleted in 0.1.1; routing now lives as data in the platform tool catalog (`aidream/services/tool_catalog/seeds/social.py`, tested by `aidream/services/tool_catalog/tests/test_social_family_is_the_old_router_as_data.py`). This package keeps only the single-provider adapters, the contracts (`ReadContext` and `DEEPER_OPTION` are in `contracts.py`) and the error types. `list_posts(profile, cursor, limit=N)` carries the TOTAL posts wanted (a hint for count-based providers; Bright Data deep posts read it, ScrapeCreators ignores it). EnsembleData (`ENSEMBLEDATA_TOKEN`) and Apify (`APIFY_API_TOKEN`) are registered in `providers/__init__.py` but have no key in any `.env` and no adapter yet.
+- **Cost**: `transport.py` calls registered observers with `SocialBillableCall(credits_charged)`. aidream wires `aidream/services/runtime/external_api_observers.py::_on_social_call` (units = credits, priced by knob `platform.external_api_prices.scrapecreators_credit_usd`).
+- **Key**: env `SCRAPECREATORS_API_KEY` (REQUIRED_ENV, credential definition `scrapecreators_api_key`).
+
+Tests: `pytest packages/matrx-social -m "not live"` — unit tests normalize REAL captured payloads (`tests/fixtures/`, captured 2026-10-09; Pinterest/Reddit/Threads-search fixtures `pin_*`, `reddit_*`, `th_search`); `-m live` makes real paid calls (~12 credits, skipped without the key; a plain `pytest` WITH the key set runs them and spends). Battery: `scripts/day_one_battery.py OUT.json` (~60 credits).
+
+## grok_x — X data through Grok (second X route, beside the direct X API)
+
+`providers/grok_x/client.py`; env `XAI_API_KEY` (already in REQUIRED_ENV); optional `GROK_X_MODEL` (default `grok-4.20-0309-non-reasoning`, the cheapest model measured to work; never a premium/reasoning model). Calls the xAI Responses API (`POST /v1/responses`) with the server-side `x_search` tool, `max_turns=2`, and a strict JSON schema — one request, no agent loop.
+- **Supports on X only:** `get_post` (status URL), `get_profile`, `list_posts` (by handle), keyword + hashtag `search`. Everything else raises `UnsupportedError`. No cursor: one page of `max_posts` (default 10; also the spend cap).
+- **Registered** in `providers/__init__.py` as `grok_x` after ScrapeCreators and Bright Data. Order between providers is the catalog's, not this package's.
+- **Honesty:** xAI does not hand back the raw X data; numbers reach us transcribed by the model. Every item carries `extras["extraction"]="grok_transcribed"`; not-shown fields are `None` (profile `following`/`post_count` came back null on NASA); posts with a non-numeric id are dropped (`effective_params.dropped_invalid`). Metrics differ slightly between two reads seconds apart (live counts) — treat as approximate and prefer ScrapeCreators for get_post/profile/list when both exist.
+- **Cost:** xAI reports exact USD per request (`usage.cost_in_usd_ticks`, 1 tick = 1e-10 USD; tokens + X Search at $5/1k posts and $10/1k profiles fetched, per docs.x.ai/developers/tools/x-search and /pricing). Surfaced as `ProviderResult.cost_usd` and `SocialBillableCall.reported_usd`; `_on_social_call` records ONE `external_api` ledger row priced by the vendor figure (no knob). It bypasses matrx-ai, so no token row exists: counted exactly once. SOC-POINTS reads that row (`provider=grok_x`). Measured: profile ~$0.033, 3 posts ~$0.02-0.04, search ~$0.04.
+- **Ruling (AI-mandate law):** this is a vendor data-retrieval call with a fixed extraction prompt and no persona, tools choice, or loop — an API adapter like ScrapeCreators, not an agent, so it does not go through a mandate/agent factory. If a future use needs Grok to reason over results, that is a separate mandate-backed agent fed by this adapter's output.
+- Rate limits: not published for x_search ("per-minute limits apply"); retried on 429/5xx with Retry-After. Terms of X-content use via xAI not reviewed — open.
+- Tests: `tests/test_grok_x.py` (real captured payload `fixtures/grok_x_responses_nasa_latest.json`); live `test_grok_x_live_search_tiny_spend` (~$0.02).
+
+## brightdata — second provider (fallback behind ScrapeCreators)
+
+`providers/brightdata/{client,normalize}.py`; env `BRIGHT_DATA_API_KEY` (read like `SCRAPECREATORS_API_KEY`; already in REQUIRED_ENV). Bright Data Web Scraper API datasets, billed **per returned record**.
+- **Wire:** `POST /datasets/v3/scrape?dataset_id=…&format=json&include_errors=true`, Bearer key, body `[{"url": …}]`. Sync 200 (records) within ~60 s, else 202 `{snapshot_id}` → poll `/datasets/v3/progress/{id}` (bounded: `poll_timeout_s`=180, announced in the error; a timed-out job is recorded at its cap, never as free) → `/datasets/v3/snapshot/{id}`. A read timeout on the POST is never retried (the job may already bill). Error records: `dead_page` → `NotFoundError(reason="missing")`; `blocked`/`crawl_error` → retryable `SocialProviderError`.
+- **🚨 Cap:** every request carries `limit_per_input` AND `limit_multiple_results`; `BrightDataTransport.scrape` raises `UncappedRequestError` before sending without a positive cap (an uncapped IG comments read returned 522 records). **Timeout:** a snapshot poll timeout POSTs `/datasets/v3/snapshot/{id}/cancel` (never retried), reports "job cancelled" or "cancel FAILED" in the error and the call detail, and still books the cap as spend. **No double jobs:** the trigger POST is retried only on connection errors before send and on 429 with `Retry-After`; a 5xx/read timeout is never re-POSTed (it may have started a billed job). Live 2026-10-11: both params at 2 → exactly 2 IG comment records. Which one alone is honoured was not isolated (testing the wrong one alone could bill hundreds of records).
+- **Cost:** `SocialBillableCall(provider="brightdata", credits_charged=records returned)`, path `/brightdata/<dataset_id>/<op>` with the operation word (`profile`/`post`/`posts`/`comments`) so `_on_social_call` classifies it; priced by knob `platform.external_api_prices.brightdata_record_usd` (0.0015). Error records count as returned records (conservative; unverified whether Bright Data bills them).
+- **Why its own transport, not matrx-scraper's Bright Data client:** a social read must be costed through the social observer (`_on_social_call`): subject attribution (`cost_subject`) and the shared-refresh cost split across tracking organizations. matrx-scraper's client feeds `_on_brightdata_call` (site data, no subject, no split). Same vendor, two cost meanings.
+- **Capabilities (only what a real record was normalized + tested for; `tests/test_brightdata.py`, fixtures `tests/fixtures/brightdata/`):** Instagram profile/post/list/comments; TikTok profile/post/list/comments (comments embedded in the post record); YouTube channel/video; LinkedIn person/post/list/comments (embedded `top_visible_comments`); Facebook page profile; X profile/post/list; Threads profile; Pinterest profile; Reddit list (subreddit discover, cap = `posts_page_size` 25). Everything else (transcripts, search, trending, ads, cursors) raises `UnsupportedError`. One page only: a cursor is refused, never re-billed.
+- **Deep posts (X, LinkedIn):** `list_posts` uses TRIGGER mode, `POST /datasets/v3/trigger?dataset_id=…&type=discover_new&discover_by=profile_url` (the sync `/scrape` returned 0 records for a discover), always snapshot-polled with its own bound `BrightDataTransport(discover_timeout_s=600)` (cancel-on-timeout, cap booked, fail-closed cap unchanged). X input carries `num_of_posts` = the requested count; the LinkedIn posts dataset REJECTS `num_of_posts` (400), so only `limit_per_input` bounds it. Count = `limit` hint, else `BrightDataProvider(deep_posts_default=50)`, ceiling `deep_posts_max=200`; one page, no cursor. Measured: 50 X posts in 27 s, 10 LinkedIn posts with like/comment counts in 34 s. Other platforms: the next sentence.
+- **list_posts** (other platforms) reads the posts embedded in the profile record; a profile record fetched in the last 10 minutes is reused (0 records). Instagram and LinkedIn embedded posts carry NO metrics (all `None`).
+- **Honesty rules:** X profiles answer `followers/following/posts_count = 0` beside real posts → all `None`, `extras.counts_withheld`. X retweets in a profile carry `likes=0, replies=0` → `None`. Ids follow ScrapeCreators (IG `pk`, Reddit id without `t3_`, LinkedIn no `platform_user_id`) so both providers land on the same rows. Every item has `extras.source_dataset`; `raw` = the record.
+- **Live proof 2026-10-11:** local server with a dead ScrapeCreators key → `/social/ingest/profile` TikTok @nasa fell back (`fallback_reason` "scrapecreators error: … HTTP 401"), stored 2,100,000 followers / 56 posts / 10.9M likes + 17 posts, 1 record = $0.0015 on one external_api row.
+
+## Data defects fixed 2026-10-11 (head-to-head)
+- ScrapeCreators Instagram `post_count`: the payload's `edge_owner_to_timeline_media.count` is the page size (12) while `has_next_page` is true; the real total is not in the payload, so it is `None` unless the count exceeds the page or the page is the whole account.
+- ScrapeCreators X reposts (`RT @…` or `retweeted_status_result`): likes, comments, saves are `None` (the vendor says 0), `extras.is_repost`.
+- Bright Data X profile counts 0 beside posts: `None` (`counts_withheld`), tested.
+- ScrapeCreators Pinterest "no public boards": a retryable `SocialProviderError`, not `NotFoundError`; the vendor answered empty for an account Bright Data reads, so it is never "the account does not exist" and never an empty profile.
+
+## Temporary: remove or disable before launch — test replay seam (REMOVE-BEFORE-LAUNCH)
+`transport.py` `register_social_replay(lookup, store)` / `ReplayHit`: a host-registered pair asked before every ScrapeCreators GET and Bright Data scrape (a hit is returned instead of the request: no guard, no observer, nothing billed) and handed each real paid answer afterwards. Unregistered (every package test, every non-aidream host) it is a no-op; a seam error is logged and treated as a miss. aidream's gate (admins/fixtures, switch, expensive list) is `aidream/services/runtime/admin_replay_cache.py`. Delete the seam with the cache before go-live (`rg REMOVE-BEFORE-LAUNCH`).

matrx_social-0.1.1/PKG-INFO

@@ -0,0 +1,65 @@
+Metadata-Version: 2.5
+Name: matrx-social
+Version: 0.1.1
+Summary: Provider-agnostic social data: contracts, provider interface, provider adapters.
+Requires-Python: >=3.13
+Requires-Dist: httpx>=0.28
+Requires-Dist: pydantic>=2.12
+Provides-Extra: dev
+Requires-Dist: pytest-asyncio>=0.25; extra == 'dev'
+Requires-Dist: pytest>=8.3; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# matrx-social
+
+Provider-agnostic social data for Social Intelligence (`common-docs/systems/marketing/social/`). Depends only on pydantic + httpx; owns no ledger, no DB, no mock mode.
+
+- **Contracts** (`contracts.py`): `Platform`, `SocialProfile`, `SocialPost`, `MetricSet`, `Comment`, `Transcript`, `Ad`, `Page[T]`, `ProviderResult[T]`. A metric the provider does not give is `None`, never `0`. Every result carries `provider`, `fallback_reason`, `cost_credits`, and the `raw` payload.
+- **Interface** (`interface.py`): `SocialProvider` — `get_post`, `get_profile`, `list_posts`, `get_comments`, `get_transcript`, `search`, `trending`, `search_ads`, `capabilities`.
+- **ScrapeCreators** (`providers/scrapecreators/`): primary. Only operations whose real payload was captured and normalized are in `capabilities()`; anything else raises `UnsupportedError`. Snapchat and comments/search on several platforms are NOT wired yet (see `common-docs/systems/marketing/social/STATE.md` for the measured picture). A lookup of a missing account returns HTTP 200 + `account_deactivated` and is billed 1 credit; the adapter raises `NotFoundError(credits_charged=1)`. A lookup the platform refused answers HTTP 200 + `success: true` with `error`/`errorStatus`/`message` (e.g. Instagram `not_found` 404 "Profile is private"); `_get` raises `NotFoundError` for it (other `errorStatus` values: `SocialProviderError`, retryable only on 5xx) — never an empty normalized profile.
+- **Pinterest** (wired 2026-10-09): no profile endpoint exists. `get_profile` reads `/v1/pinterest/user/boards` (1 credit; the owner block gives handle, id, followers; `extras.boards` holds the boards on that page; `post_count` stays `None` unless the boards listing is complete). A board URL resolves to its owner. `list_posts` = pins of the account's boards, ONE board per page (cursor = board index, 1 credit/page, boards with 0 pins skipped; `effective_params` names the board). `get_post` = `/v1/pinterest/pin`; keyword `search` = `/v1/pinterest/search` (items carry NO engagement numbers: all metrics `None`; non-pin cards skipped and counted in `effective_params.skipped_non_pins`). No comments, no transcript. **Outlier metric = saves** (`aggregatedStats.saves`, the platform's own measure, larger than `repinCount` which rides in `extras.repins`); likes = reaction total, comments = `commentCount`, shares = `shareCount`, views always `None`. Pin video shape was not captured (no video pin in any probe): a video pin without a parsable mp4 carries `extras.video_unparsed`.
+- **Reddit** (wired 2026-10-09): no user endpoint, so the trackable unit is a SUBREDDIT (`handle` = subreddit name, `display_name` = `r/name`, id = `t5_…`). `get_profile` = `/v1/reddit/subreddit/details` (`followers` = `subscribers`, `None` when the vendor sends null, as it did for r/Entrepreneur); `list_posts` = `/v1/reddit/subreddit` (sort=new, `after` cursor); `get_post` = `/v1/reddit/post`; `get_comments` = `/v1/reddit/post/comments` (cursor from `more.cursor`); keyword `search` = `/v1/reddit/search` (posts). **Outlier metric = score (net upvotes) stored as `likes`**; comments = `num_comments`; views `None`. No transcript wired (the vendor has one for video posts).
+- **Threads** now also has keyword `search` (`/v1/threads/search`, one page, no cursor).
+- **Outlier metric per platform** lives in `aidream/services/social/stats.py::PRIMARY_METRIC` (views; Pinterest saves; Reddit and Threads likes) and is mirrored in matrx-frontend `features/marketing/social/outlier.ts`. The stored `post_stat.baseline_views` column holds the baseline of that platform's primary metric; `velocity_24h` is only computed for views platforms.
+- **Transcripts** (`normalize.parse_transcript`): captions are cleaned (`clean_caption_text`: zero-width characters stripped, whitespace collapsed) and rolling/overlapping segments de-duplicated (`dedupe_caption_segments`); text is rebuilt from the segments, never taken from the vendor's `transcript_only_text` (YouTube repeats every line there: a 20-minute video counted 54,558 words, really ~11k). Test on the real payload `tests/fixtures/yt_transcript_rolling.json`.
+- **HTTP content refusals**: a structured HTTP 403 verdict naming a restricted/private/missing post or profile is classified by the same `errors.refusal_kind` as a 200 refusal. Unknown 403s and credential/IP refusals remain `AuthError`; zero stated credits stay zero. Regression fixture: `tests/fixtures/ig_post_age_restricted.json`.
+- **Mastodon** (`providers/mastodon_public/`, MKT-109): free, keyless, always configured. An account is read from ITS OWN server (`/api/v1/accounts/lookup`, `/accounts/<id>/statuses`); handles are `user@host` (a bare `user` is `mastodon.social`); profile fields ride in `extras["fields"]`, the first (verified first) link in `extras["external_url"]`. Cost is always 0. A host that does not answer the Mastodon API is `NotFoundError(reason="missing")`.
+- **Failure types**: a vendor payload missing a key the adapter needs raises `UnexpectedShapeError(endpoint, missing)` (the caller records it and may fall back). `AuthError`/`OutOfCreditsError` are the account-level failures a caller treats as critical (`ProviderAccountError` is what a caller raises when every provider fails that way). Defaults the adapter applied (trending region, Meta ads country) are reported on `ProviderResult.effective_params`.
+- **Routing is not in this package.** The provider router (chain order, fallback, chat-vs-screen `read_context`, definitive-refusal stop, credit aggregation) was deleted in 0.1.1; routing now lives as data in the platform tool catalog (`aidream/services/tool_catalog/seeds/social.py`, tested by `aidream/services/tool_catalog/tests/test_social_family_is_the_old_router_as_data.py`). This package keeps only the single-provider adapters, the contracts (`ReadContext` and `DEEPER_OPTION` are in `contracts.py`) and the error types. `list_posts(profile, cursor, limit=N)` carries the TOTAL posts wanted (a hint for count-based providers; Bright Data deep posts read it, ScrapeCreators ignores it). EnsembleData (`ENSEMBLEDATA_TOKEN`) and Apify (`APIFY_API_TOKEN`) are registered in `providers/__init__.py` but have no key in any `.env` and no adapter yet.
+- **Cost**: `transport.py` calls registered observers with `SocialBillableCall(credits_charged)`. aidream wires `aidream/services/runtime/external_api_observers.py::_on_social_call` (units = credits, priced by knob `platform.external_api_prices.scrapecreators_credit_usd`).
+- **Key**: env `SCRAPECREATORS_API_KEY` (REQUIRED_ENV, credential definition `scrapecreators_api_key`).
+
+Tests: `pytest packages/matrx-social -m "not live"` — unit tests normalize REAL captured payloads (`tests/fixtures/`, captured 2026-10-09; Pinterest/Reddit/Threads-search fixtures `pin_*`, `reddit_*`, `th_search`); `-m live` makes real paid calls (~12 credits, skipped without the key; a plain `pytest` WITH the key set runs them and spends). Battery: `scripts/day_one_battery.py OUT.json` (~60 credits).
+
+## grok_x — X data through Grok (second X route, beside the direct X API)
+
+`providers/grok_x/client.py`; env `XAI_API_KEY` (already in REQUIRED_ENV); optional `GROK_X_MODEL` (default `grok-4.20-0309-non-reasoning`, the cheapest model measured to work; never a premium/reasoning model). Calls the xAI Responses API (`POST /v1/responses`) with the server-side `x_search` tool, `max_turns=2`, and a strict JSON schema — one request, no agent loop.
+- **Supports on X only:** `get_post` (status URL), `get_profile`, `list_posts` (by handle), keyword + hashtag `search`. Everything else raises `UnsupportedError`. No cursor: one page of `max_posts` (default 10; also the spend cap).
+- **Registered** in `providers/__init__.py` as `grok_x` after ScrapeCreators and Bright Data. Order between providers is the catalog's, not this package's.
+- **Honesty:** xAI does not hand back the raw X data; numbers reach us transcribed by the model. Every item carries `extras["extraction"]="grok_transcribed"`; not-shown fields are `None` (profile `following`/`post_count` came back null on NASA); posts with a non-numeric id are dropped (`effective_params.dropped_invalid`). Metrics differ slightly between two reads seconds apart (live counts) — treat as approximate and prefer ScrapeCreators for get_post/profile/list when both exist.
+- **Cost:** xAI reports exact USD per request (`usage.cost_in_usd_ticks`, 1 tick = 1e-10 USD; tokens + X Search at $5/1k posts and $10/1k profiles fetched, per docs.x.ai/developers/tools/x-search and /pricing). Surfaced as `ProviderResult.cost_usd` and `SocialBillableCall.reported_usd`; `_on_social_call` records ONE `external_api` ledger row priced by the vendor figure (no knob). It bypasses matrx-ai, so no token row exists: counted exactly once. SOC-POINTS reads that row (`provider=grok_x`). Measured: profile ~$0.033, 3 posts ~$0.02-0.04, search ~$0.04.
+- **Ruling (AI-mandate law):** this is a vendor data-retrieval call with a fixed extraction prompt and no persona, tools choice, or loop — an API adapter like ScrapeCreators, not an agent, so it does not go through a mandate/agent factory. If a future use needs Grok to reason over results, that is a separate mandate-backed agent fed by this adapter's output.
+- Rate limits: not published for x_search ("per-minute limits apply"); retried on 429/5xx with Retry-After. Terms of X-content use via xAI not reviewed — open.
+- Tests: `tests/test_grok_x.py` (real captured payload `fixtures/grok_x_responses_nasa_latest.json`); live `test_grok_x_live_search_tiny_spend` (~$0.02).
+
+## brightdata — second provider (fallback behind ScrapeCreators)
+
+`providers/brightdata/{client,normalize}.py`; env `BRIGHT_DATA_API_KEY` (read like `SCRAPECREATORS_API_KEY`; already in REQUIRED_ENV). Bright Data Web Scraper API datasets, billed **per returned record**.
+- **Wire:** `POST /datasets/v3/scrape?dataset_id=…&format=json&include_errors=true`, Bearer key, body `[{"url": …}]`. Sync 200 (records) within ~60 s, else 202 `{snapshot_id}` → poll `/datasets/v3/progress/{id}` (bounded: `poll_timeout_s`=180, announced in the error; a timed-out job is recorded at its cap, never as free) → `/datasets/v3/snapshot/{id}`. A read timeout on the POST is never retried (the job may already bill). Error records: `dead_page` → `NotFoundError(reason="missing")`; `blocked`/`crawl_error` → retryable `SocialProviderError`.
+- **🚨 Cap:** every request carries `limit_per_input` AND `limit_multiple_results`; `BrightDataTransport.scrape` raises `UncappedRequestError` before sending without a positive cap (an uncapped IG comments read returned 522 records). **Timeout:** a snapshot poll timeout POSTs `/datasets/v3/snapshot/{id}/cancel` (never retried), reports "job cancelled" or "cancel FAILED" in the error and the call detail, and still books the cap as spend. **No double jobs:** the trigger POST is retried only on connection errors before send and on 429 with `Retry-After`; a 5xx/read timeout is never re-POSTed (it may have started a billed job). Live 2026-10-11: both params at 2 → exactly 2 IG comment records. Which one alone is honoured was not isolated (testing the wrong one alone could bill hundreds of records).
+- **Cost:** `SocialBillableCall(provider="brightdata", credits_charged=records returned)`, path `/brightdata/<dataset_id>/<op>` with the operation word (`profile`/`post`/`posts`/`comments`) so `_on_social_call` classifies it; priced by knob `platform.external_api_prices.brightdata_record_usd` (0.0015). Error records count as returned records (conservative; unverified whether Bright Data bills them).
+- **Why its own transport, not matrx-scraper's Bright Data client:** a social read must be costed through the social observer (`_on_social_call`): subject attribution (`cost_subject`) and the shared-refresh cost split across tracking organizations. matrx-scraper's client feeds `_on_brightdata_call` (site data, no subject, no split). Same vendor, two cost meanings.
+- **Capabilities (only what a real record was normalized + tested for; `tests/test_brightdata.py`, fixtures `tests/fixtures/brightdata/`):** Instagram profile/post/list/comments; TikTok profile/post/list/comments (comments embedded in the post record); YouTube channel/video; LinkedIn person/post/list/comments (embedded `top_visible_comments`); Facebook page profile; X profile/post/list; Threads profile; Pinterest profile; Reddit list (subreddit discover, cap = `posts_page_size` 25). Everything else (transcripts, search, trending, ads, cursors) raises `UnsupportedError`. One page only: a cursor is refused, never re-billed.
+- **Deep posts (X, LinkedIn):** `list_posts` uses TRIGGER mode, `POST /datasets/v3/trigger?dataset_id=…&type=discover_new&discover_by=profile_url` (the sync `/scrape` returned 0 records for a discover), always snapshot-polled with its own bound `BrightDataTransport(discover_timeout_s=600)` (cancel-on-timeout, cap booked, fail-closed cap unchanged). X input carries `num_of_posts` = the requested count; the LinkedIn posts dataset REJECTS `num_of_posts` (400), so only `limit_per_input` bounds it. Count = `limit` hint, else `BrightDataProvider(deep_posts_default=50)`, ceiling `deep_posts_max=200`; one page, no cursor. Measured: 50 X posts in 27 s, 10 LinkedIn posts with like/comment counts in 34 s. Other platforms: the next sentence.
+- **list_posts** (other platforms) reads the posts embedded in the profile record; a profile record fetched in the last 10 minutes is reused (0 records). Instagram and LinkedIn embedded posts carry NO metrics (all `None`).
+- **Honesty rules:** X profiles answer `followers/following/posts_count = 0` beside real posts → all `None`, `extras.counts_withheld`. X retweets in a profile carry `likes=0, replies=0` → `None`. Ids follow ScrapeCreators (IG `pk`, Reddit id without `t3_`, LinkedIn no `platform_user_id`) so both providers land on the same rows. Every item has `extras.source_dataset`; `raw` = the record.
+- **Live proof 2026-10-11:** local server with a dead ScrapeCreators key → `/social/ingest/profile` TikTok @nasa fell back (`fallback_reason` "scrapecreators error: … HTTP 401"), stored 2,100,000 followers / 56 posts / 10.9M likes + 17 posts, 1 record = $0.0015 on one external_api row.
+
+## Data defects fixed 2026-10-11 (head-to-head)
+- ScrapeCreators Instagram `post_count`: the payload's `edge_owner_to_timeline_media.count` is the page size (12) while `has_next_page` is true; the real total is not in the payload, so it is `None` unless the count exceeds the page or the page is the whole account.
+- ScrapeCreators X reposts (`RT @…` or `retweeted_status_result`): likes, comments, saves are `None` (the vendor says 0), `extras.is_repost`.
+- Bright Data X profile counts 0 beside posts: `None` (`counts_withheld`), tested.
+- ScrapeCreators Pinterest "no public boards": a retryable `SocialProviderError`, not `NotFoundError`; the vendor answered empty for an account Bright Data reads, so it is never "the account does not exist" and never an empty profile.
+
+## Temporary: remove or disable before launch — test replay seam (REMOVE-BEFORE-LAUNCH)
+`transport.py` `register_social_replay(lookup, store)` / `ReplayHit`: a host-registered pair asked before every ScrapeCreators GET and Bright Data scrape (a hit is returned instead of the request: no guard, no observer, nothing billed) and handed each real paid answer afterwards. Unregistered (every package test, every non-aidream host) it is a no-op; a seam error is logged and treated as a miss. aidream's gate (admins/fixtures, switch, expensive list) is `aidream/services/runtime/admin_replay_cache.py`. Delete the seam with the cache before go-live (`rg REMOVE-BEFORE-LAUNCH`).

matrx_social-0.1.1/matrx_social/__init__.py

@@ -0,0 +1,64 @@
+"""matrx-social: provider-agnostic social data (contracts, interface, adapters)."""
+
+from .contracts import (
+    DEEPER_OPTION,
+    Ad,
+    AdLibrary,
+    Attempt,
+    Capability,
+    Comment,
+    Media,
+    MediaKind,
+    MetricSet,
+    Page,
+    Platform,
+    ProviderResult,
+    ReadContext,
+    SearchType,
+    SocialPost,
+    SocialProfile,
+    Transcript,
+    TranscriptSegment,
+)
+from .errors import (
+    AllProvidersFailedError,
+    AuthError,
+    NotConfiguredError,
+    NotFoundError,
+    OutOfCreditsError,
+    ProviderAccountError,
+    SocialProviderError,
+    UnexpectedShapeError,
+    UnsupportedError,
+)
+from .interface import SocialProvider
+from .providers.brightdata import BrightDataProvider, BrightDataTransport, UncappedRequestError
+from .providers.grok_x import GrokXProvider
+from .providers.scrapecreators import ScrapeCreatorsProvider
+from .transport import (
+    AdaptiveRateLimiter,
+    RecorderMissingError,
+    ReplayHit,
+    ScrapeCreatorsTransport,
+    SocialBillableCall,
+    clear_social_call_guards,
+    clear_social_call_observers,
+    clear_social_replay,
+    ensure_recorder,
+    register_social_call_flusher,
+    register_social_call_guard,
+    register_social_call_observer,
+    register_social_replay,
+)
+from .urls import (
+    account_for_link,
+    detect_platform,
+    extract_handle,
+    is_personal_profile_url,
+    is_post_link,
+    post_owner,
+    profile_handle,
+    youtube_channel_ref,
+)
+
+__all__ = [n for n in dir() if not n.startswith("_")]