rrss-sdk-wrapper 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. rrss_sdk_wrapper-0.2.0/.env.example +22 -0
  2. rrss_sdk_wrapper-0.2.0/.gitignore +14 -0
  3. rrss_sdk_wrapper-0.2.0/CHANGELOG.md +44 -0
  4. rrss_sdk_wrapper-0.2.0/PKG-INFO +173 -0
  5. rrss_sdk_wrapper-0.2.0/README.md +146 -0
  6. rrss_sdk_wrapper-0.2.0/SECURITY.md +9 -0
  7. rrss_sdk_wrapper-0.2.0/compose.yaml +21 -0
  8. rrss_sdk_wrapper-0.2.0/docs/generated-clients.md +19 -0
  9. rrss_sdk_wrapper-0.2.0/docs/index.md +23 -0
  10. rrss_sdk_wrapper-0.2.0/docs/mapping.md +181 -0
  11. rrss_sdk_wrapper-0.2.0/docs/platforms/facebook-v1-deferral.md +17 -0
  12. rrss_sdk_wrapper-0.2.0/docs/platforms/instagram.md +32 -0
  13. rrss_sdk_wrapper-0.2.0/docs/platforms/threads.md +41 -0
  14. rrss_sdk_wrapper-0.2.0/docs/platforms/tiktok.md +33 -0
  15. rrss_sdk_wrapper-0.2.0/docs/platforms/x.md +38 -0
  16. rrss_sdk_wrapper-0.2.0/generated/go/client.go +167 -0
  17. rrss_sdk_wrapper-0.2.0/generated/go/go.mod +3 -0
  18. rrss_sdk_wrapper-0.2.0/generated/go/models.go +60 -0
  19. rrss_sdk_wrapper-0.2.0/generated/python/rrss_client.py +233 -0
  20. rrss_sdk_wrapper-0.2.0/generated/rust/Cargo.toml +8 -0
  21. rrss_sdk_wrapper-0.2.0/generated/rust/src/lib.rs +162 -0
  22. rrss_sdk_wrapper-0.2.0/mkdocs.yml +18 -0
  23. rrss_sdk_wrapper-0.2.0/pyproject.toml +52 -0
  24. rrss_sdk_wrapper-0.2.0/rrss/__init__.py +50 -0
  25. rrss_sdk_wrapper-0.2.0/rrss/adapters/base.py +80 -0
  26. rrss_sdk_wrapper-0.2.0/rrss/adapters/instagram.py +123 -0
  27. rrss_sdk_wrapper-0.2.0/rrss/adapters/threads.py +62 -0
  28. rrss_sdk_wrapper-0.2.0/rrss/adapters/tiktok.py +76 -0
  29. rrss_sdk_wrapper-0.2.0/rrss/adapters/x.py +224 -0
  30. rrss_sdk_wrapper-0.2.0/rrss/adapters/x_stub.py +5 -0
  31. rrss_sdk_wrapper-0.2.0/rrss/auth/__init__.py +19 -0
  32. rrss_sdk_wrapper-0.2.0/rrss/cache.py +209 -0
  33. rrss_sdk_wrapper-0.2.0/rrss/cli.py +78 -0
  34. rrss_sdk_wrapper-0.2.0/rrss/client.py +35 -0
  35. rrss_sdk_wrapper-0.2.0/rrss/errors.py +31 -0
  36. rrss_sdk_wrapper-0.2.0/rrss/gateway.py +306 -0
  37. rrss_sdk_wrapper-0.2.0/rrss/models.py +61 -0
  38. rrss_sdk_wrapper-0.2.0/rrss/normalize/__init__.py +1 -0
  39. rrss_sdk_wrapper-0.2.0/rrss/normalize/instagram.py +82 -0
  40. rrss_sdk_wrapper-0.2.0/rrss/normalize/threads.py +82 -0
  41. rrss_sdk_wrapper-0.2.0/rrss/normalize/tiktok.py +51 -0
  42. rrss_sdk_wrapper-0.2.0/rrss/normalize/x.py +192 -0
  43. rrss_sdk_wrapper-0.2.0/rrss/ratelimit.py +30 -0
  44. rrss_sdk_wrapper-0.2.0/rrss/serialize.py +216 -0
  45. rrss_sdk_wrapper-0.2.0/rrss/stream.py +198 -0
  46. rrss_sdk_wrapper-0.2.0/schemas/openapi.yaml +378 -0
  47. rrss_sdk_wrapper-0.2.0/schemas/rrss-v1.json +77 -0
  48. rrss_sdk_wrapper-0.2.0/scripts/generate_client_stub.py +696 -0
  49. rrss_sdk_wrapper-0.2.0/sidecar/threads/.dockerignore +2 -0
  50. rrss_sdk_wrapper-0.2.0/sidecar/threads/Dockerfile +19 -0
  51. rrss_sdk_wrapper-0.2.0/sidecar/threads/README.md +30 -0
  52. rrss_sdk_wrapper-0.2.0/sidecar/threads/package-lock.json +840 -0
  53. rrss_sdk_wrapper-0.2.0/sidecar/threads/package.json +13 -0
  54. rrss_sdk_wrapper-0.2.0/sidecar/threads/server.js +24 -0
  55. rrss_sdk_wrapper-0.2.0/tests/fakes.py +64 -0
  56. rrss_sdk_wrapper-0.2.0/tests/fixtures/instagram/media.json +10 -0
  57. rrss_sdk_wrapper-0.2.0/tests/fixtures/instagram/user.json +10 -0
  58. rrss_sdk_wrapper-0.2.0/tests/fixtures/threads/threads.json +17 -0
  59. rrss_sdk_wrapper-0.2.0/tests/fixtures/threads/user.json +6 -0
  60. rrss_sdk_wrapper-0.2.0/tests/fixtures/tiktok/user.json +7 -0
  61. rrss_sdk_wrapper-0.2.0/tests/fixtures/tiktok/video.json +8 -0
  62. rrss_sdk_wrapper-0.2.0/tests/fixtures/x/replies.json +21 -0
  63. rrss_sdk_wrapper-0.2.0/tests/fixtures/x/tweets.json +20 -0
  64. rrss_sdk_wrapper-0.2.0/tests/fixtures/x/user.json +17 -0
  65. rrss_sdk_wrapper-0.2.0/tests/test_adapter_edges.py +223 -0
  66. rrss_sdk_wrapper-0.2.0/tests/test_cache.py +132 -0
  67. rrss_sdk_wrapper-0.2.0/tests/test_cli.py +46 -0
  68. rrss_sdk_wrapper-0.2.0/tests/test_client.py +94 -0
  69. rrss_sdk_wrapper-0.2.0/tests/test_errors_retry_integration.py +176 -0
  70. rrss_sdk_wrapper-0.2.0/tests/test_gateway.py +168 -0
  71. rrss_sdk_wrapper-0.2.0/tests/test_generated_client.py +115 -0
  72. rrss_sdk_wrapper-0.2.0/tests/test_normalize.py +26 -0
  73. rrss_sdk_wrapper-0.2.0/tests/test_normalize_edges.py +191 -0
  74. rrss_sdk_wrapper-0.2.0/tests/test_stream.py +91 -0
  75. rrss_sdk_wrapper-0.2.0/tests/test_x.py +351 -0
@@ -0,0 +1,22 @@
1
+ # Instagram
2
+ INSTAGRAM_USERNAME=
3
+ INSTAGRAM_PASSWORD=
4
+ INSTAGRAM_SESSION_PATH=
5
+
6
+ # TikTok
7
+ TIKTOK_MS_TOKEN=
8
+ TIKTOK_PROXY=
9
+
10
+ # Threads sidecar (Option A)
11
+ RRSS_THREADS_SIDECAR_URL=http://127.0.0.1:1877
12
+
13
+ # X API v2 app-only bearer. Empty keeps the read adapter inert (no requests).
14
+ X_BEARER_TOKEN=
15
+
16
+ # Optional gateway (stdlib HTTP). Leave the key empty to disable the check.
17
+ RRSS_GATEWAY_API_KEY=
18
+ RRSS_GATEWAY_HOST=127.0.0.1
19
+ RRSS_GATEWAY_PORT=8080
20
+
21
+ # Optional Redis cache. Unset uses the in-memory cache.
22
+ # RRSS_REDIS_URL=redis://localhost:6379/0
@@ -0,0 +1,14 @@
1
+ __pycache__/
2
+ *.py[cod]
3
+ .pytest_cache/
4
+ .mypy_cache/
5
+ .ruff_cache/
6
+ .coverage
7
+ htmlcov/
8
+ *.egg-info/
9
+ .venv/
10
+ dist/
11
+ build/
12
+ .env
13
+ site/
14
+ node_modules/
@@ -0,0 +1,44 @@
1
+ # Changelog
2
+
3
+ All notable changes to the `rrss` PyPI package are documented here.
4
+
5
+ ## Unreleased
6
+
7
+ Not published. No release tag.
8
+
9
+ ### Added
10
+
11
+ - Platform guides for Instagram, Threads, TikTok, and X (`docs/platforms/`).
12
+ - X read-only adapter (X API v2, tweepy-shaped). Inert without `X_BEARER_TOKEN`. Optional extra `rrss[x]`.
13
+ - Threads sidecar image and `compose.yaml`. Health check is `http://127.0.0.1:1877/health`.
14
+ - GitHub bug template (platform, redacted credentials, `include_raw`) and Dependabot.
15
+ - Optional MkDocs config. `pip install -e ".[docs]"` then `mkdocs build`.
16
+
17
+ ## 0.2.0 — 2026-11-20
18
+
19
+ Not published to PyPI. No release tag.
20
+
21
+ ### Added
22
+
23
+ - Polling helpers (`poll`, `poll_user`, `poll_user_posts`) and an in-memory `WebhookStub`.
24
+ - `CachingAdapter` with `MemoryCache` and an optional `RedisCache` (`rrss[cache]`).
25
+ - CLI: `rrss pull user <platform> <ref>`.
26
+ - Stdlib HTTP gateway for the canonical read routes, plus `schemas/openapi.yaml`.
27
+ - Generated clients from `schemas/rrss-v1.json` (Python and Go HTTP stubs, Rust types and URL builders). See `docs/generated-clients.md`.
28
+
29
+ ## 0.1.0 — 2025-11-20
30
+
31
+ ### Added
32
+
33
+ - Initial PyPI release (`rrss` meta-package).
34
+ - Instagram, Threads, and TikTok adapters with golden-fixture normalization tests.
35
+ - Unified error taxonomy (`AUTH_FAILED`, `RATE_LIMITED`, `NOT_FOUND`, `NOT_SUPPORTED`, `PLATFORM_ERROR`).
36
+ - Exponential backoff retries on `RATE_LIMITED` (`rrss.ratelimit.with_retries`), wired for Instagram user fetch.
37
+ - Threads Node sidecar contract under `python/sidecar/threads/` (local HTTP on `127.0.0.1:1877`).
38
+ - X read-only stub (`NOT_SUPPORTED` until X API v2 credentials are configured).
39
+ - JSON Schema `schemas/rrss-v1.json` for canonical types.
40
+
41
+ ### Notes
42
+
43
+ - Facebook Graph adapter deferred; see `docs/platforms/facebook-v1-deferral.md`.
44
+ - Live platform credentials are optional; inject mock upstream clients for tests.
@@ -0,0 +1,173 @@
1
+ Metadata-Version: 2.5
2
+ Name: rrss-sdk-wrapper
3
+ Version: 0.2.0
4
+ Summary: Unified social media data client (Instagram, Threads, TikTok)
5
+ Author: Ferran Suils
6
+ License-Expression: MIT
7
+ Requires-Python: >=3.11
8
+ Provides-Extra: all
9
+ Requires-Dist: httpx>=0.27; extra == 'all'
10
+ Requires-Dist: tweepy>=4.14; extra == 'all'
11
+ Provides-Extra: cache
12
+ Requires-Dist: redis>=5; extra == 'cache'
13
+ Provides-Extra: dev
14
+ Requires-Dist: httpx>=0.27; extra == 'dev'
15
+ Requires-Dist: mypy>=1.11; extra == 'dev'
16
+ Requires-Dist: pytest>=8.0; extra == 'dev'
17
+ Requires-Dist: ruff>=0.6; extra == 'dev'
18
+ Provides-Extra: docs
19
+ Requires-Dist: mkdocs>=1.6; extra == 'docs'
20
+ Provides-Extra: instagram
21
+ Provides-Extra: threads
22
+ Requires-Dist: httpx>=0.27; extra == 'threads'
23
+ Provides-Extra: tiktok
24
+ Provides-Extra: x
25
+ Requires-Dist: tweepy>=4.14; extra == 'x'
26
+ Description-Content-Type: text/markdown
27
+
28
+ # rrss
29
+
30
+ Python meta-package for normalized social profile and post data across Instagram, Threads, and TikTok.
31
+
32
+ ## Install
33
+
34
+ Version 0.2.0 in this tree is not published. PyPI still serves 0.1.0 until a tagged release.
35
+
36
+ ```bash
37
+ pip install rrss-sdk-wrapper
38
+ # optional extras
39
+ pip install "rrss-sdk-wrapper[threads]"
40
+ pip install "rrss-sdk-wrapper[x]" # tweepy; live X calls only
41
+ pip install "rrss-sdk-wrapper[cache]" # redis; tests use the in-memory cache
42
+ pip install "rrss-sdk-wrapper[docs]" # mkdocs
43
+ pip install "rrss-sdk-wrapper[all]"
44
+ ```
45
+
46
+ From this repository:
47
+
48
+ ```bash
49
+ cd python
50
+ pip install -e .
51
+ pip install -e ".[dev]" # ruff, mypy, pytest
52
+ ```
53
+
54
+ ## Quick start
55
+
56
+ ```python
57
+ from rrss import Client
58
+
59
+ # Adapters accept injectable upstream clients for testing; optional deps for live usage.
60
+ ig = Client("instagram", username="u", password="p")
61
+ user = ig.get_user("someone")
62
+ for page in ig.iter_user_posts("someone", limit=50):
63
+ for post in page.items:
64
+ print(post.text)
65
+
66
+ # Re-fetch until the profile changes. Pass max_polls in tests (interval may be 0).
67
+ for profile in ig.poll_user("someone", interval=60, max_polls=5):
68
+ print(profile.follower_count)
69
+ ```
70
+
71
+ `WebhookStub` records push payloads in memory (`accept`, then iterate). The gateway also accepts `POST /v1/webhooks` into that stub. Neither path needs a public callback URL.
72
+
73
+ ## Cache
74
+
75
+ ```python
76
+ from rrss import CachingAdapter, Client, MemoryCache
77
+
78
+ cached = CachingAdapter(Client("threads"), MemoryCache())
79
+ cached.get_user("someone") # second call is served from memory
80
+ ```
81
+
82
+ Set `RRSS_REDIS_URL` and install `rrss-sdk-wrapper[cache]` to use `RedisCache`. The same `get` / `set` interface backs both. Default TTLs follow the lower bound in the plan: users 15 minutes, post lists 5 minutes, comments 1 minute.
83
+
84
+ ## CLI
85
+
86
+ ```bash
87
+ rrss pull user threads some_username
88
+ ```
89
+
90
+ The command prints canonical user JSON. Credentials come from the environment (see `.env.example`), never from flags.
91
+
92
+ ## HTTP gateway
93
+
94
+ ```bash
95
+ RRSS_GATEWAY_API_KEY=dev python -m rrss.gateway
96
+ # GET /v1/{platform}/users/{user_ref}
97
+ # GET /v1/{platform}/users/{user_ref}/posts
98
+ # GET /openapi.yaml
99
+ ```
100
+
101
+ The server is stdlib `http.server` with an injectable client factory, so tests run without live platform credentials. Spec: `schemas/openapi.yaml`.
102
+
103
+ Generated clients (Python, Go, and a Rust URL stub) are described in `docs/generated-clients.md`.
104
+
105
+ Copy `python/.env.example` to `.env` for local credentials (never commit secrets).
106
+
107
+ ## Threads sidecar (Node)
108
+
109
+ The Threads adapter calls a small **Node sidecar** that wraps `threads-wrapper` over HTTP.
110
+
111
+ ```bash
112
+ cd sidecar/threads
113
+ npm install
114
+ npm start # listens on 127.0.0.1:1877
115
+ ```
116
+
117
+ Set `RRSS_THREADS_SIDECAR_URL` if the sidecar is not on the default URL. Contract: `GET /health`, `GET /users/:username`, `GET /users/:username/threads` (see api-wrapper plan §19).
118
+
119
+ ```python
120
+ from rrss import Client
121
+
122
+ th = Client("threads") # uses http://127.0.0.1:1877 by default
123
+ user = th.get_user("threader")
124
+ ```
125
+
126
+ The sidecar is **not** published to npm; run it from this repo.
127
+
128
+ ```bash
129
+ cd python
130
+ docker compose up --build threads-sidecar
131
+ curl http://127.0.0.1:1877/health
132
+ ```
133
+
134
+ Compose publishes `127.0.0.1:1877` only. See `docs/platforms/threads.md`.
135
+
136
+ ## Errors and retries
137
+
138
+ Shared exceptions live in `rrss.errors`. Rate-limited upstream responses can be retried with `rrss.ratelimit.with_retries` (default: 3 attempts, exponential backoff with jitter). Instagram `get_user` uses this policy when the upstream raises `RateLimited`.
139
+
140
+ ## Development
141
+
142
+ ```bash
143
+ cd python
144
+ pip install -e ".[dev]"
145
+ ruff check .
146
+ mypy rrss
147
+ pytest -m "not integration"
148
+ pytest -m integration # optional: requires local Threads sidecar for health check
149
+ ```
150
+
151
+ CI on pull requests runs `pytest -m "not integration"`. Nightly/workflow-dispatch jobs may run integration markers separately.
152
+
153
+ Optional docs site (not published with the package):
154
+
155
+ ```bash
156
+ pip install -e ".[docs]"
157
+ mkdocs build # writes ./site
158
+ mkdocs serve # http://127.0.0.1:8000
159
+ ```
160
+
161
+ See `docs/index.md`.
162
+
163
+ ## Platforms
164
+
165
+ | Platform | Auth | Read / write | API | Guide |
166
+ |-----------|------------------------------|--------------|-----|-------|
167
+ | Instagram | Username/password session | Read only | Unofficial (`instagrapi`) | [docs/platforms/instagram.md](docs/platforms/instagram.md) |
168
+ | Threads | None (sidecar) | Read only | Unofficial | [docs/platforms/threads.md](docs/platforms/threads.md) |
169
+ | TikTok | `ms_token` | Read only | Unofficial | [docs/platforms/tiktok.md](docs/platforms/tiktok.md) |
170
+ | X | `X_BEARER_TOKEN` (inert if unset) | Read only | Official v2 | [docs/platforms/x.md](docs/platforms/x.md) |
171
+ | Facebook | — | Deferred | — | [docs/platforms/facebook-v1-deferral.md](docs/platforms/facebook-v1-deferral.md) |
172
+
173
+ Project specification: see the workspace `docs/api-wrapper-plan.md` for the full contract.
@@ -0,0 +1,146 @@
1
+ # rrss
2
+
3
+ Python meta-package for normalized social profile and post data across Instagram, Threads, and TikTok.
4
+
5
+ ## Install
6
+
7
+ Version 0.2.0 in this tree is not published. PyPI still serves 0.1.0 until a tagged release.
8
+
9
+ ```bash
10
+ pip install rrss-sdk-wrapper
11
+ # optional extras
12
+ pip install "rrss-sdk-wrapper[threads]"
13
+ pip install "rrss-sdk-wrapper[x]" # tweepy; live X calls only
14
+ pip install "rrss-sdk-wrapper[cache]" # redis; tests use the in-memory cache
15
+ pip install "rrss-sdk-wrapper[docs]" # mkdocs
16
+ pip install "rrss-sdk-wrapper[all]"
17
+ ```
18
+
19
+ From this repository:
20
+
21
+ ```bash
22
+ cd python
23
+ pip install -e .
24
+ pip install -e ".[dev]" # ruff, mypy, pytest
25
+ ```
26
+
27
+ ## Quick start
28
+
29
+ ```python
30
+ from rrss import Client
31
+
32
+ # Adapters accept injectable upstream clients for testing; optional deps for live usage.
33
+ ig = Client("instagram", username="u", password="p")
34
+ user = ig.get_user("someone")
35
+ for page in ig.iter_user_posts("someone", limit=50):
36
+ for post in page.items:
37
+ print(post.text)
38
+
39
+ # Re-fetch until the profile changes. Pass max_polls in tests (interval may be 0).
40
+ for profile in ig.poll_user("someone", interval=60, max_polls=5):
41
+ print(profile.follower_count)
42
+ ```
43
+
44
+ `WebhookStub` records push payloads in memory (`accept`, then iterate). The gateway also accepts `POST /v1/webhooks` into that stub. Neither path needs a public callback URL.
45
+
46
+ ## Cache
47
+
48
+ ```python
49
+ from rrss import CachingAdapter, Client, MemoryCache
50
+
51
+ cached = CachingAdapter(Client("threads"), MemoryCache())
52
+ cached.get_user("someone") # second call is served from memory
53
+ ```
54
+
55
+ Set `RRSS_REDIS_URL` and install `rrss-sdk-wrapper[cache]` to use `RedisCache`. The same `get` / `set` interface backs both. Default TTLs follow the lower bound in the plan: users 15 minutes, post lists 5 minutes, comments 1 minute.
56
+
57
+ ## CLI
58
+
59
+ ```bash
60
+ rrss pull user threads some_username
61
+ ```
62
+
63
+ The command prints canonical user JSON. Credentials come from the environment (see `.env.example`), never from flags.
64
+
65
+ ## HTTP gateway
66
+
67
+ ```bash
68
+ RRSS_GATEWAY_API_KEY=dev python -m rrss.gateway
69
+ # GET /v1/{platform}/users/{user_ref}
70
+ # GET /v1/{platform}/users/{user_ref}/posts
71
+ # GET /openapi.yaml
72
+ ```
73
+
74
+ The server is stdlib `http.server` with an injectable client factory, so tests run without live platform credentials. Spec: `schemas/openapi.yaml`.
75
+
76
+ Generated clients (Python, Go, and a Rust URL stub) are described in `docs/generated-clients.md`.
77
+
78
+ Copy `python/.env.example` to `.env` for local credentials (never commit secrets).
79
+
80
+ ## Threads sidecar (Node)
81
+
82
+ The Threads adapter calls a small **Node sidecar** that wraps `threads-wrapper` over HTTP.
83
+
84
+ ```bash
85
+ cd sidecar/threads
86
+ npm install
87
+ npm start # listens on 127.0.0.1:1877
88
+ ```
89
+
90
+ Set `RRSS_THREADS_SIDECAR_URL` if the sidecar is not on the default URL. Contract: `GET /health`, `GET /users/:username`, `GET /users/:username/threads` (see api-wrapper plan §19).
91
+
92
+ ```python
93
+ from rrss import Client
94
+
95
+ th = Client("threads") # uses http://127.0.0.1:1877 by default
96
+ user = th.get_user("threader")
97
+ ```
98
+
99
+ The sidecar is **not** published to npm; run it from this repo.
100
+
101
+ ```bash
102
+ cd python
103
+ docker compose up --build threads-sidecar
104
+ curl http://127.0.0.1:1877/health
105
+ ```
106
+
107
+ Compose publishes `127.0.0.1:1877` only. See `docs/platforms/threads.md`.
108
+
109
+ ## Errors and retries
110
+
111
+ Shared exceptions live in `rrss.errors`. Rate-limited upstream responses can be retried with `rrss.ratelimit.with_retries` (default: 3 attempts, exponential backoff with jitter). Instagram `get_user` uses this policy when the upstream raises `RateLimited`.
112
+
113
+ ## Development
114
+
115
+ ```bash
116
+ cd python
117
+ pip install -e ".[dev]"
118
+ ruff check .
119
+ mypy rrss
120
+ pytest -m "not integration"
121
+ pytest -m integration # optional: requires local Threads sidecar for health check
122
+ ```
123
+
124
+ CI on pull requests runs `pytest -m "not integration"`. Nightly/workflow-dispatch jobs may run integration markers separately.
125
+
126
+ Optional docs site (not published with the package):
127
+
128
+ ```bash
129
+ pip install -e ".[docs]"
130
+ mkdocs build # writes ./site
131
+ mkdocs serve # http://127.0.0.1:8000
132
+ ```
133
+
134
+ See `docs/index.md`.
135
+
136
+ ## Platforms
137
+
138
+ | Platform | Auth | Read / write | API | Guide |
139
+ |-----------|------------------------------|--------------|-----|-------|
140
+ | Instagram | Username/password session | Read only | Unofficial (`instagrapi`) | [docs/platforms/instagram.md](docs/platforms/instagram.md) |
141
+ | Threads | None (sidecar) | Read only | Unofficial | [docs/platforms/threads.md](docs/platforms/threads.md) |
142
+ | TikTok | `ms_token` | Read only | Unofficial | [docs/platforms/tiktok.md](docs/platforms/tiktok.md) |
143
+ | X | `X_BEARER_TOKEN` (inert if unset) | Read only | Official v2 | [docs/platforms/x.md](docs/platforms/x.md) |
144
+ | Facebook | — | Deferred | — | [docs/platforms/facebook-v1-deferral.md](docs/platforms/facebook-v1-deferral.md) |
145
+
146
+ Project specification: see the workspace `docs/api-wrapper-plan.md` for the full contract.
@@ -0,0 +1,9 @@
1
+ # Security
2
+
3
+ - Never commit credentials or session files. Use `.env` locally (see `.env.example`).
4
+ - Logs must not include passwords, tokens, or full session payloads.
5
+ - The HTTP gateway does not log request lines. Platform passwords stay in the server environment; callers send `X-RRSS-Key` only. Do not put secrets in webhook bodies you store.
6
+ - `RRSS_REDIS_URL` may embed a password. Do not print it.
7
+ - Report security issues privately to the repository owner.
8
+ - GitHub issues use the bug template: name the platform, paste an `include_raw` snippet only after stripping secrets, and replace every credential with `REDACTED`.
9
+ - Dependabot opens dependency updates. It does not receive application secrets.
@@ -0,0 +1,21 @@
1
+ # Local Threads sidecar. From python/: docker compose up --build threads-sidecar
2
+ # Host URL matches the Python default: http://127.0.0.1:1877
3
+ services:
4
+ threads-sidecar:
5
+ build:
6
+ context: ./sidecar/threads
7
+ ports:
8
+ - "127.0.0.1:1877:1877"
9
+ environment:
10
+ HOST: 0.0.0.0
11
+ PORT: "1877"
12
+ healthcheck:
13
+ test:
14
+ - CMD
15
+ - node
16
+ - -e
17
+ - "fetch('http://127.0.0.1:1877/health').then((r)=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))"
18
+ interval: 10s
19
+ timeout: 3s
20
+ retries: 3
21
+ start_period: 5s
@@ -0,0 +1,19 @@
1
+ # Generated clients
2
+
3
+ Stubs are generated from [`schemas/rrss-v1.json`](../schemas/rrss-v1.json). The HTTP paths match [`schemas/openapi.yaml`](../schemas/openapi.yaml).
4
+
5
+ Regenerate from `python/`:
6
+
7
+ ```bash
8
+ python scripts/generate_client_stub.py
9
+ ```
10
+
11
+ | Language | Path | What it does |
12
+ |----------|------|----------------|
13
+ | Python | `generated/python/rrss_client.py` | Dataclasses plus a stdlib HTTP client (`RrssClient`) |
14
+ | Go | `generated/go/` | `encoding/json` models and an `net/http` client. `go build .` |
15
+ | Rust | `generated/rust/` | Canonical structs and URL builders. `rustc --edition 2021 --crate-type lib src/lib.rs` |
16
+
17
+ Go and Rust toolchains were available when this slice landed (`go` 1.22, `rustc` 1.83). Pytest compiles both when the binaries are on `PATH` and skips that check when they are not, so CI can stay Python-only.
18
+
19
+ The Rust crate does not perform HTTP. Its standard library has no HTTP or JSON implementation, and the generator does not vendor `serde` or `reqwest`. Call the gateway with the Python or Go stub. The Rust types and `RrssClient::{get_user_url, list_user_posts_url, ...}` stay aligned with the schema for a later crate that adds those dependencies.
@@ -0,0 +1,23 @@
1
+ # rrss
2
+
3
+ Unified read client for Instagram, Threads, TikTok, and X. Facebook stays deferred.
4
+
5
+ Install and the Python quick start live in the package [README](https://github.com/FerranSuils/scraper-rrss/blob/master/python/README.md). This site is the longer reference:
6
+
7
+ - [Field mapping](mapping.md)
8
+ - [Instagram](platforms/instagram.md), [Threads](platforms/threads.md), [TikTok](platforms/tiktok.md), [X](platforms/x.md)
9
+ - [Facebook deferral](platforms/facebook-v1-deferral.md)
10
+ - [Generated clients](generated-clients.md)
11
+
12
+ ## Build this site
13
+
14
+ MkDocs is optional. The config is `mkdocs.yml` next to this `docs/` directory. The built HTML is gitignored.
15
+
16
+ ```bash
17
+ cd python
18
+ pip install -e ".[docs]"
19
+ mkdocs build # writes ./site
20
+ mkdocs serve # http://127.0.0.1:8000
21
+ ```
22
+
23
+ `mkdocs build` does not publish anywhere. PyPI upload is a separate, tagged release workflow and is not part of building docs.
@@ -0,0 +1,181 @@
1
+ # RRSS field mapping (native → canonical)
2
+
3
+ Canonical models: [rrss-v1.json](../schemas/rrss-v1.json). Platform notes: [platforms/](platforms/instagram.md).
4
+
5
+ ## Instagram (`instagrapi`)
6
+
7
+ ### User
8
+
9
+ | Canonical | Source |
10
+ |-----------|--------|
11
+ | `id` | `str(user.pk)` |
12
+ | `username` | `user.username` |
13
+ | `display_name` | `user.full_name` |
14
+ | `bio` | `user.biography` |
15
+ | `follower_count` | `user.follower_count` |
16
+ | `following_count` | `user.following_count` |
17
+ | `is_verified` | `user.is_verified` |
18
+ | `avatar_url` | `user.profile_pic_url_hd` or `profile_pic_url` |
19
+ | `profile_url` | `https://www.instagram.com/{username}/` |
20
+ | `platform` | `"instagram"` |
21
+
22
+ ### Post
23
+
24
+ | Canonical | Source |
25
+ |-----------|--------|
26
+ | `id` | `str(media.pk)` or `media.id` |
27
+ | `author_id` | `str(media.user.pk)` |
28
+ | `text` | `media.caption_text` or `media.caption.text` if present |
29
+ | `created_at` | `datetime.utcfromtimestamp(media.taken_at).isoformat() + "Z"` |
30
+ | `url` | `https://www.instagram.com/p/{media.code}/` |
31
+ | `media` | Carousel: each resource `url` + `type` image/video; else single `thumbnail_url` / `video_url` |
32
+ | `like_count` | `media.like_count` |
33
+ | `comment_count` | `media.comment_count` |
34
+ | `platform` | `"instagram"` |
35
+
36
+ ### Comment
37
+
38
+ | Canonical | Source |
39
+ |-----------|--------|
40
+ | `id` | `str(comment.pk)` |
41
+ | `post_id` | parent media id |
42
+ | `author_id` | `str(comment.user.pk)` |
43
+ | `text` | `comment.text` |
44
+ | `created_at` | from `comment.created_at_utc` if available |
45
+ | `platform` | `"instagram"` |
46
+
47
+ ### Pagination
48
+
49
+ `list_user_posts`: use `max_id` from last item; expose as `next_cursor`.
50
+
51
+ ---
52
+
53
+ ## Threads (`threads-wrapper` via sidecar)
54
+
55
+ ### User
56
+
57
+ | Canonical | Source |
58
+ |-----------|--------|
59
+ | `id` | `user_id` |
60
+ | `username` | path / lookup username |
61
+ | `display_name` | `full_name` |
62
+ | `bio` | `biography` |
63
+ | `follower_count` | `follower_count` |
64
+ | `profile_url` | `bio_links[0].url` when present |
65
+ | `platform` | `"threads"` |
66
+
67
+ ### Post
68
+
69
+ Iterate `getUserThreads` pages; for each element `thread_items[]`:
70
+
71
+ | Canonical | Source |
72
+ |-----------|--------|
73
+ | `id` | `post.pk` or `post.code` (string) |
74
+ | `author_id` | `user_id` from profile context |
75
+ | `text` | `post.caption.text` |
76
+ | `media` | `image_versions2`, `carousel_media`, or video URLs in `post` |
77
+ | `platform` | `"threads"` |
78
+
79
+ ### Pagination
80
+
81
+ Inspect raw response for `next_max_id` / cursor fields during spike; store in adapter as opaque `next_cursor`.
82
+
83
+ ---
84
+
85
+ ## TikTok (`TikTokApi`)
86
+
87
+ Keys vary by endpoint version; capture golden fixtures before release.
88
+
89
+ ### User (typical)
90
+
91
+ | Canonical | Source |
92
+ |-----------|--------|
93
+ | `id` | `user["id"]` or `userInfo.user.id` (string) |
94
+ | `username` | `uniqueId` |
95
+ | `display_name` | `nickname` |
96
+ | `bio` | `signature` |
97
+ | `avatar_url` | `avatarLarger` or `avatarMedium` |
98
+ | `follower_count` | `stats.followerCount` |
99
+ | `platform` | `"tiktok"` |
100
+
101
+ ### Post / video
102
+
103
+ | Canonical | Source |
104
+ |-----------|--------|
105
+ | `id` | `id` |
106
+ | `author_id` | `author.id` |
107
+ | `text` | `desc` |
108
+ | `created_at` | `createTime` (epoch s) → ISO UTC |
109
+ | `url` | `https://www.tiktok.com/@{uniqueId}/video/{id}` |
110
+ | `media` | `[{ "type": "video", "url": playAddr or downloadAddr }]` |
111
+ | `like_count` | `stats.diggCount` |
112
+ | `comment_count` | `stats.commentCount` |
113
+ | `platform` | `"tiktok"` |
114
+
115
+ ### Comments
116
+
117
+ Map `comment.id`, `text`, `create_time`, `user.uid`.
118
+
119
+ ---
120
+
121
+ ## X (API v2, read-only)
122
+
123
+ Responses are tweepy-shaped: `data`, `includes`, `meta`. The adapter stays inert without `X_BEARER_TOKEN`.
124
+
125
+ ### User
126
+
127
+ | Canonical | Source |
128
+ |-----------|--------|
129
+ | `id` | `data.id` (string) |
130
+ | `username` | `data.username` |
131
+ | `display_name` | `data.name` |
132
+ | `bio` | `data.description` |
133
+ | `profile_url` | `https://x.com/{username}` |
134
+ | `avatar_url` | `data.profile_image_url` |
135
+ | `follower_count` | `data.public_metrics.followers_count` |
136
+ | `following_count` | `data.public_metrics.following_count` |
137
+ | `is_verified` | `data.verified` |
138
+ | `platform` | `"x"` |
139
+
140
+ ### Post
141
+
142
+ | Canonical | Source |
143
+ |-----------|--------|
144
+ | `id` | `data.id` |
145
+ | `author_id` | `data.author_id` |
146
+ | `text` | `data.text` |
147
+ | `created_at` | `data.created_at` (ISO-8601) |
148
+ | `url` | `https://x.com/{username}/status/{id}` when `includes.users` has the author, else `https://x.com/i/web/status/{id}` |
149
+ | `media` | `includes.media` joined on `attachments.media_keys`; `video` and `animated_gif` map to `video`, other types to `image` |
150
+ | `like_count` | `data.public_metrics.like_count` |
151
+ | `comment_count` | `data.public_metrics.reply_count` |
152
+ | `platform` | `"x"` |
153
+
154
+ ### Comment
155
+
156
+ Replies come from recent search `conversation_id:{post_id} is:reply`. The root tweet id is skipped.
157
+
158
+ | Canonical | Source |
159
+ |-----------|--------|
160
+ | `id` | tweet `id` |
161
+ | `post_id` | `conversation_id` or the requested post id |
162
+ | `author_id` | `author_id` |
163
+ | `text` | `text` |
164
+ | `created_at` | `created_at` |
165
+ | `platform` | `"x"` |
166
+
167
+ Pagination: `meta.next_token` → `next_cursor`. `has_more` is true when that token is a non-empty string.
168
+
169
+ ---
170
+
171
+ ## Facebook (v1 deferred)
172
+
173
+ v1 does not ship a Facebook adapter. When added, use Graph API:
174
+
175
+ | Canonical | Source |
176
+ |-----------|--------|
177
+ | User | `/page-id?fields=id,name,about,fan_count,picture` |
178
+ | Post | `/page-id/posts` with `message`, `created_time`, `id` |
179
+ | `next_cursor` | `paging.cursors.after` |
180
+
181
+ Do not port deprecated `facebook-api-wrapper` PHP.
@@ -0,0 +1,17 @@
1
+ # Facebook adapter — v1 deferral
2
+
3
+ **Decision (2025-11-20):** The unified `rrss` package **does not** include a Facebook Graph adapter in v1.
4
+
5
+ **Rationale:**
6
+
7
+ - `facebook-api-wrapper` in this workspace is **deprecated** (no upstream changes).
8
+ - `facebook-wrapper` targets Express OAuth canvas apps, not a headless pull-data SDK.
9
+ - Meta’s official Graph API is the correct path for new work; scope belongs in a dedicated v1.1 spike with app review checklist.
10
+
11
+ **v1 behavior:** `Client("facebook")` raises `NotSupported` with a link to this document and [Meta Graph docs](https://developers.facebook.com/docs/graph-api).
12
+
13
+ **v1.1 spike exit criteria:**
14
+
15
+ - App with `pages_read_engagement` or product-appropriate scopes approved.
16
+ - Thin adapter: `get_user` (page), `list_user_posts`, `list_post_comments` only.
17
+ - No FQL; Graph v2+ JSON only.