rrss-sdk-wrapper 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rrss_sdk_wrapper-0.2.0/.env.example +22 -0
- rrss_sdk_wrapper-0.2.0/.gitignore +14 -0
- rrss_sdk_wrapper-0.2.0/CHANGELOG.md +44 -0
- rrss_sdk_wrapper-0.2.0/PKG-INFO +173 -0
- rrss_sdk_wrapper-0.2.0/README.md +146 -0
- rrss_sdk_wrapper-0.2.0/SECURITY.md +9 -0
- rrss_sdk_wrapper-0.2.0/compose.yaml +21 -0
- rrss_sdk_wrapper-0.2.0/docs/generated-clients.md +19 -0
- rrss_sdk_wrapper-0.2.0/docs/index.md +23 -0
- rrss_sdk_wrapper-0.2.0/docs/mapping.md +181 -0
- rrss_sdk_wrapper-0.2.0/docs/platforms/facebook-v1-deferral.md +17 -0
- rrss_sdk_wrapper-0.2.0/docs/platforms/instagram.md +32 -0
- rrss_sdk_wrapper-0.2.0/docs/platforms/threads.md +41 -0
- rrss_sdk_wrapper-0.2.0/docs/platforms/tiktok.md +33 -0
- rrss_sdk_wrapper-0.2.0/docs/platforms/x.md +38 -0
- rrss_sdk_wrapper-0.2.0/generated/go/client.go +167 -0
- rrss_sdk_wrapper-0.2.0/generated/go/go.mod +3 -0
- rrss_sdk_wrapper-0.2.0/generated/go/models.go +60 -0
- rrss_sdk_wrapper-0.2.0/generated/python/rrss_client.py +233 -0
- rrss_sdk_wrapper-0.2.0/generated/rust/Cargo.toml +8 -0
- rrss_sdk_wrapper-0.2.0/generated/rust/src/lib.rs +162 -0
- rrss_sdk_wrapper-0.2.0/mkdocs.yml +18 -0
- rrss_sdk_wrapper-0.2.0/pyproject.toml +52 -0
- rrss_sdk_wrapper-0.2.0/rrss/__init__.py +50 -0
- rrss_sdk_wrapper-0.2.0/rrss/adapters/base.py +80 -0
- rrss_sdk_wrapper-0.2.0/rrss/adapters/instagram.py +123 -0
- rrss_sdk_wrapper-0.2.0/rrss/adapters/threads.py +62 -0
- rrss_sdk_wrapper-0.2.0/rrss/adapters/tiktok.py +76 -0
- rrss_sdk_wrapper-0.2.0/rrss/adapters/x.py +224 -0
- rrss_sdk_wrapper-0.2.0/rrss/adapters/x_stub.py +5 -0
- rrss_sdk_wrapper-0.2.0/rrss/auth/__init__.py +19 -0
- rrss_sdk_wrapper-0.2.0/rrss/cache.py +209 -0
- rrss_sdk_wrapper-0.2.0/rrss/cli.py +78 -0
- rrss_sdk_wrapper-0.2.0/rrss/client.py +35 -0
- rrss_sdk_wrapper-0.2.0/rrss/errors.py +31 -0
- rrss_sdk_wrapper-0.2.0/rrss/gateway.py +306 -0
- rrss_sdk_wrapper-0.2.0/rrss/models.py +61 -0
- rrss_sdk_wrapper-0.2.0/rrss/normalize/__init__.py +1 -0
- rrss_sdk_wrapper-0.2.0/rrss/normalize/instagram.py +82 -0
- rrss_sdk_wrapper-0.2.0/rrss/normalize/threads.py +82 -0
- rrss_sdk_wrapper-0.2.0/rrss/normalize/tiktok.py +51 -0
- rrss_sdk_wrapper-0.2.0/rrss/normalize/x.py +192 -0
- rrss_sdk_wrapper-0.2.0/rrss/ratelimit.py +30 -0
- rrss_sdk_wrapper-0.2.0/rrss/serialize.py +216 -0
- rrss_sdk_wrapper-0.2.0/rrss/stream.py +198 -0
- rrss_sdk_wrapper-0.2.0/schemas/openapi.yaml +378 -0
- rrss_sdk_wrapper-0.2.0/schemas/rrss-v1.json +77 -0
- rrss_sdk_wrapper-0.2.0/scripts/generate_client_stub.py +696 -0
- rrss_sdk_wrapper-0.2.0/sidecar/threads/.dockerignore +2 -0
- rrss_sdk_wrapper-0.2.0/sidecar/threads/Dockerfile +19 -0
- rrss_sdk_wrapper-0.2.0/sidecar/threads/README.md +30 -0
- rrss_sdk_wrapper-0.2.0/sidecar/threads/package-lock.json +840 -0
- rrss_sdk_wrapper-0.2.0/sidecar/threads/package.json +13 -0
- rrss_sdk_wrapper-0.2.0/sidecar/threads/server.js +24 -0
- rrss_sdk_wrapper-0.2.0/tests/fakes.py +64 -0
- rrss_sdk_wrapper-0.2.0/tests/fixtures/instagram/media.json +10 -0
- rrss_sdk_wrapper-0.2.0/tests/fixtures/instagram/user.json +10 -0
- rrss_sdk_wrapper-0.2.0/tests/fixtures/threads/threads.json +17 -0
- rrss_sdk_wrapper-0.2.0/tests/fixtures/threads/user.json +6 -0
- rrss_sdk_wrapper-0.2.0/tests/fixtures/tiktok/user.json +7 -0
- rrss_sdk_wrapper-0.2.0/tests/fixtures/tiktok/video.json +8 -0
- rrss_sdk_wrapper-0.2.0/tests/fixtures/x/replies.json +21 -0
- rrss_sdk_wrapper-0.2.0/tests/fixtures/x/tweets.json +20 -0
- rrss_sdk_wrapper-0.2.0/tests/fixtures/x/user.json +17 -0
- rrss_sdk_wrapper-0.2.0/tests/test_adapter_edges.py +223 -0
- rrss_sdk_wrapper-0.2.0/tests/test_cache.py +132 -0
- rrss_sdk_wrapper-0.2.0/tests/test_cli.py +46 -0
- rrss_sdk_wrapper-0.2.0/tests/test_client.py +94 -0
- rrss_sdk_wrapper-0.2.0/tests/test_errors_retry_integration.py +176 -0
- rrss_sdk_wrapper-0.2.0/tests/test_gateway.py +168 -0
- rrss_sdk_wrapper-0.2.0/tests/test_generated_client.py +115 -0
- rrss_sdk_wrapper-0.2.0/tests/test_normalize.py +26 -0
- rrss_sdk_wrapper-0.2.0/tests/test_normalize_edges.py +191 -0
- rrss_sdk_wrapper-0.2.0/tests/test_stream.py +91 -0
- rrss_sdk_wrapper-0.2.0/tests/test_x.py +351 -0
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# Instagram
|
|
2
|
+
INSTAGRAM_USERNAME=
|
|
3
|
+
INSTAGRAM_PASSWORD=
|
|
4
|
+
INSTAGRAM_SESSION_PATH=
|
|
5
|
+
|
|
6
|
+
# TikTok
|
|
7
|
+
TIKTOK_MS_TOKEN=
|
|
8
|
+
TIKTOK_PROXY=
|
|
9
|
+
|
|
10
|
+
# Threads sidecar (Option A)
|
|
11
|
+
RRSS_THREADS_SIDECAR_URL=http://127.0.0.1:1877
|
|
12
|
+
|
|
13
|
+
# X API v2 app-only bearer. Empty keeps the read adapter inert (no requests).
|
|
14
|
+
X_BEARER_TOKEN=
|
|
15
|
+
|
|
16
|
+
# Optional gateway (stdlib HTTP). Leave the key empty to disable the check.
|
|
17
|
+
RRSS_GATEWAY_API_KEY=
|
|
18
|
+
RRSS_GATEWAY_HOST=127.0.0.1
|
|
19
|
+
RRSS_GATEWAY_PORT=8080
|
|
20
|
+
|
|
21
|
+
# Optional Redis cache. Unset uses the in-memory cache.
|
|
22
|
+
# RRSS_REDIS_URL=redis://localhost:6379/0
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to the `rrss` PyPI package are documented here.
|
|
4
|
+
|
|
5
|
+
## Unreleased
|
|
6
|
+
|
|
7
|
+
Not published. No release tag.
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
|
|
11
|
+
- Platform guides for Instagram, Threads, TikTok, and X (`docs/platforms/`).
|
|
12
|
+
- X read-only adapter (X API v2, tweepy-shaped). Inert without `X_BEARER_TOKEN`. Optional extra `rrss[x]`.
|
|
13
|
+
- Threads sidecar image and `compose.yaml`. Health check is `http://127.0.0.1:1877/health`.
|
|
14
|
+
- GitHub bug template (platform, redacted credentials, `include_raw`) and Dependabot.
|
|
15
|
+
- Optional MkDocs config. `pip install -e ".[docs]"` then `mkdocs build`.
|
|
16
|
+
|
|
17
|
+
## 0.2.0 — 2026-11-20
|
|
18
|
+
|
|
19
|
+
Not published to PyPI. No release tag.
|
|
20
|
+
|
|
21
|
+
### Added
|
|
22
|
+
|
|
23
|
+
- Polling helpers (`poll`, `poll_user`, `poll_user_posts`) and an in-memory `WebhookStub`.
|
|
24
|
+
- `CachingAdapter` with `MemoryCache` and an optional `RedisCache` (`rrss[cache]`).
|
|
25
|
+
- CLI: `rrss pull user <platform> <ref>`.
|
|
26
|
+
- Stdlib HTTP gateway for the canonical read routes, plus `schemas/openapi.yaml`.
|
|
27
|
+
- Generated clients from `schemas/rrss-v1.json` (Python and Go HTTP stubs, Rust types and URL builders). See `docs/generated-clients.md`.
|
|
28
|
+
|
|
29
|
+
## 0.1.0 — 2025-11-20
|
|
30
|
+
|
|
31
|
+
### Added
|
|
32
|
+
|
|
33
|
+
- Initial PyPI release (`rrss` meta-package).
|
|
34
|
+
- Instagram, Threads, and TikTok adapters with golden-fixture normalization tests.
|
|
35
|
+
- Unified error taxonomy (`AUTH_FAILED`, `RATE_LIMITED`, `NOT_FOUND`, `NOT_SUPPORTED`, `PLATFORM_ERROR`).
|
|
36
|
+
- Exponential backoff retries on `RATE_LIMITED` (`rrss.ratelimit.with_retries`), wired for Instagram user fetch.
|
|
37
|
+
- Threads Node sidecar contract under `python/sidecar/threads/` (local HTTP on `127.0.0.1:1877`).
|
|
38
|
+
- X read-only stub (`NOT_SUPPORTED` until X API v2 credentials are configured).
|
|
39
|
+
- JSON Schema `schemas/rrss-v1.json` for canonical types.
|
|
40
|
+
|
|
41
|
+
### Notes
|
|
42
|
+
|
|
43
|
+
- Facebook Graph adapter deferred; see `docs/platforms/facebook-v1-deferral.md`.
|
|
44
|
+
- Live platform credentials are optional; inject mock upstream clients for tests.
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: rrss-sdk-wrapper
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Unified social media data client (Instagram, Threads, TikTok)
|
|
5
|
+
Author: Ferran Suils
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Requires-Python: >=3.11
|
|
8
|
+
Provides-Extra: all
|
|
9
|
+
Requires-Dist: httpx>=0.27; extra == 'all'
|
|
10
|
+
Requires-Dist: tweepy>=4.14; extra == 'all'
|
|
11
|
+
Provides-Extra: cache
|
|
12
|
+
Requires-Dist: redis>=5; extra == 'cache'
|
|
13
|
+
Provides-Extra: dev
|
|
14
|
+
Requires-Dist: httpx>=0.27; extra == 'dev'
|
|
15
|
+
Requires-Dist: mypy>=1.11; extra == 'dev'
|
|
16
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
17
|
+
Requires-Dist: ruff>=0.6; extra == 'dev'
|
|
18
|
+
Provides-Extra: docs
|
|
19
|
+
Requires-Dist: mkdocs>=1.6; extra == 'docs'
|
|
20
|
+
Provides-Extra: instagram
|
|
21
|
+
Provides-Extra: threads
|
|
22
|
+
Requires-Dist: httpx>=0.27; extra == 'threads'
|
|
23
|
+
Provides-Extra: tiktok
|
|
24
|
+
Provides-Extra: x
|
|
25
|
+
Requires-Dist: tweepy>=4.14; extra == 'x'
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
|
|
28
|
+
# rrss
|
|
29
|
+
|
|
30
|
+
Python meta-package for normalized social profile and post data across Instagram, Threads, and TikTok.
|
|
31
|
+
|
|
32
|
+
## Install
|
|
33
|
+
|
|
34
|
+
Version 0.2.0 in this tree is not published. PyPI still serves 0.1.0 until a tagged release.
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
pip install rrss-sdk-wrapper
|
|
38
|
+
# optional extras
|
|
39
|
+
pip install "rrss-sdk-wrapper[threads]"
|
|
40
|
+
pip install "rrss-sdk-wrapper[x]" # tweepy; live X calls only
|
|
41
|
+
pip install "rrss-sdk-wrapper[cache]" # redis; tests use the in-memory cache
|
|
42
|
+
pip install "rrss-sdk-wrapper[docs]" # mkdocs
|
|
43
|
+
pip install "rrss-sdk-wrapper[all]"
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
From this repository:
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
cd python
|
|
50
|
+
pip install -e .
|
|
51
|
+
pip install -e ".[dev]" # ruff, mypy, pytest
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
## Quick start
|
|
55
|
+
|
|
56
|
+
```python
|
|
57
|
+
from rrss import Client
|
|
58
|
+
|
|
59
|
+
# Adapters accept injectable upstream clients for testing; optional deps for live usage.
|
|
60
|
+
ig = Client("instagram", username="u", password="p")
|
|
61
|
+
user = ig.get_user("someone")
|
|
62
|
+
for page in ig.iter_user_posts("someone", limit=50):
|
|
63
|
+
for post in page.items:
|
|
64
|
+
print(post.text)
|
|
65
|
+
|
|
66
|
+
# Re-fetch until the profile changes. Pass max_polls in tests (interval may be 0).
|
|
67
|
+
for profile in ig.poll_user("someone", interval=60, max_polls=5):
|
|
68
|
+
print(profile.follower_count)
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
`WebhookStub` records push payloads in memory (`accept`, then iterate). The gateway also accepts `POST /v1/webhooks` into that stub. Neither path needs a public callback URL.
|
|
72
|
+
|
|
73
|
+
## Cache
|
|
74
|
+
|
|
75
|
+
```python
|
|
76
|
+
from rrss import CachingAdapter, Client, MemoryCache
|
|
77
|
+
|
|
78
|
+
cached = CachingAdapter(Client("threads"), MemoryCache())
|
|
79
|
+
cached.get_user("someone") # second call is served from memory
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
Set `RRSS_REDIS_URL` and install `rrss-sdk-wrapper[cache]` to use `RedisCache`. The same `get` / `set` interface backs both. Default TTLs follow the lower bound in the plan: users 15 minutes, post lists 5 minutes, comments 1 minute.
|
|
83
|
+
|
|
84
|
+
## CLI
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
rrss pull user threads some_username
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
The command prints canonical user JSON. Credentials come from the environment (see `.env.example`), never from flags.
|
|
91
|
+
|
|
92
|
+
## HTTP gateway
|
|
93
|
+
|
|
94
|
+
```bash
|
|
95
|
+
RRSS_GATEWAY_API_KEY=dev python -m rrss.gateway
|
|
96
|
+
# GET /v1/{platform}/users/{user_ref}
|
|
97
|
+
# GET /v1/{platform}/users/{user_ref}/posts
|
|
98
|
+
# GET /openapi.yaml
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
The server is stdlib `http.server` with an injectable client factory, so tests run without live platform credentials. Spec: `schemas/openapi.yaml`.
|
|
102
|
+
|
|
103
|
+
Generated clients (Python, Go, and a Rust URL stub) are described in `docs/generated-clients.md`.
|
|
104
|
+
|
|
105
|
+
Copy `python/.env.example` to `.env` for local credentials (never commit secrets).
|
|
106
|
+
|
|
107
|
+
## Threads sidecar (Node)
|
|
108
|
+
|
|
109
|
+
The Threads adapter calls a small **Node sidecar** that wraps `threads-wrapper` over HTTP.
|
|
110
|
+
|
|
111
|
+
```bash
|
|
112
|
+
cd sidecar/threads
|
|
113
|
+
npm install
|
|
114
|
+
npm start # listens on 127.0.0.1:1877
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
Set `RRSS_THREADS_SIDECAR_URL` if the sidecar is not on the default URL. Contract: `GET /health`, `GET /users/:username`, `GET /users/:username/threads` (see api-wrapper plan §19).
|
|
118
|
+
|
|
119
|
+
```python
|
|
120
|
+
from rrss import Client
|
|
121
|
+
|
|
122
|
+
th = Client("threads") # uses http://127.0.0.1:1877 by default
|
|
123
|
+
user = th.get_user("threader")
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
The sidecar is **not** published to npm; run it from this repo.
|
|
127
|
+
|
|
128
|
+
```bash
|
|
129
|
+
cd python
|
|
130
|
+
docker compose up --build threads-sidecar
|
|
131
|
+
curl http://127.0.0.1:1877/health
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
Compose publishes `127.0.0.1:1877` only. See `docs/platforms/threads.md`.
|
|
135
|
+
|
|
136
|
+
## Errors and retries
|
|
137
|
+
|
|
138
|
+
Shared exceptions live in `rrss.errors`. Rate-limited upstream responses can be retried with `rrss.ratelimit.with_retries` (default: 3 attempts, exponential backoff with jitter). Instagram `get_user` uses this policy when the upstream raises `RateLimited`.
|
|
139
|
+
|
|
140
|
+
## Development
|
|
141
|
+
|
|
142
|
+
```bash
|
|
143
|
+
cd python
|
|
144
|
+
pip install -e ".[dev]"
|
|
145
|
+
ruff check .
|
|
146
|
+
mypy rrss
|
|
147
|
+
pytest -m "not integration"
|
|
148
|
+
pytest -m integration # optional: requires local Threads sidecar for health check
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
CI on pull requests runs `pytest -m "not integration"`. Nightly/workflow-dispatch jobs may run integration markers separately.
|
|
152
|
+
|
|
153
|
+
Optional docs site (not published with the package):
|
|
154
|
+
|
|
155
|
+
```bash
|
|
156
|
+
pip install -e ".[docs]"
|
|
157
|
+
mkdocs build # writes ./site
|
|
158
|
+
mkdocs serve # http://127.0.0.1:8000
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
See `docs/index.md`.
|
|
162
|
+
|
|
163
|
+
## Platforms
|
|
164
|
+
|
|
165
|
+
| Platform | Auth | Read / write | API | Guide |
|
|
166
|
+
|-----------|------------------------------|--------------|-----|-------|
|
|
167
|
+
| Instagram | Username/password session | Read only | Unofficial (`instagrapi`) | [docs/platforms/instagram.md](docs/platforms/instagram.md) |
|
|
168
|
+
| Threads | None (sidecar) | Read only | Unofficial | [docs/platforms/threads.md](docs/platforms/threads.md) |
|
|
169
|
+
| TikTok | `ms_token` | Read only | Unofficial | [docs/platforms/tiktok.md](docs/platforms/tiktok.md) |
|
|
170
|
+
| X | `X_BEARER_TOKEN` (inert if unset) | Read only | Official v2 | [docs/platforms/x.md](docs/platforms/x.md) |
|
|
171
|
+
| Facebook | — | Deferred | — | [docs/platforms/facebook-v1-deferral.md](docs/platforms/facebook-v1-deferral.md) |
|
|
172
|
+
|
|
173
|
+
Project specification: see the workspace `docs/api-wrapper-plan.md` for the full contract.
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
# rrss
|
|
2
|
+
|
|
3
|
+
Python meta-package for normalized social profile and post data across Instagram, Threads, and TikTok.
|
|
4
|
+
|
|
5
|
+
## Install
|
|
6
|
+
|
|
7
|
+
Version 0.2.0 in this tree is not published. PyPI still serves 0.1.0 until a tagged release.
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
pip install rrss-sdk-wrapper
|
|
11
|
+
# optional extras
|
|
12
|
+
pip install "rrss-sdk-wrapper[threads]"
|
|
13
|
+
pip install "rrss-sdk-wrapper[x]" # tweepy; live X calls only
|
|
14
|
+
pip install "rrss-sdk-wrapper[cache]" # redis; tests use the in-memory cache
|
|
15
|
+
pip install "rrss-sdk-wrapper[docs]" # mkdocs
|
|
16
|
+
pip install "rrss-sdk-wrapper[all]"
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
From this repository:
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
cd python
|
|
23
|
+
pip install -e .
|
|
24
|
+
pip install -e ".[dev]" # ruff, mypy, pytest
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
## Quick start
|
|
28
|
+
|
|
29
|
+
```python
|
|
30
|
+
from rrss import Client
|
|
31
|
+
|
|
32
|
+
# Adapters accept injectable upstream clients for testing; optional deps for live usage.
|
|
33
|
+
ig = Client("instagram", username="u", password="p")
|
|
34
|
+
user = ig.get_user("someone")
|
|
35
|
+
for page in ig.iter_user_posts("someone", limit=50):
|
|
36
|
+
for post in page.items:
|
|
37
|
+
print(post.text)
|
|
38
|
+
|
|
39
|
+
# Re-fetch until the profile changes. Pass max_polls in tests (interval may be 0).
|
|
40
|
+
for profile in ig.poll_user("someone", interval=60, max_polls=5):
|
|
41
|
+
print(profile.follower_count)
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
`WebhookStub` records push payloads in memory (`accept`, then iterate). The gateway also accepts `POST /v1/webhooks` into that stub. Neither path needs a public callback URL.
|
|
45
|
+
|
|
46
|
+
## Cache
|
|
47
|
+
|
|
48
|
+
```python
|
|
49
|
+
from rrss import CachingAdapter, Client, MemoryCache
|
|
50
|
+
|
|
51
|
+
cached = CachingAdapter(Client("threads"), MemoryCache())
|
|
52
|
+
cached.get_user("someone") # second call is served from memory
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
Set `RRSS_REDIS_URL` and install `rrss-sdk-wrapper[cache]` to use `RedisCache`. The same `get` / `set` interface backs both. Default TTLs follow the lower bound in the plan: users 15 minutes, post lists 5 minutes, comments 1 minute.
|
|
56
|
+
|
|
57
|
+
## CLI
|
|
58
|
+
|
|
59
|
+
```bash
|
|
60
|
+
rrss pull user threads some_username
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
The command prints canonical user JSON. Credentials come from the environment (see `.env.example`), never from flags.
|
|
64
|
+
|
|
65
|
+
## HTTP gateway
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
RRSS_GATEWAY_API_KEY=dev python -m rrss.gateway
|
|
69
|
+
# GET /v1/{platform}/users/{user_ref}
|
|
70
|
+
# GET /v1/{platform}/users/{user_ref}/posts
|
|
71
|
+
# GET /openapi.yaml
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
The server is stdlib `http.server` with an injectable client factory, so tests run without live platform credentials. Spec: `schemas/openapi.yaml`.
|
|
75
|
+
|
|
76
|
+
Generated clients (Python, Go, and a Rust URL stub) are described in `docs/generated-clients.md`.
|
|
77
|
+
|
|
78
|
+
Copy `python/.env.example` to `.env` for local credentials (never commit secrets).
|
|
79
|
+
|
|
80
|
+
## Threads sidecar (Node)
|
|
81
|
+
|
|
82
|
+
The Threads adapter calls a small **Node sidecar** that wraps `threads-wrapper` over HTTP.
|
|
83
|
+
|
|
84
|
+
```bash
|
|
85
|
+
cd sidecar/threads
|
|
86
|
+
npm install
|
|
87
|
+
npm start # listens on 127.0.0.1:1877
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
Set `RRSS_THREADS_SIDECAR_URL` if the sidecar is not on the default URL. Contract: `GET /health`, `GET /users/:username`, `GET /users/:username/threads` (see api-wrapper plan §19).
|
|
91
|
+
|
|
92
|
+
```python
|
|
93
|
+
from rrss import Client
|
|
94
|
+
|
|
95
|
+
th = Client("threads") # uses http://127.0.0.1:1877 by default
|
|
96
|
+
user = th.get_user("threader")
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
The sidecar is **not** published to npm; run it from this repo.
|
|
100
|
+
|
|
101
|
+
```bash
|
|
102
|
+
cd python
|
|
103
|
+
docker compose up --build threads-sidecar
|
|
104
|
+
curl http://127.0.0.1:1877/health
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
Compose publishes `127.0.0.1:1877` only. See `docs/platforms/threads.md`.
|
|
108
|
+
|
|
109
|
+
## Errors and retries
|
|
110
|
+
|
|
111
|
+
Shared exceptions live in `rrss.errors`. Rate-limited upstream responses can be retried with `rrss.ratelimit.with_retries` (default: 3 attempts, exponential backoff with jitter). Instagram `get_user` uses this policy when the upstream raises `RateLimited`.
|
|
112
|
+
|
|
113
|
+
## Development
|
|
114
|
+
|
|
115
|
+
```bash
|
|
116
|
+
cd python
|
|
117
|
+
pip install -e ".[dev]"
|
|
118
|
+
ruff check .
|
|
119
|
+
mypy rrss
|
|
120
|
+
pytest -m "not integration"
|
|
121
|
+
pytest -m integration # optional: requires local Threads sidecar for health check
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
CI on pull requests runs `pytest -m "not integration"`. Nightly/workflow-dispatch jobs may run integration markers separately.
|
|
125
|
+
|
|
126
|
+
Optional docs site (not published with the package):
|
|
127
|
+
|
|
128
|
+
```bash
|
|
129
|
+
pip install -e ".[docs]"
|
|
130
|
+
mkdocs build # writes ./site
|
|
131
|
+
mkdocs serve # http://127.0.0.1:8000
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
See `docs/index.md`.
|
|
135
|
+
|
|
136
|
+
## Platforms
|
|
137
|
+
|
|
138
|
+
| Platform | Auth | Read / write | API | Guide |
|
|
139
|
+
|-----------|------------------------------|--------------|-----|-------|
|
|
140
|
+
| Instagram | Username/password session | Read only | Unofficial (`instagrapi`) | [docs/platforms/instagram.md](docs/platforms/instagram.md) |
|
|
141
|
+
| Threads | None (sidecar) | Read only | Unofficial | [docs/platforms/threads.md](docs/platforms/threads.md) |
|
|
142
|
+
| TikTok | `ms_token` | Read only | Unofficial | [docs/platforms/tiktok.md](docs/platforms/tiktok.md) |
|
|
143
|
+
| X | `X_BEARER_TOKEN` (inert if unset) | Read only | Official v2 | [docs/platforms/x.md](docs/platforms/x.md) |
|
|
144
|
+
| Facebook | — | Deferred | — | [docs/platforms/facebook-v1-deferral.md](docs/platforms/facebook-v1-deferral.md) |
|
|
145
|
+
|
|
146
|
+
Project specification: see the workspace `docs/api-wrapper-plan.md` for the full contract.
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# Security
|
|
2
|
+
|
|
3
|
+
- Never commit credentials or session files. Use `.env` locally (see `.env.example`).
|
|
4
|
+
- Logs must not include passwords, tokens, or full session payloads.
|
|
5
|
+
- The HTTP gateway does not log request lines. Platform passwords stay in the server environment; callers send `X-RRSS-Key` only. Do not put secrets in webhook bodies you store.
|
|
6
|
+
- `RRSS_REDIS_URL` may embed a password. Do not print it.
|
|
7
|
+
- Report security issues privately to the repository owner.
|
|
8
|
+
- GitHub issues use the bug template: name the platform, paste an `include_raw` snippet only after stripping secrets, and replace every credential with `REDACTED`.
|
|
9
|
+
- Dependabot opens dependency updates. It does not receive application secrets.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# Local Threads sidecar. From python/: docker compose up --build threads-sidecar
|
|
2
|
+
# Host URL matches the Python default: http://127.0.0.1:1877
|
|
3
|
+
services:
|
|
4
|
+
threads-sidecar:
|
|
5
|
+
build:
|
|
6
|
+
context: ./sidecar/threads
|
|
7
|
+
ports:
|
|
8
|
+
- "127.0.0.1:1877:1877"
|
|
9
|
+
environment:
|
|
10
|
+
HOST: 0.0.0.0
|
|
11
|
+
PORT: "1877"
|
|
12
|
+
healthcheck:
|
|
13
|
+
test:
|
|
14
|
+
- CMD
|
|
15
|
+
- node
|
|
16
|
+
- -e
|
|
17
|
+
- "fetch('http://127.0.0.1:1877/health').then((r)=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))"
|
|
18
|
+
interval: 10s
|
|
19
|
+
timeout: 3s
|
|
20
|
+
retries: 3
|
|
21
|
+
start_period: 5s
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# Generated clients
|
|
2
|
+
|
|
3
|
+
Stubs are generated from [`schemas/rrss-v1.json`](../schemas/rrss-v1.json). The HTTP paths match [`schemas/openapi.yaml`](../schemas/openapi.yaml).
|
|
4
|
+
|
|
5
|
+
Regenerate from `python/`:
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
python scripts/generate_client_stub.py
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
| Language | Path | What it does |
|
|
12
|
+
|----------|------|----------------|
|
|
13
|
+
| Python | `generated/python/rrss_client.py` | Dataclasses plus a stdlib HTTP client (`RrssClient`) |
|
|
14
|
+
| Go | `generated/go/` | `encoding/json` models and an `net/http` client. `go build .` |
|
|
15
|
+
| Rust | `generated/rust/` | Canonical structs and URL builders. `rustc --edition 2021 --crate-type lib src/lib.rs` |
|
|
16
|
+
|
|
17
|
+
Go and Rust toolchains were available when this slice landed (`go` 1.22, `rustc` 1.83). Pytest compiles both when the binaries are on `PATH` and skips that check when they are not, so CI can stay Python-only.
|
|
18
|
+
|
|
19
|
+
The Rust crate does not perform HTTP. Its standard library has no HTTP or JSON implementation, and the generator does not vendor `serde` or `reqwest`. Call the gateway with the Python or Go stub. The Rust types and `RrssClient::{get_user_url, list_user_posts_url, ...}` stay aligned with the schema for a later crate that adds those dependencies.
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# rrss
|
|
2
|
+
|
|
3
|
+
Unified read client for Instagram, Threads, TikTok, and X. Facebook stays deferred.
|
|
4
|
+
|
|
5
|
+
Install and the Python quick start live in the package [README](https://github.com/FerranSuils/scraper-rrss/blob/master/python/README.md). This site is the longer reference:
|
|
6
|
+
|
|
7
|
+
- [Field mapping](mapping.md)
|
|
8
|
+
- [Instagram](platforms/instagram.md), [Threads](platforms/threads.md), [TikTok](platforms/tiktok.md), [X](platforms/x.md)
|
|
9
|
+
- [Facebook deferral](platforms/facebook-v1-deferral.md)
|
|
10
|
+
- [Generated clients](generated-clients.md)
|
|
11
|
+
|
|
12
|
+
## Build this site
|
|
13
|
+
|
|
14
|
+
MkDocs is optional. The config is `mkdocs.yml` next to this `docs/` directory. The built HTML is gitignored.
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
cd python
|
|
18
|
+
pip install -e ".[docs]"
|
|
19
|
+
mkdocs build # writes ./site
|
|
20
|
+
mkdocs serve # http://127.0.0.1:8000
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
`mkdocs build` does not publish anywhere. PyPI upload is a separate, tagged release workflow and is not part of building docs.
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
# RRSS field mapping (native → canonical)
|
|
2
|
+
|
|
3
|
+
Canonical models: [rrss-v1.json](../schemas/rrss-v1.json). Platform notes: [platforms/](platforms/instagram.md).
|
|
4
|
+
|
|
5
|
+
## Instagram (`instagrapi`)
|
|
6
|
+
|
|
7
|
+
### User
|
|
8
|
+
|
|
9
|
+
| Canonical | Source |
|
|
10
|
+
|-----------|--------|
|
|
11
|
+
| `id` | `str(user.pk)` |
|
|
12
|
+
| `username` | `user.username` |
|
|
13
|
+
| `display_name` | `user.full_name` |
|
|
14
|
+
| `bio` | `user.biography` |
|
|
15
|
+
| `follower_count` | `user.follower_count` |
|
|
16
|
+
| `following_count` | `user.following_count` |
|
|
17
|
+
| `is_verified` | `user.is_verified` |
|
|
18
|
+
| `avatar_url` | `user.profile_pic_url_hd` or `profile_pic_url` |
|
|
19
|
+
| `profile_url` | `https://www.instagram.com/{username}/` |
|
|
20
|
+
| `platform` | `"instagram"` |
|
|
21
|
+
|
|
22
|
+
### Post
|
|
23
|
+
|
|
24
|
+
| Canonical | Source |
|
|
25
|
+
|-----------|--------|
|
|
26
|
+
| `id` | `str(media.pk)` or `media.id` |
|
|
27
|
+
| `author_id` | `str(media.user.pk)` |
|
|
28
|
+
| `text` | `media.caption_text` or `media.caption.text` if present |
|
|
29
|
+
| `created_at` | `datetime.utcfromtimestamp(media.taken_at).isoformat() + "Z"` |
|
|
30
|
+
| `url` | `https://www.instagram.com/p/{media.code}/` |
|
|
31
|
+
| `media` | Carousel: each resource `url` + `type` image/video; else single `thumbnail_url` / `video_url` |
|
|
32
|
+
| `like_count` | `media.like_count` |
|
|
33
|
+
| `comment_count` | `media.comment_count` |
|
|
34
|
+
| `platform` | `"instagram"` |
|
|
35
|
+
|
|
36
|
+
### Comment
|
|
37
|
+
|
|
38
|
+
| Canonical | Source |
|
|
39
|
+
|-----------|--------|
|
|
40
|
+
| `id` | `str(comment.pk)` |
|
|
41
|
+
| `post_id` | parent media id |
|
|
42
|
+
| `author_id` | `str(comment.user.pk)` |
|
|
43
|
+
| `text` | `comment.text` |
|
|
44
|
+
| `created_at` | from `comment.created_at_utc` if available |
|
|
45
|
+
| `platform` | `"instagram"` |
|
|
46
|
+
|
|
47
|
+
### Pagination
|
|
48
|
+
|
|
49
|
+
`list_user_posts`: use `max_id` from last item; expose as `next_cursor`.
|
|
50
|
+
|
|
51
|
+
---
|
|
52
|
+
|
|
53
|
+
## Threads (`threads-wrapper` via sidecar)
|
|
54
|
+
|
|
55
|
+
### User
|
|
56
|
+
|
|
57
|
+
| Canonical | Source |
|
|
58
|
+
|-----------|--------|
|
|
59
|
+
| `id` | `user_id` |
|
|
60
|
+
| `username` | path / lookup username |
|
|
61
|
+
| `display_name` | `full_name` |
|
|
62
|
+
| `bio` | `biography` |
|
|
63
|
+
| `follower_count` | `follower_count` |
|
|
64
|
+
| `profile_url` | `bio_links[0].url` when present |
|
|
65
|
+
| `platform` | `"threads"` |
|
|
66
|
+
|
|
67
|
+
### Post
|
|
68
|
+
|
|
69
|
+
Iterate `getUserThreads` pages; for each element `thread_items[]`:
|
|
70
|
+
|
|
71
|
+
| Canonical | Source |
|
|
72
|
+
|-----------|--------|
|
|
73
|
+
| `id` | `post.pk` or `post.code` (string) |
|
|
74
|
+
| `author_id` | `user_id` from profile context |
|
|
75
|
+
| `text` | `post.caption.text` |
|
|
76
|
+
| `media` | `image_versions2`, `carousel_media`, or video URLs in `post` |
|
|
77
|
+
| `platform` | `"threads"` |
|
|
78
|
+
|
|
79
|
+
### Pagination
|
|
80
|
+
|
|
81
|
+
Inspect raw response for `next_max_id` / cursor fields during spike; store in adapter as opaque `next_cursor`.
|
|
82
|
+
|
|
83
|
+
---
|
|
84
|
+
|
|
85
|
+
## TikTok (`TikTokApi`)
|
|
86
|
+
|
|
87
|
+
Keys vary by endpoint version; capture golden fixtures before release.
|
|
88
|
+
|
|
89
|
+
### User (typical)
|
|
90
|
+
|
|
91
|
+
| Canonical | Source |
|
|
92
|
+
|-----------|--------|
|
|
93
|
+
| `id` | `user["id"]` or `userInfo.user.id` (string) |
|
|
94
|
+
| `username` | `uniqueId` |
|
|
95
|
+
| `display_name` | `nickname` |
|
|
96
|
+
| `bio` | `signature` |
|
|
97
|
+
| `avatar_url` | `avatarLarger` or `avatarMedium` |
|
|
98
|
+
| `follower_count` | `stats.followerCount` |
|
|
99
|
+
| `platform` | `"tiktok"` |
|
|
100
|
+
|
|
101
|
+
### Post / video
|
|
102
|
+
|
|
103
|
+
| Canonical | Source |
|
|
104
|
+
|-----------|--------|
|
|
105
|
+
| `id` | `id` |
|
|
106
|
+
| `author_id` | `author.id` |
|
|
107
|
+
| `text` | `desc` |
|
|
108
|
+
| `created_at` | `createTime` (epoch s) → ISO UTC |
|
|
109
|
+
| `url` | `https://www.tiktok.com/@{uniqueId}/video/{id}` |
|
|
110
|
+
| `media` | `[{ "type": "video", "url": playAddr or downloadAddr }]` |
|
|
111
|
+
| `like_count` | `stats.diggCount` |
|
|
112
|
+
| `comment_count` | `stats.commentCount` |
|
|
113
|
+
| `platform` | `"tiktok"` |
|
|
114
|
+
|
|
115
|
+
### Comments
|
|
116
|
+
|
|
117
|
+
Map `comment.id`, `text`, `create_time`, `user.uid`.
|
|
118
|
+
|
|
119
|
+
---
|
|
120
|
+
|
|
121
|
+
## X (API v2, read-only)
|
|
122
|
+
|
|
123
|
+
Responses are tweepy-shaped: `data`, `includes`, `meta`. The adapter stays inert without `X_BEARER_TOKEN`.
|
|
124
|
+
|
|
125
|
+
### User
|
|
126
|
+
|
|
127
|
+
| Canonical | Source |
|
|
128
|
+
|-----------|--------|
|
|
129
|
+
| `id` | `data.id` (string) |
|
|
130
|
+
| `username` | `data.username` |
|
|
131
|
+
| `display_name` | `data.name` |
|
|
132
|
+
| `bio` | `data.description` |
|
|
133
|
+
| `profile_url` | `https://x.com/{username}` |
|
|
134
|
+
| `avatar_url` | `data.profile_image_url` |
|
|
135
|
+
| `follower_count` | `data.public_metrics.followers_count` |
|
|
136
|
+
| `following_count` | `data.public_metrics.following_count` |
|
|
137
|
+
| `is_verified` | `data.verified` |
|
|
138
|
+
| `platform` | `"x"` |
|
|
139
|
+
|
|
140
|
+
### Post
|
|
141
|
+
|
|
142
|
+
| Canonical | Source |
|
|
143
|
+
|-----------|--------|
|
|
144
|
+
| `id` | `data.id` |
|
|
145
|
+
| `author_id` | `data.author_id` |
|
|
146
|
+
| `text` | `data.text` |
|
|
147
|
+
| `created_at` | `data.created_at` (ISO-8601) |
|
|
148
|
+
| `url` | `https://x.com/{username}/status/{id}` when `includes.users` has the author, else `https://x.com/i/web/status/{id}` |
|
|
149
|
+
| `media` | `includes.media` joined on `attachments.media_keys`; `video` and `animated_gif` map to `video`, other types to `image` |
|
|
150
|
+
| `like_count` | `data.public_metrics.like_count` |
|
|
151
|
+
| `comment_count` | `data.public_metrics.reply_count` |
|
|
152
|
+
| `platform` | `"x"` |
|
|
153
|
+
|
|
154
|
+
### Comment
|
|
155
|
+
|
|
156
|
+
Replies come from recent search `conversation_id:{post_id} is:reply`. The root tweet id is skipped.
|
|
157
|
+
|
|
158
|
+
| Canonical | Source |
|
|
159
|
+
|-----------|--------|
|
|
160
|
+
| `id` | tweet `id` |
|
|
161
|
+
| `post_id` | `conversation_id` or the requested post id |
|
|
162
|
+
| `author_id` | `author_id` |
|
|
163
|
+
| `text` | `text` |
|
|
164
|
+
| `created_at` | `created_at` |
|
|
165
|
+
| `platform` | `"x"` |
|
|
166
|
+
|
|
167
|
+
Pagination: `meta.next_token` → `next_cursor`. `has_more` is true when that token is a non-empty string.
|
|
168
|
+
|
|
169
|
+
---
|
|
170
|
+
|
|
171
|
+
## Facebook (v1 deferred)
|
|
172
|
+
|
|
173
|
+
v1 does not ship a Facebook adapter. When added, use Graph API:
|
|
174
|
+
|
|
175
|
+
| Canonical | Source |
|
|
176
|
+
|-----------|--------|
|
|
177
|
+
| User | `/page-id?fields=id,name,about,fan_count,picture` |
|
|
178
|
+
| Post | `/page-id/posts` with `message`, `created_time`, `id` |
|
|
179
|
+
| `next_cursor` | `paging.cursors.after` |
|
|
180
|
+
|
|
181
|
+
Do not port deprecated `facebook-api-wrapper` PHP.
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# Facebook adapter — v1 deferral
|
|
2
|
+
|
|
3
|
+
**Decision (2025-11-20):** The unified `rrss` package **does not** include a Facebook Graph adapter in v1.
|
|
4
|
+
|
|
5
|
+
**Rationale:**
|
|
6
|
+
|
|
7
|
+
- `facebook-api-wrapper` in this workspace is **deprecated** (no upstream changes).
|
|
8
|
+
- `facebook-wrapper` targets Express OAuth canvas apps, not a headless pull-data SDK.
|
|
9
|
+
- Meta’s official Graph API is the correct path for new work; scope belongs in a dedicated v1.1 spike with app review checklist.
|
|
10
|
+
|
|
11
|
+
**v1 behavior:** `Client("facebook")` raises `NotSupported` with a link to this document and [Meta Graph docs](https://developers.facebook.com/docs/graph-api).
|
|
12
|
+
|
|
13
|
+
**v1.1 spike exit criteria:**
|
|
14
|
+
|
|
15
|
+
- App with `pages_read_engagement` or product-appropriate scopes approved.
|
|
16
|
+
- Thin adapter: `get_user` (page), `list_user_posts`, `list_post_comments` only.
|
|
17
|
+
- No FQL; Graph v2+ JSON only.
|