socialscrape-haystack 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- socialscrape_haystack-0.1.0/.gitignore +17 -0
- socialscrape_haystack-0.1.0/LICENSE +21 -0
- socialscrape_haystack-0.1.0/PKG-INFO +179 -0
- socialscrape_haystack-0.1.0/README.md +157 -0
- socialscrape_haystack-0.1.0/pyproject.toml +50 -0
- socialscrape_haystack-0.1.0/socialscrape_haystack/__init__.py +37 -0
- socialscrape_haystack-0.1.0/socialscrape_haystack/_catalog.py +257 -0
- socialscrape_haystack-0.1.0/socialscrape_haystack/_client.py +57 -0
- socialscrape_haystack-0.1.0/socialscrape_haystack/_documents.py +79 -0
- socialscrape_haystack-0.1.0/socialscrape_haystack/endpoints.json +198 -0
- socialscrape_haystack-0.1.0/socialscrape_haystack/fetcher.py +174 -0
- socialscrape_haystack-0.1.0/socialscrape_haystack/py.typed +0 -0
- socialscrape_haystack-0.1.0/socialscrape_haystack/toolset.py +405 -0
- socialscrape_haystack-0.1.0/socialscrape_haystack/transcripts.py +146 -0
- socialscrape_haystack-0.1.0/tests/conftest.py +94 -0
- socialscrape_haystack-0.1.0/tests/fixtures/account.credits.json +15 -0
- socialscrape_haystack-0.1.0/tests/fixtures/github.user.repositories.json +106 -0
- socialscrape_haystack-0.1.0/tests/fixtures/reddit.search.json +130 -0
- socialscrape_haystack-0.1.0/tests/fixtures/tiktok.profile.json +50 -0
- socialscrape_haystack-0.1.0/tests/fixtures/transcript.json +36 -0
- socialscrape_haystack-0.1.0/tests/fixtures/youtube.channel.videos.json +110 -0
- socialscrape_haystack-0.1.0/tests/fixtures/youtube.video.comments.json +83 -0
- socialscrape_haystack-0.1.0/tests/fixtures/youtube.video.json +102 -0
- socialscrape_haystack-0.1.0/tests/test_catalog.py +34 -0
- socialscrape_haystack-0.1.0/tests/test_fetcher.py +130 -0
- socialscrape_haystack-0.1.0/tests/test_toolset.py +153 -0
- socialscrape_haystack-0.1.0/tests/test_transcripts.py +97 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 The Casey Company LLC (SocialScrape)
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: socialscrape-haystack
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Haystack components and agent tools for SocialScrape: profiles, posts, comments, search and video transcripts from social platforms as Documents.
|
|
5
|
+
Project-URL: Homepage, https://socialscrape.dev/?utm_source=pypi&utm_medium=package&utm_campaign=socialscrape-haystack
|
|
6
|
+
Project-URL: Documentation, https://socialscrape.dev/docs/haystack?utm_source=pypi&utm_medium=package&utm_campaign=socialscrape-haystack
|
|
7
|
+
Project-URL: Repository, https://github.com/SocialScraper/socialscrape-haystack
|
|
8
|
+
Project-URL: Issues, https://github.com/SocialScraper/socialscrape-haystack/issues
|
|
9
|
+
Author-email: SocialScrape <hello@socialscrape.dev>
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: agents,components,deepset,haystack,instagram,rag,social media,tiktok,tools,transcript,youtube
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Typing :: Typed
|
|
16
|
+
Requires-Python: >=3.11
|
|
17
|
+
Requires-Dist: haystack-ai>=2.14
|
|
18
|
+
Requires-Dist: socialscrape-sdk<1,>=0.1.2
|
|
19
|
+
Provides-Extra: test
|
|
20
|
+
Requires-Dist: pytest>=8; extra == 'test'
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
|
|
23
|
+
# socialscrape-haystack
|
|
24
|
+
|
|
25
|
+
[Haystack](https://haystack.deepset.ai) components and agent tools for [SocialScrape](https://socialscrape.dev/?utm_source=github&utm_medium=integration&utm_campaign=socialscrape-haystack), a social media data API: profiles, posts, comments, search results and video transcripts from YouTube, TikTok, Instagram, X, Facebook, LinkedIn, Reddit and more, as JSON in one response shape, with one API key.
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
pip install -U socialscrape-haystack
|
|
29
|
+
export SOCIALSCRAPE_API_KEY="ss_live_..." # get a key: https://socialscrape.dev/sign-up
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
Python 3.11 or newer, Haystack 2.14 or newer. The package builds on the official [`socialscrape-sdk`](https://pypi.org/project/socialscrape-sdk/) client, so retries, timeouts and long transcription jobs are handled for you. Full guide: [socialscrape.dev/docs/haystack](https://socialscrape.dev/docs/haystack?utm_source=github&utm_medium=integration&utm_campaign=socialscrape-haystack).
|
|
33
|
+
|
|
34
|
+
## Components
|
|
35
|
+
|
|
36
|
+
| Class | What it does |
|
|
37
|
+
|---|---|
|
|
38
|
+
| `SocialScrapeTranscriptFetcher` | Video URLs in, one transcript `Document` per URL out. A URL that fails goes to `errors`, not an exception. |
|
|
39
|
+
| `SocialScrapeFetcher` | Runs any endpoint by id with its parameters and returns `Document`s: one per item of a list, one otherwise. |
|
|
40
|
+
| `SocialScrapeToolset` | Eight tools for a Haystack `Agent`: profile, posts, one post, comments, transcript, search, endpoint lookup and call-any-endpoint. |
|
|
41
|
+
|
|
42
|
+
The API key is a Haystack `Secret`. The default is `Secret.from_env_var("SOCIALSCRAPE_API_KEY")`, and every component serializes with `to_dict` and `from_dict` without writing the key out.
|
|
43
|
+
|
|
44
|
+
## Transcripts into a pipeline
|
|
45
|
+
|
|
46
|
+
```python
|
|
47
|
+
from haystack import Pipeline
|
|
48
|
+
from haystack.components.builders import ChatPromptBuilder
|
|
49
|
+
from haystack.components.generators.chat import OpenAIChatGenerator
|
|
50
|
+
from haystack.components.preprocessors import DocumentSplitter
|
|
51
|
+
from haystack.dataclasses import ChatMessage
|
|
52
|
+
from socialscrape_haystack import SocialScrapeTranscriptFetcher
|
|
53
|
+
|
|
54
|
+
template = [ChatMessage.from_user(
|
|
55
|
+
"Summarize the main points of these transcripts.\n"
|
|
56
|
+
"{% for doc in documents %}\n[{{ doc.meta.url }}]\n{{ doc.content }}\n{% endfor %}"
|
|
57
|
+
)]
|
|
58
|
+
|
|
59
|
+
pipe = Pipeline()
|
|
60
|
+
pipe.add_component("fetcher", SocialScrapeTranscriptFetcher())
|
|
61
|
+
pipe.add_component("splitter", DocumentSplitter(split_by="word", split_length=300, split_overlap=30))
|
|
62
|
+
pipe.add_component("prompt", ChatPromptBuilder(template=template, required_variables=["documents"]))
|
|
63
|
+
pipe.add_component("llm", OpenAIChatGenerator(model="gpt-4o-mini"))
|
|
64
|
+
pipe.connect("fetcher.documents", "splitter.documents")
|
|
65
|
+
pipe.connect("splitter.documents", "prompt.documents")
|
|
66
|
+
pipe.connect("prompt.prompt", "llm.messages")
|
|
67
|
+
|
|
68
|
+
result = pipe.run({"fetcher": {"urls": [
|
|
69
|
+
"https://www.youtube.com/watch?v=dQw4w9WgXcQ",
|
|
70
|
+
"https://www.tiktok.com/@nasa/video/7300000000000000000",
|
|
71
|
+
]}})
|
|
72
|
+
print(result["llm"]["replies"][0].text)
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
For retrieval, put an embedder and a `DocumentWriter` after the splitter instead of the prompt builder. Each chunk keeps the meta of its video (`url`, `platform`, `language`, `duration_seconds`).
|
|
76
|
+
|
|
77
|
+
A transcript Document has the transcript text as `content` and this `meta`: `url` and `source` (the video URL), `platform`, `video_id`, `language`, `duration_seconds`, `transcript_source` (`captions`, or `ai` when the audio was transcribed) and `endpoint`.
|
|
78
|
+
|
|
79
|
+
### Errors
|
|
80
|
+
|
|
81
|
+
`SocialScrapeTranscriptFetcher` does not fail a batch for one bad link. A URL that cannot be fetched (private video, unsupported link, no captions) is left out of `documents` and listed in `errors`:
|
|
82
|
+
|
|
83
|
+
```python
|
|
84
|
+
result = SocialScrapeTranscriptFetcher().run(urls=["https://www.youtube.com/watch?v=dQw4w9WgXcQ", "https://example.com/nope"])
|
|
85
|
+
result["errors"] # [{"url": "https://example.com/nope", "error": "..."}]
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
A failed request costs no credits. Pass `raise_on_failure=True` to raise on the first failure instead. A video with no captions is transcribed from its audio up to 2 minutes; longer audio needs `allow_long_transcription=True`, which is charged per started minute (see Billing).
|
|
89
|
+
|
|
90
|
+
## Any endpoint as Documents
|
|
91
|
+
|
|
92
|
+
```python
|
|
93
|
+
from socialscrape_haystack import SocialScrapeFetcher
|
|
94
|
+
|
|
95
|
+
fetcher = SocialScrapeFetcher(max_pages=3)
|
|
96
|
+
|
|
97
|
+
# One Document per comment, up to three pages
|
|
98
|
+
comments = fetcher.run(
|
|
99
|
+
endpoint="youtube.video.comments",
|
|
100
|
+
params={"url": "https://www.youtube.com/watch?v=dQw4w9WgXcQ"},
|
|
101
|
+
)["documents"]
|
|
102
|
+
|
|
103
|
+
# One Document for a profile
|
|
104
|
+
profile = fetcher.run(endpoint="tiktok.profile", params={"handle": "nasa"})["documents"][0]
|
|
105
|
+
profile.content # name, headline and bio
|
|
106
|
+
profile.meta # {'platform': 'tiktok', 'handle': 'nasa', 'followers': ..., 'endpoint': 'tiktok.profile', ...}
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
You can also fix the endpoint and its parameters when you create the component and pass only what changes at run time: `SocialScrapeFetcher(endpoint="reddit.search", params={"sort": "top"}).run(params={"query": "mechanical keyboards"})`.
|
|
110
|
+
|
|
111
|
+
- A list endpoint gives one Document per item and is followed across pages up to `max_pages` (default 1; each page is one request). `max_items` stops early.
|
|
112
|
+
- Any other endpoint (a profile, one post, a transcript) gives one Document.
|
|
113
|
+
- The content is the item's text: a post's title and text, a comment's text, a transcript's text, a profile's name and bio. An object with no text of its own is kept whole as JSON.
|
|
114
|
+
- The item's other scalar fields, its counts and its author go to `meta`, flat, with `endpoint`, `platform` and the item's `url` (also as `source`).
|
|
115
|
+
- By default a refused request raises. With `raise_on_failure=False` it is returned in `errors`.
|
|
116
|
+
|
|
117
|
+
The endpoint ids and their parameters are in the [API reference](https://socialscrape.dev/docs/api?utm_source=github&utm_medium=integration&utm_campaign=socialscrape-haystack), and in code:
|
|
118
|
+
|
|
119
|
+
```python
|
|
120
|
+
from socialscrape_haystack import endpoints
|
|
121
|
+
|
|
122
|
+
endpoints()["tiktok.profile.videos"].signature()
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
Parameters are checked against the endpoint before the request is made, so a misspelt name is answered at once and costs nothing.
|
|
126
|
+
|
|
127
|
+
## Give an Agent the tools
|
|
128
|
+
|
|
129
|
+
```python
|
|
130
|
+
from haystack.components.agents import Agent
|
|
131
|
+
from haystack.components.generators.chat import OpenAIChatGenerator
|
|
132
|
+
from haystack.dataclasses import ChatMessage
|
|
133
|
+
from socialscrape_haystack import SocialScrapeToolset
|
|
134
|
+
|
|
135
|
+
agent = Agent(
|
|
136
|
+
chat_generator=OpenAIChatGenerator(model="gpt-4o-mini"),
|
|
137
|
+
tools=SocialScrapeToolset(), # reads SOCIALSCRAPE_API_KEY
|
|
138
|
+
system_prompt="You answer questions about social media accounts and videos. Use the tools; do not guess.",
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
result = agent.run(messages=[ChatMessage.from_user(
|
|
142
|
+
"What did NASA post on YouTube lately, and what are people saying in the comments?"
|
|
143
|
+
)])
|
|
144
|
+
print(result["last_message"].text)
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
Any chat generator that supports tool calling works. The toolset has eight tools that share one client:
|
|
148
|
+
|
|
149
|
+
| Tool | What it does |
|
|
150
|
+
|---|---|
|
|
151
|
+
| `socialscrape_get_profile` | A profile, channel or artist: name, bio, links, verification, follower counts. |
|
|
152
|
+
| `socialscrape_list_posts` | The latest posts or videos of a profile, channel or page, one page per call. |
|
|
153
|
+
| `socialscrape_get_post` | One post, video, tweet, pin or clip from its URL. |
|
|
154
|
+
| `socialscrape_list_comments` | The comments on a post or video, one page per call. |
|
|
155
|
+
| `socialscrape_get_transcript` | The transcript of a video from its URL. |
|
|
156
|
+
| `socialscrape_search` | Keyword search on one platform, one page per call. |
|
|
157
|
+
| `socialscrape_list_endpoints` | What else exists: every endpoint id with its parameters and price. Makes no request, so it is free. |
|
|
158
|
+
| `socialscrape_call_endpoint` | Runs any endpoint by id: ad libraries, followers, hashtags, playlists, music, shops and the rest. |
|
|
159
|
+
|
|
160
|
+
A tool answers with a JSON string: `data` (the object, or a list page with `items`, `cursor` and `has_more`), `credits_charged` and `credits_remaining`. A request the API refuses is free and comes back as the tool's answer (`error: not_found: ...`), so the agent can read it and try something else. Tools that list things return a `cursor` to pass back for the next page.
|
|
161
|
+
|
|
162
|
+
Toolset options: `api_key`, `max_age_days` (accept a cached result up to this many days old, which costs 0 credits), `trim` (default `True`, a smaller response that saves context), `client` and `async_client` (a `socialscrape.SocialScrape` or `AsyncSocialScrape` you configured yourself) and `base_url`.
|
|
163
|
+
|
|
164
|
+
## Billing
|
|
165
|
+
|
|
166
|
+
Most requests cost 1 credit: a profile, a post, a page of a list or of search results, a transcript from captions. AI transcription of a video with no captions costs 1 credit up to 2 minutes of audio; longer audio is transcribed only when you pass `allow_long_transcription`, at 5 credits per started minute of the whole video. A failed request and a cached repeat cost 0. A new account starts with 100 free credits, no card needed. See [pricing](https://socialscrape.dev/pricing?utm_source=github&utm_medium=integration&utm_campaign=socialscrape-haystack).
|
|
167
|
+
|
|
168
|
+
## Tests
|
|
169
|
+
|
|
170
|
+
```bash
|
|
171
|
+
pip install -e ".[test]"
|
|
172
|
+
pytest
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
The tests run offline against sample responses from the API reference.
|
|
176
|
+
|
|
177
|
+
## License
|
|
178
|
+
|
|
179
|
+
MIT, the same license as [langchain-socialscrape](https://github.com/SocialScraper/langchain-socialscrape).
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
# socialscrape-haystack
|
|
2
|
+
|
|
3
|
+
[Haystack](https://haystack.deepset.ai) components and agent tools for [SocialScrape](https://socialscrape.dev/?utm_source=github&utm_medium=integration&utm_campaign=socialscrape-haystack), a social media data API: profiles, posts, comments, search results and video transcripts from YouTube, TikTok, Instagram, X, Facebook, LinkedIn, Reddit and more, as JSON in one response shape, with one API key.
|
|
4
|
+
|
|
5
|
+
```bash
|
|
6
|
+
pip install -U socialscrape-haystack
|
|
7
|
+
export SOCIALSCRAPE_API_KEY="ss_live_..." # get a key: https://socialscrape.dev/sign-up
|
|
8
|
+
```
|
|
9
|
+
|
|
10
|
+
Python 3.11 or newer, Haystack 2.14 or newer. The package builds on the official [`socialscrape-sdk`](https://pypi.org/project/socialscrape-sdk/) client, so retries, timeouts and long transcription jobs are handled for you. Full guide: [socialscrape.dev/docs/haystack](https://socialscrape.dev/docs/haystack?utm_source=github&utm_medium=integration&utm_campaign=socialscrape-haystack).
|
|
11
|
+
|
|
12
|
+
## Components
|
|
13
|
+
|
|
14
|
+
| Class | What it does |
|
|
15
|
+
|---|---|
|
|
16
|
+
| `SocialScrapeTranscriptFetcher` | Video URLs in, one transcript `Document` per URL out. A URL that fails goes to `errors`, not an exception. |
|
|
17
|
+
| `SocialScrapeFetcher` | Runs any endpoint by id with its parameters and returns `Document`s: one per item of a list, one otherwise. |
|
|
18
|
+
| `SocialScrapeToolset` | Eight tools for a Haystack `Agent`: profile, posts, one post, comments, transcript, search, endpoint lookup and call-any-endpoint. |
|
|
19
|
+
|
|
20
|
+
The API key is a Haystack `Secret`. The default is `Secret.from_env_var("SOCIALSCRAPE_API_KEY")`, and every component serializes with `to_dict` and `from_dict` without writing the key out.
|
|
21
|
+
|
|
22
|
+
## Transcripts into a pipeline
|
|
23
|
+
|
|
24
|
+
```python
|
|
25
|
+
from haystack import Pipeline
|
|
26
|
+
from haystack.components.builders import ChatPromptBuilder
|
|
27
|
+
from haystack.components.generators.chat import OpenAIChatGenerator
|
|
28
|
+
from haystack.components.preprocessors import DocumentSplitter
|
|
29
|
+
from haystack.dataclasses import ChatMessage
|
|
30
|
+
from socialscrape_haystack import SocialScrapeTranscriptFetcher
|
|
31
|
+
|
|
32
|
+
template = [ChatMessage.from_user(
|
|
33
|
+
"Summarize the main points of these transcripts.\n"
|
|
34
|
+
"{% for doc in documents %}\n[{{ doc.meta.url }}]\n{{ doc.content }}\n{% endfor %}"
|
|
35
|
+
)]
|
|
36
|
+
|
|
37
|
+
pipe = Pipeline()
|
|
38
|
+
pipe.add_component("fetcher", SocialScrapeTranscriptFetcher())
|
|
39
|
+
pipe.add_component("splitter", DocumentSplitter(split_by="word", split_length=300, split_overlap=30))
|
|
40
|
+
pipe.add_component("prompt", ChatPromptBuilder(template=template, required_variables=["documents"]))
|
|
41
|
+
pipe.add_component("llm", OpenAIChatGenerator(model="gpt-4o-mini"))
|
|
42
|
+
pipe.connect("fetcher.documents", "splitter.documents")
|
|
43
|
+
pipe.connect("splitter.documents", "prompt.documents")
|
|
44
|
+
pipe.connect("prompt.prompt", "llm.messages")
|
|
45
|
+
|
|
46
|
+
result = pipe.run({"fetcher": {"urls": [
|
|
47
|
+
"https://www.youtube.com/watch?v=dQw4w9WgXcQ",
|
|
48
|
+
"https://www.tiktok.com/@nasa/video/7300000000000000000",
|
|
49
|
+
]}})
|
|
50
|
+
print(result["llm"]["replies"][0].text)
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
For retrieval, put an embedder and a `DocumentWriter` after the splitter instead of the prompt builder. Each chunk keeps the meta of its video (`url`, `platform`, `language`, `duration_seconds`).
|
|
54
|
+
|
|
55
|
+
A transcript Document has the transcript text as `content` and this `meta`: `url` and `source` (the video URL), `platform`, `video_id`, `language`, `duration_seconds`, `transcript_source` (`captions`, or `ai` when the audio was transcribed) and `endpoint`.
|
|
56
|
+
|
|
57
|
+
### Errors
|
|
58
|
+
|
|
59
|
+
`SocialScrapeTranscriptFetcher` does not fail a batch for one bad link. A URL that cannot be fetched (private video, unsupported link, no captions) is left out of `documents` and listed in `errors`:
|
|
60
|
+
|
|
61
|
+
```python
|
|
62
|
+
result = SocialScrapeTranscriptFetcher().run(urls=["https://www.youtube.com/watch?v=dQw4w9WgXcQ", "https://example.com/nope"])
|
|
63
|
+
result["errors"] # [{"url": "https://example.com/nope", "error": "..."}]
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
A failed request costs no credits. Pass `raise_on_failure=True` to raise on the first failure instead. A video with no captions is transcribed from its audio up to 2 minutes; longer audio needs `allow_long_transcription=True`, which is charged per started minute (see Billing).
|
|
67
|
+
|
|
68
|
+
## Any endpoint as Documents
|
|
69
|
+
|
|
70
|
+
```python
|
|
71
|
+
from socialscrape_haystack import SocialScrapeFetcher
|
|
72
|
+
|
|
73
|
+
fetcher = SocialScrapeFetcher(max_pages=3)
|
|
74
|
+
|
|
75
|
+
# One Document per comment, up to three pages
|
|
76
|
+
comments = fetcher.run(
|
|
77
|
+
endpoint="youtube.video.comments",
|
|
78
|
+
params={"url": "https://www.youtube.com/watch?v=dQw4w9WgXcQ"},
|
|
79
|
+
)["documents"]
|
|
80
|
+
|
|
81
|
+
# One Document for a profile
|
|
82
|
+
profile = fetcher.run(endpoint="tiktok.profile", params={"handle": "nasa"})["documents"][0]
|
|
83
|
+
profile.content # name, headline and bio
|
|
84
|
+
profile.meta # {'platform': 'tiktok', 'handle': 'nasa', 'followers': ..., 'endpoint': 'tiktok.profile', ...}
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
You can also fix the endpoint and its parameters when you create the component and pass only what changes at run time: `SocialScrapeFetcher(endpoint="reddit.search", params={"sort": "top"}).run(params={"query": "mechanical keyboards"})`.
|
|
88
|
+
|
|
89
|
+
- A list endpoint gives one Document per item and is followed across pages up to `max_pages` (default 1; each page is one request). `max_items` stops early.
|
|
90
|
+
- Any other endpoint (a profile, one post, a transcript) gives one Document.
|
|
91
|
+
- The content is the item's text: a post's title and text, a comment's text, a transcript's text, a profile's name and bio. An object with no text of its own is kept whole as JSON.
|
|
92
|
+
- The item's other scalar fields, its counts and its author go to `meta`, flat, with `endpoint`, `platform` and the item's `url` (also as `source`).
|
|
93
|
+
- By default a refused request raises. With `raise_on_failure=False` it is returned in `errors`.
|
|
94
|
+
|
|
95
|
+
The endpoint ids and their parameters are in the [API reference](https://socialscrape.dev/docs/api?utm_source=github&utm_medium=integration&utm_campaign=socialscrape-haystack), and in code:
|
|
96
|
+
|
|
97
|
+
```python
|
|
98
|
+
from socialscrape_haystack import endpoints
|
|
99
|
+
|
|
100
|
+
endpoints()["tiktok.profile.videos"].signature()
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
Parameters are checked against the endpoint before the request is made, so a misspelt name is answered at once and costs nothing.
|
|
104
|
+
|
|
105
|
+
## Give an Agent the tools
|
|
106
|
+
|
|
107
|
+
```python
|
|
108
|
+
from haystack.components.agents import Agent
|
|
109
|
+
from haystack.components.generators.chat import OpenAIChatGenerator
|
|
110
|
+
from haystack.dataclasses import ChatMessage
|
|
111
|
+
from socialscrape_haystack import SocialScrapeToolset
|
|
112
|
+
|
|
113
|
+
agent = Agent(
|
|
114
|
+
chat_generator=OpenAIChatGenerator(model="gpt-4o-mini"),
|
|
115
|
+
tools=SocialScrapeToolset(), # reads SOCIALSCRAPE_API_KEY
|
|
116
|
+
system_prompt="You answer questions about social media accounts and videos. Use the tools; do not guess.",
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
result = agent.run(messages=[ChatMessage.from_user(
|
|
120
|
+
"What did NASA post on YouTube lately, and what are people saying in the comments?"
|
|
121
|
+
)])
|
|
122
|
+
print(result["last_message"].text)
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
Any chat generator that supports tool calling works. The toolset has eight tools that share one client:
|
|
126
|
+
|
|
127
|
+
| Tool | What it does |
|
|
128
|
+
|---|---|
|
|
129
|
+
| `socialscrape_get_profile` | A profile, channel or artist: name, bio, links, verification, follower counts. |
|
|
130
|
+
| `socialscrape_list_posts` | The latest posts or videos of a profile, channel or page, one page per call. |
|
|
131
|
+
| `socialscrape_get_post` | One post, video, tweet, pin or clip from its URL. |
|
|
132
|
+
| `socialscrape_list_comments` | The comments on a post or video, one page per call. |
|
|
133
|
+
| `socialscrape_get_transcript` | The transcript of a video from its URL. |
|
|
134
|
+
| `socialscrape_search` | Keyword search on one platform, one page per call. |
|
|
135
|
+
| `socialscrape_list_endpoints` | What else exists: every endpoint id with its parameters and price. Makes no request, so it is free. |
|
|
136
|
+
| `socialscrape_call_endpoint` | Runs any endpoint by id: ad libraries, followers, hashtags, playlists, music, shops and the rest. |
|
|
137
|
+
|
|
138
|
+
A tool answers with a JSON string: `data` (the object, or a list page with `items`, `cursor` and `has_more`), `credits_charged` and `credits_remaining`. A request the API refuses is free and comes back as the tool's answer (`error: not_found: ...`), so the agent can read it and try something else. Tools that list things return a `cursor` to pass back for the next page.
|
|
139
|
+
|
|
140
|
+
Toolset options: `api_key`, `max_age_days` (accept a cached result up to this many days old, which costs 0 credits), `trim` (default `True`, a smaller response that saves context), `client` and `async_client` (a `socialscrape.SocialScrape` or `AsyncSocialScrape` you configured yourself) and `base_url`.
|
|
141
|
+
|
|
142
|
+
## Billing
|
|
143
|
+
|
|
144
|
+
Most requests cost 1 credit: a profile, a post, a page of a list or of search results, a transcript from captions. AI transcription of a video with no captions costs 1 credit up to 2 minutes of audio; longer audio is transcribed only when you pass `allow_long_transcription`, at 5 credits per started minute of the whole video. A failed request and a cached repeat cost 0. A new account starts with 100 free credits, no card needed. See [pricing](https://socialscrape.dev/pricing?utm_source=github&utm_medium=integration&utm_campaign=socialscrape-haystack).
|
|
145
|
+
|
|
146
|
+
## Tests
|
|
147
|
+
|
|
148
|
+
```bash
|
|
149
|
+
pip install -e ".[test]"
|
|
150
|
+
pytest
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
The tests run offline against sample responses from the API reference.
|
|
154
|
+
|
|
155
|
+
## License
|
|
156
|
+
|
|
157
|
+
MIT, the same license as [langchain-socialscrape](https://github.com/SocialScraper/langchain-socialscrape).
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "socialscrape-haystack"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Haystack components and agent tools for SocialScrape: profiles, posts, comments, search and video transcripts from social platforms as Documents."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
license-files = ["LICENSE"]
|
|
12
|
+
requires-python = ">=3.11"
|
|
13
|
+
authors = [{ name = "SocialScrape", email = "hello@socialscrape.dev" }]
|
|
14
|
+
keywords = ["haystack", "deepset", "components", "tools", "agents", "social media", "tiktok", "instagram", "youtube", "transcript", "rag"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Programming Language :: Python :: 3",
|
|
17
|
+
"License :: OSI Approved :: MIT License",
|
|
18
|
+
"Typing :: Typed",
|
|
19
|
+
]
|
|
20
|
+
dependencies = ["haystack-ai>=2.14", "socialscrape-sdk>=0.1.2,<1"]
|
|
21
|
+
|
|
22
|
+
[project.optional-dependencies]
|
|
23
|
+
test = ["pytest>=8"]
|
|
24
|
+
|
|
25
|
+
[project.urls]
|
|
26
|
+
Homepage = "https://socialscrape.dev/?utm_source=pypi&utm_medium=package&utm_campaign=socialscrape-haystack"
|
|
27
|
+
Documentation = "https://socialscrape.dev/docs/haystack?utm_source=pypi&utm_medium=package&utm_campaign=socialscrape-haystack"
|
|
28
|
+
Repository = "https://github.com/SocialScraper/socialscrape-haystack"
|
|
29
|
+
Issues = "https://github.com/SocialScraper/socialscrape-haystack/issues"
|
|
30
|
+
|
|
31
|
+
[tool.hatch.build.targets.wheel]
|
|
32
|
+
packages = ["socialscrape_haystack"]
|
|
33
|
+
|
|
34
|
+
[tool.hatch.build.targets.sdist]
|
|
35
|
+
include = ["socialscrape_haystack", "tests", "README.md", "LICENSE"]
|
|
36
|
+
|
|
37
|
+
[tool.ruff]
|
|
38
|
+
line-length = 120
|
|
39
|
+
target-version = "py311"
|
|
40
|
+
extend-exclude = [".venv"]
|
|
41
|
+
|
|
42
|
+
[tool.ruff.lint]
|
|
43
|
+
select = ["E", "F", "I", "UP", "B", "RUF"]
|
|
44
|
+
|
|
45
|
+
# Tests run offline against recorded answers:
|
|
46
|
+
# pip install -e "sdks/haystack[test]" && python -m pytest sdks/haystack/tests
|
|
47
|
+
[tool.pytest.ini_options]
|
|
48
|
+
addopts = "-q"
|
|
49
|
+
testpaths = ["tests"]
|
|
50
|
+
pythonpath = ["."]
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
"""Haystack integration for SocialScrape: social media data for pipelines and agents.
|
|
2
|
+
|
|
3
|
+
`SocialScrapeFetcher` runs any endpoint and returns Documents, `SocialScrapeTranscriptFetcher` turns video URLs
|
|
4
|
+
into transcript Documents, and `SocialScrapeToolset` gives a Haystack Agent tools for profiles, posts,
|
|
5
|
+
comments, transcripts and search across platforms.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from importlib import metadata
|
|
9
|
+
|
|
10
|
+
from ._catalog import Endpoint, endpoints
|
|
11
|
+
from .fetcher import SocialScrapeFetcher
|
|
12
|
+
from .toolset import SocialScrapeToolset
|
|
13
|
+
from .transcripts import SocialScrapeTranscriptFetcher
|
|
14
|
+
|
|
15
|
+
# Newer Haystack only loads a pipeline YAML whose classes come from trusted modules (`haystack`,
|
|
16
|
+
# `haystack_integrations`). Whoever imports this package trusts it, so say so, or Pipeline.from_dict and
|
|
17
|
+
# Pipeline.loads would refuse the components here. Older Haystack has no allowlist.
|
|
18
|
+
try:
|
|
19
|
+
from haystack.core.serialization import allow_deserialization_module
|
|
20
|
+
|
|
21
|
+
allow_deserialization_module("socialscrape_haystack")
|
|
22
|
+
except ImportError: # pragma: no cover
|
|
23
|
+
pass
|
|
24
|
+
|
|
25
|
+
try:
|
|
26
|
+
__version__ = metadata.version("socialscrape-haystack")
|
|
27
|
+
except metadata.PackageNotFoundError:
|
|
28
|
+
__version__ = ""
|
|
29
|
+
|
|
30
|
+
__all__ = [
|
|
31
|
+
"Endpoint",
|
|
32
|
+
"SocialScrapeFetcher",
|
|
33
|
+
"SocialScrapeToolset",
|
|
34
|
+
"SocialScrapeTranscriptFetcher",
|
|
35
|
+
"__version__",
|
|
36
|
+
"endpoints",
|
|
37
|
+
]
|