socialscrape-haystack 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (27) hide show
  1. socialscrape_haystack-0.1.0/.gitignore +17 -0
  2. socialscrape_haystack-0.1.0/LICENSE +21 -0
  3. socialscrape_haystack-0.1.0/PKG-INFO +179 -0
  4. socialscrape_haystack-0.1.0/README.md +157 -0
  5. socialscrape_haystack-0.1.0/pyproject.toml +50 -0
  6. socialscrape_haystack-0.1.0/socialscrape_haystack/__init__.py +37 -0
  7. socialscrape_haystack-0.1.0/socialscrape_haystack/_catalog.py +257 -0
  8. socialscrape_haystack-0.1.0/socialscrape_haystack/_client.py +57 -0
  9. socialscrape_haystack-0.1.0/socialscrape_haystack/_documents.py +79 -0
  10. socialscrape_haystack-0.1.0/socialscrape_haystack/endpoints.json +198 -0
  11. socialscrape_haystack-0.1.0/socialscrape_haystack/fetcher.py +174 -0
  12. socialscrape_haystack-0.1.0/socialscrape_haystack/py.typed +0 -0
  13. socialscrape_haystack-0.1.0/socialscrape_haystack/toolset.py +405 -0
  14. socialscrape_haystack-0.1.0/socialscrape_haystack/transcripts.py +146 -0
  15. socialscrape_haystack-0.1.0/tests/conftest.py +94 -0
  16. socialscrape_haystack-0.1.0/tests/fixtures/account.credits.json +15 -0
  17. socialscrape_haystack-0.1.0/tests/fixtures/github.user.repositories.json +106 -0
  18. socialscrape_haystack-0.1.0/tests/fixtures/reddit.search.json +130 -0
  19. socialscrape_haystack-0.1.0/tests/fixtures/tiktok.profile.json +50 -0
  20. socialscrape_haystack-0.1.0/tests/fixtures/transcript.json +36 -0
  21. socialscrape_haystack-0.1.0/tests/fixtures/youtube.channel.videos.json +110 -0
  22. socialscrape_haystack-0.1.0/tests/fixtures/youtube.video.comments.json +83 -0
  23. socialscrape_haystack-0.1.0/tests/fixtures/youtube.video.json +102 -0
  24. socialscrape_haystack-0.1.0/tests/test_catalog.py +34 -0
  25. socialscrape_haystack-0.1.0/tests/test_fetcher.py +130 -0
  26. socialscrape_haystack-0.1.0/tests/test_toolset.py +153 -0
  27. socialscrape_haystack-0.1.0/tests/test_transcripts.py +97 -0
@@ -0,0 +1,17 @@
1
+ .env
2
+ .env.*
3
+ !.env.example
4
+ node_modules/
5
+ .ref/
6
+ .venv/
7
+ .cache/
8
+ __pycache__/
9
+ *.pyc
10
+ .pytest_cache/
11
+ *.egg-info/
12
+ build/
13
+ dist/
14
+ .cache/
15
+
16
+ .wrangler/
17
+ WORKTREE.md
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 The Casey Company LLC (SocialScrape)
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,179 @@
1
+ Metadata-Version: 2.5
2
+ Name: socialscrape-haystack
3
+ Version: 0.1.0
4
+ Summary: Haystack components and agent tools for SocialScrape: profiles, posts, comments, search and video transcripts from social platforms as Documents.
5
+ Project-URL: Homepage, https://socialscrape.dev/?utm_source=pypi&utm_medium=package&utm_campaign=socialscrape-haystack
6
+ Project-URL: Documentation, https://socialscrape.dev/docs/haystack?utm_source=pypi&utm_medium=package&utm_campaign=socialscrape-haystack
7
+ Project-URL: Repository, https://github.com/SocialScraper/socialscrape-haystack
8
+ Project-URL: Issues, https://github.com/SocialScraper/socialscrape-haystack/issues
9
+ Author-email: SocialScrape <hello@socialscrape.dev>
10
+ License-Expression: MIT
11
+ License-File: LICENSE
12
+ Keywords: agents,components,deepset,haystack,instagram,rag,social media,tiktok,tools,transcript,youtube
13
+ Classifier: License :: OSI Approved :: MIT License
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Typing :: Typed
16
+ Requires-Python: >=3.11
17
+ Requires-Dist: haystack-ai>=2.14
18
+ Requires-Dist: socialscrape-sdk<1,>=0.1.2
19
+ Provides-Extra: test
20
+ Requires-Dist: pytest>=8; extra == 'test'
21
+ Description-Content-Type: text/markdown
22
+
23
+ # socialscrape-haystack
24
+
25
+ [Haystack](https://haystack.deepset.ai) components and agent tools for [SocialScrape](https://socialscrape.dev/?utm_source=github&utm_medium=integration&utm_campaign=socialscrape-haystack), a social media data API: profiles, posts, comments, search results and video transcripts from YouTube, TikTok, Instagram, X, Facebook, LinkedIn, Reddit and more, as JSON in one response shape, with one API key.
26
+
27
+ ```bash
28
+ pip install -U socialscrape-haystack
29
+ export SOCIALSCRAPE_API_KEY="ss_live_..." # get a key: https://socialscrape.dev/sign-up
30
+ ```
31
+
32
+ Python 3.11 or newer, Haystack 2.14 or newer. The package builds on the official [`socialscrape-sdk`](https://pypi.org/project/socialscrape-sdk/) client, so retries, timeouts and long transcription jobs are handled for you. Full guide: [socialscrape.dev/docs/haystack](https://socialscrape.dev/docs/haystack?utm_source=github&utm_medium=integration&utm_campaign=socialscrape-haystack).
33
+
34
+ ## Components
35
+
36
+ | Class | What it does |
37
+ |---|---|
38
+ | `SocialScrapeTranscriptFetcher` | Video URLs in, one transcript `Document` per URL out. A URL that fails goes to `errors`, not an exception. |
39
+ | `SocialScrapeFetcher` | Runs any endpoint by id with its parameters and returns `Document`s: one per item of a list, one otherwise. |
40
+ | `SocialScrapeToolset` | Eight tools for a Haystack `Agent`: profile, posts, one post, comments, transcript, search, endpoint lookup and call-any-endpoint. |
41
+
42
+ The API key is a Haystack `Secret`. The default is `Secret.from_env_var("SOCIALSCRAPE_API_KEY")`, and every component serializes with `to_dict` and `from_dict` without writing the key out.
43
+
44
+ ## Transcripts into a pipeline
45
+
46
+ ```python
47
+ from haystack import Pipeline
48
+ from haystack.components.builders import ChatPromptBuilder
49
+ from haystack.components.generators.chat import OpenAIChatGenerator
50
+ from haystack.components.preprocessors import DocumentSplitter
51
+ from haystack.dataclasses import ChatMessage
52
+ from socialscrape_haystack import SocialScrapeTranscriptFetcher
53
+
54
+ template = [ChatMessage.from_user(
55
+ "Summarize the main points of these transcripts.\n"
56
+ "{% for doc in documents %}\n[{{ doc.meta.url }}]\n{{ doc.content }}\n{% endfor %}"
57
+ )]
58
+
59
+ pipe = Pipeline()
60
+ pipe.add_component("fetcher", SocialScrapeTranscriptFetcher())
61
+ pipe.add_component("splitter", DocumentSplitter(split_by="word", split_length=300, split_overlap=30))
62
+ pipe.add_component("prompt", ChatPromptBuilder(template=template, required_variables=["documents"]))
63
+ pipe.add_component("llm", OpenAIChatGenerator(model="gpt-4o-mini"))
64
+ pipe.connect("fetcher.documents", "splitter.documents")
65
+ pipe.connect("splitter.documents", "prompt.documents")
66
+ pipe.connect("prompt.prompt", "llm.messages")
67
+
68
+ result = pipe.run({"fetcher": {"urls": [
69
+ "https://www.youtube.com/watch?v=dQw4w9WgXcQ",
70
+ "https://www.tiktok.com/@nasa/video/7300000000000000000",
71
+ ]}})
72
+ print(result["llm"]["replies"][0].text)
73
+ ```
74
+
75
+ For retrieval, put an embedder and a `DocumentWriter` after the splitter instead of the prompt builder. Each chunk keeps the meta of its video (`url`, `platform`, `language`, `duration_seconds`).
76
+
77
+ A transcript Document has the transcript text as `content` and this `meta`: `url` and `source` (the video URL), `platform`, `video_id`, `language`, `duration_seconds`, `transcript_source` (`captions`, or `ai` when the audio was transcribed) and `endpoint`.
78
+
79
+ ### Errors
80
+
81
+ `SocialScrapeTranscriptFetcher` does not fail a batch for one bad link. A URL that cannot be fetched (private video, unsupported link, no captions) is left out of `documents` and listed in `errors`:
82
+
83
+ ```python
84
+ result = SocialScrapeTranscriptFetcher().run(urls=["https://www.youtube.com/watch?v=dQw4w9WgXcQ", "https://example.com/nope"])
85
+ result["errors"] # [{"url": "https://example.com/nope", "error": "..."}]
86
+ ```
87
+
88
+ A failed request costs no credits. Pass `raise_on_failure=True` to raise on the first failure instead. A video with no captions is transcribed from its audio up to 2 minutes; longer audio needs `allow_long_transcription=True`, which is charged per started minute (see Billing).
89
+
90
+ ## Any endpoint as Documents
91
+
92
+ ```python
93
+ from socialscrape_haystack import SocialScrapeFetcher
94
+
95
+ fetcher = SocialScrapeFetcher(max_pages=3)
96
+
97
+ # One Document per comment, up to three pages
98
+ comments = fetcher.run(
99
+ endpoint="youtube.video.comments",
100
+ params={"url": "https://www.youtube.com/watch?v=dQw4w9WgXcQ"},
101
+ )["documents"]
102
+
103
+ # One Document for a profile
104
+ profile = fetcher.run(endpoint="tiktok.profile", params={"handle": "nasa"})["documents"][0]
105
+ profile.content # name, headline and bio
106
+ profile.meta # {'platform': 'tiktok', 'handle': 'nasa', 'followers': ..., 'endpoint': 'tiktok.profile', ...}
107
+ ```
108
+
109
+ You can also fix the endpoint and its parameters when you create the component and pass only what changes at run time: `SocialScrapeFetcher(endpoint="reddit.search", params={"sort": "top"}).run(params={"query": "mechanical keyboards"})`.
110
+
111
+ - A list endpoint gives one Document per item and is followed across pages up to `max_pages` (default 1; each page is one request). `max_items` stops early.
112
+ - Any other endpoint (a profile, one post, a transcript) gives one Document.
113
+ - The content is the item's text: a post's title and text, a comment's text, a transcript's text, a profile's name and bio. An object with no text of its own is kept whole as JSON.
114
+ - The item's other scalar fields, its counts and its author go to `meta`, flat, with `endpoint`, `platform` and the item's `url` (also as `source`).
115
+ - By default a refused request raises. With `raise_on_failure=False` it is returned in `errors`.
116
+
117
+ The endpoint ids and their parameters are in the [API reference](https://socialscrape.dev/docs/api?utm_source=github&utm_medium=integration&utm_campaign=socialscrape-haystack), and in code:
118
+
119
+ ```python
120
+ from socialscrape_haystack import endpoints
121
+
122
+ endpoints()["tiktok.profile.videos"].signature()
123
+ ```
124
+
125
+ Parameters are checked against the endpoint before the request is made, so a misspelt name is answered at once and costs nothing.
126
+
127
+ ## Give an Agent the tools
128
+
129
+ ```python
130
+ from haystack.components.agents import Agent
131
+ from haystack.components.generators.chat import OpenAIChatGenerator
132
+ from haystack.dataclasses import ChatMessage
133
+ from socialscrape_haystack import SocialScrapeToolset
134
+
135
+ agent = Agent(
136
+ chat_generator=OpenAIChatGenerator(model="gpt-4o-mini"),
137
+ tools=SocialScrapeToolset(), # reads SOCIALSCRAPE_API_KEY
138
+ system_prompt="You answer questions about social media accounts and videos. Use the tools; do not guess.",
139
+ )
140
+
141
+ result = agent.run(messages=[ChatMessage.from_user(
142
+ "What did NASA post on YouTube lately, and what are people saying in the comments?"
143
+ )])
144
+ print(result["last_message"].text)
145
+ ```
146
+
147
+ Any chat generator that supports tool calling works. The toolset has eight tools that share one client:
148
+
149
+ | Tool | What it does |
150
+ |---|---|
151
+ | `socialscrape_get_profile` | A profile, channel or artist: name, bio, links, verification, follower counts. |
152
+ | `socialscrape_list_posts` | The latest posts or videos of a profile, channel or page, one page per call. |
153
+ | `socialscrape_get_post` | One post, video, tweet, pin or clip from its URL. |
154
+ | `socialscrape_list_comments` | The comments on a post or video, one page per call. |
155
+ | `socialscrape_get_transcript` | The transcript of a video from its URL. |
156
+ | `socialscrape_search` | Keyword search on one platform, one page per call. |
157
+ | `socialscrape_list_endpoints` | What else exists: every endpoint id with its parameters and price. Makes no request, so it is free. |
158
+ | `socialscrape_call_endpoint` | Runs any endpoint by id: ad libraries, followers, hashtags, playlists, music, shops and the rest. |
159
+
160
+ A tool answers with a JSON string: `data` (the object, or a list page with `items`, `cursor` and `has_more`), `credits_charged` and `credits_remaining`. A request the API refuses is free and comes back as the tool's answer (`error: not_found: ...`), so the agent can read it and try something else. Tools that list things return a `cursor` to pass back for the next page.
161
+
162
+ Toolset options: `api_key`, `max_age_days` (accept a cached result up to this many days old, which costs 0 credits), `trim` (default `True`, a smaller response that saves context), `client` and `async_client` (a `socialscrape.SocialScrape` or `AsyncSocialScrape` you configured yourself) and `base_url`.
163
+
164
+ ## Billing
165
+
166
+ Most requests cost 1 credit: a profile, a post, a page of a list or of search results, a transcript from captions. AI transcription of a video with no captions costs 1 credit up to 2 minutes of audio; longer audio is transcribed only when you pass `allow_long_transcription`, at 5 credits per started minute of the whole video. A failed request and a cached repeat cost 0. A new account starts with 100 free credits, no card needed. See [pricing](https://socialscrape.dev/pricing?utm_source=github&utm_medium=integration&utm_campaign=socialscrape-haystack).
167
+
168
+ ## Tests
169
+
170
+ ```bash
171
+ pip install -e ".[test]"
172
+ pytest
173
+ ```
174
+
175
+ The tests run offline against sample responses from the API reference.
176
+
177
+ ## License
178
+
179
+ MIT, the same license as [langchain-socialscrape](https://github.com/SocialScraper/langchain-socialscrape).
@@ -0,0 +1,157 @@
1
+ # socialscrape-haystack
2
+
3
+ [Haystack](https://haystack.deepset.ai) components and agent tools for [SocialScrape](https://socialscrape.dev/?utm_source=github&utm_medium=integration&utm_campaign=socialscrape-haystack), a social media data API: profiles, posts, comments, search results and video transcripts from YouTube, TikTok, Instagram, X, Facebook, LinkedIn, Reddit and more, as JSON in one response shape, with one API key.
4
+
5
+ ```bash
6
+ pip install -U socialscrape-haystack
7
+ export SOCIALSCRAPE_API_KEY="ss_live_..." # get a key: https://socialscrape.dev/sign-up
8
+ ```
9
+
10
+ Python 3.11 or newer, Haystack 2.14 or newer. The package builds on the official [`socialscrape-sdk`](https://pypi.org/project/socialscrape-sdk/) client, so retries, timeouts and long transcription jobs are handled for you. Full guide: [socialscrape.dev/docs/haystack](https://socialscrape.dev/docs/haystack?utm_source=github&utm_medium=integration&utm_campaign=socialscrape-haystack).
11
+
12
+ ## Components
13
+
14
+ | Class | What it does |
15
+ |---|---|
16
+ | `SocialScrapeTranscriptFetcher` | Video URLs in, one transcript `Document` per URL out. A URL that fails goes to `errors`, not an exception. |
17
+ | `SocialScrapeFetcher` | Runs any endpoint by id with its parameters and returns `Document`s: one per item of a list, one otherwise. |
18
+ | `SocialScrapeToolset` | Eight tools for a Haystack `Agent`: profile, posts, one post, comments, transcript, search, endpoint lookup and call-any-endpoint. |
19
+
20
+ The API key is a Haystack `Secret`. The default is `Secret.from_env_var("SOCIALSCRAPE_API_KEY")`, and every component serializes with `to_dict` and `from_dict` without writing the key out.
21
+
22
+ ## Transcripts into a pipeline
23
+
24
+ ```python
25
+ from haystack import Pipeline
26
+ from haystack.components.builders import ChatPromptBuilder
27
+ from haystack.components.generators.chat import OpenAIChatGenerator
28
+ from haystack.components.preprocessors import DocumentSplitter
29
+ from haystack.dataclasses import ChatMessage
30
+ from socialscrape_haystack import SocialScrapeTranscriptFetcher
31
+
32
+ template = [ChatMessage.from_user(
33
+ "Summarize the main points of these transcripts.\n"
34
+ "{% for doc in documents %}\n[{{ doc.meta.url }}]\n{{ doc.content }}\n{% endfor %}"
35
+ )]
36
+
37
+ pipe = Pipeline()
38
+ pipe.add_component("fetcher", SocialScrapeTranscriptFetcher())
39
+ pipe.add_component("splitter", DocumentSplitter(split_by="word", split_length=300, split_overlap=30))
40
+ pipe.add_component("prompt", ChatPromptBuilder(template=template, required_variables=["documents"]))
41
+ pipe.add_component("llm", OpenAIChatGenerator(model="gpt-4o-mini"))
42
+ pipe.connect("fetcher.documents", "splitter.documents")
43
+ pipe.connect("splitter.documents", "prompt.documents")
44
+ pipe.connect("prompt.prompt", "llm.messages")
45
+
46
+ result = pipe.run({"fetcher": {"urls": [
47
+ "https://www.youtube.com/watch?v=dQw4w9WgXcQ",
48
+ "https://www.tiktok.com/@nasa/video/7300000000000000000",
49
+ ]}})
50
+ print(result["llm"]["replies"][0].text)
51
+ ```
52
+
53
+ For retrieval, put an embedder and a `DocumentWriter` after the splitter instead of the prompt builder. Each chunk keeps the meta of its video (`url`, `platform`, `language`, `duration_seconds`).
54
+
55
+ A transcript Document has the transcript text as `content` and this `meta`: `url` and `source` (the video URL), `platform`, `video_id`, `language`, `duration_seconds`, `transcript_source` (`captions`, or `ai` when the audio was transcribed) and `endpoint`.
56
+
57
+ ### Errors
58
+
59
+ `SocialScrapeTranscriptFetcher` does not fail a batch for one bad link. A URL that cannot be fetched (private video, unsupported link, no captions) is left out of `documents` and listed in `errors`:
60
+
61
+ ```python
62
+ result = SocialScrapeTranscriptFetcher().run(urls=["https://www.youtube.com/watch?v=dQw4w9WgXcQ", "https://example.com/nope"])
63
+ result["errors"] # [{"url": "https://example.com/nope", "error": "..."}]
64
+ ```
65
+
66
+ A failed request costs no credits. Pass `raise_on_failure=True` to raise on the first failure instead. A video with no captions is transcribed from its audio up to 2 minutes; longer audio needs `allow_long_transcription=True`, which is charged per started minute (see Billing).
67
+
68
+ ## Any endpoint as Documents
69
+
70
+ ```python
71
+ from socialscrape_haystack import SocialScrapeFetcher
72
+
73
+ fetcher = SocialScrapeFetcher(max_pages=3)
74
+
75
+ # One Document per comment, up to three pages
76
+ comments = fetcher.run(
77
+ endpoint="youtube.video.comments",
78
+ params={"url": "https://www.youtube.com/watch?v=dQw4w9WgXcQ"},
79
+ )["documents"]
80
+
81
+ # One Document for a profile
82
+ profile = fetcher.run(endpoint="tiktok.profile", params={"handle": "nasa"})["documents"][0]
83
+ profile.content # name, headline and bio
84
+ profile.meta # {'platform': 'tiktok', 'handle': 'nasa', 'followers': ..., 'endpoint': 'tiktok.profile', ...}
85
+ ```
86
+
87
+ You can also fix the endpoint and its parameters when you create the component and pass only what changes at run time: `SocialScrapeFetcher(endpoint="reddit.search", params={"sort": "top"}).run(params={"query": "mechanical keyboards"})`.
88
+
89
+ - A list endpoint gives one Document per item and is followed across pages up to `max_pages` (default 1; each page is one request). `max_items` stops early.
90
+ - Any other endpoint (a profile, one post, a transcript) gives one Document.
91
+ - The content is the item's text: a post's title and text, a comment's text, a transcript's text, a profile's name and bio. An object with no text of its own is kept whole as JSON.
92
+ - The item's other scalar fields, its counts and its author go to `meta`, flat, with `endpoint`, `platform` and the item's `url` (also as `source`).
93
+ - By default a refused request raises. With `raise_on_failure=False` it is returned in `errors`.
94
+
95
+ The endpoint ids and their parameters are in the [API reference](https://socialscrape.dev/docs/api?utm_source=github&utm_medium=integration&utm_campaign=socialscrape-haystack), and in code:
96
+
97
+ ```python
98
+ from socialscrape_haystack import endpoints
99
+
100
+ endpoints()["tiktok.profile.videos"].signature()
101
+ ```
102
+
103
+ Parameters are checked against the endpoint before the request is made, so a misspelt name is answered at once and costs nothing.
104
+
105
+ ## Give an Agent the tools
106
+
107
+ ```python
108
+ from haystack.components.agents import Agent
109
+ from haystack.components.generators.chat import OpenAIChatGenerator
110
+ from haystack.dataclasses import ChatMessage
111
+ from socialscrape_haystack import SocialScrapeToolset
112
+
113
+ agent = Agent(
114
+ chat_generator=OpenAIChatGenerator(model="gpt-4o-mini"),
115
+ tools=SocialScrapeToolset(), # reads SOCIALSCRAPE_API_KEY
116
+ system_prompt="You answer questions about social media accounts and videos. Use the tools; do not guess.",
117
+ )
118
+
119
+ result = agent.run(messages=[ChatMessage.from_user(
120
+ "What did NASA post on YouTube lately, and what are people saying in the comments?"
121
+ )])
122
+ print(result["last_message"].text)
123
+ ```
124
+
125
+ Any chat generator that supports tool calling works. The toolset has eight tools that share one client:
126
+
127
+ | Tool | What it does |
128
+ |---|---|
129
+ | `socialscrape_get_profile` | A profile, channel or artist: name, bio, links, verification, follower counts. |
130
+ | `socialscrape_list_posts` | The latest posts or videos of a profile, channel or page, one page per call. |
131
+ | `socialscrape_get_post` | One post, video, tweet, pin or clip from its URL. |
132
+ | `socialscrape_list_comments` | The comments on a post or video, one page per call. |
133
+ | `socialscrape_get_transcript` | The transcript of a video from its URL. |
134
+ | `socialscrape_search` | Keyword search on one platform, one page per call. |
135
+ | `socialscrape_list_endpoints` | What else exists: every endpoint id with its parameters and price. Makes no request, so it is free. |
136
+ | `socialscrape_call_endpoint` | Runs any endpoint by id: ad libraries, followers, hashtags, playlists, music, shops and the rest. |
137
+
138
+ A tool answers with a JSON string: `data` (the object, or a list page with `items`, `cursor` and `has_more`), `credits_charged` and `credits_remaining`. A request the API refuses is free and comes back as the tool's answer (`error: not_found: ...`), so the agent can read it and try something else. Tools that list things return a `cursor` to pass back for the next page.
139
+
140
+ Toolset options: `api_key`, `max_age_days` (accept a cached result up to this many days old, which costs 0 credits), `trim` (default `True`, a smaller response that saves context), `client` and `async_client` (a `socialscrape.SocialScrape` or `AsyncSocialScrape` you configured yourself) and `base_url`.
141
+
142
+ ## Billing
143
+
144
+ Most requests cost 1 credit: a profile, a post, a page of a list or of search results, a transcript from captions. AI transcription of a video with no captions costs 1 credit up to 2 minutes of audio; longer audio is transcribed only when you pass `allow_long_transcription`, at 5 credits per started minute of the whole video. A failed request and a cached repeat cost 0. A new account starts with 100 free credits, no card needed. See [pricing](https://socialscrape.dev/pricing?utm_source=github&utm_medium=integration&utm_campaign=socialscrape-haystack).
145
+
146
+ ## Tests
147
+
148
+ ```bash
149
+ pip install -e ".[test]"
150
+ pytest
151
+ ```
152
+
153
+ The tests run offline against sample responses from the API reference.
154
+
155
+ ## License
156
+
157
+ MIT, the same license as [langchain-socialscrape](https://github.com/SocialScraper/langchain-socialscrape).
@@ -0,0 +1,50 @@
1
+ [build-system]
2
+ requires = ["hatchling"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "socialscrape-haystack"
7
+ version = "0.1.0"
8
+ description = "Haystack components and agent tools for SocialScrape: profiles, posts, comments, search and video transcripts from social platforms as Documents."
9
+ readme = "README.md"
10
+ license = "MIT"
11
+ license-files = ["LICENSE"]
12
+ requires-python = ">=3.11"
13
+ authors = [{ name = "SocialScrape", email = "hello@socialscrape.dev" }]
14
+ keywords = ["haystack", "deepset", "components", "tools", "agents", "social media", "tiktok", "instagram", "youtube", "transcript", "rag"]
15
+ classifiers = [
16
+ "Programming Language :: Python :: 3",
17
+ "License :: OSI Approved :: MIT License",
18
+ "Typing :: Typed",
19
+ ]
20
+ dependencies = ["haystack-ai>=2.14", "socialscrape-sdk>=0.1.2,<1"]
21
+
22
+ [project.optional-dependencies]
23
+ test = ["pytest>=8"]
24
+
25
+ [project.urls]
26
+ Homepage = "https://socialscrape.dev/?utm_source=pypi&utm_medium=package&utm_campaign=socialscrape-haystack"
27
+ Documentation = "https://socialscrape.dev/docs/haystack?utm_source=pypi&utm_medium=package&utm_campaign=socialscrape-haystack"
28
+ Repository = "https://github.com/SocialScraper/socialscrape-haystack"
29
+ Issues = "https://github.com/SocialScraper/socialscrape-haystack/issues"
30
+
31
+ [tool.hatch.build.targets.wheel]
32
+ packages = ["socialscrape_haystack"]
33
+
34
+ [tool.hatch.build.targets.sdist]
35
+ include = ["socialscrape_haystack", "tests", "README.md", "LICENSE"]
36
+
37
+ [tool.ruff]
38
+ line-length = 120
39
+ target-version = "py311"
40
+ extend-exclude = [".venv"]
41
+
42
+ [tool.ruff.lint]
43
+ select = ["E", "F", "I", "UP", "B", "RUF"]
44
+
45
+ # Tests run offline against recorded answers:
46
+ # pip install -e "sdks/haystack[test]" && python -m pytest sdks/haystack/tests
47
+ [tool.pytest.ini_options]
48
+ addopts = "-q"
49
+ testpaths = ["tests"]
50
+ pythonpath = ["."]
@@ -0,0 +1,37 @@
1
+ """Haystack integration for SocialScrape: social media data for pipelines and agents.
2
+
3
+ `SocialScrapeFetcher` runs any endpoint and returns Documents, `SocialScrapeTranscriptFetcher` turns video URLs
4
+ into transcript Documents, and `SocialScrapeToolset` gives a Haystack Agent tools for profiles, posts,
5
+ comments, transcripts and search across platforms.
6
+ """
7
+
8
+ from importlib import metadata
9
+
10
+ from ._catalog import Endpoint, endpoints
11
+ from .fetcher import SocialScrapeFetcher
12
+ from .toolset import SocialScrapeToolset
13
+ from .transcripts import SocialScrapeTranscriptFetcher
14
+
15
+ # Newer Haystack only loads a pipeline YAML whose classes come from trusted modules (`haystack`,
16
+ # `haystack_integrations`). Whoever imports this package trusts it, so say so, or Pipeline.from_dict and
17
+ # Pipeline.loads would refuse the components here. Older Haystack has no allowlist.
18
+ try:
19
+ from haystack.core.serialization import allow_deserialization_module
20
+
21
+ allow_deserialization_module("socialscrape_haystack")
22
+ except ImportError: # pragma: no cover
23
+ pass
24
+
25
+ try:
26
+ __version__ = metadata.version("socialscrape-haystack")
27
+ except metadata.PackageNotFoundError:
28
+ __version__ = ""
29
+
30
+ __all__ = [
31
+ "Endpoint",
32
+ "SocialScrapeFetcher",
33
+ "SocialScrapeToolset",
34
+ "SocialScrapeTranscriptFetcher",
35
+ "__version__",
36
+ "endpoints",
37
+ ]