apify-data-scrapers 1.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (12) hide show
  1. apify_data_scrapers-1.0.0/LICENSE +21 -0
  2. apify_data_scrapers-1.0.0/PKG-INFO +281 -0
  3. apify_data_scrapers-1.0.0/README.md +257 -0
  4. apify_data_scrapers-1.0.0/apify_data_scrapers.egg-info/PKG-INFO +281 -0
  5. apify_data_scrapers-1.0.0/apify_data_scrapers.egg-info/SOURCES.txt +10 -0
  6. apify_data_scrapers-1.0.0/apify_data_scrapers.egg-info/dependency_links.txt +1 -0
  7. apify_data_scrapers-1.0.0/apify_data_scrapers.egg-info/entry_points.txt +2 -0
  8. apify_data_scrapers-1.0.0/apify_data_scrapers.egg-info/requires.txt +2 -0
  9. apify_data_scrapers-1.0.0/apify_data_scrapers.egg-info/top_level.txt +1 -0
  10. apify_data_scrapers-1.0.0/mcp_server.py +206 -0
  11. apify_data_scrapers-1.0.0/setup.cfg +4 -0
  12. apify_data_scrapers-1.0.0/setup.py +28 -0
apify_data_scrapers-1.0.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Joseph McRell
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
apify_data_scrapers-1.0.0/PKG-INFO ADDED
@@ -0,0 +1,281 @@
1
+ Metadata-Version: 2.4
2
+ Name: apify-data-scrapers
3
+ Version: 1.0.0
4
+ Summary: MCP tools and SDK for Apify public data scrapers (Google Maps, SEC EDGAR, Glassdoor, Contractor Licenses)
5
+ Home-page: https://github.com/jlucasmcrell/apify-scrapers
6
+ Author: Joseph McRell
7
+ Classifier: Programming Language :: Python :: 3
8
+ Classifier: License :: OSI Approved :: MIT License
9
+ Classifier: Operating System :: OS Independent
10
+ Requires-Python: >=3.9
11
+ Description-Content-Type: text/markdown
12
+ License-File: LICENSE
13
+ Requires-Dist: apify-client>=1.8.0
14
+ Requires-Dist: mcp>=1.0.0
15
+ Dynamic: author
16
+ Dynamic: classifier
17
+ Dynamic: description
18
+ Dynamic: description-content-type
19
+ Dynamic: home-page
20
+ Dynamic: license-file
21
+ Dynamic: requires-dist
22
+ Dynamic: requires-python
23
+ Dynamic: summary
24
+
25
+ # Apify Public Data Scrapers & Extractors
26
+
27
+ [![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
28
+ [![Node.js 18+](https://img.shields.io/badge/node-18+-green.svg)](https://nodejs.org/)
29
+ [![Apify Verified](https://img.shields.io/badge/apify-store-orange.svg)](https://apify.com/captainhandsome)
30
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
31
+
32
+ A curated collection of reliable, production-ready scrapers and public-data extractors hosted on the **[Apify Store](https://apify.com/captainhandsome)**.
33
+
34
+ Each actor is built with strict schema validation, deterministic field mapping, self-healing DOM selectors, and pay-per-event pricing starting at **$0.0002 / start**.
35
+
36
+ ---
37
+
38
+ ## Quick Navigation
39
+
40
+ - [Available Extractors & Store Listings](#available-extractors--store-listings)
41
+ - [Python Quickstart](#python-quickstart)
42
+ - [Node.js Quickstart](#nodejs-quickstart)
43
+ - [No-Code & Automation Workflows (n8n, Sheets, Slack)](#no-code--automation-workflows)
44
+ - [Pre-Built Example Tasks (Zero Code)](#pre-built-example-tasks-zero-code)
45
+ - [Free Sample Datasets](#free-sample-datasets)
46
+ - [AI Agent & MCP Integration (Claude Desktop, Cursor, Custom Agent)](#ai-agent--mcp-integration)
47
+ - [In-Depth Engineering Guides](#in-depth-engineering-guides)
48
+ - [Repository Structure](#repository-structure)
49
+ - [Contributing & Author](#author--support)
50
+
51
+ ---
52
+
53
+ ## Available Extractors & Store Listings
54
+
55
+ | Tool | Store Link | Key Output Fields | Best For |
56
+ |---|---|---|---|
57
+ | **Google Maps Business Leads** | [`captainhandsome/google-maps-business-search`](https://apify.com/captainhandsome/google-maps-business-search) | Name, phone, website, rating, reviews, address, coordinates, hours | B2B lead generation, local agency prospecting |
58
+ | **Glassdoor Jobs & Salaries** | [`captainhandsome/glassdoor-jobs-scraper`](https://apify.com/captainhandsome/glassdoor-jobs-scraper) | Title, company, salary estimate, rating, location, job URL, posting date | Hiring intelligence, compensation benchmarking |
59
+ | **Airbnb Vacation Rentals** | [`captainhandsome/airbnb-listings-search`](https://apify.com/captainhandsome/airbnb-listings-search) | Title, room type, nightly price, rating, reviews count, listing URL | Real estate research, market rate tracking |
60
+ | **SEC EDGAR Corporate Filings** | [`captainhandsome/sec-edgar-filings-search`](https://apify.com/captainhandsome/sec-edgar-filings-search) | Ticker, CIK, form (10-K, 10-Q, 8-K), filing date, primary document URL | Financial diligence, equity research, compliance |
61
+ | **USAspending Federal Awards** | [`captainhandsome/usaspending-federal-awards`](https://apify.com/captainhandsome/usaspending-federal-awards) | Recipient vendor, award amount, awarding agency, description, dates | Government contracting, procurement intel |
62
+ | **LinkedIn Public Jobs** | [`captainhandsome/linkedin-public-jobs-search`](https://apify.com/captainhandsome/linkedin-public-jobs-search) | Job title, employer, location, direct apply URL, posting age | Recruitment, tech talent monitoring |
63
+ | **Google Play App Reviews** | [`captainhandsome/google-play-reviews-scraper`](https://apify.com/captainhandsome/google-play-reviews-scraper) | Review text, star score, thumbs up, date, reviewer name | App store sentiment, competitor feedback |
64
+ | **YouTube Video Search** | [`captainhandsome/youtube-search-scraper`](https://apify.com/captainhandsome/youtube-search-scraper) | Title, video URL, channel, views count, duration, publish date | Content tracking, creator outreach |
65
+ | **Twitch Live Streams** | [`captainhandsome/twitch-live-streams-scraper`](https://apify.com/captainhandsome/twitch-live-streams-scraper) | Streamer username, title, viewer count, language, category | Esports analytics, live stream monitoring |
66
+ | **US Contractor Licenses** | [`captainhandsome/us-contractor-license-search`](https://apify.com/captainhandsome/us-contractor-license-search) | Contractor name, license number, classification, status, state | Trades verification, subcontractor diligence |
67
+ | **US Business Entity Registries** | [`captainhandsome/us-business-entity-search`](https://apify.com/captainhandsome/us-business-entity-search) | Legal entity name, filing number, jurisdiction, status | Legal due diligence, corporate registration checks |
68
+
69
+ ---
70
+
71
+ ## Python Quickstart
72
+
73
+ ### 1. Install dependencies
74
+
75
+ ```bash
76
+ pip install apify-client pandas python-dotenv
77
+ ```
78
+
79
+ ### 2. Export 50 Google Maps Leads to CSV
80
+
81
+ ```python
82
+ import os
83
+ from apify_client import ApifyClient
84
+ import pandas as pd
85
+
86
+ # Get your API token from https://console.apify.com/account/integrations
87
+ client = ApifyClient(os.getenv("APIFY_TOKEN"))
88
+
89
+ # Run the actor
90
+ run = client.actor("captainhandsome/google-maps-business-search").call(run_input={
91
+ "search_query": "commercial electricians",
92
+ "location": "Dallas, Texas",
93
+ "max_items": 50,
94
+ "include_details": True,
95
+ })
96
+
97
+ # Fetch dataset items and export to CSV
98
+ items = list(client.dataset(run["defaultDatasetId"]).iterate_items())
99
+ df = pd.DataFrame(items)
100
+ df.to_csv("dallas_electricians.csv", index=False)
101
+ print(f"Exported {len(df)} leads to dallas_electricians.csv")
102
+ ```
103
+
104
+ See [examples/google_maps_leads_to_csv.py](examples/google_maps_leads_to_csv.py) for the full script.
105
+
106
+ ---
107
+
108
+ ## Node.js Quickstart
109
+
110
+ ### 1. Install dependencies
111
+
112
+ ```bash
113
+ npm install apify-client
114
+ ```
115
+
116
+ ### 2. Query SEC EDGAR Filings
117
+
118
+ ```javascript
119
+ import { ApifyClient } from 'apify-client';
120
+
121
+ const client = new ApifyClient({ token: process.env.APIFY_TOKEN });
122
+
123
+ const run = await client.actor('captainhandsome/sec-edgar-filings-search').call({
124
+ companies: ['AAPL', 'NVDA', 'MSFT'],
125
+ forms: ['10-K'],
126
+ max_items: 15,
127
+ });
128
+
129
+ const { items } = await client.dataset(run.defaultDatasetId).listItems();
130
+ items.forEach(filing => {
131
+ console.log(`[${filing.ticker}] ${filing.form} (${filing.filing_date}): ${filing.primary_document_url}`);
132
+ });
133
+ ```
134
+
135
+ See [examples/sec_filings.js](examples/sec_filings.js) for the full script.
136
+
137
+ ---
138
+
139
+ ## No-Code & Automation Workflows
140
+
141
+ If you automate via n8n, Make, Zapier, or Google Sheets, ready-to-import blueprints are included in [`workflows/`](workflows/):
142
+
143
+ - **[Google Maps Leads to Google Sheets (n8n)](workflows/n8n_google_maps_to_sheets.json):** Daily automated cron scrape piping HVAC/trade leads directly into Google Sheets with deduplication.
144
+ - **[SEC EDGAR 10-K & 8-K Alerts to Slack (n8n)](workflows/n8n_sec_edgar_to_slack.json):** Hourly monitor alerting Slack or Discord when watchlisted public companies drop new filings.
145
+
146
+ ---
147
+
148
+ ## Pre-Built Example Tasks (Zero Code)
149
+
150
+ If you prefer runnable web UI tasks without writing any code, each actor includes pre-configured tasks published on Apify Store:
151
+
152
+ ### Google Maps Leads
153
+ - [Phoenix HVAC Company Leads](https://apify.com/captainhandsome/google-maps-business-search/tasks/phoenix-hvac-company-leads)
154
+ - [Dallas Commercial Electrician Leads](https://apify.com/captainhandsome/google-maps-business-search/tasks/dallas-commercial-electrician-leads)
155
+ - [Chicago Italian Restaurants & Reviews](https://apify.com/captainhandsome/google-maps-business-search/tasks/chicago-italian-restaurants)
156
+
157
+ ### Glassdoor Jobs
158
+ - [Austin Software Engineer Jobs](https://apify.com/captainhandsome/glassdoor-jobs-scraper/tasks/austin-software-engineer-jobs)
159
+ - [Remote Product Manager Jobs](https://apify.com/captainhandsome/glassdoor-jobs-scraper/tasks/remote-product-manager-jobs)
160
+ - [New York Data Scientist Postings](https://apify.com/captainhandsome/glassdoor-jobs-scraper/tasks/new-york-data-scientist-jobs)
161
+
162
+ ### Airbnb Rentals
163
+ - [Nashville Vacation Rental Listings](https://apify.com/captainhandsome/airbnb-listings-search/tasks/nashville-vacation-rentals)
164
+ - [Miami Beach Condos & Apartments](https://apify.com/captainhandsome/airbnb-listings-search/tasks/miami-beach-condos)
165
+ - [Austin Downtown Rental Market](https://apify.com/captainhandsome/airbnb-listings-search/tasks/austin-airbnb-listings)
166
+
167
+ ### YouTube & Google Play
168
+ - [Small Business Marketing Videos](https://apify.com/captainhandsome/youtube-search-scraper/tasks/small-business-marketing-videos)
169
+ - [Python Web Scraping Tutorials](https://apify.com/captainhandsome/youtube-search-scraper/tasks/python-web-scraping-tutorials)
170
+ - [Instagram 1-Star Play Store Reviews](https://apify.com/captainhandsome/google-play-reviews-scraper/tasks/instagram-one-star-reviews)
171
+
172
+ ---
173
+
174
+ ## Free Sample Datasets
175
+
176
+ Looking for clean data to benchmark, analyze, or train models? Verified sample bundles with metadata schemas are available in [`datasets/`](datasets/) and hosted publicly on Hugging Face Datasets:
177
+
178
+ 1. **Phoenix HVAC Contractor Leads:** [`datasets/phoenix_hvac_leads/`](datasets/phoenix_hvac_leads/) | [Hugging Face Hub](https://huggingface.co/datasets/joeygambino/phoenix-hvac-contractor-leads) (20 verified HVAC contractor profiles with ratings, addresses, and phone numbers).
179
+ 2. **California Licensed Contractors:** [`datasets/california_solar_contractors/`](datasets/california_solar_contractors/) | [Hugging Face Hub](https://huggingface.co/datasets/joeygambino/california-licensed-contractors) (Active C-46 and B licensed solar installers with state verification numbers).
180
+ 3. **Austin Software Engineer Postings:** [`datasets/austin_software_jobs/`](datasets/austin_software_jobs/) | [Hugging Face Hub](https://huggingface.co/datasets/joeygambino/austin-software-engineer-jobs) (Normalized job listings with estimated posting dates and salary ranges).
181
+
182
+ ---
183
+
184
+ ## AI Agent & MCP Integration
185
+
186
+ All actors in this repository conform to OpenAPI and JSON Schema standards, making them directly callable by AI agents via the Model Context Protocol (MCP):
187
+
188
+ ### Option A: Hosted Apify MCP Server (Claude Desktop / Cursor)
189
+
190
+ Add this to your `claude_desktop_config.json` or Cursor MCP settings:
191
+
192
+ ```json
193
+ {
194
+ \"mcpServers\": {
195
+ \"apify\": {
196
+ \"command\": \"npx\",
197
+ \"args\": [\"-y\", \"@apify/mcp-server\"],
198
+ \"env\": {
199
+ \"APIFY_TOKEN\": \"YOUR_APIFY_API_TOKEN\"
200
+ }
201
+ }
202
+ }
203
+ }
204
+ ```
205
+
206
+ ### Option B: Local Lightweight Python MCP Server
207
+
208
+ For local agent workflows without Node.js dependencies, a direct Python MCP server is included:
209
+
210
+ ```bash
211
+ export APIFY_TOKEN=\"your_token_here\"
212
+ python mcp_server.py
213
+ ```
214
+
215
+ Inspect tools and capabilities via [`mcp.json`](mcp.json).
216
+
217
+ ### Agent Prompts That Work Out-of-the-Box:
218
+ - *\"Search Google Maps for 50 commercial roofers in Atlanta with phone numbers and websites.\"*
219
+ - *\"Retrieve Apple and Microsoft Form 10-K filings from SEC EDGAR for the last 2 years.\"*
220
+ - *\"Find the 30 newest reviews for Duolingo on Google Play and analyze negative feedback.\"*
221
+
222
+ ---
223
+
224
+ ## In-Depth Engineering Guides
225
+
226
+ Technical case studies and problem-solution writeups are located in [`articles/`](articles/):
227
+
228
+ - **[Bypassing Playwright Headless Pagination Hurdles on Airbnb](articles/airbnb_playwright_pagination_guide.md):** How to solve sticky overlay modal interruptions and viewport boundary clipping in large headless browser crawls.
229
+ - **[Extracting & Normalizing Clean Job Posting Dates from Glassdoor](articles/glassdoor_posting_dates_guide.md):** Overcoming relative timestamp drift (\"24h\", \"3d\", \"30d+\") with deterministic parsing and ISO-8601 boundary tracking.
230
+
231
+ ---
232
+
233
+ ## Repository Structure
234
+
235
+ ```text
236
+ apify-scrapers/
237
+ README.md # Documentation and quickstart
238
+ LICENSE # MIT License
239
+ requirements.txt # Python client dependencies
240
+ package.json # Node.js dependencies
241
+ mcp.json # MCP tool registry specification
242
+ mcp_server.py # Native Python stdio MCP server
243
+ articles/ # In-depth engineering case studies
244
+ airbnb_playwright_pagination_guide.md
245
+ glassdoor_posting_dates_guide.md
246
+ reddit_community_responses.md # Reference technical answers for forums
247
+ datasets/ # Sample benchmark datasets
248
+ phoenix_hvac_leads/
249
+ california_solar_contractors/
250
+ austin_software_jobs/
251
+ workflows/ # No-code automation templates
252
+ n8n_google_maps_to_sheets.json
253
+ n8n_sec_edgar_to_slack.json
254
+ README.md
255
+ examples/ # Standalone developer scripts
256
+ google_maps_leads_to_csv.py
257
+ sec_edgar_filings_downloader.py
258
+ glassdoor_jobs_tracker.py
259
+ airbnb_market_scraper.py
260
+ usaspending_defense_awards.py
261
+ twitch_live_stream_monitor.py
262
+ google_maps_leads.js
263
+ sec_filings.js
264
+ ```
265
+
266
+ ---
267
+
268
+ ## Author & Support
269
+
270
+ Maintained by **[Joseph McRell](https://apify.com/captainhandsome)**.
271
+
272
+ - **Apify Store:** [https://apify.com/captainhandsome](https://apify.com/captainhandsome)
273
+ - **GitHub:** [@jlucasmcrell](https://github.com/jlucasmcrell)
274
+ - **Hugging Face:** [@joeygambino](https://huggingface.co/joeygambino)
275
+ - **Issues & Requests:** Please open an issue on this repository or submit a ticket on the respective Apify Actor Store page.
276
+
277
+ ---
278
+
279
+ ## License
280
+
281
+ This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
apify_data_scrapers-1.0.0/README.md ADDED
@@ -0,0 +1,257 @@
1
+ # Apify Public Data Scrapers & Extractors
2
+
3
+ [![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
4
+ [![Node.js 18+](https://img.shields.io/badge/node-18+-green.svg)](https://nodejs.org/)
5
+ [![Apify Verified](https://img.shields.io/badge/apify-store-orange.svg)](https://apify.com/captainhandsome)
6
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
7
+
8
+ A curated collection of reliable, production-ready scrapers and public-data extractors hosted on the **[Apify Store](https://apify.com/captainhandsome)**.
9
+
10
+ Each actor is built with strict schema validation, deterministic field mapping, self-healing DOM selectors, and pay-per-event pricing starting at **$0.0002 / start**.
11
+
12
+ ---
13
+
14
+ ## Quick Navigation
15
+
16
+ - [Available Extractors & Store Listings](#available-extractors--store-listings)
17
+ - [Python Quickstart](#python-quickstart)
18
+ - [Node.js Quickstart](#nodejs-quickstart)
19
+ - [No-Code & Automation Workflows (n8n, Sheets, Slack)](#no-code--automation-workflows)
20
+ - [Pre-Built Example Tasks (Zero Code)](#pre-built-example-tasks-zero-code)
21
+ - [Free Sample Datasets](#free-sample-datasets)
22
+ - [AI Agent & MCP Integration (Claude Desktop, Cursor, Custom Agent)](#ai-agent--mcp-integration)
23
+ - [In-Depth Engineering Guides](#in-depth-engineering-guides)
24
+ - [Repository Structure](#repository-structure)
25
+ - [Contributing & Author](#author--support)
26
+
27
+ ---
28
+
29
+ ## Available Extractors & Store Listings
30
+
31
+ | Tool | Store Link | Key Output Fields | Best For |
32
+ |---|---|---|---|
33
+ | **Google Maps Business Leads** | [`captainhandsome/google-maps-business-search`](https://apify.com/captainhandsome/google-maps-business-search) | Name, phone, website, rating, reviews, address, coordinates, hours | B2B lead generation, local agency prospecting |
34
+ | **Glassdoor Jobs & Salaries** | [`captainhandsome/glassdoor-jobs-scraper`](https://apify.com/captainhandsome/glassdoor-jobs-scraper) | Title, company, salary estimate, rating, location, job URL, posting date | Hiring intelligence, compensation benchmarking |
35
+ | **Airbnb Vacation Rentals** | [`captainhandsome/airbnb-listings-search`](https://apify.com/captainhandsome/airbnb-listings-search) | Title, room type, nightly price, rating, reviews count, listing URL | Real estate research, market rate tracking |
36
+ | **SEC EDGAR Corporate Filings** | [`captainhandsome/sec-edgar-filings-search`](https://apify.com/captainhandsome/sec-edgar-filings-search) | Ticker, CIK, form (10-K, 10-Q, 8-K), filing date, primary document URL | Financial diligence, equity research, compliance |
37
+ | **USAspending Federal Awards** | [`captainhandsome/usaspending-federal-awards`](https://apify.com/captainhandsome/usaspending-federal-awards) | Recipient vendor, award amount, awarding agency, description, dates | Government contracting, procurement intel |
38
+ | **LinkedIn Public Jobs** | [`captainhandsome/linkedin-public-jobs-search`](https://apify.com/captainhandsome/linkedin-public-jobs-search) | Job title, employer, location, direct apply URL, posting age | Recruitment, tech talent monitoring |
39
+ | **Google Play App Reviews** | [`captainhandsome/google-play-reviews-scraper`](https://apify.com/captainhandsome/google-play-reviews-scraper) | Review text, star score, thumbs up, date, reviewer name | App store sentiment, competitor feedback |
40
+ | **YouTube Video Search** | [`captainhandsome/youtube-search-scraper`](https://apify.com/captainhandsome/youtube-search-scraper) | Title, video URL, channel, views count, duration, publish date | Content tracking, creator outreach |
41
+ | **Twitch Live Streams** | [`captainhandsome/twitch-live-streams-scraper`](https://apify.com/captainhandsome/twitch-live-streams-scraper) | Streamer username, title, viewer count, language, category | Esports analytics, live stream monitoring |
42
+ | **US Contractor Licenses** | [`captainhandsome/us-contractor-license-search`](https://apify.com/captainhandsome/us-contractor-license-search) | Contractor name, license number, classification, status, state | Trades verification, subcontractor diligence |
43
+ | **US Business Entity Registries** | [`captainhandsome/us-business-entity-search`](https://apify.com/captainhandsome/us-business-entity-search) | Legal entity name, filing number, jurisdiction, status | Legal due diligence, corporate registration checks |
44
+
45
+ ---
46
+
47
+ ## Python Quickstart
48
+
49
+ ### 1. Install dependencies
50
+
51
+ ```bash
52
+ pip install apify-client pandas python-dotenv
53
+ ```
54
+
55
+ ### 2. Export 50 Google Maps Leads to CSV
56
+
57
+ ```python
58
+ import os
59
+ from apify_client import ApifyClient
60
+ import pandas as pd
61
+
62
+ # Get your API token from https://console.apify.com/account/integrations
63
+ client = ApifyClient(os.getenv("APIFY_TOKEN"))
64
+
65
+ # Run the actor
66
+ run = client.actor("captainhandsome/google-maps-business-search").call(run_input={
67
+ "search_query": "commercial electricians",
68
+ "location": "Dallas, Texas",
69
+ "max_items": 50,
70
+ "include_details": True,
71
+ })
72
+
73
+ # Fetch dataset items and export to CSV
74
+ items = list(client.dataset(run["defaultDatasetId"]).iterate_items())
75
+ df = pd.DataFrame(items)
76
+ df.to_csv("dallas_electricians.csv", index=False)
77
+ print(f"Exported {len(df)} leads to dallas_electricians.csv")
78
+ ```
79
+
80
+ See [examples/google_maps_leads_to_csv.py](examples/google_maps_leads_to_csv.py) for the full script.
81
+
82
+ ---
83
+
84
+ ## Node.js Quickstart
85
+
86
+ ### 1. Install dependencies
87
+
88
+ ```bash
89
+ npm install apify-client
90
+ ```
91
+
92
+ ### 2. Query SEC EDGAR Filings
93
+
94
+ ```javascript
95
+ import { ApifyClient } from 'apify-client';
96
+
97
+ const client = new ApifyClient({ token: process.env.APIFY_TOKEN });
98
+
99
+ const run = await client.actor('captainhandsome/sec-edgar-filings-search').call({
100
+ companies: ['AAPL', 'NVDA', 'MSFT'],
101
+ forms: ['10-K'],
102
+ max_items: 15,
103
+ });
104
+
105
+ const { items } = await client.dataset(run.defaultDatasetId).listItems();
106
+ items.forEach(filing => {
107
+ console.log(`[${filing.ticker}] ${filing.form} (${filing.filing_date}): ${filing.primary_document_url}`);
108
+ });
109
+ ```
110
+
111
+ See [examples/sec_filings.js](examples/sec_filings.js) for the full script.
112
+
113
+ ---
114
+
115
+ ## No-Code & Automation Workflows
116
+
117
+ If you automate via n8n, Make, Zapier, or Google Sheets, ready-to-import blueprints are included in [`workflows/`](workflows/):
118
+
119
+ - **[Google Maps Leads to Google Sheets (n8n)](workflows/n8n_google_maps_to_sheets.json):** Daily automated cron scrape piping HVAC/trade leads directly into Google Sheets with deduplication.
120
+ - **[SEC EDGAR 10-K & 8-K Alerts to Slack (n8n)](workflows/n8n_sec_edgar_to_slack.json):** Hourly monitor alerting Slack or Discord when watchlisted public companies drop new filings.
121
+
122
+ ---
123
+
124
+ ## Pre-Built Example Tasks (Zero Code)
125
+
126
+ If you prefer runnable web UI tasks without writing any code, each actor includes pre-configured tasks published on Apify Store:
127
+
128
+ ### Google Maps Leads
129
+ - [Phoenix HVAC Company Leads](https://apify.com/captainhandsome/google-maps-business-search/tasks/phoenix-hvac-company-leads)
130
+ - [Dallas Commercial Electrician Leads](https://apify.com/captainhandsome/google-maps-business-search/tasks/dallas-commercial-electrician-leads)
131
+ - [Chicago Italian Restaurants & Reviews](https://apify.com/captainhandsome/google-maps-business-search/tasks/chicago-italian-restaurants)
132
+
133
+ ### Glassdoor Jobs
134
+ - [Austin Software Engineer Jobs](https://apify.com/captainhandsome/glassdoor-jobs-scraper/tasks/austin-software-engineer-jobs)
135
+ - [Remote Product Manager Jobs](https://apify.com/captainhandsome/glassdoor-jobs-scraper/tasks/remote-product-manager-jobs)
136
+ - [New York Data Scientist Postings](https://apify.com/captainhandsome/glassdoor-jobs-scraper/tasks/new-york-data-scientist-jobs)
137
+
138
+ ### Airbnb Rentals
139
+ - [Nashville Vacation Rental Listings](https://apify.com/captainhandsome/airbnb-listings-search/tasks/nashville-vacation-rentals)
140
+ - [Miami Beach Condos & Apartments](https://apify.com/captainhandsome/airbnb-listings-search/tasks/miami-beach-condos)
141
+ - [Austin Downtown Rental Market](https://apify.com/captainhandsome/airbnb-listings-search/tasks/austin-airbnb-listings)
142
+
143
+ ### YouTube & Google Play
144
+ - [Small Business Marketing Videos](https://apify.com/captainhandsome/youtube-search-scraper/tasks/small-business-marketing-videos)
145
+ - [Python Web Scraping Tutorials](https://apify.com/captainhandsome/youtube-search-scraper/tasks/python-web-scraping-tutorials)
146
+ - [Instagram 1-Star Play Store Reviews](https://apify.com/captainhandsome/google-play-reviews-scraper/tasks/instagram-one-star-reviews)
147
+
148
+ ---
149
+
150
+ ## Free Sample Datasets
151
+
152
+ Looking for clean data to benchmark, analyze, or train models? Verified sample bundles with metadata schemas are available in [`datasets/`](datasets/) and hosted publicly on Hugging Face Datasets:
153
+
154
+ 1. **Phoenix HVAC Contractor Leads:** [`datasets/phoenix_hvac_leads/`](datasets/phoenix_hvac_leads/) | [Hugging Face Hub](https://huggingface.co/datasets/joeygambino/phoenix-hvac-contractor-leads) (20 verified HVAC contractor profiles with ratings, addresses, and phone numbers).
155
+ 2. **California Licensed Contractors:** [`datasets/california_solar_contractors/`](datasets/california_solar_contractors/) | [Hugging Face Hub](https://huggingface.co/datasets/joeygambino/california-licensed-contractors) (Active C-46 and B licensed solar installers with state verification numbers).
156
+ 3. **Austin Software Engineer Postings:** [`datasets/austin_software_jobs/`](datasets/austin_software_jobs/) | [Hugging Face Hub](https://huggingface.co/datasets/joeygambino/austin-software-engineer-jobs) (Normalized job listings with estimated posting dates and salary ranges).
157
+
158
+ ---
159
+
160
+ ## AI Agent & MCP Integration
161
+
162
+ All actors in this repository conform to OpenAPI and JSON Schema standards, making them directly callable by AI agents via the Model Context Protocol (MCP):
163
+
164
+ ### Option A: Hosted Apify MCP Server (Claude Desktop / Cursor)
165
+
166
+ Add this to your `claude_desktop_config.json` or Cursor MCP settings:
167
+
168
+ ```json
169
+ {
170
+ \"mcpServers\": {
171
+ \"apify\": {
172
+ \"command\": \"npx\",
173
+ \"args\": [\"-y\", \"@apify/mcp-server\"],
174
+ \"env\": {
175
+ \"APIFY_TOKEN\": \"YOUR_APIFY_API_TOKEN\"
176
+ }
177
+ }
178
+ }
179
+ }
180
+ ```
181
+
182
+ ### Option B: Local Lightweight Python MCP Server
183
+
184
+ For local agent workflows without Node.js dependencies, a direct Python MCP server is included:
185
+
186
+ ```bash
187
+ export APIFY_TOKEN=\"your_token_here\"
188
+ python mcp_server.py
189
+ ```
190
+
191
+ Inspect tools and capabilities via [`mcp.json`](mcp.json).
192
+
193
+ ### Agent Prompts That Work Out-of-the-Box:
194
+ - *\"Search Google Maps for 50 commercial roofers in Atlanta with phone numbers and websites.\"*
195
+ - *\"Retrieve Apple and Microsoft Form 10-K filings from SEC EDGAR for the last 2 years.\"*
196
+ - *\"Find the 30 newest reviews for Duolingo on Google Play and analyze negative feedback.\"*
197
+
198
+ ---
199
+
200
+ ## In-Depth Engineering Guides
201
+
202
+ Technical case studies and problem-solution writeups are located in [`articles/`](articles/):
203
+
204
+ - **[Bypassing Playwright Headless Pagination Hurdles on Airbnb](articles/airbnb_playwright_pagination_guide.md):** How to solve sticky overlay modal interruptions and viewport boundary clipping in large headless browser crawls.
205
+ - **[Extracting & Normalizing Clean Job Posting Dates from Glassdoor](articles/glassdoor_posting_dates_guide.md):** Overcoming relative timestamp drift (\"24h\", \"3d\", \"30d+\") with deterministic parsing and ISO-8601 boundary tracking.
206
+
207
+ ---
208
+
209
+ ## Repository Structure
210
+
211
+ ```text
212
+ apify-scrapers/
213
+ README.md # Documentation and quickstart
214
+ LICENSE # MIT License
215
+ requirements.txt # Python client dependencies
216
+ package.json # Node.js dependencies
217
+ mcp.json # MCP tool registry specification
218
+ mcp_server.py # Native Python stdio MCP server
219
+ articles/ # In-depth engineering case studies
220
+ airbnb_playwright_pagination_guide.md
221
+ glassdoor_posting_dates_guide.md
222
+ reddit_community_responses.md # Reference technical answers for forums
223
+ datasets/ # Sample benchmark datasets
224
+ phoenix_hvac_leads/
225
+ california_solar_contractors/
226
+ austin_software_jobs/
227
+ workflows/ # No-code automation templates
228
+ n8n_google_maps_to_sheets.json
229
+ n8n_sec_edgar_to_slack.json
230
+ README.md
231
+ examples/ # Standalone developer scripts
232
+ google_maps_leads_to_csv.py
233
+ sec_edgar_filings_downloader.py
234
+ glassdoor_jobs_tracker.py
235
+ airbnb_market_scraper.py
236
+ usaspending_defense_awards.py
237
+ twitch_live_stream_monitor.py
238
+ google_maps_leads.js
239
+ sec_filings.js
240
+ ```
241
+
242
+ ---
243
+
244
+ ## Author & Support
245
+
246
+ Maintained by **[Joseph McRell](https://apify.com/captainhandsome)**.
247
+
248
+ - **Apify Store:** [https://apify.com/captainhandsome](https://apify.com/captainhandsome)
249
+ - **GitHub:** [@jlucasmcrell](https://github.com/jlucasmcrell)
250
+ - **Hugging Face:** [@joeygambino](https://huggingface.co/joeygambino)
251
+ - **Issues & Requests:** Please open an issue on this repository or submit a ticket on the respective Apify Actor Store page.
252
+
253
+ ---
254
+
255
+ ## License
256
+
257
+ This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
apify_data_scrapers-1.0.0/apify_data_scrapers.egg-info/PKG-INFO ADDED
@@ -0,0 +1,281 @@
1
+ Metadata-Version: 2.4
2
+ Name: apify-data-scrapers
3
+ Version: 1.0.0
4
+ Summary: MCP tools and SDK for Apify public data scrapers (Google Maps, SEC EDGAR, Glassdoor, Contractor Licenses)
5
+ Home-page: https://github.com/jlucasmcrell/apify-scrapers
6
+ Author: Joseph McRell
7
+ Classifier: Programming Language :: Python :: 3
8
+ Classifier: License :: OSI Approved :: MIT License
9
+ Classifier: Operating System :: OS Independent
10
+ Requires-Python: >=3.9
11
+ Description-Content-Type: text/markdown
12
+ License-File: LICENSE
13
+ Requires-Dist: apify-client>=1.8.0
14
+ Requires-Dist: mcp>=1.0.0
15
+ Dynamic: author
16
+ Dynamic: classifier
17
+ Dynamic: description
18
+ Dynamic: description-content-type
19
+ Dynamic: home-page
20
+ Dynamic: license-file
21
+ Dynamic: requires-dist
22
+ Dynamic: requires-python
23
+ Dynamic: summary
24
+
25
+ # Apify Public Data Scrapers & Extractors
26
+
27
+ [![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
28
+ [![Node.js 18+](https://img.shields.io/badge/node-18+-green.svg)](https://nodejs.org/)
29
+ [![Apify Verified](https://img.shields.io/badge/apify-store-orange.svg)](https://apify.com/captainhandsome)
30
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
31
+
32
+ A curated collection of reliable, production-ready scrapers and public-data extractors hosted on the **[Apify Store](https://apify.com/captainhandsome)**.
33
+
34
+ Each actor is built with strict schema validation, deterministic field mapping, self-healing DOM selectors, and pay-per-event pricing starting at **$0.0002 / start**.
35
+
36
+ ---
37
+
38
+ ## Quick Navigation
39
+
40
+ - [Available Extractors & Store Listings](#available-extractors--store-listings)
41
+ - [Python Quickstart](#python-quickstart)
42
+ - [Node.js Quickstart](#nodejs-quickstart)
43
+ - [No-Code & Automation Workflows (n8n, Sheets, Slack)](#no-code--automation-workflows)
44
+ - [Pre-Built Example Tasks (Zero Code)](#pre-built-example-tasks-zero-code)
45
+ - [Free Sample Datasets](#free-sample-datasets)
46
+ - [AI Agent & MCP Integration (Claude Desktop, Cursor, Custom Agent)](#ai-agent--mcp-integration)
47
+ - [In-Depth Engineering Guides](#in-depth-engineering-guides)
48
+ - [Repository Structure](#repository-structure)
49
+ - [Contributing & Author](#author--support)
50
+
51
+ ---
52
+
53
+ ## Available Extractors & Store Listings
54
+
55
+ | Tool | Store Link | Key Output Fields | Best For |
56
+ |---|---|---|---|
57
+ | **Google Maps Business Leads** | [`captainhandsome/google-maps-business-search`](https://apify.com/captainhandsome/google-maps-business-search) | Name, phone, website, rating, reviews, address, coordinates, hours | B2B lead generation, local agency prospecting |
58
+ | **Glassdoor Jobs & Salaries** | [`captainhandsome/glassdoor-jobs-scraper`](https://apify.com/captainhandsome/glassdoor-jobs-scraper) | Title, company, salary estimate, rating, location, job URL, posting date | Hiring intelligence, compensation benchmarking |
59
+ | **Airbnb Vacation Rentals** | [`captainhandsome/airbnb-listings-search`](https://apify.com/captainhandsome/airbnb-listings-search) | Title, room type, nightly price, rating, reviews count, listing URL | Real estate research, market rate tracking |
60
+ | **SEC EDGAR Corporate Filings** | [`captainhandsome/sec-edgar-filings-search`](https://apify.com/captainhandsome/sec-edgar-filings-search) | Ticker, CIK, form (10-K, 10-Q, 8-K), filing date, primary document URL | Financial diligence, equity research, compliance |
61
+ | **USAspending Federal Awards** | [`captainhandsome/usaspending-federal-awards`](https://apify.com/captainhandsome/usaspending-federal-awards) | Recipient vendor, award amount, awarding agency, description, dates | Government contracting, procurement intel |
62
+ | **LinkedIn Public Jobs** | [`captainhandsome/linkedin-public-jobs-search`](https://apify.com/captainhandsome/linkedin-public-jobs-search) | Job title, employer, location, direct apply URL, posting age | Recruitment, tech talent monitoring |
63
+ | **Google Play App Reviews** | [`captainhandsome/google-play-reviews-scraper`](https://apify.com/captainhandsome/google-play-reviews-scraper) | Review text, star score, thumbs up, date, reviewer name | App store sentiment, competitor feedback |
64
+ | **YouTube Video Search** | [`captainhandsome/youtube-search-scraper`](https://apify.com/captainhandsome/youtube-search-scraper) | Title, video URL, channel, views count, duration, publish date | Content tracking, creator outreach |
65
+ | **Twitch Live Streams** | [`captainhandsome/twitch-live-streams-scraper`](https://apify.com/captainhandsome/twitch-live-streams-scraper) | Streamer username, title, viewer count, language, category | Esports analytics, live stream monitoring |
66
+ | **US Contractor Licenses** | [`captainhandsome/us-contractor-license-search`](https://apify.com/captainhandsome/us-contractor-license-search) | Contractor name, license number, classification, status, state | Trades verification, subcontractor diligence |
67
+ | **US Business Entity Registries** | [`captainhandsome/us-business-entity-search`](https://apify.com/captainhandsome/us-business-entity-search) | Legal entity name, filing number, jurisdiction, status | Legal due diligence, corporate registration checks |
68
+
69
+ ---
70
+
71
+ ## Python Quickstart
72
+
73
+ ### 1. Install dependencies
74
+
75
+ ```bash
76
+ pip install apify-client pandas python-dotenv
77
+ ```
78
+
79
+ ### 2. Export 50 Google Maps Leads to CSV
80
+
81
+ ```python
82
+ import os
83
+ from apify_client import ApifyClient
84
+ import pandas as pd
85
+
86
+ # Get your API token from https://console.apify.com/account/integrations
87
+ client = ApifyClient(os.getenv("APIFY_TOKEN"))
88
+
89
+ # Run the actor
90
+ run = client.actor("captainhandsome/google-maps-business-search").call(run_input={
91
+ "search_query": "commercial electricians",
92
+ "location": "Dallas, Texas",
93
+ "max_items": 50,
94
+ "include_details": True,
95
+ })
96
+
97
+ # Fetch dataset items and export to CSV
98
+ items = list(client.dataset(run["defaultDatasetId"]).iterate_items())
99
+ df = pd.DataFrame(items)
100
+ df.to_csv("dallas_electricians.csv", index=False)
101
+ print(f"Exported {len(df)} leads to dallas_electricians.csv")
102
+ ```
103
+
104
+ See [examples/google_maps_leads_to_csv.py](examples/google_maps_leads_to_csv.py) for the full script.
105
+
106
+ ---
107
+
108
+ ## Node.js Quickstart
109
+
110
+ ### 1. Install dependencies
111
+
112
+ ```bash
113
+ npm install apify-client
114
+ ```
115
+
116
+ ### 2. Query SEC EDGAR Filings
117
+
118
+ ```javascript
119
+ import { ApifyClient } from 'apify-client';
120
+
121
+ const client = new ApifyClient({ token: process.env.APIFY_TOKEN });
122
+
123
+ const run = await client.actor('captainhandsome/sec-edgar-filings-search').call({
124
+ companies: ['AAPL', 'NVDA', 'MSFT'],
125
+ forms: ['10-K'],
126
+ max_items: 15,
127
+ });
128
+
129
+ const { items } = await client.dataset(run.defaultDatasetId).listItems();
130
+ items.forEach(filing => {
131
+ console.log(`[${filing.ticker}] ${filing.form} (${filing.filing_date}): ${filing.primary_document_url}`);
132
+ });
133
+ ```
134
+
135
+ See [examples/sec_filings.js](examples/sec_filings.js) for the full script.
136
+
137
+ ---
138
+
139
+ ## No-Code & Automation Workflows
140
+
141
+ If you automate via n8n, Make, Zapier, or Google Sheets, ready-to-import blueprints are included in [`workflows/`](workflows/):
142
+
143
+ - **[Google Maps Leads to Google Sheets (n8n)](workflows/n8n_google_maps_to_sheets.json):** Daily automated cron scrape piping HVAC/trade leads directly into Google Sheets with deduplication.
144
+ - **[SEC EDGAR 10-K & 8-K Alerts to Slack (n8n)](workflows/n8n_sec_edgar_to_slack.json):** Hourly monitor alerting Slack or Discord when watchlisted public companies drop new filings.
145
+
146
+ ---
147
+
148
+ ## Pre-Built Example Tasks (Zero Code)
149
+
150
+ If you prefer runnable web UI tasks without writing any code, each actor includes pre-configured tasks published on Apify Store:
151
+
152
+ ### Google Maps Leads
153
+ - [Phoenix HVAC Company Leads](https://apify.com/captainhandsome/google-maps-business-search/tasks/phoenix-hvac-company-leads)
154
+ - [Dallas Commercial Electrician Leads](https://apify.com/captainhandsome/google-maps-business-search/tasks/dallas-commercial-electrician-leads)
155
+ - [Chicago Italian Restaurants & Reviews](https://apify.com/captainhandsome/google-maps-business-search/tasks/chicago-italian-restaurants)
156
+
157
+ ### Glassdoor Jobs
158
+ - [Austin Software Engineer Jobs](https://apify.com/captainhandsome/glassdoor-jobs-scraper/tasks/austin-software-engineer-jobs)
159
+ - [Remote Product Manager Jobs](https://apify.com/captainhandsome/glassdoor-jobs-scraper/tasks/remote-product-manager-jobs)
160
+ - [New York Data Scientist Postings](https://apify.com/captainhandsome/glassdoor-jobs-scraper/tasks/new-york-data-scientist-jobs)
161
+
162
+ ### Airbnb Rentals
163
+ - [Nashville Vacation Rental Listings](https://apify.com/captainhandsome/airbnb-listings-search/tasks/nashville-vacation-rentals)
164
+ - [Miami Beach Condos & Apartments](https://apify.com/captainhandsome/airbnb-listings-search/tasks/miami-beach-condos)
165
+ - [Austin Downtown Rental Market](https://apify.com/captainhandsome/airbnb-listings-search/tasks/austin-airbnb-listings)
166
+
167
+ ### YouTube & Google Play
168
+ - [Small Business Marketing Videos](https://apify.com/captainhandsome/youtube-search-scraper/tasks/small-business-marketing-videos)
169
+ - [Python Web Scraping Tutorials](https://apify.com/captainhandsome/youtube-search-scraper/tasks/python-web-scraping-tutorials)
170
+ - [Instagram 1-Star Play Store Reviews](https://apify.com/captainhandsome/google-play-reviews-scraper/tasks/instagram-one-star-reviews)
171
+
172
+ ---
173
+
174
+ ## Free Sample Datasets
175
+
176
+ Looking for clean data to benchmark, analyze, or train models? Verified sample bundles with metadata schemas are available in [`datasets/`](datasets/) and hosted publicly on Hugging Face Datasets:
177
+
178
+ 1. **Phoenix HVAC Contractor Leads:** [`datasets/phoenix_hvac_leads/`](datasets/phoenix_hvac_leads/) | [Hugging Face Hub](https://huggingface.co/datasets/joeygambino/phoenix-hvac-contractor-leads) (20 verified HVAC contractor profiles with ratings, addresses, and phone numbers).
179
+ 2. **California Licensed Contractors:** [`datasets/california_solar_contractors/`](datasets/california_solar_contractors/) | [Hugging Face Hub](https://huggingface.co/datasets/joeygambino/california-licensed-contractors) (Active C-46 and B licensed solar installers with state verification numbers).
180
+ 3. **Austin Software Engineer Postings:** [`datasets/austin_software_jobs/`](datasets/austin_software_jobs/) | [Hugging Face Hub](https://huggingface.co/datasets/joeygambino/austin-software-engineer-jobs) (Normalized job listings with estimated posting dates and salary ranges).
181
+
182
+ ---
183
+
184
+ ## AI Agent & MCP Integration
185
+
186
+ All actors in this repository conform to OpenAPI and JSON Schema standards, making them directly callable by AI agents via the Model Context Protocol (MCP):
187
+
188
+ ### Option A: Hosted Apify MCP Server (Claude Desktop / Cursor)
189
+
190
+ Add this to your `claude_desktop_config.json` or Cursor MCP settings:
191
+
192
+ ```json
193
+ {
194
+ \"mcpServers\": {
195
+ \"apify\": {
196
+ \"command\": \"npx\",
197
+ \"args\": [\"-y\", \"@apify/mcp-server\"],
198
+ \"env\": {
199
+ \"APIFY_TOKEN\": \"YOUR_APIFY_API_TOKEN\"
200
+ }
201
+ }
202
+ }
203
+ }
204
+ ```
205
+
206
+ ### Option B: Local Lightweight Python MCP Server
207
+
208
+ For local agent workflows without Node.js dependencies, a direct Python MCP server is included:
209
+
210
+ ```bash
211
+ export APIFY_TOKEN=\"your_token_here\"
212
+ python mcp_server.py
213
+ ```
214
+
215
+ Inspect tools and capabilities via [`mcp.json`](mcp.json).
216
+
217
+ ### Agent Prompts That Work Out-of-the-Box:
218
+ - *\"Search Google Maps for 50 commercial roofers in Atlanta with phone numbers and websites.\"*
219
+ - *\"Retrieve Apple and Microsoft Form 10-K filings from SEC EDGAR for the last 2 years.\"*
220
+ - *\"Find the 30 newest reviews for Duolingo on Google Play and analyze negative feedback.\"*
221
+
222
+ ---
223
+
224
+ ## In-Depth Engineering Guides
225
+
226
+ Technical case studies and problem-solution writeups are located in [`articles/`](articles/):
227
+
228
+ - **[Bypassing Playwright Headless Pagination Hurdles on Airbnb](articles/airbnb_playwright_pagination_guide.md):** How to solve sticky overlay modal interruptions and viewport boundary clipping in large headless browser crawls.
229
+ - **[Extracting & Normalizing Clean Job Posting Dates from Glassdoor](articles/glassdoor_posting_dates_guide.md):** Overcoming relative timestamp drift (\"24h\", \"3d\", \"30d+\") with deterministic parsing and ISO-8601 boundary tracking.
230
+
231
+ ---
232
+
233
+ ## Repository Structure
234
+
235
+ ```text
236
+ apify-scrapers/
237
+ README.md # Documentation and quickstart
238
+ LICENSE # MIT License
239
+ requirements.txt # Python client dependencies
240
+ package.json # Node.js dependencies
241
+ mcp.json # MCP tool registry specification
242
+ mcp_server.py # Native Python stdio MCP server
243
+ articles/ # In-depth engineering case studies
244
+ airbnb_playwright_pagination_guide.md
245
+ glassdoor_posting_dates_guide.md
246
+ reddit_community_responses.md # Reference technical answers for forums
247
+ datasets/ # Sample benchmark datasets
248
+ phoenix_hvac_leads/
249
+ california_solar_contractors/
250
+ austin_software_jobs/
251
+ workflows/ # No-code automation templates
252
+ n8n_google_maps_to_sheets.json
253
+ n8n_sec_edgar_to_slack.json
254
+ README.md
255
+ examples/ # Standalone developer scripts
256
+ google_maps_leads_to_csv.py
257
+ sec_edgar_filings_downloader.py
258
+ glassdoor_jobs_tracker.py
259
+ airbnb_market_scraper.py
260
+ usaspending_defense_awards.py
261
+ twitch_live_stream_monitor.py
262
+ google_maps_leads.js
263
+ sec_filings.js
264
+ ```
265
+
266
+ ---
267
+
268
+ ## Author & Support
269
+
270
+ Maintained by **[Joseph McRell](https://apify.com/captainhandsome)**.
271
+
272
+ - **Apify Store:** [https://apify.com/captainhandsome](https://apify.com/captainhandsome)
273
+ - **GitHub:** [@jlucasmcrell](https://github.com/jlucasmcrell)
274
+ - **Hugging Face:** [@joeygambino](https://huggingface.co/joeygambino)
275
+ - **Issues & Requests:** Please open an issue on this repository or submit a ticket on the respective Apify Actor Store page.
276
+
277
+ ---
278
+
279
+ ## License
280
+
281
+ This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
apify_data_scrapers-1.0.0/apify_data_scrapers.egg-info/SOURCES.txt ADDED
@@ -0,0 +1,10 @@
1
+ LICENSE
2
+ README.md
3
+ mcp_server.py
4
+ setup.py
5
+ apify_data_scrapers.egg-info/PKG-INFO
6
+ apify_data_scrapers.egg-info/SOURCES.txt
7
+ apify_data_scrapers.egg-info/dependency_links.txt
8
+ apify_data_scrapers.egg-info/entry_points.txt
9
+ apify_data_scrapers.egg-info/requires.txt
10
+ apify_data_scrapers.egg-info/top_level.txt
apify_data_scrapers-1.0.0/apify_data_scrapers.egg-info/entry_points.txt ADDED
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ apify-data-scrapers = mcp_server:main
apify_data_scrapers-1.0.0/apify_data_scrapers.egg-info/requires.txt ADDED
@@ -0,0 +1,2 @@
1
+ apify-client>=1.8.0
2
+ mcp>=1.0.0
apify_data_scrapers-1.0.0/mcp_server.py ADDED
@@ -0,0 +1,206 @@
1
+ import os
2
+ import sys
3
+ import json
4
+ import asyncio
5
+ from typing import Any, Dict, List, Optional
6
+ import urllib.request
7
+ import urllib.error
8
+
9
+ # MCP server protocol specification for Apify Scrapers Fleet
10
+ # Author: Joseph McRell (@jlucasmcrell)
11
+ # Tools provided:
12
+ # 1. google_maps_search: Extract B2B leads, businesses, phone numbers, ratings
13
+ # 2. glassdoor_jobs_search: Extract job postings with normalized posting dates
14
+ # 3. sec_edgar_filings: Query SEC 10-K, 10-Q, 8-K filings by ticker/company
15
+ # 4. usaspending_contracts: Search federal awards, defense contracts, prime obligations
16
+ # 5. twitch_live_streams: Live stream monitoring by game or language
17
+ # 6. airbnb_listings_search: Real estate & short-term rental market rates
18
+
19
+ APIFY_API_BASE = "https://api.apify.com/v2"
20
+
21
+ ACTORS = {
22
+ "google_maps": "captainhandsome/google-maps-business-search",
23
+ "glassdoor_jobs": "captainhandsome/glassdoor-jobs-scraper",
24
+ "sec_edgar": "captainhandsome/sec-edgar-filings-search",
25
+ "usaspending": "captainhandsome/usaspending-federal-awards",
26
+ "twitch_streams": "captainhandsome/twitch-live-streams-scraper",
27
+ "airbnb": "captainhandsome/airbnb-listings-search",
28
+ }
29
+
30
+ def get_token() -> str:
31
+ token = os.environ.get("APIFY_TOKEN")
32
+ if not token:
33
+ raise ValueError("APIFY_TOKEN environment variable is required to run Apify tools.")
34
+ return token
35
+
36
+ def run_actor_sync(actor_id: str, run_input: Dict[str, Any], timeout_secs: int = 120) -> List[Dict[str, Any]]:
37
+ """Runs an Apify actor synchronously and returns the output dataset items."""
38
+ token = get_token()
39
+ clean_id = actor_id.replace("/", "~")
40
+ url = f"{APIFY_API_BASE}/acts/{clean_id}/run-sync-get-dataset-items?token={token}&timeout={timeout_secs}"
41
+
42
+ payload = json.dumps(run_input).encode("utf-8")
43
+ req = urllib.request.Request(
44
+ url,
45
+ data=payload,
46
+ headers={"Content-Type": "application/json"},
47
+ method="POST"
48
+ )
49
+
50
+ try:
51
+ with urllib.request.urlopen(req, timeout=timeout_secs + 10) as response:
52
+ data = json.loads(response.read().decode("utf-8"))
53
+ return data
54
+ except urllib.error.HTTPError as e:
55
+ err_msg = e.read().decode("utf-8", errors="ignore")
56
+ raise RuntimeError(f"Apify API error ({e.code}): {err_msg}")
57
+ except Exception as e:
58
+ raise RuntimeError(f"Actor run failed: {str(e)}")
59
+
60
+ # Standard JSON-RPC MCP server loop (stdio)
61
+ async def handle_stdio():
62
+ reader = asyncio.StreamReader()
63
+ protocol = asyncio.StreamReaderProtocol(reader)
64
+ loop = asyncio.get_event_loop()
65
+ await loop.connect_read_pipe(lambda: protocol, sys.stdin)
66
+ writer = sys.stdout
67
+
68
+ while True:
69
+ line = await reader.readline()
70
+ if not line:
71
+ break
72
+ try:
73
+ req = json.loads(line.decode("utf-8"))
74
+ except Exception:
75
+ continue
76
+
77
+ req_id = req.get("id")
78
+ method = req.get("method")
79
+ params = req.get("params", {})
80
+
81
+ if method == "initialize":
82
+ res = {
83
+ "jsonrpc": "2.0",
84
+ "id": req_id,
85
+ "result": {
86
+ "protocolVersion": "2024-11-05",
87
+ "capabilities": {"tools": {}},
88
+ "serverInfo": {
89
+ "name": "apify-scrapers-mcp",
90
+ "version": "1.0.0"
91
+ }
92
+ }
93
+ }
94
+ writer.write(json.dumps(res) + "\n")
95
+ writer.flush()
96
+
97
+ elif method == "tools/list":
98
+ res = {
99
+ "jsonrpc": "2.0",
100
+ "id": req_id,
101
+ "result": {
102
+ "tools": [
103
+ {
104
+ "name": "google_maps_search",
105
+ "description": "Extract verified business leads, addresses, ratings, and phone numbers from Google Maps. Backed by captainhandsome/google-maps-business-search.",
106
+ "inputSchema": {
107
+ "type": "object",
108
+ "properties": {
109
+ "search_query": {"type": "string", "description": "e.g. 'HVAC contractors in Phoenix, AZ'"},
110
+ "max_results": {"type": "integer", "default": 10, "description": "Number of leads to retrieve (1-100)"}
111
+ },
112
+ "required": ["search_query"]
113
+ }
114
+ },
115
+ {
116
+ "name": "glassdoor_jobs_search",
117
+ "description": "Scrape Glassdoor job openings with canonical listing URLs and normalized posting dates. Backed by captainhandsome/glassdoor-jobs-scraper.",
118
+ "inputSchema": {
119
+ "type": "object",
120
+ "properties": {
121
+ "job_title": {"type": "string", "description": "e.g. 'Software Engineer'"},
122
+ "location": {"type": "string", "description": "e.g. 'Austin, TX' or 'Remote'"},
123
+ "max_results": {"type": "integer", "default": 10}
124
+ },
125
+ "required": ["job_title"]
126
+ }
127
+ },
128
+ {
129
+ "name": "sec_edgar_filings",
130
+ "description": "Retrieve corporate SEC filings (10-K, 10-Q, 8-K) by company name or stock ticker. Backed by captainhandsome/sec-edgar-filings-search.",
131
+ "inputSchema": {
132
+ "type": "object",
133
+ "properties": {
134
+ "ticker": {"type": "string", "description": "e.g. 'AAPL' or 'NVDA'"},
135
+ "form_type": {"type": "string", "default": "10-K", "description": "10-K, 10-Q, or 8-K"},
136
+ "max_results": {"type": "integer", "default": 5}
137
+ },
138
+ "required": ["ticker"]
139
+ }
140
+ },
141
+ {
142
+ "name": "usaspending_contracts",
143
+ "description": "Search US federal procurement, defense awards, and prime agency obligations. Backed by captainhandsome/usaspending-federal-awards.",
144
+ "inputSchema": {
145
+ "type": "object",
146
+ "properties": {
147
+ "recipient_name": {"type": "string", "description": "e.g. 'Lockheed Martin' or 'Palantir'"},
148
+ "max_results": {"type": "integer", "default": 10}
149
+ },
150
+ "required": ["recipient_name"]
151
+ }
152
+ }
153
+ ]
154
+ }
155
+ }
156
+ writer.write(json.dumps(res) + "\n")
157
+ writer.flush()
158
+
159
+ elif method == "tools/call":
160
+ tool_name = params.get("name")
161
+ tool_args = params.get("arguments", {})
162
+ try:
163
+ if tool_name == "google_maps_search":
164
+ query = tool_args.get("search_query")
165
+ limit = int(tool_args.get("max_results", 10))
166
+ data = run_actor_sync(ACTORS["google_maps"], {"search_terms": [query], "max_items": limit})
167
+ elif tool_name == "glassdoor_jobs_search":
168
+ title = tool_args.get("job_title")
169
+ loc = tool_args.get("location", "")
170
+ limit = int(tool_args.get("max_results", 10))
171
+ data = run_actor_sync(ACTORS["glassdoor_jobs"], {"keyword": title, "location": loc, "max_results": limit})
172
+ elif tool_name == "sec_edgar_filings":
173
+ ticker = tool_args.get("ticker")
174
+ form = tool_args.get("form_type", "10-K")
175
+ limit = int(tool_args.get("max_results", 5))
176
+ data = run_actor_sync(ACTORS["sec_edgar"], {"company_or_ticker": ticker, "form_type": form, "max_items": limit})
177
+ elif tool_name == "usaspending_contracts":
178
+ rec = tool_args.get("recipient_name")
179
+ limit = int(tool_args.get("max_results", 10))
180
+ data = run_actor_sync(ACTORS["usaspending"], {"award_type": "contracts", "recipient_search_text": rec, "max_items": limit})
181
+ else:
182
+ raise ValueError(f"Unknown tool: {tool_name}")
183
+
184
+ res = {
185
+ "jsonrpc": "2.0",
186
+ "id": req_id,
187
+ "result": {
188
+ "content": [
189
+ {"type": "text", "text": json.dumps(data, indent=2)}
190
+ ]
191
+ }
192
+ }
193
+ except Exception as err:
194
+ res = {
195
+ "jsonrpc": "2.0",
196
+ "id": req_id,
197
+ "error": {
198
+ "code": -32603,
199
+ "message": str(err)
200
+ }
201
+ }
202
+ writer.write(json.dumps(res) + "\n")
203
+ writer.flush()
204
+
205
+ if __name__ == "__main__":
206
+ asyncio.run(handle_stdio())
apify_data_scrapers-1.0.0/setup.cfg ADDED
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
apify_data_scrapers-1.0.0/setup.py ADDED
@@ -0,0 +1,28 @@
1
+ import setuptools
2
+
3
+ setuptools.setup(
4
+ name="apify-data-scrapers",
5
+ version="1.0.0",
6
+ author="Joseph McRell",
7
+ description="MCP tools and SDK for Apify public data scrapers (Google Maps, SEC EDGAR, Glassdoor, Contractor Licenses)",
8
+ long_description=open("README.md", "r", encoding="utf-8").read(),
9
+ long_description_content_type="text/markdown",
10
+ url="https://github.com/jlucasmcrell/apify-scrapers",
11
+ packages=setuptools.find_packages(),
12
+ py_modules=["mcp_server"],
13
+ classifiers=[
14
+ "Programming Language :: Python :: 3",
15
+ "License :: OSI Approved :: MIT License",
16
+ "Operating System :: OS Independent",
17
+ ],
18
+ python_requires=">=3.9",
19
+ install_requires=[
20
+ "apify-client>=1.8.0",
21
+ "mcp>=1.0.0",
22
+ ],
23
+ entry_points={
24
+ "console_scripts": [
25
+ "apify-data-scrapers=mcp_server:main",
26
+ ],
27
+ },
28
+ )