cliqtomedia-apify 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cliqtomedia_apify-0.1.0/.gitignore +5 -0
- cliqtomedia_apify-0.1.0/LICENSE +21 -0
- cliqtomedia_apify-0.1.0/PKG-INFO +181 -0
- cliqtomedia_apify-0.1.0/README.md +155 -0
- cliqtomedia_apify-0.1.0/pyproject.toml +43 -0
- cliqtomedia_apify-0.1.0/src/cliqtomedia_apify/__init__.py +6 -0
- cliqtomedia_apify-0.1.0/src/cliqtomedia_apify/__main__.py +5 -0
- cliqtomedia_apify-0.1.0/src/cliqtomedia_apify/cli.py +86 -0
- cliqtomedia_apify-0.1.0/src/cliqtomedia_apify/client.py +138 -0
- cliqtomedia_apify-0.1.0/tests/test_client.py +156 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Cliqto Media
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: cliqtomedia-apify
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Run any public Apify Actor from the command line or Python and get its dataset items. No dependencies.
|
|
5
|
+
Project-URL: Homepage, https://cliqtomedia.com/actors/?utm_source=pypi&utm_medium=template&utm_campaign=promo-2026-11
|
|
6
|
+
Project-URL: Documentation, https://docs.apify.com/api/v2
|
|
7
|
+
Author: Cliqto Media
|
|
8
|
+
Maintainer: Cliqto Media
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: api-client,apify,automation,data-extraction,lead-generation,scraper,web-scraping
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Topic :: Internet :: WWW/HTTP
|
|
22
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
23
|
+
Classifier: Topic :: Utilities
|
|
24
|
+
Requires-Python: >=3.9
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
|
|
27
|
+
# cliqtomedia-apify: run any Apify Actor from Python or the command line
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
Run any public Apify Actor and get its dataset items as JSON, JSONL or CSV. One command, no dependencies.
|
|
32
|
+
|
|
33
|
+
[](https://pypi.org/project/cliqtomedia-apify/)
|
|
34
|
+
[](https://pypi.org/project/cliqtomedia-apify/)
|
|
35
|
+
[](https://pypi.org/project/cliqtomedia-apify/)
|
|
36
|
+
|
|
37
|
+
## What you get
|
|
38
|
+
|
|
39
|
+
- A `cliqtomedia-apify run` command that starts an Actor, waits for it to finish and prints the results.
|
|
40
|
+
- A `run_actor()` function that returns the dataset items as a Python list.
|
|
41
|
+
- Output as `json`, `jsonl` or `csv`.
|
|
42
|
+
- Clear errors and exit codes for a missing token, a wrong Actor name and a failed run.
|
|
43
|
+
- Standard library only. It needs Python 3.9 or newer and installs no other packages.
|
|
44
|
+
|
|
45
|
+
The package works with any public Actor on Apify, not only ours. It reads dataset items with `clean=true`, so hidden helper fields are dropped.
|
|
46
|
+
|
|
47
|
+
## Install
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
pip install cliqtomedia-apify
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
## Get your Apify token
|
|
54
|
+
|
|
55
|
+
The package uses your own Apify account. It never ships a token.
|
|
56
|
+
|
|
57
|
+
1. Create a free account at [apify.com](https://apify.com).
|
|
58
|
+
2. In Apify Console, open Settings, then API & Integrations. Copy the Personal API token ([how to find it](https://docs.apify.com/platform/integrations/api#api-token)).
|
|
59
|
+
3. Set it as the `APIFY_TOKEN` environment variable:
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
export APIFY_TOKEN=your_token_here
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
You can also pass the token with `--token` on the command line or `token=` in Python.
|
|
66
|
+
|
|
67
|
+
## Quick start: command line
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
cliqtomedia-apify run cliqtomedia/youtube-transcript-scraper \
|
|
71
|
+
--input '{"videoUrls": ["jNQXAC9IVRw"]}' --format json
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
Read the input from a file:
|
|
75
|
+
|
|
76
|
+
```bash
|
|
77
|
+
cliqtomedia-apify run cliqtomedia/youtube-transcript-scraper --input-file input.json --format jsonl > items.jsonl
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
| Option | Meaning |
|
|
81
|
+
|---|---|
|
|
82
|
+
| `--input` | Actor input as a JSON string |
|
|
83
|
+
| `--input-file` | Path to a JSON file with the Actor input |
|
|
84
|
+
| `--limit N` | Fetch at most N items |
|
|
85
|
+
| `--format` | `json` (default), `jsonl` or `csv` |
|
|
86
|
+
| `--timeout SECONDS` | How long to wait for the run (default 300) |
|
|
87
|
+
| `--token` | Apify token (default: `APIFY_TOKEN`) |
|
|
88
|
+
| `--version` | Print the package version |
|
|
89
|
+
|
|
90
|
+
Exit codes: `0` success, `1` Apify or run error, `2` usage error such as a missing token or bad input JSON.
|
|
91
|
+
|
|
92
|
+
`python -m cliqtomedia_apify run ...` works the same way.
|
|
93
|
+
|
|
94
|
+
## Quick start: Python
|
|
95
|
+
|
|
96
|
+
```python
|
|
97
|
+
from cliqtomedia_apify import run_actor
|
|
98
|
+
|
|
99
|
+
items = run_actor(
|
|
100
|
+
"cliqtomedia/youtube-transcript-scraper",
|
|
101
|
+
{"videoUrls": ["jNQXAC9IVRw"]},
|
|
102
|
+
limit=10,
|
|
103
|
+
)
|
|
104
|
+
print(items[0]["status"], items[0]["text"][:80])
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
`run_actor(actor, run_input=None, *, token=None, limit=None, timeout=300)` returns a list of dicts. It raises `ApifyError` (or the subclasses `ApifyHTTPError` and `MissingTokenError`) on failure.
|
|
108
|
+
|
|
109
|
+
## Example
|
|
110
|
+
|
|
111
|
+
Input for the [YouTube Transcript Scraper](https://cliqtomedia.com/actors/youtube-transcript-scraper/?utm_source=pypi&utm_medium=template&utm_campaign=promo-2026-11) Actor:
|
|
112
|
+
|
|
113
|
+
```json
|
|
114
|
+
{
|
|
115
|
+
"videoUrls": ["jNQXAC9IVRw"]
|
|
116
|
+
}
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
Output, shortened to the main fields of the first dataset item:
|
|
120
|
+
|
|
121
|
+
```json
|
|
122
|
+
[
|
|
123
|
+
{
|
|
124
|
+
"status": "SUCCEEDED",
|
|
125
|
+
"complete": true,
|
|
126
|
+
"videoId": "jNQXAC9IVRw",
|
|
127
|
+
"canonicalUrl": "https://www.youtube.com/watch?v=jNQXAC9IVRw",
|
|
128
|
+
"track": {
|
|
129
|
+
"sourceLanguageCode": "en",
|
|
130
|
+
"sourceIsGenerated": false,
|
|
131
|
+
"isTranslated": false
|
|
132
|
+
},
|
|
133
|
+
"segments": [
|
|
134
|
+
{
|
|
135
|
+
"sequence": 0,
|
|
136
|
+
"text": "All right, so here we are, in front of the elephants",
|
|
137
|
+
"startSeconds": 1.2,
|
|
138
|
+
"durationSeconds": 2.16
|
|
139
|
+
}
|
|
140
|
+
],
|
|
141
|
+
"segmentCount": 6
|
|
142
|
+
}
|
|
143
|
+
]
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
Each Actor has its own input and output fields. Read them on the Actor page.
|
|
147
|
+
|
|
148
|
+
## Pricing
|
|
149
|
+
|
|
150
|
+
This package is free (MIT license). Apify charges only for your own runs, under your account. The exact price of each Actor is on its Apify page. Check it before you start.
|
|
151
|
+
|
|
152
|
+
## Limits
|
|
153
|
+
|
|
154
|
+
- It runs one Actor per call and returns when the run ends. It does not stream partial results.
|
|
155
|
+
- It waits up to `--timeout` seconds (default 300). If the run is not done by then, the package aborts it and reports an error.
|
|
156
|
+
- It reads at most the first `--limit` dataset items. Without `--limit`, it reads the whole dataset.
|
|
157
|
+
- It does not manage tasks, schedules, webhooks or key-value stores. Use the full [Apify API](https://docs.apify.com/api/v2) for those.
|
|
158
|
+
- The run must succeed. If the Actor fails, you get an error, not partial items.
|
|
159
|
+
|
|
160
|
+
## FAQ
|
|
161
|
+
|
|
162
|
+
**Does it work with Actors from other authors?**
|
|
163
|
+
Yes. Use any Actor name in the form `username/actor-name`.
|
|
164
|
+
|
|
165
|
+
**Where does the token go?**
|
|
166
|
+
Into the `APIFY_TOKEN` environment variable, or `--token` / `token=`. The package sends it only to `api.apify.com`.
|
|
167
|
+
|
|
168
|
+
**Why do I get HTTP 401?**
|
|
169
|
+
Apify rejected the token. Copy it again from Console, Settings, API & Integrations.
|
|
170
|
+
|
|
171
|
+
**Why do I get HTTP 404?**
|
|
172
|
+
The Actor name is wrong or the Actor is private. Check the spelling, for example `cliqtomedia/youtube-transcript-scraper`.
|
|
173
|
+
|
|
174
|
+
**Can I use it in a script or a cron job?**
|
|
175
|
+
Yes. It prints data to stdout, errors to stderr, and sets the exit code.
|
|
176
|
+
|
|
177
|
+
## Links
|
|
178
|
+
|
|
179
|
+
- [Cliqto Media Actor catalog](https://cliqtomedia.com/actors/?utm_source=pypi&utm_medium=template&utm_campaign=promo-2026-11)
|
|
180
|
+
- [Apify API reference](https://docs.apify.com/api/v2)
|
|
181
|
+
- [Apify API token docs](https://docs.apify.com/platform/integrations/api#api-token)
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
# cliqtomedia-apify: run any Apify Actor from Python or the command line
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
Run any public Apify Actor and get its dataset items as JSON, JSONL or CSV. One command, no dependencies.
|
|
6
|
+
|
|
7
|
+
[](https://pypi.org/project/cliqtomedia-apify/)
|
|
8
|
+
[](https://pypi.org/project/cliqtomedia-apify/)
|
|
9
|
+
[](https://pypi.org/project/cliqtomedia-apify/)
|
|
10
|
+
|
|
11
|
+
## What you get
|
|
12
|
+
|
|
13
|
+
- A `cliqtomedia-apify run` command that starts an Actor, waits for it to finish and prints the results.
|
|
14
|
+
- A `run_actor()` function that returns the dataset items as a Python list.
|
|
15
|
+
- Output as `json`, `jsonl` or `csv`.
|
|
16
|
+
- Clear errors and exit codes for a missing token, a wrong Actor name and a failed run.
|
|
17
|
+
- Standard library only. It needs Python 3.9 or newer and installs no other packages.
|
|
18
|
+
|
|
19
|
+
The package works with any public Actor on Apify, not only ours. It reads dataset items with `clean=true`, so hidden helper fields are dropped.
|
|
20
|
+
|
|
21
|
+
## Install
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
pip install cliqtomedia-apify
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
## Get your Apify token
|
|
28
|
+
|
|
29
|
+
The package uses your own Apify account. It never ships a token.
|
|
30
|
+
|
|
31
|
+
1. Create a free account at [apify.com](https://apify.com).
|
|
32
|
+
2. In Apify Console, open Settings, then API & Integrations. Copy the Personal API token ([how to find it](https://docs.apify.com/platform/integrations/api#api-token)).
|
|
33
|
+
3. Set it as the `APIFY_TOKEN` environment variable:
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
export APIFY_TOKEN=your_token_here
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
You can also pass the token with `--token` on the command line or `token=` in Python.
|
|
40
|
+
|
|
41
|
+
## Quick start: command line
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
cliqtomedia-apify run cliqtomedia/youtube-transcript-scraper \
|
|
45
|
+
--input '{"videoUrls": ["jNQXAC9IVRw"]}' --format json
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
Read the input from a file:
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
cliqtomedia-apify run cliqtomedia/youtube-transcript-scraper --input-file input.json --format jsonl > items.jsonl
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
| Option | Meaning |
|
|
55
|
+
|---|---|
|
|
56
|
+
| `--input` | Actor input as a JSON string |
|
|
57
|
+
| `--input-file` | Path to a JSON file with the Actor input |
|
|
58
|
+
| `--limit N` | Fetch at most N items |
|
|
59
|
+
| `--format` | `json` (default), `jsonl` or `csv` |
|
|
60
|
+
| `--timeout SECONDS` | How long to wait for the run (default 300) |
|
|
61
|
+
| `--token` | Apify token (default: `APIFY_TOKEN`) |
|
|
62
|
+
| `--version` | Print the package version |
|
|
63
|
+
|
|
64
|
+
Exit codes: `0` success, `1` Apify or run error, `2` usage error such as a missing token or bad input JSON.
|
|
65
|
+
|
|
66
|
+
`python -m cliqtomedia_apify run ...` works the same way.
|
|
67
|
+
|
|
68
|
+
## Quick start: Python
|
|
69
|
+
|
|
70
|
+
```python
|
|
71
|
+
from cliqtomedia_apify import run_actor
|
|
72
|
+
|
|
73
|
+
items = run_actor(
|
|
74
|
+
"cliqtomedia/youtube-transcript-scraper",
|
|
75
|
+
{"videoUrls": ["jNQXAC9IVRw"]},
|
|
76
|
+
limit=10,
|
|
77
|
+
)
|
|
78
|
+
print(items[0]["status"], items[0]["text"][:80])
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
`run_actor(actor, run_input=None, *, token=None, limit=None, timeout=300)` returns a list of dicts. It raises `ApifyError` (or the subclasses `ApifyHTTPError` and `MissingTokenError`) on failure.
|
|
82
|
+
|
|
83
|
+
## Example
|
|
84
|
+
|
|
85
|
+
Input for the [YouTube Transcript Scraper](https://cliqtomedia.com/actors/youtube-transcript-scraper/?utm_source=pypi&utm_medium=template&utm_campaign=promo-2026-11) Actor:
|
|
86
|
+
|
|
87
|
+
```json
|
|
88
|
+
{
|
|
89
|
+
"videoUrls": ["jNQXAC9IVRw"]
|
|
90
|
+
}
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
Output, shortened to the main fields of the first dataset item:
|
|
94
|
+
|
|
95
|
+
```json
|
|
96
|
+
[
|
|
97
|
+
{
|
|
98
|
+
"status": "SUCCEEDED",
|
|
99
|
+
"complete": true,
|
|
100
|
+
"videoId": "jNQXAC9IVRw",
|
|
101
|
+
"canonicalUrl": "https://www.youtube.com/watch?v=jNQXAC9IVRw",
|
|
102
|
+
"track": {
|
|
103
|
+
"sourceLanguageCode": "en",
|
|
104
|
+
"sourceIsGenerated": false,
|
|
105
|
+
"isTranslated": false
|
|
106
|
+
},
|
|
107
|
+
"segments": [
|
|
108
|
+
{
|
|
109
|
+
"sequence": 0,
|
|
110
|
+
"text": "All right, so here we are, in front of the elephants",
|
|
111
|
+
"startSeconds": 1.2,
|
|
112
|
+
"durationSeconds": 2.16
|
|
113
|
+
}
|
|
114
|
+
],
|
|
115
|
+
"segmentCount": 6
|
|
116
|
+
}
|
|
117
|
+
]
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
Each Actor has its own input and output fields. Read them on the Actor page.
|
|
121
|
+
|
|
122
|
+
## Pricing
|
|
123
|
+
|
|
124
|
+
This package is free (MIT license). Apify charges only for your own runs, under your account. The exact price of each Actor is on its Apify page. Check it before you start.
|
|
125
|
+
|
|
126
|
+
## Limits
|
|
127
|
+
|
|
128
|
+
- It runs one Actor per call and returns when the run ends. It does not stream partial results.
|
|
129
|
+
- It waits up to `--timeout` seconds (default 300). If the run is not done by then, the package aborts it and reports an error.
|
|
130
|
+
- It reads at most the first `--limit` dataset items. Without `--limit`, it reads the whole dataset.
|
|
131
|
+
- It does not manage tasks, schedules, webhooks or key-value stores. Use the full [Apify API](https://docs.apify.com/api/v2) for those.
|
|
132
|
+
- The run must succeed. If the Actor fails, you get an error, not partial items.
|
|
133
|
+
|
|
134
|
+
## FAQ
|
|
135
|
+
|
|
136
|
+
**Does it work with Actors from other authors?**
|
|
137
|
+
Yes. Use any Actor name in the form `username/actor-name`.
|
|
138
|
+
|
|
139
|
+
**Where does the token go?**
|
|
140
|
+
Into the `APIFY_TOKEN` environment variable, or `--token` / `token=`. The package sends it only to `api.apify.com`.
|
|
141
|
+
|
|
142
|
+
**Why do I get HTTP 401?**
|
|
143
|
+
Apify rejected the token. Copy it again from Console, Settings, API & Integrations.
|
|
144
|
+
|
|
145
|
+
**Why do I get HTTP 404?**
|
|
146
|
+
The Actor name is wrong or the Actor is private. Check the spelling, for example `cliqtomedia/youtube-transcript-scraper`.
|
|
147
|
+
|
|
148
|
+
**Can I use it in a script or a cron job?**
|
|
149
|
+
Yes. It prints data to stdout, errors to stderr, and sets the exit code.
|
|
150
|
+
|
|
151
|
+
## Links
|
|
152
|
+
|
|
153
|
+
- [Cliqto Media Actor catalog](https://cliqtomedia.com/actors/?utm_source=pypi&utm_medium=template&utm_campaign=promo-2026-11)
|
|
154
|
+
- [Apify API reference](https://docs.apify.com/api/v2)
|
|
155
|
+
- [Apify API token docs](https://docs.apify.com/platform/integrations/api#api-token)
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "cliqtomedia-apify"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Run any public Apify Actor from the command line or Python and get its dataset items. No dependencies."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{ name = "Cliqto Media" }]
|
|
14
|
+
maintainers = [{ name = "Cliqto Media" }]
|
|
15
|
+
keywords = ["apify", "web-scraping", "scraper", "data-extraction", "lead-generation", "automation", "api-client"]
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Development Status :: 4 - Beta",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"Operating System :: OS Independent",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3.9",
|
|
22
|
+
"Programming Language :: Python :: 3.10",
|
|
23
|
+
"Programming Language :: Python :: 3.11",
|
|
24
|
+
"Programming Language :: Python :: 3.12",
|
|
25
|
+
"Programming Language :: Python :: 3.13",
|
|
26
|
+
"Topic :: Internet :: WWW/HTTP",
|
|
27
|
+
"Topic :: Software Development :: Libraries",
|
|
28
|
+
"Topic :: Utilities",
|
|
29
|
+
]
|
|
30
|
+
dependencies = []
|
|
31
|
+
|
|
32
|
+
[project.urls]
|
|
33
|
+
Homepage = "https://cliqtomedia.com/actors/?utm_source=pypi&utm_medium=template&utm_campaign=promo-2026-11"
|
|
34
|
+
Documentation = "https://docs.apify.com/api/v2"
|
|
35
|
+
|
|
36
|
+
[project.scripts]
|
|
37
|
+
cliqtomedia-apify = "cliqtomedia_apify.cli:main"
|
|
38
|
+
|
|
39
|
+
[tool.hatch.build.targets.wheel]
|
|
40
|
+
packages = ["src/cliqtomedia_apify"]
|
|
41
|
+
|
|
42
|
+
[tool.hatch.build.targets.sdist]
|
|
43
|
+
include = ["src", "tests", "README.md", "LICENSE", "pyproject.toml"]
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
"""Run any public Apify Actor and get its dataset items."""
|
|
2
|
+
|
|
3
|
+
from .client import ApifyError, ApifyHTTPError, MissingTokenError, normalize_actor_id, run_actor
|
|
4
|
+
|
|
5
|
+
__version__ = "0.1.0"
|
|
6
|
+
__all__ = ["run_actor", "normalize_actor_id", "ApifyError", "ApifyHTTPError", "MissingTokenError", "__version__"]
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
"""Command line interface: cliqtomedia-apify run <user/actor> --input '{...}'."""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import csv
|
|
5
|
+
import io
|
|
6
|
+
import json
|
|
7
|
+
import sys
|
|
8
|
+
|
|
9
|
+
from . import __version__
|
|
10
|
+
from .client import ApifyError, MissingTokenError, run_actor
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def format_items(items, fmt):
|
|
14
|
+
if fmt == "jsonl":
|
|
15
|
+
return "\n".join(json.dumps(i, ensure_ascii=False) for i in items) + ("\n" if items else "")
|
|
16
|
+
if fmt == "csv":
|
|
17
|
+
columns = []
|
|
18
|
+
for item in items:
|
|
19
|
+
for key in item:
|
|
20
|
+
if key not in columns:
|
|
21
|
+
columns.append(key)
|
|
22
|
+
buf = io.StringIO()
|
|
23
|
+
writer = csv.writer(buf, lineterminator="\n")
|
|
24
|
+
writer.writerow(columns)
|
|
25
|
+
for item in items:
|
|
26
|
+
writer.writerow(
|
|
27
|
+
[v if isinstance(v, (str, int, float)) or v is None else json.dumps(v, ensure_ascii=False)
|
|
28
|
+
for v in (item.get(c) for c in columns)]
|
|
29
|
+
)
|
|
30
|
+
return buf.getvalue()
|
|
31
|
+
return json.dumps(items, indent=2, ensure_ascii=False) + "\n"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def build_parser():
|
|
35
|
+
parser = argparse.ArgumentParser(
|
|
36
|
+
prog="cliqtomedia-apify",
|
|
37
|
+
description="Run a public Apify Actor and print its dataset items. "
|
|
38
|
+
"Your own Apify token is read from APIFY_TOKEN or --token.",
|
|
39
|
+
)
|
|
40
|
+
parser.add_argument("--version", action="version", version="%(prog)s " + __version__)
|
|
41
|
+
sub = parser.add_subparsers(dest="command", metavar="command")
|
|
42
|
+
run = sub.add_parser("run", help="run an Actor and print its items")
|
|
43
|
+
run.add_argument("actor", help="Actor name, for example username/actor-name")
|
|
44
|
+
src = run.add_mutually_exclusive_group()
|
|
45
|
+
src.add_argument("--input", help="Actor input as a JSON string")
|
|
46
|
+
src.add_argument("--input-file", help="path to a JSON file with the Actor input")
|
|
47
|
+
run.add_argument("--limit", type=int, help="maximum number of items to fetch")
|
|
48
|
+
run.add_argument("--format", choices=["json", "jsonl", "csv"], default="json", help="output format (default: json)")
|
|
49
|
+
run.add_argument("--timeout", type=int, default=300, help="seconds to wait for the run (default: 300)")
|
|
50
|
+
run.add_argument("--token", help="Apify API token (default: APIFY_TOKEN environment variable)")
|
|
51
|
+
return parser
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _load_input(args):
|
|
55
|
+
if args.input_file:
|
|
56
|
+
with open(args.input_file, encoding="utf-8") as fh:
|
|
57
|
+
return json.load(fh)
|
|
58
|
+
if args.input:
|
|
59
|
+
return json.loads(args.input)
|
|
60
|
+
return {}
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def main(argv=None):
|
|
64
|
+
parser = build_parser()
|
|
65
|
+
args = parser.parse_args(argv)
|
|
66
|
+
if args.command != "run":
|
|
67
|
+
parser.print_help(sys.stderr)
|
|
68
|
+
return 2
|
|
69
|
+
try:
|
|
70
|
+
run_input = _load_input(args)
|
|
71
|
+
except (OSError, ValueError) as err:
|
|
72
|
+
print("error: cannot read the input JSON: %s" % err, file=sys.stderr)
|
|
73
|
+
return 2
|
|
74
|
+
try:
|
|
75
|
+
items = run_actor(args.actor, run_input, token=args.token, limit=args.limit, timeout=args.timeout)
|
|
76
|
+
except MissingTokenError as err:
|
|
77
|
+
print("error: %s" % err, file=sys.stderr)
|
|
78
|
+
return 2
|
|
79
|
+
except ApifyError as err:
|
|
80
|
+
print("error: %s" % err, file=sys.stderr)
|
|
81
|
+
return 1
|
|
82
|
+
except KeyboardInterrupt:
|
|
83
|
+
print("interrupted: the Apify run may still be running in Console", file=sys.stderr)
|
|
84
|
+
return 130
|
|
85
|
+
sys.stdout.write(format_items(items, args.format))
|
|
86
|
+
return 0
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
"""Minimal Apify API v2 runner built on the standard library."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
import time
|
|
6
|
+
import urllib.error
|
|
7
|
+
import urllib.parse
|
|
8
|
+
import urllib.request
|
|
9
|
+
|
|
10
|
+
API = "https://api.apify.com/v2"
|
|
11
|
+
TERMINAL = ("SUCCEEDED", "FAILED", "ABORTED", "TIMED-OUT")
|
|
12
|
+
MAX_SERVER_WAIT = 60 # Apify allows waitForFinish up to 60 seconds per request.
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class ApifyError(Exception):
|
|
16
|
+
"""Any failure while running an Actor."""
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class MissingTokenError(ApifyError):
|
|
20
|
+
"""No Apify API token was given."""
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class ApifyHTTPError(ApifyError):
|
|
24
|
+
"""The Apify API answered with an error status."""
|
|
25
|
+
|
|
26
|
+
def __init__(self, status, message):
|
|
27
|
+
super().__init__(message)
|
|
28
|
+
self.status = status
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def normalize_actor_id(actor):
|
|
32
|
+
"""Turn ``user/name`` into the API form ``user~name``."""
|
|
33
|
+
actor = (actor or "").strip()
|
|
34
|
+
if not actor:
|
|
35
|
+
raise ApifyError("Actor id is empty. Use the form username/actor-name.")
|
|
36
|
+
return actor.replace("/", "~")
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def resolve_token(token=None):
|
|
40
|
+
token = (token or os.environ.get("APIFY_TOKEN") or "").strip()
|
|
41
|
+
if not token:
|
|
42
|
+
raise MissingTokenError(
|
|
43
|
+
"No Apify API token. Set the APIFY_TOKEN environment variable or pass --token. "
|
|
44
|
+
"Find your token in Apify Console -> Settings -> API & Integrations."
|
|
45
|
+
)
|
|
46
|
+
return token
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _request(method, path, token, body=None, query=None):
|
|
50
|
+
url = API + path
|
|
51
|
+
if query:
|
|
52
|
+
url += "?" + urllib.parse.urlencode(query)
|
|
53
|
+
data = None
|
|
54
|
+
headers = {
|
|
55
|
+
"Authorization": "Bearer " + token,
|
|
56
|
+
"Accept": "application/json",
|
|
57
|
+
"User-Agent": "cliqtomedia-apify/0.1.0",
|
|
58
|
+
}
|
|
59
|
+
if body is not None:
|
|
60
|
+
data = json.dumps(body).encode("utf-8")
|
|
61
|
+
headers["Content-Type"] = "application/json"
|
|
62
|
+
req = urllib.request.Request(url, data=data, headers=headers, method=method)
|
|
63
|
+
try:
|
|
64
|
+
with urllib.request.urlopen(req, timeout=MAX_SERVER_WAIT + 30) as res:
|
|
65
|
+
raw = res.read().decode("utf-8")
|
|
66
|
+
except urllib.error.HTTPError as err:
|
|
67
|
+
raise _http_error(err, path) from None
|
|
68
|
+
except urllib.error.URLError as err:
|
|
69
|
+
raise ApifyError("Could not reach the Apify API: %s" % err.reason) from None
|
|
70
|
+
return json.loads(raw) if raw else None
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _http_error(err, path):
|
|
74
|
+
detail = ""
|
|
75
|
+
try:
|
|
76
|
+
payload = json.loads(err.read().decode("utf-8"))
|
|
77
|
+
detail = (payload.get("error") or {}).get("message", "")
|
|
78
|
+
except Exception:
|
|
79
|
+
pass
|
|
80
|
+
if err.code == 401:
|
|
81
|
+
msg = "Apify rejected the token (HTTP 401). Check APIFY_TOKEN or --token."
|
|
82
|
+
elif err.code == 404:
|
|
83
|
+
what = "Actor" if path.startswith("/acts/") else "Resource"
|
|
84
|
+
msg = "%s not found (HTTP 404). Check the name, for example username/actor-name." % what
|
|
85
|
+
elif err.code == 402:
|
|
86
|
+
msg = "Apify asks for a payment or a higher plan (HTTP 402)."
|
|
87
|
+
elif err.code == 429:
|
|
88
|
+
msg = "Apify rate limit reached (HTTP 429). Wait a moment and retry."
|
|
89
|
+
else:
|
|
90
|
+
msg = "Apify API returned HTTP %s." % err.code
|
|
91
|
+
if detail:
|
|
92
|
+
msg += " " + detail
|
|
93
|
+
return ApifyHTTPError(err.code, msg)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def run_actor(actor, run_input=None, *, token=None, limit=None, timeout=300, poll_interval=2.0):
|
|
97
|
+
"""Run an Actor, wait for it to finish and return its dataset items as a list.
|
|
98
|
+
|
|
99
|
+
``actor`` is ``username/actor-name``. ``timeout`` is the number of seconds
|
|
100
|
+
to wait; when it runs out the run is aborted and ApifyError is raised.
|
|
101
|
+
"""
|
|
102
|
+
token = resolve_token(token)
|
|
103
|
+
actor_id = normalize_actor_id(actor)
|
|
104
|
+
deadline = time.monotonic() + timeout
|
|
105
|
+
wait = int(max(0, min(MAX_SERVER_WAIT, timeout)))
|
|
106
|
+
|
|
107
|
+
run = _request(
|
|
108
|
+
"POST", "/acts/%s/runs" % urllib.parse.quote(actor_id, safe="~"), token,
|
|
109
|
+
body=run_input if run_input is not None else {}, query={"waitForFinish": wait},
|
|
110
|
+
)["data"]
|
|
111
|
+
|
|
112
|
+
while run["status"] not in TERMINAL:
|
|
113
|
+
remaining = deadline - time.monotonic()
|
|
114
|
+
if remaining <= 0:
|
|
115
|
+
_abort(run["id"], token)
|
|
116
|
+
raise ApifyError(
|
|
117
|
+
"Run %s did not finish within %s seconds and was aborted. Raise --timeout." % (run["id"], timeout)
|
|
118
|
+
)
|
|
119
|
+
time.sleep(min(poll_interval, remaining))
|
|
120
|
+
run = _request(
|
|
121
|
+
"GET", "/actor-runs/%s" % run["id"], token,
|
|
122
|
+
query={"waitForFinish": int(max(0, min(MAX_SERVER_WAIT, remaining)))},
|
|
123
|
+
)["data"]
|
|
124
|
+
|
|
125
|
+
if run["status"] != "SUCCEEDED":
|
|
126
|
+
raise ApifyError("Run %s ended with status %s." % (run["id"], run["status"]))
|
|
127
|
+
|
|
128
|
+
query = {"clean": "true", "format": "json"}
|
|
129
|
+
if limit:
|
|
130
|
+
query["limit"] = int(limit)
|
|
131
|
+
return _request("GET", "/datasets/%s/items" % run["defaultDatasetId"], token, query=query)
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _abort(run_id, token):
|
|
135
|
+
try:
|
|
136
|
+
_request("POST", "/actor-runs/%s/abort" % run_id, token)
|
|
137
|
+
except ApifyError:
|
|
138
|
+
pass
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
import io
|
|
2
|
+
import json
|
|
3
|
+
import os
|
|
4
|
+
import unittest
|
|
5
|
+
import urllib.error
|
|
6
|
+
from unittest import mock
|
|
7
|
+
|
|
8
|
+
from cliqtomedia_apify import ApifyError, ApifyHTTPError, MissingTokenError, normalize_actor_id, run_actor
|
|
9
|
+
from cliqtomedia_apify import cli
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class Resp:
|
|
13
|
+
def __init__(self, payload):
|
|
14
|
+
self.payload = json.dumps(payload).encode()
|
|
15
|
+
|
|
16
|
+
def read(self):
|
|
17
|
+
return self.payload
|
|
18
|
+
|
|
19
|
+
def __enter__(self):
|
|
20
|
+
return self
|
|
21
|
+
|
|
22
|
+
def __exit__(self, *a):
|
|
23
|
+
return False
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def run_obj(status, rid="RUN1"):
|
|
27
|
+
return Resp({"data": {"id": rid, "status": status, "defaultDatasetId": "DS1"}})
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def http_error(code, message="boom"):
|
|
31
|
+
body = io.BytesIO(json.dumps({"error": {"message": message}}).encode())
|
|
32
|
+
return urllib.error.HTTPError("https://api.apify.com/v2/x", code, "err", {}, body)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class Calls:
|
|
36
|
+
"""Side effect for urlopen that returns queued responses and records requests."""
|
|
37
|
+
|
|
38
|
+
def __init__(self, *responses):
|
|
39
|
+
self.responses = list(responses)
|
|
40
|
+
self.requests = []
|
|
41
|
+
|
|
42
|
+
def __call__(self, req, timeout=None):
|
|
43
|
+
self.requests.append(req)
|
|
44
|
+
item = self.responses.pop(0)
|
|
45
|
+
if isinstance(item, Exception):
|
|
46
|
+
raise item
|
|
47
|
+
return item
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class RunActorTests(unittest.TestCase):
|
|
51
|
+
def go(self, calls, **kw):
|
|
52
|
+
with mock.patch("urllib.request.urlopen", calls), mock.patch("time.sleep"):
|
|
53
|
+
return run_actor("user/actor", {"a": 1}, token="T", **kw)
|
|
54
|
+
|
|
55
|
+
def test_start_fetch(self):
|
|
56
|
+
calls = Calls(run_obj("SUCCEEDED"), Resp([{"x": 1}]))
|
|
57
|
+
items = self.go(calls, limit=5)
|
|
58
|
+
self.assertEqual(items, [{"x": 1}])
|
|
59
|
+
start, fetch = calls.requests
|
|
60
|
+
self.assertEqual(start.get_method(), "POST")
|
|
61
|
+
self.assertIn("/acts/user~actor/runs?waitForFinish=", start.full_url)
|
|
62
|
+
self.assertEqual(json.loads(start.data), {"a": 1})
|
|
63
|
+
self.assertEqual(start.get_header("Authorization"), "Bearer T")
|
|
64
|
+
self.assertIn("/datasets/DS1/items", fetch.full_url)
|
|
65
|
+
self.assertIn("limit=5", fetch.full_url)
|
|
66
|
+
self.assertIn("clean=true", fetch.full_url)
|
|
67
|
+
|
|
68
|
+
def test_polls_until_terminal(self):
|
|
69
|
+
calls = Calls(run_obj("RUNNING"), run_obj("RUNNING"), run_obj("SUCCEEDED"), Resp([]))
|
|
70
|
+
self.assertEqual(self.go(calls), [])
|
|
71
|
+
self.assertEqual(len(calls.requests), 4)
|
|
72
|
+
self.assertIn("/actor-runs/RUN1", calls.requests[1].full_url)
|
|
73
|
+
|
|
74
|
+
def test_failed_run(self):
|
|
75
|
+
with self.assertRaisesRegex(ApifyError, "FAILED"):
|
|
76
|
+
self.go(Calls(run_obj("FAILED")))
|
|
77
|
+
|
|
78
|
+
def test_timeout_aborts(self):
|
|
79
|
+
calls = Calls(run_obj("RUNNING"), run_obj("RUNNING"), Resp({}))
|
|
80
|
+
with mock.patch("urllib.request.urlopen", calls), mock.patch("time.sleep"), \
|
|
81
|
+
mock.patch("time.monotonic", side_effect=[0, 1, 1000]):
|
|
82
|
+
with self.assertRaisesRegex(ApifyError, "aborted"):
|
|
83
|
+
run_actor("user/actor", token="T", timeout=10)
|
|
84
|
+
self.assertTrue(calls.requests[-1].full_url.endswith("/actor-runs/RUN1/abort"))
|
|
85
|
+
|
|
86
|
+
def test_missing_token(self):
|
|
87
|
+
with mock.patch.dict(os.environ, {}, clear=True):
|
|
88
|
+
with self.assertRaises(MissingTokenError):
|
|
89
|
+
run_actor("user/actor")
|
|
90
|
+
|
|
91
|
+
def test_token_from_env(self):
|
|
92
|
+
calls = Calls(run_obj("SUCCEEDED"), Resp([]))
|
|
93
|
+
with mock.patch.dict(os.environ, {"APIFY_TOKEN": "ENVTOK"}), mock.patch("urllib.request.urlopen", calls):
|
|
94
|
+
run_actor("user/actor")
|
|
95
|
+
self.assertEqual(calls.requests[0].get_header("Authorization"), "Bearer ENVTOK")
|
|
96
|
+
|
|
97
|
+
def test_http_401(self):
|
|
98
|
+
with self.assertRaisesRegex(ApifyHTTPError, "rejected the token") as ctx:
|
|
99
|
+
self.go(Calls(http_error(401)))
|
|
100
|
+
self.assertEqual(ctx.exception.status, 401)
|
|
101
|
+
|
|
102
|
+
def test_http_404(self):
|
|
103
|
+
with self.assertRaisesRegex(ApifyHTTPError, "Actor not found"):
|
|
104
|
+
self.go(Calls(http_error(404, "Actor was not found")))
|
|
105
|
+
|
|
106
|
+
def test_normalize(self):
|
|
107
|
+
self.assertEqual(normalize_actor_id("user/name"), "user~name")
|
|
108
|
+
self.assertEqual(normalize_actor_id("user~name"), "user~name")
|
|
109
|
+
with self.assertRaises(ApifyError):
|
|
110
|
+
normalize_actor_id(" ")
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
class CliTests(unittest.TestCase):
|
|
114
|
+
def run_cli(self, argv, items=None, env=None):
|
|
115
|
+
out, err = io.StringIO(), io.StringIO()
|
|
116
|
+
with mock.patch.dict(os.environ, env or {}, clear=True), \
|
|
117
|
+
mock.patch("sys.stdout", out), mock.patch("sys.stderr", err), \
|
|
118
|
+
mock.patch.object(cli, "run_actor", return_value=items or []) as ra:
|
|
119
|
+
code = cli.main(argv)
|
|
120
|
+
return code, out.getvalue(), err.getvalue(), ra
|
|
121
|
+
|
|
122
|
+
def test_json_output(self):
|
|
123
|
+
code, out, _, ra = self.run_cli(["run", "u/a", "--input", '{"q": 1}', "--limit", "3", "--token", "T"], [{"k": "v"}])
|
|
124
|
+
self.assertEqual(code, 0)
|
|
125
|
+
self.assertEqual(json.loads(out), [{"k": "v"}])
|
|
126
|
+
self.assertEqual(ra.call_args.args, ("u/a", {"q": 1}))
|
|
127
|
+
self.assertEqual(ra.call_args.kwargs["limit"], 3)
|
|
128
|
+
|
|
129
|
+
def test_jsonl_and_csv(self):
|
|
130
|
+
items = [{"a": 1, "b": [1, 2]}, {"a": 2}]
|
|
131
|
+
_, out, _, _ = self.run_cli(["run", "u/a", "--format", "jsonl"], items)
|
|
132
|
+
self.assertEqual([json.loads(x) for x in out.splitlines()], items)
|
|
133
|
+
_, out, _, _ = self.run_cli(["run", "u/a", "--format", "csv"], items)
|
|
134
|
+
self.assertEqual(out.splitlines(), ["a,b", '1,"[1, 2]"', "2,"])
|
|
135
|
+
|
|
136
|
+
def test_missing_token_exit_code(self):
|
|
137
|
+
out, err = io.StringIO(), io.StringIO()
|
|
138
|
+
with mock.patch.dict(os.environ, {}, clear=True), mock.patch("sys.stderr", err):
|
|
139
|
+
code = cli.main(["run", "u/a"])
|
|
140
|
+
self.assertEqual(code, 2)
|
|
141
|
+
self.assertIn("APIFY_TOKEN", err.getvalue())
|
|
142
|
+
|
|
143
|
+
def test_api_error_exit_code(self):
|
|
144
|
+
err = io.StringIO()
|
|
145
|
+
with mock.patch.object(cli, "run_actor", side_effect=ApifyError("nope")), mock.patch("sys.stderr", err):
|
|
146
|
+
self.assertEqual(cli.main(["run", "u/a", "--token", "T"]), 1)
|
|
147
|
+
self.assertIn("error: nope", err.getvalue())
|
|
148
|
+
|
|
149
|
+
def test_bad_input_json(self):
|
|
150
|
+
code, _, err, _ = self.run_cli(["run", "u/a", "--input", "{bad"])
|
|
151
|
+
self.assertEqual(code, 2)
|
|
152
|
+
self.assertIn("input JSON", err)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
if __name__ == "__main__":
|
|
156
|
+
unittest.main()
|