launchtape 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- launchtape-0.1.0/.gitignore +13 -0
- launchtape-0.1.0/LICENSE +21 -0
- launchtape-0.1.0/PKG-INFO +125 -0
- launchtape-0.1.0/README.md +95 -0
- launchtape-0.1.0/pyproject.toml +52 -0
- launchtape-0.1.0/src/launchtape/__init__.py +18 -0
- launchtape-0.1.0/src/launchtape/__main__.py +3 -0
- launchtape-0.1.0/src/launchtape/_catalog.py +54 -0
- launchtape-0.1.0/src/launchtape/_download.py +216 -0
- launchtape-0.1.0/src/launchtape/_read.py +98 -0
- launchtape-0.1.0/src/launchtape/_version.py +1 -0
- launchtape-0.1.0/src/launchtape/cli.py +80 -0
- launchtape-0.1.0/src/launchtape/py.typed +0 -0
- launchtape-0.1.0/src/launchtape/schemas/gradtrades.md +40 -0
- launchtape-0.1.0/src/launchtape/schemas/meteora-pools.md +17 -0
- launchtape-0.1.0/src/launchtape/schemas/padfeed.md +19 -0
- launchtape-0.1.0/src/launchtape/schemas/pregrad.md +39 -0
- launchtape-0.1.0/tests/conftest.py +105 -0
- launchtape-0.1.0/tests/test_catalog_cli.py +80 -0
- launchtape-0.1.0/tests/test_download.py +177 -0
- launchtape-0.1.0/tests/test_read.py +104 -0
launchtape-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Launch Tape
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: launchtape
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Load Launch Tape's daily Solana launch data (pump.fun graduations and launches, other launchpads, Meteora pools) in Python
|
|
5
|
+
Project-URL: Homepage, https://launchtape.io
|
|
6
|
+
Project-URL: Datasets, https://launchtape.io/#datasets
|
|
7
|
+
Project-URL: Pricing, https://launchtape.io/#buy
|
|
8
|
+
Project-URL: Data licence, https://launchtape.io/#licence
|
|
9
|
+
Project-URL: Source, https://github.com/launchtape-io/launchtape
|
|
10
|
+
Author-email: Launch Tape <launchio48@gmail.com>
|
|
11
|
+
License-Expression: MIT
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Keywords: crypto,dataset,market data,meteora,pump.fun,pumpswap,raydium,solana
|
|
14
|
+
Classifier: Development Status :: 4 - Beta
|
|
15
|
+
Classifier: Intended Audience :: Financial and Insurance Industry
|
|
16
|
+
Classifier: Intended Audience :: Science/Research
|
|
17
|
+
Classifier: Operating System :: OS Independent
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
20
|
+
Classifier: Topic :: Office/Business :: Financial :: Investment
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
22
|
+
Classifier: Typing :: Typed
|
|
23
|
+
Requires-Python: >=3.9
|
|
24
|
+
Requires-Dist: requests>=2.20
|
|
25
|
+
Provides-Extra: pandas
|
|
26
|
+
Requires-Dist: pandas>=1.3; extra == 'pandas'
|
|
27
|
+
Provides-Extra: test
|
|
28
|
+
Requires-Dist: pytest>=7; extra == 'test'
|
|
29
|
+
Description-Content-Type: text/markdown
|
|
30
|
+
|
|
31
|
+
# launchtape
|
|
32
|
+
|
|
33
|
+
Python access to [Launch Tape](https://launchtape.io): daily files of Solana launch data, one gzipped JSON-lines
|
|
34
|
+
file per dataset per UTC day.
|
|
35
|
+
|
|
36
|
+
| Dataset | What it is |
|
|
37
|
+
|---|---|
|
|
38
|
+
| `gradtrades` | Every PumpSwap buy and sell in the first 30 minutes after each pump.fun graduation (wallet, amounts, price, pool reserves, fees) |
|
|
39
|
+
| `pregrad` | Who is behind each pump.fun launch: creator and its funding, curve and graduation-slot buyers, the cap path, and a +10 min outcome |
|
|
40
|
+
| `padfeed` | Graduations on Meteora DBC and Raydium LaunchLab: migration facts, pool balances every few seconds, top holders, US$ quote prices |
|
|
41
|
+
| `meteora-pools` | Per-minute price, TVL, volume and exact fees for ~200 Meteora DLMM pools |
|
|
42
|
+
|
|
43
|
+
A free 2-hour sample of every dataset is on the site; `launchtape` downloads it, checks its SHA-256 and reads it.
|
|
44
|
+
|
|
45
|
+
## Install
|
|
46
|
+
|
|
47
|
+
```sh
|
|
48
|
+
pip install launchtape # needs only requests
|
|
49
|
+
pip install "launchtape[pandas]" # adds pandas for launchtape.frame()
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
Python 3.9 or newer.
|
|
53
|
+
|
|
54
|
+
## Quickstart
|
|
55
|
+
|
|
56
|
+
Crash rate of pump.fun graduates, split by "instant" launches (graduated within 5 s of creation):
|
|
57
|
+
|
|
58
|
+
```python
|
|
59
|
+
import launchtape as lt
|
|
60
|
+
|
|
61
|
+
path = lt.sample("pregrad") # free sample, SHA-256 checked, cached
|
|
62
|
+
rec = lt.frame(path, types="record")[["mint", "sym", "instant", "ageS"]]
|
|
63
|
+
out = lt.frame(path, types="outcome")[["mint", "crashed", "maxGain"]]
|
|
64
|
+
df = rec.merge(out, on="mint") # one row per graduate seen in both
|
|
65
|
+
print(df.groupby("instant")["crashed"].agg(["mean", "count"]))
|
|
66
|
+
print(lt.schema("pregrad")) # every row type and field
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
The sample is two hours, so the counts are small: it shows the shape of the data, not a result.
|
|
70
|
+
|
|
71
|
+
## API
|
|
72
|
+
|
|
73
|
+
| Call | Does |
|
|
74
|
+
|---|---|
|
|
75
|
+
| `lt.datasets()` | the datasets as `(name, description)` pairs |
|
|
76
|
+
| `lt.schema(dataset)` | the schema as Markdown text (bundled, works offline) |
|
|
77
|
+
| `lt.sample(dataset)` | downloads the free sample into `~/.cache/launchtape`, checks it against the site's `SHA256SUMS`, returns its path; a good cached copy is reused |
|
|
78
|
+
| `lt.load(path_or_dataset, types=None, limit=None)` | streams the rows as dicts; `types` keeps only rows whose `type` is one of them, `limit` stops after that many rows; a dataset name reads its sample |
|
|
79
|
+
| `lt.frame(path_or_dataset, types=None, limit=None)` | the same rows as a pandas DataFrame (nested fields stay dicts/lists) |
|
|
80
|
+
| `lt.fetch(link_or_token, dataset, day, dest=None)` | downloads a paid day file from your delivery link, checks it against its `.manifest.json`, returns its path |
|
|
81
|
+
|
|
82
|
+
```python
|
|
83
|
+
for row in lt.load("gradtrades", types="grad", limit=5): # streams, so a 5-million-row day is fine
|
|
84
|
+
print(row["mint"], row["capSol"])
|
|
85
|
+
|
|
86
|
+
path = lt.fetch("https://launchtape.io/dl/<token>", "pregrad", "2026-10-11", dest="data/")
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
Times: `t` is milliseconds since 1970 UTC (`pd.to_datetime(df["t"], unit="ms", utc=True)`). Coverage gaps are rows
|
|
90
|
+
in the files (`down`, `up`, `resume`, `end`), never filled in: see each schema.
|
|
91
|
+
|
|
92
|
+
A full `gradtrades` day is about 5 million rows; for a DataFrame, pass `types=` and/or `limit=`, or stream with
|
|
93
|
+
`load()`. A download that does not match its SHA-256 raises `launchtape.ChecksumError` and is not kept.
|
|
94
|
+
Environment: `LAUNCHTAPE_CACHE` changes the cache folder.
|
|
95
|
+
|
|
96
|
+
## Command line
|
|
97
|
+
|
|
98
|
+
```sh
|
|
99
|
+
launchtape datasets
|
|
100
|
+
launchtape sample gradtrades # prints the cached path
|
|
101
|
+
launchtape head gradtrades -n 5 # first rows of the sample, as JSON lines
|
|
102
|
+
launchtape head pregrad -t record -n 2 # only rows of one type
|
|
103
|
+
launchtape schema pregrad
|
|
104
|
+
launchtape fetch https://launchtape.io/dl/<token> pregrad 2026-10-11 -o data/
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
`python -m launchtape ...` works too.
|
|
108
|
+
|
|
109
|
+
## Full days: prices and how to buy
|
|
110
|
+
|
|
111
|
+
Day file (one dataset, one UTC day): 0.05 SOL or 6 USDC. Month of history (one dataset, 30 days): 0.5 SOL or
|
|
112
|
+
60 USDC. Daily feed (every dataset, each new day as it closes): 1 SOL or 120 USDC a month. Send SOL or USDC on
|
|
113
|
+
Solana to `5vf1PzK8LrF9D5bemYJYPsFYp4nHULSTA1biBampgU58` with a memo giving your email address or Telegram handle
|
|
114
|
+
and the datasets and days you want; within 24 hours you get a private download link
|
|
115
|
+
(`https://launchtape.io/dl/<token>`, valid 7 days) that `launchtape fetch` reads. Or buy through
|
|
116
|
+
[launchtape.io](https://launchtape.io/#buy), which has a Solana Pay QR code and a contact form.
|
|
117
|
+
|
|
118
|
+
## Licences
|
|
119
|
+
|
|
120
|
+
- **This package** (the code): MIT, see [LICENSE](LICENSE).
|
|
121
|
+
- **The data** it downloads, the free samples included, has its own licence: personal and research use inside
|
|
122
|
+
your own team. No resale, republishing or redistribution of the files, in whole or in part. Results you derive
|
|
123
|
+
from it (charts, papers, models, strategies) are yours. Full text: [launchtape.io](https://launchtape.io/#licence).
|
|
124
|
+
|
|
125
|
+
The data is market data recorded from public sources, provided as is. It is not financial advice.
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
# launchtape
|
|
2
|
+
|
|
3
|
+
Python access to [Launch Tape](https://launchtape.io): daily files of Solana launch data, one gzipped JSON-lines
|
|
4
|
+
file per dataset per UTC day.
|
|
5
|
+
|
|
6
|
+
| Dataset | What it is |
|
|
7
|
+
|---|---|
|
|
8
|
+
| `gradtrades` | Every PumpSwap buy and sell in the first 30 minutes after each pump.fun graduation (wallet, amounts, price, pool reserves, fees) |
|
|
9
|
+
| `pregrad` | Who is behind each pump.fun launch: creator and its funding, curve and graduation-slot buyers, the cap path, and a +10 min outcome |
|
|
10
|
+
| `padfeed` | Graduations on Meteora DBC and Raydium LaunchLab: migration facts, pool balances every few seconds, top holders, US$ quote prices |
|
|
11
|
+
| `meteora-pools` | Per-minute price, TVL, volume and exact fees for ~200 Meteora DLMM pools |
|
|
12
|
+
|
|
13
|
+
A free 2-hour sample of every dataset is on the site; `launchtape` downloads it, checks its SHA-256 and reads it.
|
|
14
|
+
|
|
15
|
+
## Install
|
|
16
|
+
|
|
17
|
+
```sh
|
|
18
|
+
pip install launchtape # needs only requests
|
|
19
|
+
pip install "launchtape[pandas]" # adds pandas for launchtape.frame()
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
Python 3.9 or newer.
|
|
23
|
+
|
|
24
|
+
## Quickstart
|
|
25
|
+
|
|
26
|
+
Crash rate of pump.fun graduates, split by "instant" launches (graduated within 5 s of creation):
|
|
27
|
+
|
|
28
|
+
```python
|
|
29
|
+
import launchtape as lt
|
|
30
|
+
|
|
31
|
+
path = lt.sample("pregrad") # free sample, SHA-256 checked, cached
|
|
32
|
+
rec = lt.frame(path, types="record")[["mint", "sym", "instant", "ageS"]]
|
|
33
|
+
out = lt.frame(path, types="outcome")[["mint", "crashed", "maxGain"]]
|
|
34
|
+
df = rec.merge(out, on="mint") # one row per graduate seen in both
|
|
35
|
+
print(df.groupby("instant")["crashed"].agg(["mean", "count"]))
|
|
36
|
+
print(lt.schema("pregrad")) # every row type and field
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
The sample is two hours, so the counts are small: it shows the shape of the data, not a result.
|
|
40
|
+
|
|
41
|
+
## API
|
|
42
|
+
|
|
43
|
+
| Call | Does |
|
|
44
|
+
|---|---|
|
|
45
|
+
| `lt.datasets()` | the datasets as `(name, description)` pairs |
|
|
46
|
+
| `lt.schema(dataset)` | the schema as Markdown text (bundled, works offline) |
|
|
47
|
+
| `lt.sample(dataset)` | downloads the free sample into `~/.cache/launchtape`, checks it against the site's `SHA256SUMS`, returns its path; a good cached copy is reused |
|
|
48
|
+
| `lt.load(path_or_dataset, types=None, limit=None)` | streams the rows as dicts; `types` keeps only rows whose `type` is one of them, `limit` stops after that many rows; a dataset name reads its sample |
|
|
49
|
+
| `lt.frame(path_or_dataset, types=None, limit=None)` | the same rows as a pandas DataFrame (nested fields stay dicts/lists) |
|
|
50
|
+
| `lt.fetch(link_or_token, dataset, day, dest=None)` | downloads a paid day file from your delivery link, checks it against its `.manifest.json`, returns its path |
|
|
51
|
+
|
|
52
|
+
```python
|
|
53
|
+
for row in lt.load("gradtrades", types="grad", limit=5): # streams, so a 5-million-row day is fine
|
|
54
|
+
print(row["mint"], row["capSol"])
|
|
55
|
+
|
|
56
|
+
path = lt.fetch("https://launchtape.io/dl/<token>", "pregrad", "2026-10-11", dest="data/")
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
Times: `t` is milliseconds since 1970 UTC (`pd.to_datetime(df["t"], unit="ms", utc=True)`). Coverage gaps are rows
|
|
60
|
+
in the files (`down`, `up`, `resume`, `end`), never filled in: see each schema.
|
|
61
|
+
|
|
62
|
+
A full `gradtrades` day is about 5 million rows; for a DataFrame, pass `types=` and/or `limit=`, or stream with
|
|
63
|
+
`load()`. A download that does not match its SHA-256 raises `launchtape.ChecksumError` and is not kept.
|
|
64
|
+
Environment: `LAUNCHTAPE_CACHE` changes the cache folder.
|
|
65
|
+
|
|
66
|
+
## Command line
|
|
67
|
+
|
|
68
|
+
```sh
|
|
69
|
+
launchtape datasets
|
|
70
|
+
launchtape sample gradtrades # prints the cached path
|
|
71
|
+
launchtape head gradtrades -n 5 # first rows of the sample, as JSON lines
|
|
72
|
+
launchtape head pregrad -t record -n 2 # only rows of one type
|
|
73
|
+
launchtape schema pregrad
|
|
74
|
+
launchtape fetch https://launchtape.io/dl/<token> pregrad 2026-10-11 -o data/
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
`python -m launchtape ...` works too.
|
|
78
|
+
|
|
79
|
+
## Full days: prices and how to buy
|
|
80
|
+
|
|
81
|
+
Day file (one dataset, one UTC day): 0.05 SOL or 6 USDC. Month of history (one dataset, 30 days): 0.5 SOL or
|
|
82
|
+
60 USDC. Daily feed (every dataset, each new day as it closes): 1 SOL or 120 USDC a month. Send SOL or USDC on
|
|
83
|
+
Solana to `5vf1PzK8LrF9D5bemYJYPsFYp4nHULSTA1biBampgU58` with a memo giving your email address or Telegram handle
|
|
84
|
+
and the datasets and days you want; within 24 hours you get a private download link
|
|
85
|
+
(`https://launchtape.io/dl/<token>`, valid 7 days) that `launchtape fetch` reads. Or buy through
|
|
86
|
+
[launchtape.io](https://launchtape.io/#buy), which has a Solana Pay QR code and a contact form.
|
|
87
|
+
|
|
88
|
+
## Licences
|
|
89
|
+
|
|
90
|
+
- **This package** (the code): MIT, see [LICENSE](LICENSE).
|
|
91
|
+
- **The data** it downloads, the free samples included, has its own licence: personal and research use inside
|
|
92
|
+
your own team. No resale, republishing or redistribution of the files, in whole or in part. Results you derive
|
|
93
|
+
from it (charts, papers, models, strategies) are yours. Full text: [launchtape.io](https://launchtape.io/#licence).
|
|
94
|
+
|
|
95
|
+
The data is market data recorded from public sources, provided as is. It is not financial advice.
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.27"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "launchtape"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "Load Launch Tape's daily Solana launch data (pump.fun graduations and launches, other launchpads, Meteora pools) in Python"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{ name = "Launch Tape", email = "launchio48@gmail.com" }]
|
|
14
|
+
keywords = ["solana", "pump.fun", "pumpswap", "meteora", "raydium", "dataset", "market data", "crypto"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 4 - Beta",
|
|
17
|
+
"Intended Audience :: Science/Research",
|
|
18
|
+
"Intended Audience :: Financial and Insurance Industry",
|
|
19
|
+
"Operating System :: OS Independent",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
22
|
+
"Topic :: Office/Business :: Financial :: Investment",
|
|
23
|
+
"Topic :: Scientific/Engineering :: Information Analysis",
|
|
24
|
+
"Typing :: Typed",
|
|
25
|
+
]
|
|
26
|
+
dependencies = ["requests>=2.20"]
|
|
27
|
+
|
|
28
|
+
[project.optional-dependencies]
|
|
29
|
+
pandas = ["pandas>=1.3"]
|
|
30
|
+
test = ["pytest>=7"]
|
|
31
|
+
|
|
32
|
+
[project.urls]
|
|
33
|
+
Homepage = "https://launchtape.io"
|
|
34
|
+
Datasets = "https://launchtape.io/#datasets"
|
|
35
|
+
Pricing = "https://launchtape.io/#buy"
|
|
36
|
+
"Data licence" = "https://launchtape.io/#licence"
|
|
37
|
+
Source = "https://github.com/launchtape-io/launchtape"
|
|
38
|
+
|
|
39
|
+
[project.scripts]
|
|
40
|
+
launchtape = "launchtape.cli:main"
|
|
41
|
+
|
|
42
|
+
[tool.hatch.version]
|
|
43
|
+
path = "src/launchtape/_version.py"
|
|
44
|
+
|
|
45
|
+
[tool.hatch.build.targets.wheel]
|
|
46
|
+
packages = ["src/launchtape"]
|
|
47
|
+
|
|
48
|
+
[tool.hatch.build.targets.sdist]
|
|
49
|
+
include = ["src/launchtape", "tests", "README.md", "LICENSE", "pyproject.toml"]
|
|
50
|
+
|
|
51
|
+
[tool.pytest.ini_options]
|
|
52
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""Launch Tape: daily files of Solana launch data (https://launchtape.io) in Python.
|
|
2
|
+
|
|
3
|
+
import launchtape as lt
|
|
4
|
+
path = lt.sample("pregrad") # free 2-hour sample, SHA-256 checked
|
|
5
|
+
for row in lt.load(path, types="record", limit=3):
|
|
6
|
+
print(row["sym"], row["instant"])
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from ._catalog import DATASETS, Dataset, datasets, schema
|
|
10
|
+
from ._download import ChecksumError, LaunchTapeError, cache_dir, fetch, sample
|
|
11
|
+
from ._read import frame, load
|
|
12
|
+
from ._version import __version__
|
|
13
|
+
|
|
14
|
+
__all__ = [
|
|
15
|
+
"DATASETS", "Dataset", "datasets", "schema",
|
|
16
|
+
"sample", "fetch", "cache_dir", "LaunchTapeError", "ChecksumError",
|
|
17
|
+
"load", "frame", "__version__",
|
|
18
|
+
]
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""The datasets, their one-line descriptions and their bundled schema docs."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from importlib.resources import files
|
|
6
|
+
from typing import List, NamedTuple
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class Dataset(NamedTuple):
|
|
10
|
+
name: str
|
|
11
|
+
description: str
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
_DATASETS = (
|
|
15
|
+
Dataset(
|
|
16
|
+
"gradtrades",
|
|
17
|
+
"Every PumpSwap buy and sell in the first 30 minutes after each pump.fun graduation "
|
|
18
|
+
"(wallet, amounts, price, pool reserves, fees)",
|
|
19
|
+
),
|
|
20
|
+
Dataset(
|
|
21
|
+
"pregrad",
|
|
22
|
+
"Who is behind each pump.fun launch: creator and its funding, curve and graduation-slot "
|
|
23
|
+
"buyers, the cap path, and a +10 min outcome",
|
|
24
|
+
),
|
|
25
|
+
Dataset(
|
|
26
|
+
"padfeed",
|
|
27
|
+
"Graduations on Meteora DBC and Raydium LaunchLab: migration facts, pool balances every "
|
|
28
|
+
"few seconds, top holders, US$ quote prices",
|
|
29
|
+
),
|
|
30
|
+
Dataset(
|
|
31
|
+
"meteora-pools",
|
|
32
|
+
"Per-minute price, TVL, volume and exact fees for ~200 Meteora DLMM pools",
|
|
33
|
+
),
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
DATASETS = tuple(d.name for d in _DATASETS)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def check_dataset(dataset: str) -> str:
|
|
40
|
+
if dataset not in DATASETS:
|
|
41
|
+
raise ValueError(f"unknown dataset {dataset!r}; choose one of: {', '.join(DATASETS)}")
|
|
42
|
+
return dataset
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def datasets() -> List[Dataset]:
|
|
46
|
+
"""The datasets as (name, description) pairs."""
|
|
47
|
+
return list(_DATASETS)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def schema(dataset: str) -> str:
|
|
51
|
+
"""The schema of a dataset as Markdown text (every row type and field)."""
|
|
52
|
+
check_dataset(dataset)
|
|
53
|
+
# one name per joinpath: Python 3.9's Traversable takes a single argument
|
|
54
|
+
return files("launchtape").joinpath("schemas").joinpath(f"{dataset}.md").read_text(encoding="utf-8")
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
"""Downloads: the free samples and paid day files. A file is kept only after its SHA-256 matches the published one."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import datetime as _dt
|
|
6
|
+
import hashlib
|
|
7
|
+
import json
|
|
8
|
+
import os
|
|
9
|
+
import re
|
|
10
|
+
import sys
|
|
11
|
+
import warnings
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Dict, Optional, Union
|
|
14
|
+
from urllib.parse import urlsplit
|
|
15
|
+
|
|
16
|
+
import requests
|
|
17
|
+
|
|
18
|
+
from ._catalog import check_dataset
|
|
19
|
+
from ._version import __version__
|
|
20
|
+
|
|
21
|
+
DEFAULT_URL = "https://launchtape.io"
|
|
22
|
+
CHUNK = 1 << 20
|
|
23
|
+
TIMEOUT = (15, 120) # connect, read (seconds)
|
|
24
|
+
|
|
25
|
+
PathLike = Union[str, "os.PathLike[str]"]
|
|
26
|
+
|
|
27
|
+
_HEX64 = re.compile(r"[0-9a-fA-F]{64}")
|
|
28
|
+
_TOKEN = re.compile(r"[0-9a-f]{32}")
|
|
29
|
+
_NAME = re.compile(r"[a-z][a-z0-9-]*")
|
|
30
|
+
_DAY = re.compile(r"\d{4}-\d{2}-\d{2}")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class LaunchTapeError(Exception):
|
|
34
|
+
"""A download could not be completed or checked."""
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class ChecksumError(LaunchTapeError):
|
|
38
|
+
"""A downloaded file's SHA-256 does not match the published one (the file is not kept)."""
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def base_url() -> str:
|
|
42
|
+
"""The site to download from: $LAUNCHTAPE_URL, else https://launchtape.io."""
|
|
43
|
+
return os.environ.get("LAUNCHTAPE_URL", DEFAULT_URL).rstrip("/")
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def cache_dir() -> Path:
|
|
47
|
+
"""Where samples are kept: $LAUNCHTAPE_CACHE, else ~/.cache/launchtape."""
|
|
48
|
+
env = os.environ.get("LAUNCHTAPE_CACHE")
|
|
49
|
+
return Path(env).expanduser() if env else Path.home() / ".cache" / "launchtape"
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def sha256_file(path: PathLike) -> str:
|
|
53
|
+
h = hashlib.sha256()
|
|
54
|
+
with open(path, "rb") as f:
|
|
55
|
+
for block in iter(lambda: f.read(CHUNK), b""):
|
|
56
|
+
h.update(block)
|
|
57
|
+
return h.hexdigest()
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _get(url: str, stream: bool = False) -> requests.Response:
|
|
61
|
+
try:
|
|
62
|
+
r = requests.get(url, stream=stream, timeout=TIMEOUT,
|
|
63
|
+
headers={"User-Agent": f"launchtape-python/{__version__}"})
|
|
64
|
+
except requests.RequestException as e:
|
|
65
|
+
raise LaunchTapeError(f"could not reach {url}: {e}") from e
|
|
66
|
+
if r.status_code != 200:
|
|
67
|
+
r.close()
|
|
68
|
+
hint = ""
|
|
69
|
+
if r.status_code == 404 and "/dl/" in url:
|
|
70
|
+
hint = " (download links last 7 days and list only the files you bought; write to us through launchtape.io)"
|
|
71
|
+
raise LaunchTapeError(f"{url}: HTTP {r.status_code}{hint}")
|
|
72
|
+
return r
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _progress(name: str, done: int, total: Optional[int]) -> None:
|
|
76
|
+
size = f"{done / 1e6:,.1f} / {total / 1e6:,.1f} MB" if total else f"{done / 1e6:,.1f} MB"
|
|
77
|
+
sys.stderr.write(f"\r{name}: {size}")
|
|
78
|
+
sys.stderr.flush()
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _download(url: str, target: Path, sha256: str, total: Optional[int] = None, progress: bool = False) -> Path:
|
|
82
|
+
"""Stream url to target, hashing on the way; the file appears under its name only if the hash matches."""
|
|
83
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
84
|
+
part = target.with_name(f"{target.name}.{os.getpid()}.part")
|
|
85
|
+
h = hashlib.sha256()
|
|
86
|
+
done = 0
|
|
87
|
+
try:
|
|
88
|
+
with _get(url, stream=True) as r, open(part, "wb") as out:
|
|
89
|
+
total = total or int(r.headers.get("content-length") or 0) or None
|
|
90
|
+
for chunk in r.iter_content(CHUNK):
|
|
91
|
+
out.write(chunk)
|
|
92
|
+
h.update(chunk)
|
|
93
|
+
done += len(chunk)
|
|
94
|
+
if progress:
|
|
95
|
+
_progress(target.name, done, total)
|
|
96
|
+
if progress:
|
|
97
|
+
sys.stderr.write("\n")
|
|
98
|
+
got = h.hexdigest()
|
|
99
|
+
if got != sha256.lower():
|
|
100
|
+
raise ChecksumError(f"{target.name}: SHA-256 {got} does not match the published {sha256.lower()}; "
|
|
101
|
+
"the download was discarded, please try again")
|
|
102
|
+
os.replace(part, target)
|
|
103
|
+
except BaseException:
|
|
104
|
+
try:
|
|
105
|
+
part.unlink()
|
|
106
|
+
except FileNotFoundError:
|
|
107
|
+
pass
|
|
108
|
+
raise
|
|
109
|
+
return target
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _parse_sums(text: str) -> Dict[str, str]:
|
|
113
|
+
"""`sha256sum` output -> {file name: hex digest}."""
|
|
114
|
+
sums = {}
|
|
115
|
+
for line in text.splitlines():
|
|
116
|
+
parts = line.strip().split(None, 1)
|
|
117
|
+
if len(parts) == 2 and _HEX64.fullmatch(parts[0]):
|
|
118
|
+
sums[parts[1].lstrip("*")] = parts[0].lower()
|
|
119
|
+
return sums
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def sample(dataset: str, *, cache: Optional[PathLike] = None, progress: bool = False) -> Path:
|
|
123
|
+
"""Download the free 2-hour sample of a dataset (checked against the site's SHA256SUMS) and return its path.
|
|
124
|
+
|
|
125
|
+
A sample already in the cache is reused when its hash still matches. If the site cannot be reached, a sample
|
|
126
|
+
already in the cache (checked when it was downloaded) is returned with a warning.
|
|
127
|
+
"""
|
|
128
|
+
check_dataset(dataset)
|
|
129
|
+
folder = Path(cache).expanduser() if cache is not None else cache_dir()
|
|
130
|
+
pattern = re.compile(rf"{re.escape(dataset)}-\d{{4}}-\d{{2}}-\d{{2}}-\d{{4}}-\d{{4}}utc\.jsonl\.gz")
|
|
131
|
+
kept = sorted(p for p in folder.glob(f"{dataset}-*utc.jsonl.gz") if pattern.fullmatch(p.name))
|
|
132
|
+
base = base_url()
|
|
133
|
+
try:
|
|
134
|
+
with _get(f"{base}/sample/SHA256SUMS") as r:
|
|
135
|
+
sums = _parse_sums(r.text)
|
|
136
|
+
except LaunchTapeError as e:
|
|
137
|
+
if kept:
|
|
138
|
+
warnings.warn(f"{e}; using the {dataset} sample already in {folder}", stacklevel=2)
|
|
139
|
+
return kept[-1]
|
|
140
|
+
raise
|
|
141
|
+
names = sorted(n for n in sums if pattern.fullmatch(n))
|
|
142
|
+
if not names:
|
|
143
|
+
raise LaunchTapeError(f"no {dataset} sample is listed in {base}/sample/SHA256SUMS")
|
|
144
|
+
name = names[-1]
|
|
145
|
+
target = folder / name
|
|
146
|
+
if not (target.is_file() and sha256_file(target) == sums[name]):
|
|
147
|
+
_download(f"{base}/sample/{name}", target, sums[name], progress=progress)
|
|
148
|
+
for old in kept: # an older sample window of the same dataset
|
|
149
|
+
if old != target:
|
|
150
|
+
old.unlink()
|
|
151
|
+
return target
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _delivery_base(url_or_token: str) -> str:
|
|
155
|
+
s = str(url_or_token).strip()
|
|
156
|
+
if _TOKEN.fullmatch(s):
|
|
157
|
+
return f"{base_url()}/dl/{s}"
|
|
158
|
+
parts = urlsplit(s)
|
|
159
|
+
m = re.match(r"/dl/([0-9a-f]{32})(?:/|$)", parts.path)
|
|
160
|
+
if parts.scheme not in ("http", "https") or not parts.netloc or not m:
|
|
161
|
+
raise ValueError(f"not a Launch Tape delivery link or token: {url_or_token!r} "
|
|
162
|
+
"(expected https://launchtape.io/dl/<token> or the 32-character token)")
|
|
163
|
+
return f"{parts.scheme}://{parts.netloc}/dl/{m.group(1)}"
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _day(day: Union[str, _dt.date]) -> str:
|
|
167
|
+
if isinstance(day, _dt.datetime):
|
|
168
|
+
day = day.date()
|
|
169
|
+
if isinstance(day, _dt.date):
|
|
170
|
+
return day.isoformat()
|
|
171
|
+
s = str(day).strip()
|
|
172
|
+
if _DAY.fullmatch(s):
|
|
173
|
+
try:
|
|
174
|
+
_dt.date.fromisoformat(s)
|
|
175
|
+
return s
|
|
176
|
+
except ValueError:
|
|
177
|
+
pass
|
|
178
|
+
raise ValueError(f"day must be a UTC date like 2026-10-11, got {day!r}")
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def fetch(url_or_token: str, dataset: str, day: Union[str, _dt.date], dest: Optional[PathLike] = None,
|
|
182
|
+
*, progress: bool = False) -> Path:
|
|
183
|
+
"""Download one paid day file from a delivery link and return its path.
|
|
184
|
+
|
|
185
|
+
url_or_token: the link from your delivery mail (https://launchtape.io/dl/<token>) or just its token.
|
|
186
|
+
day: the UTC day, "YYYY-MM-DD" or a date. dest: a folder (default: the current folder) or a file path ending
|
|
187
|
+
in .gz. The file is checked against the SHA-256 in <dataset>-<day>.manifest.json, which is saved next to it.
|
|
188
|
+
A file already at the destination with the right hash is not downloaded again.
|
|
189
|
+
"""
|
|
190
|
+
if not _NAME.fullmatch(str(dataset)):
|
|
191
|
+
raise ValueError(f"not a dataset name: {dataset!r}")
|
|
192
|
+
stem = f"{dataset}-{_day(day)}"
|
|
193
|
+
fname = f"{stem}.jsonl.gz"
|
|
194
|
+
base = _delivery_base(url_or_token)
|
|
195
|
+
with _get(f"{base}/{stem}.manifest.json") as r:
|
|
196
|
+
raw = r.content
|
|
197
|
+
try:
|
|
198
|
+
manifest = json.loads(raw)
|
|
199
|
+
except ValueError as e:
|
|
200
|
+
raise LaunchTapeError(f"{stem}.manifest.json is not valid JSON: {e}") from None
|
|
201
|
+
files = manifest.get("files") if isinstance(manifest, dict) else None
|
|
202
|
+
entry = next((f for f in files or [] if isinstance(f, dict) and f.get("file") == fname), None)
|
|
203
|
+
sha = entry.get("sha256") if entry else None
|
|
204
|
+
if not isinstance(sha, str) or not _HEX64.fullmatch(sha):
|
|
205
|
+
raise LaunchTapeError(f"{stem}.manifest.json has no SHA-256 for {fname}")
|
|
206
|
+
size = entry.get("bytes")
|
|
207
|
+
if dest is None:
|
|
208
|
+
target = Path.cwd() / fname
|
|
209
|
+
else:
|
|
210
|
+
target = Path(dest).expanduser()
|
|
211
|
+
if target.is_dir() or not target.name.endswith(".gz"):
|
|
212
|
+
target = target / fname
|
|
213
|
+
if not (target.is_file() and sha256_file(target) == sha.lower()):
|
|
214
|
+
_download(f"{base}/{fname}", target, sha, total=size if isinstance(size, int) else None, progress=progress)
|
|
215
|
+
(target.parent / f"{stem}.manifest.json").write_bytes(raw)
|
|
216
|
+
return target
|