fusou-datasets 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fusou_datasets-0.1.0/PKG-INFO +95 -0
- fusou_datasets-0.1.0/README.md +62 -0
- fusou_datasets-0.1.0/fusou_datasets/__init__.py +499 -0
- fusou_datasets-0.1.0/fusou_datasets.egg-info/PKG-INFO +95 -0
- fusou_datasets-0.1.0/fusou_datasets.egg-info/SOURCES.txt +9 -0
- fusou_datasets-0.1.0/fusou_datasets.egg-info/dependency_links.txt +1 -0
- fusou_datasets-0.1.0/fusou_datasets.egg-info/entry_points.txt +2 -0
- fusou_datasets-0.1.0/fusou_datasets.egg-info/requires.txt +8 -0
- fusou_datasets-0.1.0/fusou_datasets.egg-info/top_level.txt +1 -0
- fusou_datasets-0.1.0/pyproject.toml +54 -0
- fusou_datasets-0.1.0/setup.cfg +4 -0
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: fusou-datasets
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Secure data loader for FUSOU research datasets with Device Trust authentication
|
|
5
|
+
Author-email: FUSOU Team <dev@fusou.dev>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/tsukasa-u/FUSOU
|
|
8
|
+
Project-URL: Documentation, https://github.com/tsukasa-u/FUSOU/docs
|
|
9
|
+
Project-URL: Repository, https://github.com/tsukasa-u/FUSOU
|
|
10
|
+
Project-URL: Bug Tracker, https://github.com/tsukasa-u/FUSOU/issues
|
|
11
|
+
Keywords: fusou,datasets,avro,pandas,research,kancolle
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Intended Audience :: Science/Research
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
23
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
24
|
+
Requires-Python: >=3.8
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
Requires-Dist: requests>=2.25.0
|
|
27
|
+
Requires-Dist: pandas>=1.3.0
|
|
28
|
+
Requires-Dist: fastavro>=1.4.0
|
|
29
|
+
Requires-Dist: tqdm>=4.60.0
|
|
30
|
+
Provides-Extra: dev
|
|
31
|
+
Requires-Dist: pytest>=6.0; extra == "dev"
|
|
32
|
+
Requires-Dist: pytest-cov>=2.0; extra == "dev"
|
|
33
|
+
|
|
34
|
+
# Fusou Datasets
|
|
35
|
+
|
|
36
|
+
Secure data loader for FUSOU research datasets.
|
|
37
|
+
|
|
38
|
+
## Installation
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
pip install fusou-datasets
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
Or from source:
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
pip install -e packages/fusou-datasets/python
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
## Quick Start
|
|
51
|
+
|
|
52
|
+
```python
|
|
53
|
+
import fusou_datasets
|
|
54
|
+
|
|
55
|
+
# API key loaded automatically from FUSOU_API_KEY env var
|
|
56
|
+
tables = fusou_datasets.list_tables()
|
|
57
|
+
df = fusou_datasets.load("ship_type")
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
## Configuration
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
# Set API key (recommended)
|
|
64
|
+
export FUSOU_API_KEY="your_key"
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
Or save to config:
|
|
68
|
+
|
|
69
|
+
```python
|
|
70
|
+
fusou_datasets.save_api_key("your_key")
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
## API
|
|
74
|
+
|
|
75
|
+
| Function | Description |
|
|
76
|
+
| ---------------------------------- | -------------------------- |
|
|
77
|
+
| `list_tables()` | Get available table names |
|
|
78
|
+
| `list_period_tags()` | Get period tags and latest |
|
|
79
|
+
| `load(table, period_tag="latest")` | Load data as DataFrame |
|
|
80
|
+
|
|
81
|
+
## Period Tags
|
|
82
|
+
|
|
83
|
+
```python
|
|
84
|
+
df = fusou_datasets.load("ship_type") # latest
|
|
85
|
+
df = fusou_datasets.load("ship_type", period_tag="2024-12")
|
|
86
|
+
df = fusou_datasets.load("ship_type", period_tag="all")
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
## CLI
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
fusou-datasets --tables
|
|
93
|
+
fusou-datasets --period-tags
|
|
94
|
+
fusou-datasets --client-id
|
|
95
|
+
```
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
# Fusou Datasets
|
|
2
|
+
|
|
3
|
+
Secure data loader for FUSOU research datasets.
|
|
4
|
+
|
|
5
|
+
## Installation
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
pip install fusou-datasets
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
Or from source:
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
pip install -e packages/fusou-datasets/python
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
## Quick Start
|
|
18
|
+
|
|
19
|
+
```python
|
|
20
|
+
import fusou_datasets
|
|
21
|
+
|
|
22
|
+
# API key loaded automatically from FUSOU_API_KEY env var
|
|
23
|
+
tables = fusou_datasets.list_tables()
|
|
24
|
+
df = fusou_datasets.load("ship_type")
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
## Configuration
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
# Set API key (recommended)
|
|
31
|
+
export FUSOU_API_KEY="your_key"
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
Or save to config:
|
|
35
|
+
|
|
36
|
+
```python
|
|
37
|
+
fusou_datasets.save_api_key("your_key")
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
## API
|
|
41
|
+
|
|
42
|
+
| Function | Description |
|
|
43
|
+
| ---------------------------------- | -------------------------- |
|
|
44
|
+
| `list_tables()` | Get available table names |
|
|
45
|
+
| `list_period_tags()` | Get period tags and latest |
|
|
46
|
+
| `load(table, period_tag="latest")` | Load data as DataFrame |
|
|
47
|
+
|
|
48
|
+
## Period Tags
|
|
49
|
+
|
|
50
|
+
```python
|
|
51
|
+
df = fusou_datasets.load("ship_type") # latest
|
|
52
|
+
df = fusou_datasets.load("ship_type", period_tag="2024-12")
|
|
53
|
+
df = fusou_datasets.load("ship_type", period_tag="all")
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
## CLI
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
fusou-datasets --tables
|
|
60
|
+
fusou-datasets --period-tags
|
|
61
|
+
fusou-datasets --client-id
|
|
62
|
+
```
|
|
@@ -0,0 +1,499 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Fusou Datasets
|
|
3
|
+
==============
|
|
4
|
+
|
|
5
|
+
Secure data loader for FUSOU research datasets with Device Trust authentication.
|
|
6
|
+
|
|
7
|
+
Usage:
|
|
8
|
+
import fusou_datasets
|
|
9
|
+
|
|
10
|
+
# API key is loaded automatically from:
|
|
11
|
+
# 1. Environment variable: FUSOU_API_KEY
|
|
12
|
+
# 2. Config file: ~/.fusou_loader/settings.json
|
|
13
|
+
|
|
14
|
+
# List available tables
|
|
15
|
+
tables = fusou_datasets.list_tables()
|
|
16
|
+
|
|
17
|
+
# List period tags
|
|
18
|
+
tags = fusou_datasets.list_period_tags()
|
|
19
|
+
|
|
20
|
+
# Load data
|
|
21
|
+
df = fusou_datasets.load("ship_type") # latest period
|
|
22
|
+
df = fusou_datasets.load("ship_type", period_tag="all") # all periods
|
|
23
|
+
|
|
24
|
+
Google Colab:
|
|
25
|
+
In Google Colab, if your Google account email matches the API key email,
|
|
26
|
+
device verification will be automatic (no code input required).
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
import json
|
|
30
|
+
import os
|
|
31
|
+
import sys
|
|
32
|
+
import uuid
|
|
33
|
+
from io import BytesIO
|
|
34
|
+
from pathlib import Path
|
|
35
|
+
from typing import Optional, List, Dict, Any
|
|
36
|
+
from urllib.parse import urljoin
|
|
37
|
+
|
|
38
|
+
import fastavro
|
|
39
|
+
import pandas as pd
|
|
40
|
+
import requests
|
|
41
|
+
from tqdm import tqdm
|
|
42
|
+
|
|
43
|
+
# =============================================================================
|
|
44
|
+
# Configuration
|
|
45
|
+
# =============================================================================
|
|
46
|
+
|
|
47
|
+
__version__ = "1.0.0"
|
|
48
|
+
__author__ = "FUSOU Team"
|
|
49
|
+
__all__ = [
|
|
50
|
+
"configure",
|
|
51
|
+
"save_api_key",
|
|
52
|
+
"list_tables",
|
|
53
|
+
"list_period_tags",
|
|
54
|
+
"load",
|
|
55
|
+
"get_client_id",
|
|
56
|
+
"FusouDatasetsError",
|
|
57
|
+
"AuthenticationError",
|
|
58
|
+
"DeviceUnverifiedError",
|
|
59
|
+
"DatasetNotFoundError",
|
|
60
|
+
]
|
|
61
|
+
|
|
62
|
+
DEFAULT_API_URL = "https://fusou.pages.dev/api/data-loader"
|
|
63
|
+
SETTINGS_DIR = Path.home() / ".fusou_loader"
|
|
64
|
+
SETTINGS_FILE = SETTINGS_DIR / "settings.json"
|
|
65
|
+
REQUEST_TIMEOUT = 30
|
|
66
|
+
DOWNLOAD_TIMEOUT = 300
|
|
67
|
+
|
|
68
|
+
_config: Dict[str, Any] = {"api_key": None, "api_url": DEFAULT_API_URL}
|
|
69
|
+
|
|
70
|
+
# =============================================================================
|
|
71
|
+
# Terms of Service (shown on import)
|
|
72
|
+
# =============================================================================
|
|
73
|
+
|
|
74
|
+
_TERMS = """
|
|
75
|
+
================================================================================
|
|
76
|
+
Fusou Datasets v{version}
|
|
77
|
+
================================================================================
|
|
78
|
+
[EN] By using this library, you agree to use data for research purposes only.
|
|
79
|
+
Redistribution of raw data is prohibited. Visit: https://fusou.dev/terms
|
|
80
|
+
[JP] このライブラリを使用することで、データを研究目的のみに使用することに同意します。
|
|
81
|
+
生データの再配布は禁止です。詳細: https://fusou.dev/terms
|
|
82
|
+
================================================================================
|
|
83
|
+
"""
|
|
84
|
+
if not os.getenv("FUSOU_DATASETS_SILENT"):
|
|
85
|
+
print(_TERMS.format(version=__version__), file=sys.stderr)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
# =============================================================================
|
|
90
|
+
# Exceptions
|
|
91
|
+
# =============================================================================
|
|
92
|
+
|
|
93
|
+
class FusouDatasetsError(Exception):
|
|
94
|
+
"""Base exception."""
|
|
95
|
+
pass
|
|
96
|
+
|
|
97
|
+
class AuthenticationError(FusouDatasetsError):
|
|
98
|
+
"""Invalid or missing API key."""
|
|
99
|
+
pass
|
|
100
|
+
|
|
101
|
+
class DeviceUnverifiedError(FusouDatasetsError):
|
|
102
|
+
"""Device requires verification."""
|
|
103
|
+
pass
|
|
104
|
+
|
|
105
|
+
class DatasetNotFoundError(FusouDatasetsError):
|
|
106
|
+
"""Dataset not found."""
|
|
107
|
+
pass
|
|
108
|
+
|
|
109
|
+
class VerificationError(FusouDatasetsError):
|
|
110
|
+
"""Verification failed."""
|
|
111
|
+
pass
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
# =============================================================================
|
|
115
|
+
# Environment Detection
|
|
116
|
+
# =============================================================================
|
|
117
|
+
|
|
118
|
+
def _is_colab() -> bool:
|
|
119
|
+
"""Check if running in Google Colab."""
|
|
120
|
+
try:
|
|
121
|
+
import google.colab # noqa: F401
|
|
122
|
+
return True
|
|
123
|
+
except ImportError:
|
|
124
|
+
return False
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _get_colab_credentials() -> Optional[Dict[str, str]]:
|
|
128
|
+
"""
|
|
129
|
+
Get Google account credentials from Colab.
|
|
130
|
+
Returns dict with 'email' and optionally 'token'.
|
|
131
|
+
"""
|
|
132
|
+
if not _is_colab():
|
|
133
|
+
return None
|
|
134
|
+
|
|
135
|
+
try:
|
|
136
|
+
from google.colab import auth
|
|
137
|
+
auth.authenticate_user()
|
|
138
|
+
|
|
139
|
+
# Get credentials
|
|
140
|
+
import google.auth
|
|
141
|
+
from google.auth.transport.requests import Request
|
|
142
|
+
creds, _ = google.auth.default()
|
|
143
|
+
|
|
144
|
+
# Refresh token if needed
|
|
145
|
+
if creds.expired and creds.refresh_token:
|
|
146
|
+
creds.refresh(Request())
|
|
147
|
+
|
|
148
|
+
# Get user info
|
|
149
|
+
resp = requests.get(
|
|
150
|
+
"https://www.googleapis.com/oauth2/v1/userinfo",
|
|
151
|
+
headers={"Authorization": f"Bearer {creds.token}"},
|
|
152
|
+
timeout=10,
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
if resp.ok:
|
|
156
|
+
user_info = resp.json()
|
|
157
|
+
return {
|
|
158
|
+
"email": user_info.get("email"),
|
|
159
|
+
"token": creds.token,
|
|
160
|
+
}
|
|
161
|
+
except Exception as e:
|
|
162
|
+
print(f"[fusou_datasets] Colab auth failed: {e}", file=sys.stderr)
|
|
163
|
+
|
|
164
|
+
return None
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
# =============================================================================
|
|
168
|
+
# Configuration
|
|
169
|
+
# =============================================================================
|
|
170
|
+
|
|
171
|
+
def configure(api_key: Optional[str] = None, api_url: Optional[str] = None) -> None:
|
|
172
|
+
"""Configure API credentials."""
|
|
173
|
+
if api_key:
|
|
174
|
+
_config["api_key"] = api_key
|
|
175
|
+
if api_url:
|
|
176
|
+
_config["api_url"] = api_url
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def save_api_key(api_key: str) -> None:
|
|
180
|
+
"""Save API key to config file for persistent use."""
|
|
181
|
+
_ensure_settings_dir()
|
|
182
|
+
settings = _load_settings()
|
|
183
|
+
settings["api_key"] = api_key
|
|
184
|
+
_save_settings(settings)
|
|
185
|
+
_config["api_key"] = api_key
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _ensure_settings_dir() -> None:
|
|
189
|
+
SETTINGS_DIR.mkdir(parents=True, exist_ok=True)
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def _load_settings() -> Dict[str, Any]:
|
|
193
|
+
if SETTINGS_FILE.exists():
|
|
194
|
+
try:
|
|
195
|
+
with open(SETTINGS_FILE, "r", encoding="utf-8") as f:
|
|
196
|
+
return json.load(f)
|
|
197
|
+
except (json.JSONDecodeError, IOError):
|
|
198
|
+
pass
|
|
199
|
+
return {}
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _save_settings(settings: Dict[str, Any]) -> None:
|
|
203
|
+
_ensure_settings_dir()
|
|
204
|
+
with open(SETTINGS_FILE, "w", encoding="utf-8") as f:
|
|
205
|
+
json.dump(settings, f, indent=2)
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def _get_api_key() -> str:
|
|
209
|
+
if _config.get("api_key"):
|
|
210
|
+
return _config["api_key"]
|
|
211
|
+
|
|
212
|
+
env_key = os.environ.get("FUSOU_API_KEY")
|
|
213
|
+
if env_key:
|
|
214
|
+
return env_key
|
|
215
|
+
|
|
216
|
+
settings = _load_settings()
|
|
217
|
+
if settings.get("api_key"):
|
|
218
|
+
return settings["api_key"]
|
|
219
|
+
|
|
220
|
+
raise AuthenticationError(
|
|
221
|
+
"API key not configured. Set FUSOU_API_KEY environment variable "
|
|
222
|
+
"or call fusou_datasets.save_api_key('your_key')"
|
|
223
|
+
)
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def _get_client_id() -> str:
|
|
227
|
+
settings = _load_settings()
|
|
228
|
+
if "client_id" in settings:
|
|
229
|
+
return settings["client_id"]
|
|
230
|
+
|
|
231
|
+
client_id = str(uuid.uuid4())
|
|
232
|
+
settings["client_id"] = client_id
|
|
233
|
+
_save_settings(settings)
|
|
234
|
+
return client_id
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def get_client_id() -> str:
|
|
238
|
+
"""Get the current device's client ID."""
|
|
239
|
+
return _get_client_id()
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
# =============================================================================
|
|
243
|
+
# API Client
|
|
244
|
+
# =============================================================================
|
|
245
|
+
|
|
246
|
+
def _request(method: str, endpoint: str, json_data: Optional[dict] = None, timeout: int = REQUEST_TIMEOUT) -> requests.Response:
|
|
247
|
+
api_key = _get_api_key()
|
|
248
|
+
client_id = _get_client_id()
|
|
249
|
+
api_url = _config.get("api_url", DEFAULT_API_URL)
|
|
250
|
+
url = f"{api_url.rstrip('/')}/{endpoint.lstrip('/')}"
|
|
251
|
+
|
|
252
|
+
try:
|
|
253
|
+
return requests.request(
|
|
254
|
+
method=method,
|
|
255
|
+
url=url,
|
|
256
|
+
headers={
|
|
257
|
+
"X-API-KEY": api_key,
|
|
258
|
+
"X-CLIENT-ID": client_id,
|
|
259
|
+
"Content-Type": "application/json",
|
|
260
|
+
"User-Agent": f"FusouDatasets/{__version__}",
|
|
261
|
+
},
|
|
262
|
+
json=json_data,
|
|
263
|
+
timeout=timeout,
|
|
264
|
+
)
|
|
265
|
+
except requests.exceptions.RequestException as e:
|
|
266
|
+
raise FusouDatasetsError(f"Request failed: {e}")
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def _verify_device_colab() -> bool:
|
|
270
|
+
"""Try to verify device using Google Colab credentials."""
|
|
271
|
+
creds = _get_colab_credentials()
|
|
272
|
+
if not creds or not creds.get("email"):
|
|
273
|
+
return False
|
|
274
|
+
|
|
275
|
+
print(f"[fusou_datasets] Attempting Colab verification with: {creds['email']}", file=sys.stderr)
|
|
276
|
+
|
|
277
|
+
try:
|
|
278
|
+
resp = _request("POST", "/verify-google", {
|
|
279
|
+
"email": creds["email"],
|
|
280
|
+
"google_token": creds.get("token"),
|
|
281
|
+
})
|
|
282
|
+
|
|
283
|
+
if resp.status_code == 200:
|
|
284
|
+
print(f"✓ Device verified via Google account: {creds['email']}", file=sys.stderr)
|
|
285
|
+
return True
|
|
286
|
+
|
|
287
|
+
try:
|
|
288
|
+
error_data = resp.json()
|
|
289
|
+
if error_data.get("error") == "EMAIL_MISMATCH":
|
|
290
|
+
print(
|
|
291
|
+
f"✗ Google account ({creds['email']}) does not match API key email.",
|
|
292
|
+
file=sys.stderr,
|
|
293
|
+
)
|
|
294
|
+
print(" Falling back to code verification...", file=sys.stderr)
|
|
295
|
+
except json.JSONDecodeError:
|
|
296
|
+
# Non-JSON error response; ignore structured parsing and fall back to generic failure.
|
|
297
|
+
pass
|
|
298
|
+
except Exception as e:
|
|
299
|
+
print(f"[fusou_datasets] Colab verification failed: {e}", file=sys.stderr)
|
|
300
|
+
|
|
301
|
+
return False
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def _verify_device_code() -> bool:
|
|
305
|
+
"""Verify device using email code (interactive)."""
|
|
306
|
+
print("\n" + "=" * 50, file=sys.stderr)
|
|
307
|
+
print("DEVICE VERIFICATION / デバイス認証", file=sys.stderr)
|
|
308
|
+
print("Check your email for the verification code.", file=sys.stderr)
|
|
309
|
+
print("メールで認証コードを確認してください。", file=sys.stderr)
|
|
310
|
+
print("=" * 50, file=sys.stderr)
|
|
311
|
+
|
|
312
|
+
for attempt in range(3):
|
|
313
|
+
try:
|
|
314
|
+
code = input(f"Code ({attempt+1}/3): ").strip()
|
|
315
|
+
except (EOFError, KeyboardInterrupt):
|
|
316
|
+
raise VerificationError("Verification cancelled")
|
|
317
|
+
|
|
318
|
+
if not code:
|
|
319
|
+
continue
|
|
320
|
+
|
|
321
|
+
resp = _request("POST", "/verify", {"code": code})
|
|
322
|
+
if resp.status_code == 200:
|
|
323
|
+
print("✓ Device verified!", file=sys.stderr)
|
|
324
|
+
return True
|
|
325
|
+
print("✗ Invalid code", file=sys.stderr)
|
|
326
|
+
|
|
327
|
+
raise VerificationError("Max attempts exceeded")
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
def _verify_device() -> bool:
|
|
331
|
+
"""
|
|
332
|
+
Verify device - tries Colab auth first, then falls back to code input.
|
|
333
|
+
"""
|
|
334
|
+
# Try Colab verification first
|
|
335
|
+
if _is_colab():
|
|
336
|
+
if _verify_device_colab():
|
|
337
|
+
return True
|
|
338
|
+
|
|
339
|
+
# Fall back to code verification
|
|
340
|
+
return _verify_device_code()
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
def _handle_403(response: requests.Response, retry_func, *args, **kwargs):
|
|
344
|
+
try:
|
|
345
|
+
data = response.json()
|
|
346
|
+
if data.get("error") == "DEVICE_UNVERIFIED":
|
|
347
|
+
_verify_device()
|
|
348
|
+
return retry_func(*args, **kwargs, _retry=False)
|
|
349
|
+
elif data.get("error") == "INVALID_API_KEY":
|
|
350
|
+
raise AuthenticationError("Invalid API key")
|
|
351
|
+
except json.JSONDecodeError:
|
|
352
|
+
# If the response body is not valid JSON, treat it as a generic access denial.
|
|
353
|
+
pass
|
|
354
|
+
raise AuthenticationError("Access denied")
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def _download_avro(url: str, _retry: bool = True) -> pd.DataFrame:
|
|
358
|
+
api_key = _get_api_key()
|
|
359
|
+
client_id = _get_client_id()
|
|
360
|
+
api_url = _config.get("api_url", DEFAULT_API_URL)
|
|
361
|
+
|
|
362
|
+
if url.startswith("/"):
|
|
363
|
+
# Use urljoin for robust URL construction
|
|
364
|
+
base_url = api_url.rsplit("/api/", 1)[0] if "/api/" in api_url else api_url
|
|
365
|
+
url = urljoin(base_url + "/", url.lstrip("/"))
|
|
366
|
+
|
|
367
|
+
try:
|
|
368
|
+
resp = requests.get(
|
|
369
|
+
url,
|
|
370
|
+
headers={"X-API-KEY": api_key, "X-CLIENT-ID": client_id},
|
|
371
|
+
timeout=DOWNLOAD_TIMEOUT,
|
|
372
|
+
)
|
|
373
|
+
|
|
374
|
+
# Handle device verification required
|
|
375
|
+
if resp.status_code == 403 and _retry:
|
|
376
|
+
return _handle_403(resp, lambda: _download_avro(url, _retry=False))
|
|
377
|
+
|
|
378
|
+
resp.raise_for_status()
|
|
379
|
+
return pd.DataFrame.from_records(list(fastavro.reader(BytesIO(resp.content))))
|
|
380
|
+
except Exception as e:
|
|
381
|
+
raise FusouDatasetsError(f"Download failed: {e}")
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
# =============================================================================
|
|
385
|
+
# Public API
|
|
386
|
+
# =============================================================================
|
|
387
|
+
|
|
388
|
+
def list_tables(_retry: bool = True) -> List[str]:
|
|
389
|
+
"""List available tables."""
|
|
390
|
+
resp = _request("GET", "/tables")
|
|
391
|
+
if resp.status_code == 403 and _retry:
|
|
392
|
+
return _handle_403(resp, list_tables)
|
|
393
|
+
if resp.status_code != 200:
|
|
394
|
+
raise FusouDatasetsError(f"Failed (HTTP {resp.status_code})")
|
|
395
|
+
return resp.json().get("tables", [])
|
|
396
|
+
|
|
397
|
+
|
|
398
|
+
def list_period_tags(_retry: bool = True) -> Dict[str, Any]:
|
|
399
|
+
"""
|
|
400
|
+
List available period tags.
|
|
401
|
+
|
|
402
|
+
Returns:
|
|
403
|
+
Dict with 'period_tags' (list) and 'latest' (str)
|
|
404
|
+
"""
|
|
405
|
+
resp = _request("GET", "/period-tags")
|
|
406
|
+
if resp.status_code == 403 and _retry:
|
|
407
|
+
return _handle_403(resp, list_period_tags)
|
|
408
|
+
if resp.status_code != 200:
|
|
409
|
+
raise FusouDatasetsError(f"Failed (HTTP {resp.status_code})")
|
|
410
|
+
data = resp.json()
|
|
411
|
+
return {"period_tags": data.get("period_tags", []), "latest": data.get("latest")}
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
def _load_impl(table: str, period_tag: str = "latest", limit: int = 100, show_progress: bool = True, _retry: bool = True) -> pd.DataFrame:
|
|
415
|
+
"""Internal implementation of load() with retry parameter."""
|
|
416
|
+
if not table:
|
|
417
|
+
raise ValueError("Table name required")
|
|
418
|
+
|
|
419
|
+
resp = _request("GET", f"/data/{table}?period_tag={period_tag}&limit={limit}")
|
|
420
|
+
|
|
421
|
+
if resp.status_code == 403 and _retry:
|
|
422
|
+
return _handle_403(resp, lambda: _load_impl(table, period_tag, limit, show_progress, _retry=False))
|
|
423
|
+
if resp.status_code == 404:
|
|
424
|
+
raise DatasetNotFoundError(f"No data for '{table}' with period_tag='{period_tag}'")
|
|
425
|
+
if resp.status_code != 200:
|
|
426
|
+
raise FusouDatasetsError(f"Failed (HTTP {resp.status_code})")
|
|
427
|
+
|
|
428
|
+
files = resp.json().get("files", [])
|
|
429
|
+
if not files:
|
|
430
|
+
raise DatasetNotFoundError(f"No files for '{table}'")
|
|
431
|
+
|
|
432
|
+
dfs = []
|
|
433
|
+
file_iter = tqdm(files, desc=f"Loading {table}", unit="file", disable=not show_progress)
|
|
434
|
+
for f in file_iter:
|
|
435
|
+
url = f.get("download_url")
|
|
436
|
+
if url:
|
|
437
|
+
try:
|
|
438
|
+
dfs.append(_download_avro(url))
|
|
439
|
+
except Exception as e:
|
|
440
|
+
print(f"Warning: {f.get('file_path')}: {e}", file=sys.stderr)
|
|
441
|
+
|
|
442
|
+
if not dfs:
|
|
443
|
+
raise FusouDatasetsError("No files downloaded")
|
|
444
|
+
return pd.concat(dfs, ignore_index=True)
|
|
445
|
+
|
|
446
|
+
|
|
447
|
+
def load(table: str, period_tag: str = "latest", limit: int = 100, show_progress: bool = True) -> pd.DataFrame:
|
|
448
|
+
"""
|
|
449
|
+
Load data for a table.
|
|
450
|
+
|
|
451
|
+
Args:
|
|
452
|
+
table: Table name (use list_tables() to see options)
|
|
453
|
+
period_tag: "latest", "all", or specific tag
|
|
454
|
+
limit: Max files to load
|
|
455
|
+
show_progress: Show download progress bar
|
|
456
|
+
|
|
457
|
+
Returns:
|
|
458
|
+
pd.DataFrame: Combined data from all matching files
|
|
459
|
+
"""
|
|
460
|
+
return _load_impl(table, period_tag, limit, show_progress, _retry=True)
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
# =============================================================================
|
|
464
|
+
# CLI
|
|
465
|
+
# =============================================================================
|
|
466
|
+
|
|
467
|
+
def main():
|
|
468
|
+
import argparse
|
|
469
|
+
parser = argparse.ArgumentParser(description="Fusou Datasets CLI")
|
|
470
|
+
parser.add_argument("--version", action="version", version=f"fusou-datasets {__version__}")
|
|
471
|
+
parser.add_argument("--client-id", action="store_true", help="Show client ID")
|
|
472
|
+
parser.add_argument("--tables", action="store_true", help="List tables")
|
|
473
|
+
parser.add_argument("--period-tags", action="store_true", help="List period tags")
|
|
474
|
+
args = parser.parse_args()
|
|
475
|
+
|
|
476
|
+
if args.client_id:
|
|
477
|
+
print(f"Client ID: {get_client_id()}")
|
|
478
|
+
elif args.tables:
|
|
479
|
+
for t in list_tables():
|
|
480
|
+
print(t)
|
|
481
|
+
elif args.period_tags:
|
|
482
|
+
info = list_period_tags()
|
|
483
|
+
period_tags = info.get("period_tags") or []
|
|
484
|
+
latest = info.get("latest")
|
|
485
|
+
if period_tags:
|
|
486
|
+
print("Available period tags:")
|
|
487
|
+
for tag in period_tags:
|
|
488
|
+
marker = " (latest)" if latest is not None and tag == latest else ""
|
|
489
|
+
print(f" {tag}{marker}")
|
|
490
|
+
elif latest:
|
|
491
|
+
print(f"Latest period tag: {latest}")
|
|
492
|
+
else:
|
|
493
|
+
print("No period tags available")
|
|
494
|
+
else:
|
|
495
|
+
parser.print_help()
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
if __name__ == "__main__":
|
|
499
|
+
main()
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: fusou-datasets
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Secure data loader for FUSOU research datasets with Device Trust authentication
|
|
5
|
+
Author-email: FUSOU Team <dev@fusou.dev>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/tsukasa-u/FUSOU
|
|
8
|
+
Project-URL: Documentation, https://github.com/tsukasa-u/FUSOU/docs
|
|
9
|
+
Project-URL: Repository, https://github.com/tsukasa-u/FUSOU
|
|
10
|
+
Project-URL: Bug Tracker, https://github.com/tsukasa-u/FUSOU/issues
|
|
11
|
+
Keywords: fusou,datasets,avro,pandas,research,kancolle
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Intended Audience :: Science/Research
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
23
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
24
|
+
Requires-Python: >=3.8
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
Requires-Dist: requests>=2.25.0
|
|
27
|
+
Requires-Dist: pandas>=1.3.0
|
|
28
|
+
Requires-Dist: fastavro>=1.4.0
|
|
29
|
+
Requires-Dist: tqdm>=4.60.0
|
|
30
|
+
Provides-Extra: dev
|
|
31
|
+
Requires-Dist: pytest>=6.0; extra == "dev"
|
|
32
|
+
Requires-Dist: pytest-cov>=2.0; extra == "dev"
|
|
33
|
+
|
|
34
|
+
# Fusou Datasets
|
|
35
|
+
|
|
36
|
+
Secure data loader for FUSOU research datasets.
|
|
37
|
+
|
|
38
|
+
## Installation
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
pip install fusou-datasets
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
Or from source:
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
pip install -e packages/fusou-datasets/python
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
## Quick Start
|
|
51
|
+
|
|
52
|
+
```python
|
|
53
|
+
import fusou_datasets
|
|
54
|
+
|
|
55
|
+
# API key loaded automatically from FUSOU_API_KEY env var
|
|
56
|
+
tables = fusou_datasets.list_tables()
|
|
57
|
+
df = fusou_datasets.load("ship_type")
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
## Configuration
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
# Set API key (recommended)
|
|
64
|
+
export FUSOU_API_KEY="your_key"
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
Or save to config:
|
|
68
|
+
|
|
69
|
+
```python
|
|
70
|
+
fusou_datasets.save_api_key("your_key")
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
## API
|
|
74
|
+
|
|
75
|
+
| Function | Description |
|
|
76
|
+
| ---------------------------------- | -------------------------- |
|
|
77
|
+
| `list_tables()` | Get available table names |
|
|
78
|
+
| `list_period_tags()` | Get period tags and latest |
|
|
79
|
+
| `load(table, period_tag="latest")` | Load data as DataFrame |
|
|
80
|
+
|
|
81
|
+
## Period Tags
|
|
82
|
+
|
|
83
|
+
```python
|
|
84
|
+
df = fusou_datasets.load("ship_type") # latest
|
|
85
|
+
df = fusou_datasets.load("ship_type", period_tag="2024-12")
|
|
86
|
+
df = fusou_datasets.load("ship_type", period_tag="all")
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
## CLI
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
fusou-datasets --tables
|
|
93
|
+
fusou-datasets --period-tags
|
|
94
|
+
fusou-datasets --client-id
|
|
95
|
+
```
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
README.md
|
|
2
|
+
pyproject.toml
|
|
3
|
+
fusou_datasets/__init__.py
|
|
4
|
+
fusou_datasets.egg-info/PKG-INFO
|
|
5
|
+
fusou_datasets.egg-info/SOURCES.txt
|
|
6
|
+
fusou_datasets.egg-info/dependency_links.txt
|
|
7
|
+
fusou_datasets.egg-info/entry_points.txt
|
|
8
|
+
fusou_datasets.egg-info/requires.txt
|
|
9
|
+
fusou_datasets.egg-info/top_level.txt
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
fusou_datasets
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61.0", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "fusou-datasets"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Secure data loader for FUSOU research datasets with Device Trust authentication"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = {text = "MIT"}
|
|
11
|
+
authors = [
|
|
12
|
+
{name = "FUSOU Team", email = "dev@fusou.dev"}
|
|
13
|
+
]
|
|
14
|
+
requires-python = ">=3.8"
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 4 - Beta",
|
|
17
|
+
"Intended Audience :: Developers",
|
|
18
|
+
"Intended Audience :: Science/Research",
|
|
19
|
+
"License :: OSI Approved :: MIT License",
|
|
20
|
+
"Operating System :: OS Independent",
|
|
21
|
+
"Programming Language :: Python :: 3",
|
|
22
|
+
"Programming Language :: Python :: 3.8",
|
|
23
|
+
"Programming Language :: Python :: 3.9",
|
|
24
|
+
"Programming Language :: Python :: 3.10",
|
|
25
|
+
"Programming Language :: Python :: 3.11",
|
|
26
|
+
"Programming Language :: Python :: 3.12",
|
|
27
|
+
"Topic :: Scientific/Engineering :: Information Analysis",
|
|
28
|
+
]
|
|
29
|
+
keywords = ["fusou", "datasets", "avro", "pandas", "research", "kancolle"]
|
|
30
|
+
dependencies = [
|
|
31
|
+
"requests>=2.25.0",
|
|
32
|
+
"pandas>=1.3.0",
|
|
33
|
+
"fastavro>=1.4.0",
|
|
34
|
+
"tqdm>=4.60.0",
|
|
35
|
+
]
|
|
36
|
+
|
|
37
|
+
[project.optional-dependencies]
|
|
38
|
+
dev = [
|
|
39
|
+
"pytest>=6.0",
|
|
40
|
+
"pytest-cov>=2.0",
|
|
41
|
+
]
|
|
42
|
+
|
|
43
|
+
[project.urls]
|
|
44
|
+
Homepage = "https://github.com/tsukasa-u/FUSOU"
|
|
45
|
+
Documentation = "https://github.com/tsukasa-u/FUSOU/docs"
|
|
46
|
+
Repository = "https://github.com/tsukasa-u/FUSOU"
|
|
47
|
+
"Bug Tracker" = "https://github.com/tsukasa-u/FUSOU/issues"
|
|
48
|
+
|
|
49
|
+
[project.scripts]
|
|
50
|
+
fusou-datasets = "fusou_datasets:main"
|
|
51
|
+
|
|
52
|
+
[tool.setuptools.packages.find]
|
|
53
|
+
where = ["."]
|
|
54
|
+
include = ["fusou_datasets*"]
|