tinet-data-foundry-cli 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tinet_data_foundry_cli-0.1.0/PKG-INFO +64 -0
- tinet_data_foundry_cli-0.1.0/README.md +47 -0
- tinet_data_foundry_cli-0.1.0/pyproject.toml +31 -0
- tinet_data_foundry_cli-0.1.0/setup.cfg +4 -0
- tinet_data_foundry_cli-0.1.0/src/tinet_data_foundry_cli/__init__.py +1 -0
- tinet_data_foundry_cli-0.1.0/src/tinet_data_foundry_cli/__main__.py +4 -0
- tinet_data_foundry_cli-0.1.0/src/tinet_data_foundry_cli/auth.py +55 -0
- tinet_data_foundry_cli-0.1.0/src/tinet_data_foundry_cli/cli.py +204 -0
- tinet_data_foundry_cli-0.1.0/src/tinet_data_foundry_cli/client.py +67 -0
- tinet_data_foundry_cli-0.1.0/src/tinet_data_foundry_cli.egg-info/PKG-INFO +64 -0
- tinet_data_foundry_cli-0.1.0/src/tinet_data_foundry_cli.egg-info/SOURCES.txt +14 -0
- tinet_data_foundry_cli-0.1.0/src/tinet_data_foundry_cli.egg-info/dependency_links.txt +1 -0
- tinet_data_foundry_cli-0.1.0/src/tinet_data_foundry_cli.egg-info/entry_points.txt +3 -0
- tinet_data_foundry_cli-0.1.0/src/tinet_data_foundry_cli.egg-info/top_level.txt +1 -0
- tinet_data_foundry_cli-0.1.0/tests/test_cli.py +24 -0
- tinet_data_foundry_cli-0.1.0/tests/test_sign.py +34 -0
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: tinet-data-foundry-cli
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Tinet Data Foundry 取数 CLI(catalog / query / schema)
|
|
5
|
+
Author: Tinet
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://data-foundry.tinetcloud.com
|
|
8
|
+
Project-URL: Documentation, https://data-foundry.tinetcloud.com
|
|
9
|
+
Keywords: data-foundry,tinet,cli
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Environment :: Console
|
|
15
|
+
Requires-Python: >=3.10
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
|
|
18
|
+
# tinet-data-foundry-cli
|
|
19
|
+
|
|
20
|
+
Tinet Data Foundry 取数 CLI。包名 `tinet-data-foundry-cli`,命令 **`df`**(同包 **`tdf`**)。
|
|
21
|
+
|
|
22
|
+
鉴权只用环境变量,**不要**把 AccessKey / SecretKey 写入本地文件。
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
pip install -U tinet-data-foundry-cli
|
|
26
|
+
|
|
27
|
+
export DF_ENDPOINT="https://data-foundry-data-api.tinetcloud.com"
|
|
28
|
+
export DF_AK="你的_AccessKey"
|
|
29
|
+
export DF_SK="你的_SecretKey"
|
|
30
|
+
|
|
31
|
+
df catalog
|
|
32
|
+
df schema --dataset "DATASET_ID"
|
|
33
|
+
df query \
|
|
34
|
+
--dataset "DATASET_ID" \
|
|
35
|
+
--fields "id" \
|
|
36
|
+
--filter '{"op":"eq","field":"id","value":"sample"}' \
|
|
37
|
+
--sort '[{"field":"id","direction":"DESC"}]' \
|
|
38
|
+
--page-no 1 \
|
|
39
|
+
--page-size 10
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
需要 Python **≥ 3.10**。
|
|
43
|
+
|
|
44
|
+
| 命令 | 接口 |
|
|
45
|
+
| --- | --- |
|
|
46
|
+
| `df catalog` | `GET /data-api/v1/catalog` |
|
|
47
|
+
| `df query` | `POST /data-api/v1/datasets/{id}/query` |
|
|
48
|
+
| `df schema` | `GET /data-api/v1/datasets/{id}/schema` |
|
|
49
|
+
|
|
50
|
+
签名:`canonical = METHOD + "\n" + PATH + "\n" + TIMESTAMP + "\n" + NONCE + "\n" + hex(SHA256(body))`,`X-DF-Signature = Base64(HMAC-SHA256(canonical, SK))`。GET 的 body 为空字节。PATH 不含域名、不含 query。
|
|
51
|
+
|
|
52
|
+
## 发布到 PyPI
|
|
53
|
+
|
|
54
|
+
版本号改 `pyproject.toml` 与 `src/tinet_data_foundry_cli/__init__.py`。
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
python -m pip install -U build twine
|
|
58
|
+
python -m build
|
|
59
|
+
python -m twine check dist/*
|
|
60
|
+
python -m twine upload --repository testpypi dist/*
|
|
61
|
+
python -m twine upload dist/*
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
已发布版本号不可覆盖。
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# tinet-data-foundry-cli
|
|
2
|
+
|
|
3
|
+
Tinet Data Foundry 取数 CLI。包名 `tinet-data-foundry-cli`,命令 **`df`**(同包 **`tdf`**)。
|
|
4
|
+
|
|
5
|
+
鉴权只用环境变量,**不要**把 AccessKey / SecretKey 写入本地文件。
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
pip install -U tinet-data-foundry-cli
|
|
9
|
+
|
|
10
|
+
export DF_ENDPOINT="https://data-foundry-data-api.tinetcloud.com"
|
|
11
|
+
export DF_AK="你的_AccessKey"
|
|
12
|
+
export DF_SK="你的_SecretKey"
|
|
13
|
+
|
|
14
|
+
df catalog
|
|
15
|
+
df schema --dataset "DATASET_ID"
|
|
16
|
+
df query \
|
|
17
|
+
--dataset "DATASET_ID" \
|
|
18
|
+
--fields "id" \
|
|
19
|
+
--filter '{"op":"eq","field":"id","value":"sample"}' \
|
|
20
|
+
--sort '[{"field":"id","direction":"DESC"}]' \
|
|
21
|
+
--page-no 1 \
|
|
22
|
+
--page-size 10
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
需要 Python **≥ 3.10**。
|
|
26
|
+
|
|
27
|
+
| 命令 | 接口 |
|
|
28
|
+
| --- | --- |
|
|
29
|
+
| `df catalog` | `GET /data-api/v1/catalog` |
|
|
30
|
+
| `df query` | `POST /data-api/v1/datasets/{id}/query` |
|
|
31
|
+
| `df schema` | `GET /data-api/v1/datasets/{id}/schema` |
|
|
32
|
+
|
|
33
|
+
签名:`canonical = METHOD + "\n" + PATH + "\n" + TIMESTAMP + "\n" + NONCE + "\n" + hex(SHA256(body))`,`X-DF-Signature = Base64(HMAC-SHA256(canonical, SK))`。GET 的 body 为空字节。PATH 不含域名、不含 query。
|
|
34
|
+
|
|
35
|
+
## 发布到 PyPI
|
|
36
|
+
|
|
37
|
+
版本号改 `pyproject.toml` 与 `src/tinet_data_foundry_cli/__init__.py`。
|
|
38
|
+
|
|
39
|
+
```bash
|
|
40
|
+
python -m pip install -U build twine
|
|
41
|
+
python -m build
|
|
42
|
+
python -m twine check dist/*
|
|
43
|
+
python -m twine upload --repository testpypi dist/*
|
|
44
|
+
python -m twine upload dist/*
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
已发布版本号不可覆盖。
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "tinet-data-foundry-cli"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Tinet Data Foundry 取数 CLI(catalog / query / schema)"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
authors = [{ name = "Tinet" }]
|
|
13
|
+
keywords = ["data-foundry", "tinet", "cli"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Programming Language :: Python :: 3",
|
|
16
|
+
"Programming Language :: Python :: 3.10",
|
|
17
|
+
"Programming Language :: Python :: 3.11",
|
|
18
|
+
"Programming Language :: Python :: 3.12",
|
|
19
|
+
"Environment :: Console",
|
|
20
|
+
]
|
|
21
|
+
|
|
22
|
+
[project.urls]
|
|
23
|
+
Homepage = "https://data-foundry.tinetcloud.com"
|
|
24
|
+
Documentation = "https://data-foundry.tinetcloud.com"
|
|
25
|
+
|
|
26
|
+
[project.scripts]
|
|
27
|
+
df = "tinet_data_foundry_cli.cli:main"
|
|
28
|
+
tdf = "tinet_data_foundry_cli.cli:main"
|
|
29
|
+
|
|
30
|
+
[tool.setuptools.packages.find]
|
|
31
|
+
where = ["src"]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"""HMAC-SHA256 签名,与 data-api SignatureVerifier 一致。"""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import base64
|
|
6
|
+
import hashlib
|
|
7
|
+
import hmac
|
|
8
|
+
import os
|
|
9
|
+
import time
|
|
10
|
+
import uuid
|
|
11
|
+
from typing import Tuple
|
|
12
|
+
|
|
13
|
+
DEFAULT_ENDPOINT = "https://data-foundry-data-api.tinetcloud.com"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def sha256_hex(body: bytes) -> str:
|
|
17
|
+
return hashlib.sha256(body or b"").hexdigest()
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def canonical(method: str, path: str, timestamp: str, nonce: str, body: bytes) -> str:
|
|
21
|
+
return "\n".join(
|
|
22
|
+
[
|
|
23
|
+
method.upper(),
|
|
24
|
+
path,
|
|
25
|
+
timestamp,
|
|
26
|
+
nonce,
|
|
27
|
+
sha256_hex(body),
|
|
28
|
+
]
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def sign_base64(method: str, path: str, timestamp: str, nonce: str, body: bytes, secret: str) -> str:
|
|
33
|
+
raw = hmac.new(
|
|
34
|
+
secret.encode("utf-8"),
|
|
35
|
+
canonical(method, path, timestamp, nonce, body).encode("utf-8"),
|
|
36
|
+
hashlib.sha256,
|
|
37
|
+
).digest()
|
|
38
|
+
return base64.b64encode(raw).decode("ascii")
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def new_nonce() -> str:
|
|
42
|
+
return uuid.uuid4().hex
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def now_ts() -> str:
|
|
46
|
+
return str(int(time.time()))
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def credentials(endpoint: str | None = None) -> Tuple[str, str, str]:
|
|
50
|
+
base = (endpoint or os.environ.get("DF_ENDPOINT") or DEFAULT_ENDPOINT).rstrip("/")
|
|
51
|
+
ak = (os.environ.get("DF_AK") or "").strip()
|
|
52
|
+
sk = (os.environ.get("DF_SK") or "").strip()
|
|
53
|
+
if not ak or not sk:
|
|
54
|
+
raise SystemExit("缺少 DF_AK / DF_SK 环境变量(不要写入本地配置文件)")
|
|
55
|
+
return base, ak, sk
|
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import json
|
|
5
|
+
import sys
|
|
6
|
+
from typing import Any, List, Mapping, Optional
|
|
7
|
+
|
|
8
|
+
from tinet_data_foundry_cli import __version__
|
|
9
|
+
from tinet_data_foundry_cli.auth import credentials
|
|
10
|
+
from tinet_data_foundry_cli.client import ApiError, request
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _parse_json(raw: Optional[str], flag: str) -> Any:
|
|
14
|
+
if not raw:
|
|
15
|
+
return None
|
|
16
|
+
try:
|
|
17
|
+
return json.loads(raw)
|
|
18
|
+
except json.JSONDecodeError as ex:
|
|
19
|
+
raise SystemExit(f"{flag} 不是合法 JSON:{ex}") from ex
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _print_json(data: Any) -> None:
|
|
23
|
+
json.dump(data, sys.stdout, ensure_ascii=False, indent=2)
|
|
24
|
+
sys.stdout.write("\n")
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _cell(value: Any, width: int) -> str:
|
|
28
|
+
text = "—" if value is None or value == "" else str(value).replace("\n", " ")
|
|
29
|
+
if len(text) > width:
|
|
30
|
+
text = text[: max(0, width - 1)] + "…"
|
|
31
|
+
return text.ljust(width)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _print_catalog_table(rows: List[Mapping[str, Any]]) -> None:
|
|
35
|
+
if not rows:
|
|
36
|
+
print("当前凭证没有可查询的 Dataset。")
|
|
37
|
+
return
|
|
38
|
+
cols = [
|
|
39
|
+
("id", "datasetId", 36),
|
|
40
|
+
("name", "名称", 28),
|
|
41
|
+
("tableComment", "中文描述", 24),
|
|
42
|
+
("businessDomainName", "业务域", 12),
|
|
43
|
+
("layer", "分层", 6),
|
|
44
|
+
("schemaVersion", "版本", 8),
|
|
45
|
+
("fieldCount", "列数", 6),
|
|
46
|
+
("rlsColumn", "行级", 16),
|
|
47
|
+
]
|
|
48
|
+
header = " ".join(_cell(title, w) for _, title, w in cols)
|
|
49
|
+
print(header)
|
|
50
|
+
print("-" * len(header))
|
|
51
|
+
for row in rows:
|
|
52
|
+
print(" ".join(_cell(row.get(key), w) for key, _, w in cols))
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _exit_api(err: ApiError) -> None:
|
|
56
|
+
payload = err.payload if isinstance(err.payload, dict) else {"message": str(err.payload)}
|
|
57
|
+
_print_json(payload)
|
|
58
|
+
code = err.http_status
|
|
59
|
+
if code == 401:
|
|
60
|
+
raise SystemExit(2)
|
|
61
|
+
if code == 403:
|
|
62
|
+
raise SystemExit(3)
|
|
63
|
+
if code == 422:
|
|
64
|
+
raise SystemExit(4)
|
|
65
|
+
raise SystemExit(1)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _ok_data(parsed: Any) -> Any:
|
|
69
|
+
if isinstance(parsed, dict) and parsed.get("code") not in (None, 0, "0"):
|
|
70
|
+
_print_json(parsed)
|
|
71
|
+
raise SystemExit(1)
|
|
72
|
+
return parsed.get("data") if isinstance(parsed, dict) else parsed
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def cmd_catalog(args: argparse.Namespace) -> int:
|
|
76
|
+
base, ak, sk = credentials(args.endpoint)
|
|
77
|
+
query = {
|
|
78
|
+
"keyword": args.keyword or "",
|
|
79
|
+
"layer": args.layer or "",
|
|
80
|
+
"domain": args.domain or "",
|
|
81
|
+
}
|
|
82
|
+
try:
|
|
83
|
+
_, parsed, _ = request(
|
|
84
|
+
"GET",
|
|
85
|
+
base,
|
|
86
|
+
"/data-api/v1/catalog",
|
|
87
|
+
ak,
|
|
88
|
+
sk,
|
|
89
|
+
query=query,
|
|
90
|
+
timeout=args.timeout,
|
|
91
|
+
insecure=args.insecure,
|
|
92
|
+
dry_run=args.dry_run,
|
|
93
|
+
)
|
|
94
|
+
except ApiError as ex:
|
|
95
|
+
_exit_api(ex)
|
|
96
|
+
if args.dry_run or args.format == "json":
|
|
97
|
+
_print_json(parsed)
|
|
98
|
+
return 0
|
|
99
|
+
data = _ok_data(parsed) or {}
|
|
100
|
+
rows = data.get("list") if isinstance(data, dict) else []
|
|
101
|
+
_print_catalog_table(list(rows or []))
|
|
102
|
+
return 0
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def cmd_query(args: argparse.Namespace) -> int:
|
|
106
|
+
base, ak, sk = credentials(args.endpoint)
|
|
107
|
+
payload: dict[str, Any] = {
|
|
108
|
+
"pageNo": args.page_no,
|
|
109
|
+
"pageSize": args.page_size,
|
|
110
|
+
}
|
|
111
|
+
if args.fields:
|
|
112
|
+
payload["fields"] = [p.strip() for p in args.fields.split(",") if p.strip()]
|
|
113
|
+
filt = _parse_json(args.filter, "--filter")
|
|
114
|
+
if filt is not None:
|
|
115
|
+
payload["filter"] = filt
|
|
116
|
+
sort = _parse_json(args.sort, "--sort")
|
|
117
|
+
if sort is not None:
|
|
118
|
+
payload["sort"] = sort
|
|
119
|
+
visitor = _parse_json(args.visitor, "--visitor")
|
|
120
|
+
if visitor is not None:
|
|
121
|
+
payload["visitor"] = visitor
|
|
122
|
+
body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
|
|
123
|
+
path = f"/data-api/v1/datasets/{args.dataset}/query"
|
|
124
|
+
try:
|
|
125
|
+
_, parsed, _ = request(
|
|
126
|
+
"POST",
|
|
127
|
+
base,
|
|
128
|
+
path,
|
|
129
|
+
ak,
|
|
130
|
+
sk,
|
|
131
|
+
body=body,
|
|
132
|
+
timeout=args.timeout,
|
|
133
|
+
insecure=args.insecure,
|
|
134
|
+
dry_run=args.dry_run,
|
|
135
|
+
)
|
|
136
|
+
except ApiError as ex:
|
|
137
|
+
_exit_api(ex)
|
|
138
|
+
_print_json(parsed)
|
|
139
|
+
if isinstance(parsed, dict) and parsed.get("code") not in (None, 0, "0") and not args.dry_run:
|
|
140
|
+
return 1
|
|
141
|
+
return 0
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def cmd_schema(args: argparse.Namespace) -> int:
|
|
145
|
+
base, ak, sk = credentials(args.endpoint)
|
|
146
|
+
path = f"/data-api/v1/datasets/{args.dataset}/schema"
|
|
147
|
+
try:
|
|
148
|
+
_, parsed, _ = request(
|
|
149
|
+
"GET",
|
|
150
|
+
base,
|
|
151
|
+
path,
|
|
152
|
+
ak,
|
|
153
|
+
sk,
|
|
154
|
+
timeout=args.timeout,
|
|
155
|
+
insecure=args.insecure,
|
|
156
|
+
dry_run=args.dry_run,
|
|
157
|
+
)
|
|
158
|
+
except ApiError as ex:
|
|
159
|
+
_exit_api(ex)
|
|
160
|
+
_print_json(parsed)
|
|
161
|
+
if isinstance(parsed, dict) and parsed.get("code") not in (None, 0, "0") and not args.dry_run:
|
|
162
|
+
return 1
|
|
163
|
+
return 0
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
167
|
+
p = argparse.ArgumentParser(
|
|
168
|
+
prog="df",
|
|
169
|
+
description="Tinet Data Foundry 取数 CLI。凭证用环境变量 DF_ENDPOINT / DF_AK / DF_SK。",
|
|
170
|
+
)
|
|
171
|
+
p.add_argument("--version", action="version", version=f"tinet-data-foundry-cli {__version__}")
|
|
172
|
+
p.add_argument("--endpoint", help="覆盖 DF_ENDPOINT")
|
|
173
|
+
p.add_argument("--timeout", type=float, default=30)
|
|
174
|
+
p.add_argument("--insecure", action="store_true", help="跳过 TLS 校验(仅联调)")
|
|
175
|
+
p.add_argument("--dry-run", action="store_true", help="只打印将发送的请求,不访问网关")
|
|
176
|
+
sub = p.add_subparsers(dest="cmd", required=True)
|
|
177
|
+
|
|
178
|
+
c = sub.add_parser("catalog", help="列出当前凭证有权访问的数据目录")
|
|
179
|
+
c.add_argument("--keyword")
|
|
180
|
+
c.add_argument("--layer", choices=["ods", "dwd"])
|
|
181
|
+
c.add_argument("--domain", help="业务域 ID 或名称")
|
|
182
|
+
c.add_argument("--format", choices=["table", "json"], default="table")
|
|
183
|
+
c.set_defaults(func=cmd_catalog)
|
|
184
|
+
|
|
185
|
+
q = sub.add_parser("query", help="查询 Dataset")
|
|
186
|
+
q.add_argument("--dataset", required=True)
|
|
187
|
+
q.add_argument("--fields", help="逗号分隔列名")
|
|
188
|
+
q.add_argument("--filter", help="filter AST JSON")
|
|
189
|
+
q.add_argument("--sort", help="sort JSON 数组")
|
|
190
|
+
q.add_argument("--page-no", type=int, default=1)
|
|
191
|
+
q.add_argument("--page-size", type=int, default=10)
|
|
192
|
+
q.add_argument("--visitor", help="应用凭证 visitor JSON")
|
|
193
|
+
q.set_defaults(func=cmd_query)
|
|
194
|
+
|
|
195
|
+
s = sub.add_parser("schema", help="拉取 Dataset Schema")
|
|
196
|
+
s.add_argument("--dataset", required=True)
|
|
197
|
+
s.set_defaults(func=cmd_schema)
|
|
198
|
+
return p
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def main(argv: Optional[List[str]] = None) -> int:
|
|
202
|
+
parser = build_parser()
|
|
203
|
+
args = parser.parse_args(argv)
|
|
204
|
+
return int(args.func(args) or 0)
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import ssl
|
|
5
|
+
import urllib.error
|
|
6
|
+
import urllib.parse
|
|
7
|
+
import urllib.request
|
|
8
|
+
from typing import Any, Dict, Optional, Tuple
|
|
9
|
+
|
|
10
|
+
from tinet_data_foundry_cli.auth import new_nonce, now_ts, sign_base64
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class ApiError(Exception):
|
|
14
|
+
def __init__(self, http_status: int, payload: Any):
|
|
15
|
+
self.http_status = http_status
|
|
16
|
+
self.payload = payload
|
|
17
|
+
super().__init__(f"HTTP {http_status}: {payload}")
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def request(
|
|
21
|
+
method: str,
|
|
22
|
+
endpoint: str,
|
|
23
|
+
path: str,
|
|
24
|
+
ak: str,
|
|
25
|
+
sk: str,
|
|
26
|
+
body: Optional[bytes] = None,
|
|
27
|
+
query: Optional[Dict[str, str]] = None,
|
|
28
|
+
timeout: float = 30,
|
|
29
|
+
insecure: bool = False,
|
|
30
|
+
dry_run: bool = False,
|
|
31
|
+
) -> Tuple[int, Any, Dict[str, str]]:
|
|
32
|
+
payload = body or b""
|
|
33
|
+
ts = now_ts()
|
|
34
|
+
nonce = new_nonce()
|
|
35
|
+
signature = sign_base64(method, path, ts, nonce, payload, sk)
|
|
36
|
+
headers = {
|
|
37
|
+
"X-DF-Access-Key": ak,
|
|
38
|
+
"X-DF-Timestamp": ts,
|
|
39
|
+
"X-DF-Nonce": nonce,
|
|
40
|
+
"X-DF-Signature": signature,
|
|
41
|
+
"Accept": "application/json",
|
|
42
|
+
}
|
|
43
|
+
if method.upper() == "POST":
|
|
44
|
+
headers["Content-Type"] = "application/json; charset=utf-8"
|
|
45
|
+
url = endpoint.rstrip("/") + path
|
|
46
|
+
if query:
|
|
47
|
+
filtered = {k: v for k, v in query.items() if v}
|
|
48
|
+
if filtered:
|
|
49
|
+
url += "?" + urllib.parse.urlencode(filtered)
|
|
50
|
+
meta = {"method": method.upper(), "url": url, "path": path, "headers": dict(headers)}
|
|
51
|
+
if dry_run:
|
|
52
|
+
return 0, {"dryRun": True, **meta, "body": payload.decode("utf-8") if payload else ""}, headers
|
|
53
|
+
|
|
54
|
+
ctx = ssl._create_unverified_context() if insecure else None
|
|
55
|
+
req = urllib.request.Request(url, data=payload if method.upper() == "POST" else None, headers=headers, method=method.upper())
|
|
56
|
+
try:
|
|
57
|
+
with urllib.request.urlopen(req, timeout=timeout, context=ctx) as resp:
|
|
58
|
+
raw = resp.read()
|
|
59
|
+
parsed = json.loads(raw.decode("utf-8")) if raw else {}
|
|
60
|
+
return resp.status, parsed, headers
|
|
61
|
+
except urllib.error.HTTPError as ex:
|
|
62
|
+
raw = ex.read() if ex.fp else b""
|
|
63
|
+
try:
|
|
64
|
+
parsed = json.loads(raw.decode("utf-8")) if raw else {"message": str(ex)}
|
|
65
|
+
except json.JSONDecodeError:
|
|
66
|
+
parsed = {"message": raw.decode("utf-8", errors="replace")}
|
|
67
|
+
raise ApiError(ex.code, parsed) from ex
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: tinet-data-foundry-cli
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Tinet Data Foundry 取数 CLI(catalog / query / schema)
|
|
5
|
+
Author: Tinet
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://data-foundry.tinetcloud.com
|
|
8
|
+
Project-URL: Documentation, https://data-foundry.tinetcloud.com
|
|
9
|
+
Keywords: data-foundry,tinet,cli
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Environment :: Console
|
|
15
|
+
Requires-Python: >=3.10
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
|
|
18
|
+
# tinet-data-foundry-cli
|
|
19
|
+
|
|
20
|
+
Tinet Data Foundry 取数 CLI。包名 `tinet-data-foundry-cli`,命令 **`df`**(同包 **`tdf`**)。
|
|
21
|
+
|
|
22
|
+
鉴权只用环境变量,**不要**把 AccessKey / SecretKey 写入本地文件。
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
pip install -U tinet-data-foundry-cli
|
|
26
|
+
|
|
27
|
+
export DF_ENDPOINT="https://data-foundry-data-api.tinetcloud.com"
|
|
28
|
+
export DF_AK="你的_AccessKey"
|
|
29
|
+
export DF_SK="你的_SecretKey"
|
|
30
|
+
|
|
31
|
+
df catalog
|
|
32
|
+
df schema --dataset "DATASET_ID"
|
|
33
|
+
df query \
|
|
34
|
+
--dataset "DATASET_ID" \
|
|
35
|
+
--fields "id" \
|
|
36
|
+
--filter '{"op":"eq","field":"id","value":"sample"}' \
|
|
37
|
+
--sort '[{"field":"id","direction":"DESC"}]' \
|
|
38
|
+
--page-no 1 \
|
|
39
|
+
--page-size 10
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
需要 Python **≥ 3.10**。
|
|
43
|
+
|
|
44
|
+
| 命令 | 接口 |
|
|
45
|
+
| --- | --- |
|
|
46
|
+
| `df catalog` | `GET /data-api/v1/catalog` |
|
|
47
|
+
| `df query` | `POST /data-api/v1/datasets/{id}/query` |
|
|
48
|
+
| `df schema` | `GET /data-api/v1/datasets/{id}/schema` |
|
|
49
|
+
|
|
50
|
+
签名:`canonical = METHOD + "\n" + PATH + "\n" + TIMESTAMP + "\n" + NONCE + "\n" + hex(SHA256(body))`,`X-DF-Signature = Base64(HMAC-SHA256(canonical, SK))`。GET 的 body 为空字节。PATH 不含域名、不含 query。
|
|
51
|
+
|
|
52
|
+
## 发布到 PyPI
|
|
53
|
+
|
|
54
|
+
版本号改 `pyproject.toml` 与 `src/tinet_data_foundry_cli/__init__.py`。
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
python -m pip install -U build twine
|
|
58
|
+
python -m build
|
|
59
|
+
python -m twine check dist/*
|
|
60
|
+
python -m twine upload --repository testpypi dist/*
|
|
61
|
+
python -m twine upload dist/*
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
已发布版本号不可覆盖。
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
README.md
|
|
2
|
+
pyproject.toml
|
|
3
|
+
src/tinet_data_foundry_cli/__init__.py
|
|
4
|
+
src/tinet_data_foundry_cli/__main__.py
|
|
5
|
+
src/tinet_data_foundry_cli/auth.py
|
|
6
|
+
src/tinet_data_foundry_cli/cli.py
|
|
7
|
+
src/tinet_data_foundry_cli/client.py
|
|
8
|
+
src/tinet_data_foundry_cli.egg-info/PKG-INFO
|
|
9
|
+
src/tinet_data_foundry_cli.egg-info/SOURCES.txt
|
|
10
|
+
src/tinet_data_foundry_cli.egg-info/dependency_links.txt
|
|
11
|
+
src/tinet_data_foundry_cli.egg-info/entry_points.txt
|
|
12
|
+
src/tinet_data_foundry_cli.egg-info/top_level.txt
|
|
13
|
+
tests/test_cli.py
|
|
14
|
+
tests/test_sign.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
tinet_data_foundry_cli
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import unittest
|
|
2
|
+
from unittest.mock import patch
|
|
3
|
+
|
|
4
|
+
from tinet_data_foundry_cli.cli import build_parser, main
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class CliParserTest(unittest.TestCase):
|
|
8
|
+
def test_subcommands(self):
|
|
9
|
+
parser = build_parser()
|
|
10
|
+
catalog = parser.parse_args(["catalog", "--format", "json"])
|
|
11
|
+
self.assertEqual(catalog.cmd, "catalog")
|
|
12
|
+
query = parser.parse_args(["query", "--dataset", "ds_1", "--page-no", "1", "--page-size", "10"])
|
|
13
|
+
self.assertEqual(query.cmd, "query")
|
|
14
|
+
schema = parser.parse_args(["schema", "--dataset", "ds_1"])
|
|
15
|
+
self.assertEqual(schema.cmd, "schema")
|
|
16
|
+
|
|
17
|
+
def test_dry_run_catalog_does_not_need_network(self):
|
|
18
|
+
with patch.dict("os.environ", {"DF_AK": "ak", "DF_SK": "sk" * 8}, clear=False):
|
|
19
|
+
code = main(["--dry-run", "catalog"])
|
|
20
|
+
self.assertEqual(code, 0)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
if __name__ == "__main__":
|
|
24
|
+
unittest.main()
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import base64
|
|
2
|
+
import hashlib
|
|
3
|
+
import hmac
|
|
4
|
+
import unittest
|
|
5
|
+
|
|
6
|
+
from tinet_data_foundry_cli.auth import canonical, sha256_hex, sign_base64
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class SignTest(unittest.TestCase):
|
|
10
|
+
def test_empty_body_hash(self):
|
|
11
|
+
self.assertEqual(
|
|
12
|
+
sha256_hex(b""),
|
|
13
|
+
hashlib.sha256(b"").hexdigest(),
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
def test_canonical_matches_java_contract(self):
|
|
17
|
+
body = b'{"pageNo":1}'
|
|
18
|
+
text = canonical("POST", "/data-api/v1/datasets/d1/query", "2000000000", "nonce-1", body)
|
|
19
|
+
self.assertTrue(text.startswith("POST\n/data-api/v1/datasets/d1/query\n"))
|
|
20
|
+
self.assertIn(sha256_hex(body), text)
|
|
21
|
+
|
|
22
|
+
def test_sign_is_hmac_sha256_base64(self):
|
|
23
|
+
secret = "12345678901234567890123456789012"
|
|
24
|
+
sig = sign_base64("GET", "/data-api/v1/catalog", "2000000000", "n1", b"", secret)
|
|
25
|
+
expected = hmac.new(
|
|
26
|
+
secret.encode("utf-8"),
|
|
27
|
+
canonical("GET", "/data-api/v1/catalog", "2000000000", "n1", b"").encode("utf-8"),
|
|
28
|
+
hashlib.sha256,
|
|
29
|
+
).digest()
|
|
30
|
+
self.assertEqual(sig, base64.b64encode(expected).decode("ascii"))
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
if __name__ == "__main__":
|
|
34
|
+
unittest.main()
|