kestrel-data-sdk 0.3.2__py3-none-any.whl

kestrel_data_sdk/__init__.py

@@ -0,0 +1,5 @@
+from .client import DownloadResult, Kestrel
+from .cache import CacheCheckResult, CacheInfo
+
+__all__ = ["CacheCheckResult", "CacheInfo", "DownloadResult", "Kestrel"]
+__version__ = "0.3.1"

kestrel_data_sdk/cache.py

@@ -0,0 +1,318 @@
+from __future__ import annotations
+
+import hashlib
+import sqlite3
+from contextlib import contextmanager
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Iterator
+
+
+@dataclass(frozen=True)
+class CacheInfo:
+    path: Path
+    files: int
+    partitions: int
+    total_bytes: int
+
+
+@dataclass(frozen=True)
+class CacheCheckResult:
+    path: Path
+    checked: int
+    valid: int
+    missing: int
+    changed: int
+    corrupt: int
+    invalidated: int
+    deep: bool
+
+
+@dataclass(frozen=True)
+class CachedFile:
+    product: str
+    partition_key: str
+    version: int
+    relative_path: str
+    path: Path
+    file_size: int
+    sha256: str
+
+
+class LocalCache:
+    """Per-output-root SQLite index for immutable SDK release files."""
+
+    def __init__(self, output: str | Path):
+        self.root = Path(output).expanduser().resolve()
+        metadata = self.root / ".kestrel"
+        metadata.mkdir(parents=True, exist_ok=True)
+        self.path = metadata / "sdk-cache.sqlite3"
+        self._initialize()
+
+    def completed_partition(self, product: str, partition_key: str,
+                            version: int) -> list[CachedFile] | None:
+        with self._connect() as db:
+            marker = db.execute("""
+                SELECT file_count FROM partition_cache
+                WHERE product_id=? AND partition_key=? AND version=?
+                """, (product, partition_key, version)).fetchone()
+            if marker is None:
+                return None
+            expected_files = int(marker[0])
+            rows = self._rows(db, product, partition_key, version)
+            if len(rows) != expected_files:
+                self._invalidate_partition(db, product, partition_key, version)
+                return None
+            return [self._trusted_file(row) for row in rows]
+
+    def cached_file(self, product: str, partition_key: str, version: int,
+                    relative_path: str) -> CachedFile | None:
+        with self._connect() as db:
+            row = db.execute("""
+                SELECT relative_path,local_path,file_size,sha256,mtime_ns,product_id,partition_key,version
+                FROM file_cache WHERE product_id=? AND partition_key=? AND version=? AND relative_path=?
+                """, (product, partition_key, version, relative_path)).fetchone()
+            if row is None:
+                return None
+            return self._trusted_file(row)
+
+    def record(self, product: str, partition_key: str, version: int,
+               relative_path: str, target: Path, file_size: int, sha256: str) -> None:
+        stat = target.stat()
+        local_path = target.resolve().relative_to(self.root).as_posix()
+        now = _now()
+        with self._connect() as db:
+            db.execute("""
+                INSERT INTO file_cache(product_id,partition_key,version,relative_path,local_path,
+                  file_size,sha256,mtime_ns,verified_at)
+                VALUES(?,?,?,?,?,?,?,?,?)
+                ON CONFLICT(product_id,partition_key,version,relative_path) DO UPDATE SET
+                  local_path=excluded.local_path,file_size=excluded.file_size,sha256=excluded.sha256,
+                  mtime_ns=excluded.mtime_ns,verified_at=excluded.verified_at
+                """, (product, partition_key, version, relative_path, local_path, file_size,
+                      sha256, stat.st_mtime_ns, now))
+
+    def mark_complete(self, product: str, partition_key: str, version: int, file_count: int) -> None:
+        with self._connect() as db:
+            actual = db.execute("""
+                SELECT count(*) FROM file_cache
+                WHERE product_id=? AND partition_key=? AND version=?
+                """, (product, partition_key, version)).fetchone()[0]
+            if int(actual) != file_count:
+                self._invalidate_partition(db, product, partition_key, version)
+                return
+            db.execute("""
+                INSERT INTO partition_cache(product_id,partition_key,version,file_count,completed_at)
+                VALUES(?,?,?,?,?)
+                ON CONFLICT(product_id,partition_key,version) DO UPDATE SET
+                  file_count=excluded.file_count,completed_at=excluded.completed_at
+                """, (product, partition_key, version, file_count, _now()))
+            db.execute("""
+                DELETE FROM partition_cache
+                WHERE product_id=? AND partition_key=? AND version<>?
+                """, (product, partition_key, version))
+
+    def retain_files(self, product: str, partition_key: str, version: int,
+                     relative_paths: list[str]) -> None:
+        keep = set(relative_paths)
+        with self._connect() as db:
+            rows = db.execute("""
+                SELECT relative_path,local_path FROM file_cache
+                WHERE product_id=? AND partition_key=? AND version=?
+                """, (product, partition_key, version)).fetchall()
+            for row in rows:
+                if str(row[0]) not in keep:
+                    self._unlink_relative(str(row[1]))
+                    self._invalidate_file(db, product, partition_key, version, str(row[0]))
+            current_paths = {str(row[0]) for row in db.execute("""
+                SELECT local_path FROM file_cache
+                WHERE product_id=? AND partition_key=? AND version=?
+                """, (product, partition_key, version)).fetchall()}
+            old_paths = {str(row[0]) for row in db.execute("""
+                SELECT local_path FROM file_cache
+                WHERE product_id=? AND partition_key=? AND version<>?
+                """, (product, partition_key, version)).fetchall()}
+            for local_path in old_paths-current_paths:
+                self._unlink_relative(local_path)
+            db.execute("""
+                DELETE FROM file_cache
+                WHERE product_id=? AND partition_key=? AND version<>?
+                """, (product, partition_key, version))
+
+    def remove_partition(self, product: str, partition_key: str) -> int:
+        """Apply a server tombstone by deleting only files tracked by this cache."""
+        removed = 0
+        with self._connect() as db:
+            rows = db.execute("""
+                SELECT DISTINCT local_path FROM file_cache
+                WHERE product_id=? AND partition_key=?
+                """, (product, partition_key)).fetchall()
+            for row in rows:
+                if self._unlink_relative(str(row[0])):
+                    removed += 1
+            db.execute("DELETE FROM file_cache WHERE product_id=? AND partition_key=?",
+                       (product, partition_key))
+            db.execute("DELETE FROM partition_cache WHERE product_id=? AND partition_key=?",
+                       (product, partition_key))
+        return removed
+
+    def _unlink_relative(self, local_path: str) -> bool:
+        target = (self.root / local_path).resolve()
+        if not target.is_relative_to(self.root):
+            raise ValueError("缓存路径越界")
+        try:
+            target.unlink()
+            return True
+        except FileNotFoundError:
+            return False
+
+    def info(self, product: str | None = None) -> CacheInfo:
+        where = " WHERE product_id=?" if product else ""
+        args = (product,) if product else ()
+        with self._connect() as db:
+            files, total = db.execute(
+                "SELECT count(*),COALESCE(sum(file_size),0) FROM file_cache" + where, args).fetchone()
+            partitions = db.execute(
+                "SELECT count(*) FROM partition_cache" + where, args).fetchone()[0]
+        return CacheInfo(self.path, int(files), int(partitions), int(total))
+
+    def revisions(self, product: str) -> dict[str, int]:
+        """Return the trusted local revision vector without touching product files."""
+        with self._connect() as db:
+            rows=db.execute("""
+                SELECT partition_key,version FROM partition_cache
+                WHERE product_id=? ORDER BY partition_key
+                """,(product,)).fetchall()
+        return {str(row[0]):int(row[1]) for row in rows}
+
+    def check(self, product: str | None = None, *, deep: bool = False,
+              repair: bool = True) -> CacheCheckResult:
+        where = " WHERE product_id=?" if product else ""
+        args = (product,) if product else ()
+        checked = valid = missing = changed = corrupt = invalidated = 0
+        with self._connect() as db:
+            rows = db.execute("""
+                SELECT relative_path,local_path,file_size,sha256,mtime_ns,product_id,partition_key,version
+                FROM file_cache
+                """ + where + " ORDER BY product_id,partition_key,relative_path", args).fetchall()
+            for row in rows:
+                checked += 1
+                target = self.root / str(row[1])
+                reason: str | None = None
+                try:
+                    stat = target.stat()
+                    if not target.is_file():
+                        reason = "missing"
+                    elif stat.st_size != int(row[2]):
+                        reason = "changed"
+                    elif deep:
+                        if _sha256(target) != str(row[3]):
+                            reason = "corrupt"
+                        elif stat.st_mtime_ns != int(row[4]):
+                            db.execute("""
+                                UPDATE file_cache SET mtime_ns=?,verified_at=?
+                                WHERE product_id=? AND partition_key=? AND version=? AND relative_path=?
+                                """, (stat.st_mtime_ns, _now(), row[5], row[6], row[7], row[0]))
+                    elif stat.st_mtime_ns != int(row[4]):
+                        reason = "changed"
+                except FileNotFoundError:
+                    reason = "missing"
+                if reason is None:
+                    valid += 1
+                    continue
+                if reason == "missing": missing += 1
+                elif reason == "corrupt": corrupt += 1
+                else: changed += 1
+                if repair:
+                    self._invalidate_file(db, str(row[5]), str(row[6]), int(row[7]), str(row[0]))
+                    invalidated += 1
+        return CacheCheckResult(self.path, checked, valid, missing, changed, corrupt, invalidated, deep)
+
+    def _trusted_file(self, row: sqlite3.Row) -> CachedFile:
+        """Return SQLite state without touching the filesystem; check() is explicit reconciliation."""
+        return CachedFile(str(row[5]), str(row[6]), int(row[7]), str(row[0]),
+                          self.root / str(row[1]),
+                          int(row[2]), str(row[3]))
+
+    @staticmethod
+    def _rows(db: sqlite3.Connection, product: str, partition_key: str,
+              version: int) -> list[sqlite3.Row]:
+        return db.execute("""
+            SELECT relative_path,local_path,file_size,sha256,mtime_ns,product_id,partition_key,version
+            FROM file_cache WHERE product_id=? AND partition_key=? AND version=?
+            ORDER BY relative_path
+            """, (product, partition_key, version)).fetchall()
+
+    @staticmethod
+    def _invalidate_partition(db: sqlite3.Connection, product: str,
+                              partition_key: str, version: int) -> None:
+        db.execute("""
+            DELETE FROM partition_cache WHERE product_id=? AND partition_key=? AND version=?
+            """, (product, partition_key, version))
+
+    @classmethod
+    def _invalidate_file(cls, db: sqlite3.Connection, product: str, partition_key: str,
+                         version: int, relative_path: str) -> None:
+        db.execute("""
+            DELETE FROM file_cache
+            WHERE product_id=? AND partition_key=? AND version=? AND relative_path=?
+            """, (product, partition_key, version, relative_path))
+        cls._invalidate_partition(db, product, partition_key, version)
+
+    @contextmanager
+    def _connect(self) -> Iterator[sqlite3.Connection]:
+        db = sqlite3.connect(self.path, timeout=30)
+        db.row_factory = sqlite3.Row
+        db.execute("PRAGMA foreign_keys=ON")
+        db.execute("PRAGMA busy_timeout=30000")
+        try:
+            yield db
+            db.commit()
+        except Exception:
+            db.rollback()
+            raise
+        finally:
+            db.close()
+
+    def _initialize(self) -> None:
+        with self._connect() as db:
+            db.execute("PRAGMA journal_mode=WAL")
+            db.execute("""
+                CREATE TABLE IF NOT EXISTS file_cache (
+                  product_id TEXT NOT NULL,
+                  partition_key TEXT NOT NULL,
+                  version INTEGER NOT NULL CHECK(version>0),
+                  relative_path TEXT NOT NULL,
+                  local_path TEXT NOT NULL,
+                  file_size INTEGER NOT NULL CHECK(file_size>=0),
+                  sha256 TEXT NOT NULL CHECK(length(sha256)=64),
+                  mtime_ns INTEGER NOT NULL,
+                  verified_at TEXT NOT NULL,
+                  PRIMARY KEY(product_id,partition_key,version,relative_path)
+                )
+                """)
+            db.execute("""
+                CREATE TABLE IF NOT EXISTS partition_cache (
+                  product_id TEXT NOT NULL,
+                  partition_key TEXT NOT NULL,
+                  version INTEGER NOT NULL CHECK(version>0),
+                  file_count INTEGER NOT NULL CHECK(file_count>=0),
+                  completed_at TEXT NOT NULL,
+                  PRIMARY KEY(product_id,partition_key,version)
+                )
+                """)
+            db.execute("CREATE INDEX IF NOT EXISTS idx_sdk_cache_product ON file_cache(product_id)")
+            db.execute("PRAGMA user_version=1")
+
+
+def _sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as source:
+        while block := source.read(4 * 1024 * 1024):
+            digest.update(block)
+    return digest.hexdigest()
+
+
+def _now() -> str:
+    return datetime.now(timezone.utc).isoformat()

kestrel_data_sdk/certs/kestrel-root-ca.crt

@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIFhjCCA26gAwIBAgIUBUFoV1/TVqWxP2z+IYI1AHxR0tAwDQYJKoZIhvcNAQEL
+BQAwSTELMAkGA1UEBhMCQ04xFTATBgNVBAoMDEtlc3RyZWwgRGF0YTEjMCEGA1UE
+AwwaS2VzdHJlbCBQcml2YXRlIFJvb3QgQ0EgVjEwHhcNMjYwODEwMTAzOTU0WhcN
+MzYwODEwMTAzOTU0WjBJMQswCQYDVQQGEwJDTjEVMBMGA1UECgwMS2VzdHJlbCBE
+YXRhMSMwIQYDVQQDDBpLZXN0cmVsIFByaXZhdGUgUm9vdCBDQSBWMTCCAiIwDQYJ
+KoZIhvcNAQEBBQADggIPADCCAgoCggIBAMRyHZUWclK+nUE6i/gkHTynk30tBWjc
+AYXNJ7jp6iA1vlobr2hHNG6gyPLefmXGl+GFLYL4eTA3foOZzuk1flJrlIFM18hH
+tELrSkrTTIRUgfI+n9JR3XKKuJHAN0ZeD5vcgmg8F7ECkHE20/etth8Qw0IulwoP
+g0qOP4qyM+pZtjrF+2Kq9i0AJYzt8oV/xIjYSCikv4FeuijSkxejEQ+9hfwGfyBv
+/NKietIXn29aurzUbY3TzoFz6c9ItHZxiEz3r/cHgW0IRODrracpsppGKhWAVSqN
+nwdw9KlY0ZI6H3sV+rTjDNjDJcdQmyoRH9fTWAYENz4kzx9ohnThgDCnnhgUDdXX
+RgtBEyVy226pmJ6m/sdFM9baHFyddo4+V67XQHAwKGWxnniGdWjeoVpkWgeuF/cd
+jHlFlXQlq2xlXKWPbYDOkiPUbJHXRTRZSxQFTkARdTQZct8qGClLQbN7b6bRZ1tA
+IelfvyOwg0kBpouzYEQyJK+FOja+QK7bfdZhhDcvwRHUiPkSikK7Ik7T+3x7J59s
+fTJysdrHD8c1USiCCK1RjVT85Zp00Oyz29LGHzOjtGC0pXaSQQ+bxY1i0I7mwuD2
+uT5knczW5Do2r73uLbaZHvZ5RwxaN376uTiIGy9/mA+54E041Y49hi3efVxBqCHX
+PcisurOujvKZAgMBAAGjZjBkMB8GA1UdIwQYMBaAFEMu8xhrgmAyJmsu3TkND3I/
+05fnMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQW
+BBRDLvMYa4JgMiZrLt05DQ9yP9OX5zANBgkqhkiG9w0BAQsFAAOCAgEAjqVG659Z
+MJFuIDanwVj17J0REf9wo7dqUJRjQJS8w/p1yLT3r93kzsuAGZ+XjwPWfTqJrPCQ
+rHMUSyrblwXibu3MAEbzIqgyWurWAMQFldcar4366HolC1u4WocVlEO+Ic1Yms1e
+OCNHLngzdkLKZheZHUzBhAxQIQh05oooew0kLRIiyzZx+z5WwhxptLFV8c9WV00t
+VeHoflAmDsNyOftsY1unOjv4PY2Pi8xgTV248ZXspNvjf42NkGYu7pTRKKCd+ryc
+ydzZElB50xNAMf+Ru+PE8otox1GmdO5tQ7DH3/Bi1FzUPirIZ+EFO+t+XOzoIBGM
+MxScCXkoi0EMV1iz+djANQk61M4a77W2lp45e8Z4P6qZMgP/fSvDCt5pcHluTm4U
+R41v4fclEgz3v1KpeVsDREGXFoCfCPy2S/qnmW9rll/FlG1eyEdenVTM0oTXu6uo
+XlR24HqZaJFZmjJVWiDUt4DvNlZRm/0iTSkt6HfFShoCGQPUE5EHeH5Xd6MdoUFu
+pt8fkifjnq23wvb2q9XhVRGxyUYJSAwFq+XRm3uKty0/SUmQC2QIxtW1hsZNybOi
+9zLycutC1nZMJJ4F0QGhyBqaAmut96+7GVO/6Bh5F1E8ULZRr5rtbTXZmJc0HOtt
+avJDshZQdZcY5HWrRSmd/8kkYPpk6Ep5/RI=
+-----END CERTIFICATE-----

kestrel_data_sdk/client.py

@@ -0,0 +1,283 @@
+from __future__ import annotations
+
+import calendar
+import hashlib
+import http.client
+import json
+import os
+import socket
+import sys
+import time
+from dataclasses import dataclass
+from datetime import date
+from pathlib import Path, PurePosixPath
+from urllib.parse import quote, urlencode
+
+from .cache import CacheCheckResult, CacheInfo, LocalCache
+from .endpoint import GatewayEndpoint, discover_endpoint, ssl_context
+
+@dataclass(frozen=True)
+class DownloadResult:
+    path: Path
+    file_size: int
+    sha256: str
+    resumed_from: int
+    reused: bool
+
+class _Connection(http.client.HTTPSConnection):
+    def __init__(self,endpoint: GatewayEndpoint,timeout: float):
+        super().__init__(endpoint.logical_host,endpoint.port,timeout=timeout,context=ssl_context())
+        self.endpoint_ip=endpoint.ip
+    def connect(self) -> None:
+        raw=socket.create_connection((self.endpoint_ip,self.port),self.timeout)
+        try: self.sock=self._context.wrap_socket(raw,server_hostname=self.host)
+        except Exception: raw.close();raise
+
+class Kestrel:
+    def __init__(self,username: str,password: str,*,timeout: float=120.0,retries: int=5,
+                 endpoint: GatewayEndpoint|None=None,cache: bool=True):
+        if not username or not password: raise ValueError("用户名和密码必填")
+        if retries<1: raise ValueError("retries 必须大于等于 1")
+        self.username=username;self.password=password;self.timeout=timeout
+        self.retries=retries;self.cache_enabled=cache
+        self.endpoint=endpoint or discover_endpoint(timeout=min(timeout,8));self.token: str|None=None
+
+    def login(self) -> dict:
+        body=self._json("POST","/api/auth/login",{"username":self.username,"password":self.password},auth=False)
+        data=self._data(body)
+        if data.get("accountType")!="api": raise PermissionError("该账户不是 API 工具账户")
+        self.token=str(data["token"]);return {k:v for k,v in data.items() if k!="token"}
+
+    def products(self) -> list[dict]: return list(self._data(self._json("GET","/api/sdk/products")))
+
+    def partitions(self,product: str,*,start: str|date|None=None,end: str|date|None=None) -> list[dict]:
+        query={};start=_date_value(start,False);end=_date_value(end,True)
+        if start: query["from"]=start
+        if end: query["to"]=end
+        path=f"/api/sdk/products/{quote(product,safe='')}/partitions"
+        if query: path+="?"+urlencode(query)
+        return list(self._data(self._json("GET",path)))
+
+    def files(self,product: str,partition_key: str,version: int,*,plan_id: str|None=None) -> list[dict]:
+        args={"partitionKey":partition_key,"version":version}
+        if plan_id is not None:args["planId"]=plan_id
+        query=urlencode(args)
+        return list(self._data(self._json("GET",f"/api/sdk/products/{quote(product,safe='')}/files?{query}")))
+
+    def sync_plan(self,product: str,*,start: str|date|None=None,end: str|date|None=None,
+                  local_versions: dict[str,int]|None=None) -> dict:
+        payload={"productId":product,"localVersions":local_versions or {}}
+        start=_date_value(start,False);end=_date_value(end,True)
+        if start: payload["from"]=start
+        if end: payload["to"]=end
+        return dict(self._data(self._json("POST","/api/sdk/sync-plans",payload)))
+
+    def download(self,product: str,output: str|Path,*,start: str|date|None=None,
+                 end: str|date|None=None,force: bool=False,progress: bool=True) -> list[DownloadResult]:
+        results=[];local_cache=LocalCache(output) if self.cache_enabled else None
+        local_versions={} if force or local_cache is None else local_cache.revisions(product)
+        if progress:
+            sys.stderr.write(f"[检查更新] {product}  本地已有 {len(local_versions)} 个分区\n");sys.stderr.flush()
+        plan=self.sync_plan(product,start=start,end=end,local_versions=local_versions)
+        plan_id=str(plan["planId"])
+        actions=list(plan.get("actions",[]))
+        if progress:
+            total=sum(int(x.get("parquetBytes",0)) for x in actions if x.get("action")=="add_replace")
+            sys.stderr.write(f"[同步计划] 更新 {sum(x.get('action')=='add_replace' for x in actions)} 个分区"
+                             f"，删除 {sum(x.get('action')=='delete' for x in actions)} 个分区"
+                             f"，预计 {_size(total)}\n");sys.stderr.flush()
+        for index,action in enumerate(actions,1):
+            partition_key=str(action["partitionKey"]);version=int(action["version"])
+            if action["action"]=="delete":
+                if local_cache is not None:
+                    removed=local_cache.remove_partition(product,partition_key)
+                    if progress: sys.stderr.write(f"[服务端删除] {product}  {partition_key}  清理 {removed} 个本地文件\n")
+                continue
+            if progress:sys.stderr.write(f"[准备分区 {index}/{len(actions)}] {partition_key}\n");sys.stderr.flush()
+            plan_files=list(action.get("files",[]))
+            results.extend(self.download_partition(product,partition_key,version,output,force=force,
+                                                    progress=progress,_cache=local_cache,
+                                                    _items=plan_files or None,_plan_id=plan_id))
+        return results
+
+    def download_partition(self,product: str,partition_key: str,version: int,
+                           output: str|Path,*,force: bool=False,progress: bool=True,
+                           _cache: LocalCache|None=None,_items: list[dict]|None=None,
+                           _plan_id: str|None=None) -> list[DownloadResult]:
+        output_root=Path(output).expanduser().resolve();root=output_root/product
+        local_cache=_cache if _cache is not None else (LocalCache(output_root) if self.cache_enabled else None)
+        results=[];items=self.files(product,partition_key,version,plan_id=_plan_id) if _items is None else _items;manifest_paths=[]
+        for item in items:
+            manifest_path=str(item["path"]);manifest_paths.append(manifest_path)
+            relative=_safe_relative(f"{partition_key}/{item['path']}")
+            target=root.joinpath(*relative.parts)
+            size=int(item["fileSize"]);sha=str(item["sha256"])
+            if local_cache is not None and not force:
+                cached=local_cache.cached_file(product,partition_key,version,manifest_path)
+                if cached is not None and cached.path==target and cached.file_size==size and cached.sha256==sha:
+                    if progress: _ProgressBar(f"{product}  {partition_key}/{item['path']}",size,size).finish(reused=True)
+                    results.append(DownloadResult(target,size,sha,size,True));continue
+            query_args={"partitionKey":partition_key,"version":version,"path":item["path"]}
+            if _plan_id is not None:query_args["planId"]=_plan_id
+            query=urlencode(query_args)
+            url=f"/api/sdk/products/{quote(product,safe='')}/download?{query}"
+            label=f"{product}  {partition_key}/{item['path']}"
+            result=self._download(url,target,size,sha,force=force,progress=progress,label=label)
+            results.append(result)
+            if local_cache is not None:
+                local_cache.record(product,partition_key,version,manifest_path,target,size,sha)
+        if local_cache is not None:
+            local_cache.retain_files(product,partition_key,version,manifest_paths)
+            local_cache.mark_complete(product,partition_key,version,len(items))
+        return results
+
+    def cache_info(self,output: str|Path,*,product: str|None=None) -> CacheInfo:
+        return LocalCache(output).info(product)
+
+    def check_cache(self,output: str|Path,*,product: str|None=None,deep: bool=False,
+                    repair: bool=True) -> CacheCheckResult:
+        """Validate local cache; deep=True recalculates SHA-256, repair removes invalid entries."""
+        return LocalCache(output).check(product,deep=deep,repair=repair)
+
+    def _download(self,url: str,target: Path,size: int,sha: str,*,force: bool=False,
+                  progress: bool=False,label: str="") -> DownloadResult:
+        target.parent.mkdir(parents=True,exist_ok=True);partial=target.with_name(target.name+".kestrel.part")
+        if not force and target.is_file() and target.stat().st_size==size and _sha256(target)==sha:
+            if progress: _ProgressBar(label or target.name,size,size).finish(reused=True)
+            return DownloadResult(target,size,sha,size,True)
+        if force: partial.unlink(missing_ok=True)
+        offset=partial.stat().st_size if partial.is_file() else 0
+        if offset>size: partial.unlink();offset=0
+        resumed=offset;bar=_ProgressBar(label or target.name,size,offset) if progress else None
+        if bar: bar.update(offset)
+        last_error: Exception|None=None
+        for attempt in range(1,self.retries+1):
+            offset=partial.stat().st_size if partial.is_file() else 0
+            headers=self._headers();headers["Accept"]="application/octet-stream"
+            if offset: headers["Range"]=f"bytes={offset}-"
+            connection=_Connection(self.endpoint,self.timeout)
+            try:
+                connection.request("GET",url,headers=headers);response=connection.getresponse()
+                if offset and response.status==200:
+                    offset=0;resumed=0
+                    if bar: bar=_ProgressBar(label or target.name,size,0);bar.update(0)
+                elif offset and response.status!=206:
+                    error=f"断点续传失败: HTTP {response.status}"
+                    if response.status in (408,429,500,502,503,504): raise _RetryableDownloadError(error)
+                    raise _PermanentDownloadError(error)
+                elif not offset and response.status not in (200,206):
+                    error=f"下载失败: HTTP {response.status}"
+                    if response.status not in (408,429,500,502,503,504): raise _PermanentDownloadError(error)
+                    raise _RetryableDownloadError(error)
+                with partial.open("ab" if offset else "wb") as sink:
+                    while block:=response.read(1024*1024):
+                        sink.write(block);offset+=len(block)
+                        if bar: bar.update(offset)
+                    sink.flush();os.fsync(sink.fileno())
+                actual=partial.stat().st_size
+                if actual!=size:
+                    raise _RetryableDownloadError(f"文件未下载完整: {actual}/{size} B")
+                if _sha256(partial)!=sha:
+                    partial.unlink(missing_ok=True)
+                    raise _RetryableDownloadError("文件 SHA-256 校验失败，已从头重试")
+                last_error=None
+                break
+            except _PermanentDownloadError:
+                raise
+            except (OSError,http.client.HTTPException,_RetryableDownloadError) as error:
+                last_error=error
+                if attempt>=self.retries: break
+                if bar: bar.retry(attempt,self.retries,error)
+                time.sleep(min(2**(attempt-1),8))
+            finally: connection.close()
+        if last_error is not None:
+            raise RuntimeError(f"下载失败，已断点重试 {self.retries} 次: {target}: {last_error}") from last_error
+        os.replace(partial,target)
+        if bar: bar.finish()
+        return DownloadResult(target,size,sha,resumed,False)
+
+    def _json(self,method: str,path: str,payload: dict|None=None,*,auth: bool=True) -> object:
+        if auth and not self.token: self.login()
+        data=None if payload is None else json.dumps(payload,separators=(",",":"),ensure_ascii=False).encode()
+        headers=self._headers() if auth else {"Accept":"application/json"}
+        if data is not None: headers["Content-Type"]="application/json"
+        connection=_Connection(self.endpoint,self.timeout)
+        try:
+            connection.request(method,path,body=data,headers=headers);response=connection.getresponse()
+            raw=response.read(4*1024*1024)
+            if response.status==401 and auth:
+                self.token=None;connection.close();self.login();return self._json(method,path,payload)
+            if response.status not in (200,201): raise RuntimeError(f"API 请求失败: HTTP {response.status}")
+            return json.loads(raw.decode("utf-8"))
+        finally: connection.close()
+
+    def _headers(self) -> dict[str,str]:
+        return {"Accept":"application/json","Authorization":f"Bearer {self.token}"}
+
+    @staticmethod
+    def _data(body: object):
+        if not isinstance(body,dict) or body.get("code")!=0: raise RuntimeError(str(body.get("msg") if isinstance(body,dict) else "API 响应无效"))
+        return body.get("data")
+
+def _date_value(value: str|date|None,end: bool) -> str|None:
+    if value is None: return None
+    if isinstance(value,date): return value.isoformat()
+    text=str(value).strip().replace("-","")
+    if len(text)==6 and text.isdigit():
+        year,month=int(text[:4]),int(text[4:]);day=calendar.monthrange(year,month)[1] if end else 1
+        return date(year,month,day).isoformat()
+    if len(text)==8 and text.isdigit(): return date(int(text[:4]),int(text[4:6]),int(text[6:])).isoformat()
+    raise ValueError("日期必须是 YYYYMM、YYYYMMDD 或 date")
+
+def _safe_relative(value: str) -> PurePosixPath:
+    path=PurePosixPath(value)
+    if path.is_absolute() or ".." in path.parts or path.suffix!=".parquet": raise ValueError("服务端文件路径无效")
+    return path
+
+def _sha256(path: Path) -> str:
+    digest=hashlib.sha256()
+    with path.open("rb") as source:
+        while block:=source.read(4*1024*1024): digest.update(block)
+    return digest.hexdigest()
+
+class _ProgressBar:
+    def __init__(self,label: str,total: int,initial: int=0):
+        self.label=label;self.total=max(0,total);self.initial=max(0,initial)
+        self.started=time.monotonic();self.last=0.0;self.tty=sys.stderr.isatty()
+    def update(self,current: int,*,final: bool=False) -> None:
+        now=time.monotonic()
+        if not final and now-self.last<0.15:return
+        self.last=now;current=min(max(0,current),self.total)
+        ratio=current/self.total if self.total else 1.0;filled=min(24,int(ratio*24))
+        elapsed=max(.001,now-self.started);speed=max(0,(current-self.initial)/elapsed)
+        eta=(self.total-current)/speed if speed>0 else None
+        line=(f"[{('='*filled)+(' '*(24-filled))}] {ratio*100:6.2f}%  "
+              f"{_size(current)} / {_size(self.total)}  {_size(speed)}/s  ETA {_eta(eta)}  {self.label}")
+        if self.tty: sys.stderr.write("\r\033[2K"+line)
+        elif final: sys.stderr.write(line)
+        sys.stderr.flush()
+    def finish(self,*,reused: bool=False) -> None:
+        if reused:
+            line=f"[本地复用] {_size(self.total)}  {self.label}"
+            if self.tty:sys.stderr.write("\r\033[2K"+line+"\n")
+            else:sys.stderr.write(line+"\n")
+            sys.stderr.flush();return
+        self.update(self.total,final=True);sys.stderr.write("\n");sys.stderr.flush()
+    def retry(self,attempt: int,total: int,error: Exception) -> None:
+        message=f"\n[连接中断，自动续传 {attempt}/{total}] {self.label}: {error}\n"
+        sys.stderr.write(message);sys.stderr.flush()
+
+class _RetryableDownloadError(Exception): pass
+class _PermanentDownloadError(Exception): pass
+
+def _size(value: float|int) -> str:
+    number=float(value)
+    for unit in ("B","KiB","MiB","GiB","TiB"):
+        if abs(number)<1024 or unit=="TiB":return f"{number:.1f} {unit}"
+        number/=1024
+    return f"{number:.1f} TiB"
+
+def _eta(value: float|None) -> str:
+    if value is None:return "--:--"
+    seconds=max(0,int(value));hours,seconds=divmod(seconds,3600);minutes,seconds=divmod(seconds,60)
+    return f"{hours:02d}:{minutes:02d}:{seconds:02d}" if hours else f"{minutes:02d}:{seconds:02d}"

kestrel_data_sdk/endpoint.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+import ipaddress
+import json
+import os
+import socket
+import ssl
+import urllib.request
+from dataclasses import dataclass
+from importlib.resources import files
+
+DEFAULT_CLOUD_ENDPOINT = "https://47.116.211.215:15688/ip/endpoint"
+DEFAULT_LOGICAL_HOST = "data.kestrel.internal"
+DEFAULT_GATEWAY_PORT = 15688
+DEFAULT_INTERNAL_GATEWAY_IP = "192.168.100.118"
+DEFAULT_INTERNAL_CIDRS = "192.168.100.0/24"
+
+@dataclass(frozen=True)
+class GatewayEndpoint:
+    ip: str
+    logical_host: str = DEFAULT_LOGICAL_HOST
+    port: int = DEFAULT_GATEWAY_PORT
+
+def ca_certificate() -> str:
+    override=os.environ.get("KESTREL_CA_FILE")
+    if override:
+        return open(override,"r",encoding="ascii").read()
+    return files("kestrel_data_sdk").joinpath("certs/kestrel-root-ca.crt").read_text(encoding="ascii")
+
+def ssl_context() -> ssl.SSLContext:
+    context=ssl.create_default_context(cadata=ca_certificate())
+    context.minimum_version=ssl.TLSVersion.TLSv1_2
+    return context
+
+def discover_endpoint(timeout: float=8.0) -> GatewayEndpoint:
+    override=os.environ.get("KESTREL_GATEWAY_IP")
+    if override: return _endpoint(override)
+    internal=_internal_endpoint(min(timeout,1.0))
+    if internal: return internal
+    request=urllib.request.Request(os.environ.get("KESTREL_CLOUD_ENDPOINT",DEFAULT_CLOUD_ENDPOINT),
+                                   headers={"Accept":"application/json","User-Agent":"kestrel-data-sdk"})
+    try:
+        with urllib.request.urlopen(request,timeout=timeout,context=ssl_context()) as response:
+            body=json.loads(response.read(16*1024).decode("utf-8"))
+    except Exception as error:
+        raise RuntimeError(f"无法获取 Kestrel Data 公网入口: {error}") from error
+    if not body.get("ok") or body.get("stale"): raise RuntimeError("公网入口尚未就绪或已过期")
+    return _endpoint(str(body.get("ip","")))
+
+def _internal_endpoint(timeout: float) -> GatewayEndpoint|None:
+    endpoint=_endpoint(os.environ.get("KESTREL_INTERNAL_GATEWAY_IP",DEFAULT_INTERNAL_GATEWAY_IP))
+    try:
+        with socket.socket(socket.AF_INET,socket.SOCK_DGRAM) as probe:
+            probe.connect((endpoint.ip,endpoint.port));local=ipaddress.ip_address(probe.getsockname()[0])
+        cidrs=os.environ.get("KESTREL_INTERNAL_CIDRS",DEFAULT_INTERNAL_CIDRS).split(",")
+        if not any(local in ipaddress.ip_network(x.strip()) for x in cidrs if x.strip()): return None
+        with socket.create_connection((endpoint.ip,endpoint.port),timeout=max(.2,timeout)) as raw:
+            with ssl_context().wrap_socket(raw,server_hostname=endpoint.logical_host): return endpoint
+    except (OSError,ValueError,ssl.SSLError): return None
+
+def _endpoint(value: str) -> GatewayEndpoint:
+    try: ip=str(ipaddress.ip_address(value.strip()))
+    except ValueError as error: raise RuntimeError("入口服务返回了无效 IP") from error
+    port=int(os.environ.get("KESTREL_GATEWAY_PORT",str(DEFAULT_GATEWAY_PORT)))
+    if not 1<=port<=65535: raise RuntimeError("网关端口无效")
+    return GatewayEndpoint(ip,os.environ.get("KESTREL_GATEWAY_HOST",DEFAULT_LOGICAL_HOST),port)

kestrel_data_sdk-0.3.2.dist-info/METADATA

@@ -0,0 +1,52 @@
+Metadata-Version: 2.1
+Name: kestrel-data-sdk
+Version: 0.3.2
+Summary: Kestrel Data on-demand download SDK
+Author: Kestrel Data
+License: Proprietary
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+
+# kestrel-data-sdk
+
+Kestrel Data 按需数据下载 SDK。适用于已获授权的 API 账户，无 Web UI、无常驻服务、
+无自动下发，支持 Linux、macOS 和 Windows 的 Python 3.10+。
+
+```python
+from kestrel_data_sdk import Kestrel
+
+kd = Kestrel("api-user", "password")
+print(kd.products())
+kd.download("ods.wind.ashareeodprices", "/data/kestrel", start="202601", end="202608")
+```
+
+授权范围内没有下载次数配额。特定 API 账户还可由管理员开启“全部数据权限”。默认在输出
+目录的 `.kestrel/sdk-cache.sqlite3` 记录已完成的发布版本。每次同步只把本地版本号集合提交给
+服务端，由服务端立即返回不可变的 24 小时同步计划；只有需要更新的分区才逐个获取文件清单，
+避免在首次下载前等待整批历史文件扫描。
+需要主动重新下载时使用：
+
+```python
+kd.download("ods.wind.ashareeodprices", "/data/kestrel", start="202608", end="202608", force=True)
+```
+
+下载默认在终端显示文件进度、已下载/总大小、速度和 ETA。非交互脚本可使用 `progress=False`
+关闭。网络中断时会保留 `.kestrel.part` 临时文件并自动断点续传，默认最多尝试 5 次。
+
+缓存查询和校验：
+
+```python
+print(kd.cache_info("/data/kestrel"))
+
+# 显式检查存在性、大小和修改时间；失效索引会被清除，下次下载自动补齐
+print(kd.check_cache("/data/kestrel"))
+
+# 精确检查会读取全部已缓存文件并重算 SHA-256
+print(kd.check_cache("/data/kestrel", deep=True))
+```
+
+`check_cache(..., repair=True)` 只修复缓存索引，不会删除本地文件。特殊脚本可以使用
+`Kestrel(..., cache=False)` 禁用持久缓存。
+
+账户必须由管理员设为 `API 工具`。普通 API 账户受产品授权、数据范围和授权有效期约束；
+开启“全部数据权限”的账户不受这些约束。

kestrel_data_sdk-0.3.2.dist-info/RECORD

@@ -0,0 +1,9 @@
+kestrel_data_sdk/__init__.py,sha256=CUdSZ-s5JGg_AGoSEPUI9JCafQII5mvGoWDyIznRcWY,187
+kestrel_data_sdk/cache.py,sha256=swRCCnm5L_jjacnR2J1Yh3_L4oxfUGrQDUfh2x2ORcg,14026
+kestrel_data_sdk/client.py,sha256=yo3EDkTu8MPcdeRLUFDS9fl4zhMO4Q8IdvQeHOJLFqI,15966
+kestrel_data_sdk/endpoint.py,sha256=OS2hz0WXCBMPGLYV8HmhATCVgaREEJbpHHhcXFxG9s8,3131
+kestrel_data_sdk/certs/kestrel-root-ca.crt,sha256=EK1VVsoqkrrkiRC89_SeizQ40lT4Fozhm2B6wItTrUI,1976
+kestrel_data_sdk-0.3.2.dist-info/METADATA,sha256=A4A3E5PNG9Bz5y4-N8EQ2ZXLOfGBByG3aqCCcQjVsFk,2110
+kestrel_data_sdk-0.3.2.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+kestrel_data_sdk-0.3.2.dist-info/top_level.txt,sha256=gLsKiu_Dql888wgJnkcys1zlXG_EZPZseotpX9NMYds,17
+kestrel_data_sdk-0.3.2.dist-info/RECORD,,

kestrel_data_sdk-0.3.2.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.42.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

kestrel_data_sdk-0.3.2.dist-info/top_level.txt

@@ -0,0 +1 @@
+kestrel_data_sdk