intellif-aihub 0.1.3__py3-none-any.whl → 0.1.5__py3-none-any.whl

aihub/utils/download.py

@@ -1,69 +1,69 @@
-from __future__ import annotations
-
-import concurrent.futures
-import os
-import tempfile
-from typing import List, TypedDict
-
-import pyarrow.parquet as pq
-from tqdm import tqdm
-
-from .http import http_download_file
-from .s3 import s3_to_url
-
-
-class DatasetParquetMeta(TypedDict):
-    parent_dir: str
-    name: str
-    s3path: str
-    type: int  # 0=file, 1=dir
-
-
-_ENUM_FILE = 0
-
-
-def _read_parquet_index(file_path: str) -> List[DatasetParquetMeta]:
-    table = pq.read_table(file_path)
-    return table.to_pylist()  # 每行转 dict
-
-
-def _safe_rel(part: str) -> str:
-    if not part:
-        return ""
-    drive, tail = os.path.splitdrive(part)
-    return tail.lstrip(r"\/")
-
-
-def dataset_download(index_url: str, local_dir: str, worker: int = 4) -> None:
-    with tempfile.TemporaryDirectory() as tmpdir:
-        tmp_file = os.path.join(tmpdir, "index.parquet")
-        http_download_file(index_url, tmp_file)
-        rows = _read_parquet_index(tmp_file)
-
-    host = (index_url.split("//", 1)[-1]).split("/", 1)[0]
-
-    files = [
-        (
-            os.path.join(
-                local_dir,
-                _safe_rel(row["parent_dir"]),
-                _safe_rel(row["name"]),
-            ),
-            s3_to_url(row["s3path"], host),
-        )
-        for row in rows if row["type"] == _ENUM_FILE
-    ]
-
-    if worker < 1:
-        worker = 1
-
-    with tqdm(total=len(files), desc="Downloading dataset") as bar, \
-            concurrent.futures.ThreadPoolExecutor(max_workers=worker) as pool:
-
-        def _one(flocal: str, furl: str):
-            http_download_file(furl, flocal)
-            bar.update()
-
-        futures = [pool.submit(_one, p, u) for p, u in files]
-        for fut in concurrent.futures.as_completed(futures):
-            fut.result()
+from __future__ import annotations
+
+import concurrent.futures
+import os
+import tempfile
+from typing import List, TypedDict
+
+import pyarrow.parquet as pq
+from tqdm import tqdm
+
+from .http import http_download_file
+from .s3 import s3_to_url
+
+
+class DatasetParquetMeta(TypedDict):
+    parent_dir: str
+    name: str
+    s3path: str
+    type: int  # 0=file, 1=dir
+
+
+_ENUM_FILE = 0
+
+
+def _read_parquet_index(file_path: str) -> List[DatasetParquetMeta]:
+    table = pq.read_table(file_path)
+    return table.to_pylist()  # 每行转 dict
+
+
+def _safe_rel(part: str) -> str:
+    if not part:
+        return ""
+    drive, tail = os.path.splitdrive(part)
+    return tail.lstrip(r"\/")
+
+
+def dataset_download(index_url: str, local_dir: str, worker: int = 4) -> None:
+    with tempfile.TemporaryDirectory() as tmpdir:
+        tmp_file = os.path.join(tmpdir, "index.parquet")
+        http_download_file(index_url, tmp_file)
+        rows = _read_parquet_index(tmp_file)
+
+    host = (index_url.split("//", 1)[-1]).split("/", 1)[0]
+
+    files = [
+        (
+            os.path.join(
+                local_dir,
+                _safe_rel(row["parent_dir"]),
+                _safe_rel(row["name"]),
+            ),
+            s3_to_url(row["s3path"], host),
+        )
+        for row in rows if row["type"] == _ENUM_FILE
+    ]
+
+    if worker < 1:
+        worker = 1
+
+    with tqdm(total=len(files), desc="Downloading dataset") as bar, \
+            concurrent.futures.ThreadPoolExecutor(max_workers=worker) as pool:
+
+        def _one(flocal: str, furl: str):
+            http_download_file(furl, flocal)
+            bar.update()
+
+        futures = [pool.submit(_one, p, u) for p, u in files]
+        for fut in concurrent.futures.as_completed(futures):
+            fut.result()

aihub/utils/http.py

@@ -1,13 +1,13 @@
-from __future__ import annotations
-
-import httpx
-import os
-
-
-def http_download_file(url: str, dst_path: str, chunk: int = 1 << 16) -> None:
-    os.makedirs(os.path.dirname(dst_path), exist_ok=True)
-    with httpx.stream("GET", url, follow_redirects=True, timeout=None) as r:
-        r.raise_for_status()
-        with open(dst_path, "wb") as f:
-            for block in r.iter_bytes(chunk):
-                f.write(block)
+from __future__ import annotations
+
+import httpx
+import os
+
+
+def http_download_file(url: str, dst_path: str, chunk: int = 1 << 16) -> None:
+    os.makedirs(os.path.dirname(dst_path), exist_ok=True)
+    with httpx.stream("GET", url, follow_redirects=True, timeout=None) as r:
+        r.raise_for_status()
+        with open(dst_path, "wb") as f:
+            for block in r.iter_bytes(chunk):
+                f.write(block)

aihub/utils/s3.py

@@ -1,77 +1,77 @@
-from __future__ import annotations
-
-import os
-import re
-from pathlib import Path
-from typing import Any
-
-from loguru import logger
-from minio import Minio
-
-
-def s3_to_url(s3_path: str, host: str) -> str:
-    key = s3_path.replace("s3://", "").lstrip("/")
-    return f"http://{host.rstrip('/')}/{key}"
-
-
-def S3_path_to_info(s3_path) -> tuple[str | Any, str | Any] | None:
-    if not s3_path.startswith("s3://"):
-        return None
-
-    pattern = r"s3://(?P<bucket>\w+)/(?P<objectname>.+)"
-
-    match = re.match(pattern, s3_path)
-
-    if match:
-        bucket = match.group("bucket")
-        objectname = match.group("objectname")
-        return bucket, objectname
-    return None
-
-
-def local_path_to_s3_key(work_dir: str, local_path: str) -> str:
-    work_dir = Path(work_dir)
-    local_path = Path(local_path)
-    s3_key = str(local_path.relative_to(work_dir))
-    return s3_key
-
-
-def upload_dir_to_s3(
-    s3_client: Minio, local_dir: str, bucket: str, object_prefix: str
-) -> None:
-    logger.info(
-        f"Uploading directory {local_dir} to S3 bucket {bucket} with prefix {object_prefix}"
-    )
-
-    for root, dirs, files in os.walk(local_dir):
-        for file in files:
-            local_path = Path(root) / file
-            s3_key = local_path_to_s3_key(local_dir, str(local_path))
-            s3_client.fput_object(
-                bucket, os.path.join(object_prefix, s3_key), str(local_path)
-            )
-
-    logger.info(
-        f"Uploaded directory {local_dir} to S3 bucket {bucket} with prefix {object_prefix}"
-    )
-    return
-
-
-def download_dir_from_s3(
-    s3_client: Minio, bucket: str, object_prefix: str, local_dir: str
-) -> None:
-    logger.info(
-        f"Downloading directory from S3 bucket {bucket} with prefix {object_prefix} to {local_dir}"
-    )
-    objs = s3_client.list_objects(bucket, object_prefix, recursive=True)
-
-    for obj in objs:
-        file_name = Path(obj.object_name).relative_to(object_prefix)
-        s3_client.fget_object(
-            bucket, obj.object_name, os.path.join(local_dir, file_name)
-        )
-
-    logger.info(
-        f"Downloaded directory from S3 bucket {bucket} with prefix {object_prefix} to {local_dir}"
-    )
-    return
+from __future__ import annotations
+
+import os
+import re
+from pathlib import Path
+from typing import Any
+
+from loguru import logger
+from minio import Minio
+
+
+def s3_to_url(s3_path: str, host: str) -> str:
+    key = s3_path.replace("s3://", "").lstrip("/")
+    return f"http://{host.rstrip('/')}/{key}"
+
+
+def S3_path_to_info(s3_path) -> tuple[str | Any, str | Any] | None:
+    if not s3_path.startswith("s3://"):
+        return None
+
+    pattern = r"s3://(?P<bucket>\w+)/(?P<objectname>.+)"
+
+    match = re.match(pattern, s3_path)
+
+    if match:
+        bucket = match.group("bucket")
+        objectname = match.group("objectname")
+        return bucket, objectname
+    return None
+
+
+def local_path_to_s3_key(work_dir: str, local_path: str) -> str:
+    work_dir = Path(work_dir)
+    local_path = Path(local_path)
+    s3_key = str(local_path.relative_to(work_dir))
+    return s3_key
+
+
+def upload_dir_to_s3(
+    s3_client: Minio, local_dir: str, bucket: str, object_prefix: str
+) -> None:
+    logger.info(
+        f"Uploading directory {local_dir} to S3 bucket {bucket} with prefix {object_prefix}"
+    )
+
+    for root, dirs, files in os.walk(local_dir):
+        for file in files:
+            local_path = Path(root) / file
+            s3_key = local_path_to_s3_key(local_dir, str(local_path))
+            s3_client.fput_object(
+                bucket, os.path.join(object_prefix, s3_key), str(local_path)
+            )
+
+    logger.info(
+        f"Uploaded directory {local_dir} to S3 bucket {bucket} with prefix {object_prefix}"
+    )
+    return
+
+
+def download_dir_from_s3(
+    s3_client: Minio, bucket: str, object_prefix: str, local_dir: str
+) -> None:
+    logger.info(
+        f"Downloading directory from S3 bucket {bucket} with prefix {object_prefix} to {local_dir}"
+    )
+    objs = s3_client.list_objects(bucket, object_prefix, recursive=True)
+
+    for obj in objs:
+        file_name = Path(obj.object_name).relative_to(object_prefix)
+        s3_client.fget_object(
+            bucket, obj.object_name, os.path.join(local_dir, file_name)
+        )
+
+    logger.info(
+        f"Downloaded directory from S3 bucket {bucket} with prefix {object_prefix} to {local_dir}"
+    )
+    return

{intellif_aihub-0.1.3.dist-info → intellif_aihub-0.1.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: intellif-aihub
-Version: 0.1.3
+Version: 0.1.5
 Summary: Intellif AI-hub SDK.
 Author-email: Platform Team <aihub@example.com>
 License-Expression: Apache-2.0

intellif_aihub-0.1.5.dist-info/RECORD

@@ -0,0 +1,36 @@
+aihub/__init__.py,sha256=rPSfWgIeq2YWVPyESOAwCBt8vftsTpIkuLAGDEzyRQc,22
+aihub/client.py,sha256=Eu0evMLB_aWyqL0VOFTfNh0XYQb9Ohob4dnaLzVrfNI,3779
+aihub/exceptions.py,sha256=l2cMAvipTqQOio3o11fXsCCSCevbuK4PTsxofkobFjk,500
+aihub/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+aihub/models/artifact.py,sha256=V1rPNn9pUwSP0M7FvLIqR9Q7429JYbTU2tB-hUEJ04w,3533
+aihub/models/common.py,sha256=qmabc2LkAdQJXIcpT1P35zxd0Lc8yDYdD4ame1iF4Bs,241
+aihub/models/dataset_management.py,sha256=lP92aOsZJihg4SEhf1jeITcXp-N8l_YzHYf1l9Zq7-g,3381
+aihub/models/document_center.py,sha256=xmAk_JIY3GjuVDZurMUonmSz3Siy3TAxhj3ewIJ6dUQ,489
+aihub/models/eval.py,sha256=5h25jR4-JeZ-Sbq0ifp6fd1IVrq0I3w-S8YV_BFUDao,326
+aihub/models/labelfree.py,sha256=RgSCWPQ-_XHbCjalaTDQcgGAQXCKObVs38h2kTpRm88,1755
+aihub/models/model_training_platform.py,sha256=IWgGZ60A9qjLxj1srC3dUc6a-i8gLe8D2yASJ3HeNRY,6521
+aihub/models/quota_schedule_management.py,sha256=E_q6x5KaJOyrOPlYqBumwDgKB1uRpfOHSsYgMxqI6MA,6801
+aihub/models/tag_resource_management.py,sha256=httqeAjCypqykkmhJXSjJfL4Qlfqz21y3w2qXEYtT6g,1104
+aihub/models/task_center.py,sha256=0Sbxsi7mwKUsotO5_4EpQ1dE8goYcI9jnPWLJ5YzfpA,4030
+aihub/models/user_system.py,sha256=4hhOw6DJVjqoJKLkhPbRZnzMOITH2s-t40id1d409zE,7433
+aihub/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+aihub/services/artifact.py,sha256=Ye2T78KZFwDLFK7ngvqw712HxSfoNwoNdJ71-wjd4YI,11201
+aihub/services/dataset_management.py,sha256=fqGJATO1kisPfjk7GxKlzh9fU2w42FeeWEAh-WRDJto,8669
+aihub/services/document_center.py,sha256=DClItxWXMMFPnKMR7kyNGohhXP3wowImj-Lm8vzBgNo,1339
+aihub/services/eval.py,sha256=X6NBV6PTsczoeulV2y1ySrr9GB9R5EBaXgjEmiiUiCE,1963
+aihub/services/labelfree.py,sha256=UUuq519us1fma0ERXM-sXfLPWRHB1f6BqE2XvCq-3Pw,1215
+aihub/services/model_training_platform.py,sha256=fsCO4BEuja_1ePl0hwt32kpnN99AArjzBv_e459p0X4,7560
+aihub/services/quota_schedule_management.py,sha256=4MdtEP2Ziz6hh3tXmd6nbLXxtQ46Z_eT2VtjE3Kr-0I,6803
+aihub/services/reporter.py,sha256=ot93SmhxgwDJOzlHSCwlxDOuSydTWUEUQ-Ctp97wJBQ,669
+aihub/services/tag_resource_management.py,sha256=GcuVMlwGlfZkvUKvLUTKtteHNjurlLr2FUy2Eg8cfyE,1811
+aihub/services/task_center.py,sha256=xhyMdmhFjeL9QZ5YZ0eGz6WIgclECitG8Yp_dXO7OEY,6211
+aihub/services/user_system.py,sha256=vEcwvSX28lMp7YBUm1AWRDujEa73NqOtwMQi-t7xPSM,13654
+aihub/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+aihub/utils/download.py,sha256=Rh1m3VpMlw8-Kl36sowJ7M0dpB68u-9V4Vo3GQChq1I,1758
+aihub/utils/http.py,sha256=rSNh4uNP7E3YGm3H1indRHctxC5Wu5xNBPvDrb9UHt4,421
+aihub/utils/s3.py,sha256=ISIBP-XdBPkURpXnN56ZnIWokOOg2SRUh_qvxJk-G1Q,2187
+intellif_aihub-0.1.5.dist-info/licenses/LICENSE,sha256=QwcOLU5TJoTeUhuIXzhdCEEDDvorGiC6-3YTOl4TecE,11356
+intellif_aihub-0.1.5.dist-info/METADATA,sha256=2TgCnrq8SWHiEnohwrD2ZaktHoHDIu2W7Q2Zoy_9yWA,2922
+intellif_aihub-0.1.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+intellif_aihub-0.1.5.dist-info/top_level.txt,sha256=vIvTtSIN73xv46BpYM-ctVGnyOiUQ9EWP_6ngvdIlvw,6
+intellif_aihub-0.1.5.dist-info/RECORD,,