unitlab 2.3.46__tar.gz → 2.3.48__tar.gz

{unitlab-2.3.46/src/unitlab.egg-info → unitlab-2.3.48}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: unitlab
-Version: 2.3.46
+Version: 2.3.48
 Home-page: https://github.com/teamunitlab/unitlab-sdk
 Author: Unitlab Inc.
 Author-email: team@unitlab.ai

{unitlab-2.3.46 → unitlab-2.3.48}/setup.py

@@ -2,11 +2,11 @@ from setuptools import find_packages, setup
 
 setup(
     name="unitlab",
-    version="2.3.46",
+    version="2.3.48",
     license="MIT",
     author="Unitlab Inc.",
     author_email="team@unitlab.ai",
-    packages=find_packages("src"),
+    packages=find_packages(where="src"),
     include_package_data=True,
     package_data={"static": ["*"], "Potato": ["*.so"]},
     classifiers=[

{unitlab-2.3.46 → unitlab-2.3.48}/src/unitlab/client.py

@@ -113,24 +113,43 @@ class UnitlabClient:
     def project_members(self, project_id, pretty=0):
         return self._get(f"/api/sdk/projects/{project_id}/members/?pretty={pretty}")
 
-    def project_upload_data(self, project_id, directory, batch_size=100):
+    def project_upload_info(self, project_id):
+        return self._get(f"/api/sdk/projects/{project_id}/upload-info/")
+
+    def project_upload_data(
+        self,
+        project_id,
+        directory,
+        batch_size=100,
+        sentences_per_chunk=10,
+        fps=1.0,
+    ):
         if not os.path.isdir(directory):
             raise ValueError(f"Directory {directory} does not exist")
 
+        upload_info = self.project_upload_info(project_id)
+        accepted_formats = upload_info["accepted_formats"]
+        max_file_size_bytes = upload_info["max_file_size"]
+        generic_type = upload_info["generic_type"]
+
         files = [
             file
             for files_list in (
-                glob.glob(os.path.join(directory, "") + extension)
-                for extension in ["*jpg", "*png", "*jpeg", "*webp"]
+                glob.glob(os.path.join(directory, "") + f"*.{ext}")
+                for ext in accepted_formats
             )
             for file in files_list
         ]
+
         filtered_files = []
+        max_file_size_mb = max_file_size_bytes / 1024 / 1024
         for file in files:
-            file_size = os.path.getsize(file) / 1024 / 1024
-            if file_size > 6:
+            file_size_bytes = os.path.getsize(file)
+            if file_size_bytes > max_file_size_bytes:
+                file_size_mb = file_size_bytes / 1024 / 1024
                 logger.warning(
-                    f"File {file} is too large ({file_size:.4f} megabytes) skipping, max size is 6 MB"
+                    f"File {file} is too large ({file_size_mb:.4f} MB) skipping, "
+                    f"max size is {max_file_size_mb:.2f} MB"
                 )
                 continue
             filtered_files.append(file)
@@ -141,14 +160,20 @@ class UnitlabClient:
         async def post_file(session: aiohttp.ClientSession, file: str, project_id: str):
             async with aiofiles.open(file, "rb") as f:
                 form_data = aiohttp.FormData()
-                form_data.add_field("project", project_id)
                 form_data.add_field(
                     "file", await f.read(), filename=os.path.basename(file)
                 )
+                if generic_type == "text":
+                    form_data.add_field("sentences_per_chunk", str(sentences_per_chunk))
+                elif generic_type == "video":
+                    form_data.add_field("fps", str(fps))
+
                 try:
                     await asyncio.sleep(0.1)
                     async with session.post(
-                        urllib.parse.urljoin(self.api_url, "/api/sdk/upload-data/"),
+                        urllib.parse.urljoin(
+                            self.api_url, f"/api/sdk/projects/{project_id}/upload-data/"
+                        ),
                         data=form_data,
                     ) as response:
                         response.raise_for_status()
@@ -204,16 +229,44 @@ class UnitlabClient:
             return os.path.abspath(filename)
 
     def dataset_download_files(self, dataset_id):
+        """Download files from a dataset.
+
+        Args:
+            dataset_id: UUID of the dataset/release to download.
+
+        Returns:
+            Path to the download folder.
+        """
         response = self._post(
             f"/api/sdk/datasets/{dataset_id}/", data={"download_type": "files"}
         )
-        folder = f"dataset-files-{dataset_id}"
-        os.makedirs(folder, exist_ok=True)
-        dataset_files = [
-            dataset_file
-            for dataset_file in response
-            if not os.path.isfile(os.path.join(folder, dataset_file["file_name"]))
-        ]
+        base_folder = f"dataset-files-{dataset_id}"
+        os.makedirs(base_folder, exist_ok=True)
+
+        files_to_download = []
+        for dataset_file in response:
+            file_name = dataset_file["file_name"]
+            file_path = os.path.join(base_folder, file_name)
+
+            parent_dir = os.path.dirname(file_path)
+            if parent_dir:
+                os.makedirs(parent_dir, exist_ok=True)
+
+            if "content" in dataset_file:
+                if not os.path.isfile(file_path):
+                    with open(file_path, "w", encoding="utf-8") as f:
+                        f.write(dataset_file["content"])
+            elif "source" in dataset_file and not os.path.isfile(file_path):
+                files_to_download.append(
+                    {
+                        "file_name": file_name,
+                        "source": dataset_file["source"],
+                        "file_path": file_path,
+                    }
+                )
+
+        if not files_to_download:
+            return base_folder
 
         async def download_file(session: aiohttp.ClientSession, dataset_file: dict):
             async with session.get(url=dataset_file["source"]) as r:
@@ -224,21 +277,20 @@ class UnitlabClient:
                         f"Error downloading file {dataset_file['file_name']} - {e}"
                     )
                     return 0
-                async with aiofiles.open(
-                    os.path.join(folder, dataset_file["file_name"]), "wb"
-                ) as f:
+                async with aiofiles.open(dataset_file["file_path"], "wb") as f:
                     async for chunk in r.content.iter_any():
                         await f.write(chunk)
                     return 1
 
         async def main():
-            with tqdm.tqdm(total=len(dataset_files), ncols=80) as pbar:
+            with tqdm.tqdm(total=len(files_to_download), ncols=80) as pbar:
                 async with aiohttp.ClientSession() as session:
                     tasks = [
-                        download_file(session=session, dataset_file=dataset_file)
-                        for dataset_file in dataset_files
+                        download_file(session=session, dataset_file=df)
+                        for df in files_to_download
                     ]
                     for f in asyncio.as_completed(tasks):
                         pbar.update(await f)
 
         asyncio.run(main())
+        return base_folder

{unitlab-2.3.46 → unitlab-2.3.48}/src/unitlab/main.py

@@ -72,8 +72,25 @@ def upload(
     directory: Annotated[
         Path, typer.Option(help="Directory containing the data to be uploaded")
     ],
+    sentences_per_chunk: Annotated[
+        int,
+        typer.Option(
+            help="Number of sentences per chunk for text projects (default: 10, range: 3-50)"
+        ),
+    ] = 10,
+    fps: Annotated[
+        float,
+        typer.Option(
+            help="Frames per second for video projects (default: 1.0, min: 0.25)"
+        ),
+    ] = 1.0,
 ):
-    get_client(api_key).project_upload_data(str(pk), directory=directory)
+    get_client(api_key).project_upload_data(
+        str(pk),
+        directory=directory,
+        sentences_per_chunk=sentences_per_chunk,
+        fps=fps,
+    )
 
 
 @dataset_app.command(name="list", help="List datasets")
@@ -105,8 +122,9 @@ def dataset_download(
             raise typer.BadParameter(
                 "Split type is required when download type is annotation"
             )
-        return get_client(api_key).dataset_download(pk, export_type, split_type)
-    get_client(api_key).dataset_download_files(pk)
+        print(get_client(api_key).dataset_download(pk, export_type, split_type))
+    else:
+        print(get_client(api_key).dataset_download_files(pk))
 
 
 if __name__ == "__main__":

{unitlab-2.3.46 → unitlab-2.3.48/src/unitlab.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: unitlab
-Version: 2.3.46
+Version: 2.3.48
 Home-page: https://github.com/teamunitlab/unitlab-sdk
 Author: Unitlab Inc.
 Author-email: team@unitlab.ai