datamarket 0.9.47__tar.gz → 0.9.49__tar.gz

{datamarket-0.9.47 → datamarket-0.9.49}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.9.47
+Version: 0.9.49
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket
@@ -34,6 +34,7 @@ Provides-Extra: html2text
 Provides-Extra: httpx
 Provides-Extra: json5
 Provides-Extra: lxml
+Provides-Extra: matplotlib
 Provides-Extra: nodriver
 Provides-Extra: openpyxl
 Provides-Extra: pandarallel
@@ -90,6 +91,7 @@ Requires-Dist: jellyfish (>=1.0.0,<2.0.0)
 Requires-Dist: jinja2 (>=3.0.0,<4.0.0)
 Requires-Dist: json5 (>=0.10.0,<0.11.0) ; extra == "json5"
 Requires-Dist: lxml[html-clean] (>=5.0.0,<6.0.0) ; extra == "lxml"
+Requires-Dist: matplotlib (>=3.0.0,<4.0.0) ; extra == "matplotlib"
 Requires-Dist: nodriver (>=0.44,<0.45) ; extra == "nodriver"
 Requires-Dist: numpy (>=2.0.0,<3.0.0)
 Requires-Dist: openpyxl (>=3.0.0,<4.0.0) ; extra == "openpyxl"

{datamarket-0.9.47 → datamarket-0.9.49}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "datamarket"
-version = "0.9.47"
+version = "0.9.49"
 description = "Utilities that integrate advanced scraping knowledge into just one library."
 authors = ["DataMarket <techsupport@datamarket.es>"]
 license = "GPL-3.0-or-later"
@@ -87,6 +87,7 @@ pyrate-limiter = { version = "^3.0.0", optional = true }
 pyproj = { version = "^3.0.0", optional = true }
 sqlparse = { version = "~0.5.0", optional = true }
 rnet = { version = "^2.0.0", optional = true }
+matplotlib = { version = "^3.0.0", optional = true }
 
 [tool.poetry.extras]
 boto3 = ["boto3"]
@@ -135,6 +136,7 @@ pyrate-limiter = ["pyrate-limiter"]
 pyproj = ["pyproj"]
 sqlparse = ["sqlparse"]
 rnet = ["rnet"]
+matplotlib = ["matplotlib"]
 
 # Interface groups
 aws = ["boto3"]

{datamarket-0.9.47 → datamarket-0.9.49}/src/datamarket/interfaces/drive.py

@@ -18,13 +18,9 @@ class DriveInterface:
         if "drive" in config:
             self.config = config["drive"]
 
-            GoogleAuth.DEFAULT_SETTINGS[
-                "client_config_file"
-            ] = f'{self.config["config_path"]}/credentials.json'
+            GoogleAuth.DEFAULT_SETTINGS["client_config_file"] = f"{self.config['config_path']}/credentials.json"
 
-            self.gauth = GoogleAuth(
-                settings_file=f'{self.config["config_path"]}/settings.yaml'
-            )
+            self.gauth = GoogleAuth(settings_file=f"{self.config['config_path']}/settings.yaml")
             self.gauth.LocalWebserverAuth()
 
             self.drive = GoogleDrive(self.gauth)
@@ -48,24 +44,78 @@ class DriveInterface:
                 logger.info(f"deleting old {filename}...")
                 drive_file.Delete(param={"supportsTeamDrives": True})
 
+    def _create_remote_dir_tree(self, base_folder_id, path_parts):
+        """
+        Ensure the nested folders described by path_parts exist under base_folder_id.
+        Returns the folder_id of the deepest folder (or base_folder_id if path_parts is empty).
+        """
+        parent_id = base_folder_id
+        for part in path_parts:
+            part = part.strip()
+            if not part:
+                continue
+
+            query = (
+                f"'{parent_id}' in parents and title = '{part}'"
+                " and mimeType = 'application/vnd.google-apps.folder' and trashed=false"
+            )
+            results = self.drive.ListFile(
+                {
+                    "q": query,
+                    "corpora": "teamDrive",
+                    "teamDriveId": self.team_id,
+                    "includeTeamDriveItems": True,
+                    "supportsTeamDrives": True,
+                }
+            ).GetList()
+
+            if results:
+                parent_id = results[0]["id"]
+            else:
+                folder_metadata = {
+                    "title": part,
+                    "mimeType": "application/vnd.google-apps.folder",
+                    "parents": [
+                        {
+                            "kind": "drive#fileLink",
+                            "teamDriveId": self.team_id,
+                            "id": parent_id,
+                        }
+                    ],
+                }
+                folder = self.drive.CreateFile(folder_metadata)
+                folder.Upload(param={"supportsTeamDrives": True})
+                parent_id = folder["id"]
+
+        return parent_id
+
     def upload_file(self, local_filename, drive_filename, folder_id):
-        self.delete_old_files(drive_filename, folder_id)
+        drive_filename = drive_filename.strip("/")
+        parts = drive_filename.split("/")
+        if len(parts) > 1:
+            *folders, filename = parts
+            target_folder_id = self._create_remote_dir_tree(folder_id, folders)
+        else:
+            filename = parts[0]
+            target_folder_id = folder_id
+
+        self.delete_old_files(filename, target_folder_id)
 
         f = self.drive.CreateFile(
             {
-                "title": drive_filename,
+                "title": filename,
                 "parents": [
                     {
                         "kind": "drive#fileLink",
                         "teamDriveId": self.team_id,
-                        "id": folder_id,
+                        "id": target_folder_id,
                     }
                 ],
             }
         )
         f.SetContentFile(local_filename)
 
-        logger.info(f"uploading {drive_filename} to folder: {folder_id}...")
+        logger.info(f"uploading {drive_filename} to folder: {target_folder_id}...")
         f.Upload(param={"supportsTeamDrives": True})
 
     def validate_file(self, filename, folder_id):