localdex 0.1.1a6__py3-none-any.whl → 0.1.1a10__py3-none-any.whl

localdex/download_data.py

@@ -12,6 +12,8 @@ import requests
 from pathlib import Path
 from typing import Dict, List, Any, Optional
 import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+import threading
 
 from .exceptions import DataLoadError
 
@@ -24,12 +26,13 @@ class DataDownloader:
     then converts it to the format expected by LocalDex.
     """
     
-    def __init__(self, output_dir: str = "localdex/data"):
+    def __init__(self, output_dir: str = "localdex/data", max_workers: int = 5):
         """
         Initialize the data downloader.
         
         Args:
             output_dir: Directory to save downloaded data
+            max_workers: Maximum number of concurrent downloads
         """
         self.output_dir = Path(output_dir)
         self.base_url = "https://pokeapi.co/api/v2"
@@ -43,6 +46,14 @@ class DataDownloader:
         
         for directory in [self.pokemon_dir, self.moves_dir, self.abilities_dir, self.items_dir]:
             directory.mkdir(parents=True, exist_ok=True)
+        
+        self.max_workers = max_workers
+        self._session_lock = threading.Lock()
+    
+    def _get_session(self) -> requests.Session:
+        """Get a thread-safe session."""
+        with self._session_lock:
+            return self.session
     
     def download_pokemon_data(self, limit: Optional[int] = None) -> None:
         """
@@ -58,7 +69,8 @@ class DataDownloader:
         if limit:
             url += f"?limit={limit}"
         
-        response = self.session.get(url)
+        session = self._get_session()
+        response = session.get(url)
         response.raise_for_status()
         
         pokemon_list = response.json()["results"]
@@ -66,13 +78,16 @@ class DataDownloader:
         
         print(f"Found {total} Pokemon to download")
         
-        for i, pokemon_info in enumerate(pokemon_list, 1):
+        # Thread-safe counter for progress tracking
+        completed_count = 0
+        lock = threading.Lock()
+        
+        def download_and_save_pokemon(pokemon_info):
+            nonlocal completed_count
             try:
                 pokemon_id = pokemon_info["url"].split("/")[-2]
                 pokemon_name = pokemon_info["name"]
                 
-                print(f"Downloading {pokemon_name} ({i}/{total})")
-                
                 # Download detailed Pokemon data
                 pokemon_data = self._download_pokemon_detail(pokemon_id)
                 
@@ -81,12 +96,23 @@ class DataDownloader:
                 with open(output_file, 'w', encoding='utf-8') as f:
                     json.dump(pokemon_data, f, indent=2, ensure_ascii=False)
                 
-                # Rate limiting
-                time.sleep(0.1)
+                # Update progress
+                with lock:
+                    completed_count += 1
+                    print(f"Downloaded {pokemon_name} ({completed_count}/{total})")
+                
+                return pokemon_name
                 
             except Exception as e:
-                print(f"Error downloading {pokemon_info['name']}: {e}")
-                continue
+                with lock:
+                    completed_count += 1
+                    print(f"Error downloading {pokemon_info['name']}: {e}")
+                return None
+        
+        with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
+            futures = [executor.submit(download_and_save_pokemon, pokemon_info) for pokemon_info in pokemon_list]
+            for future in as_completed(futures):
+                future.result()  # Wait for completion and handle any exceptions
     
     def download_move_data(self, limit: Optional[int] = None) -> None:
         """
@@ -102,7 +128,8 @@ class DataDownloader:
         if limit:
             url += f"?limit={limit}"
         
-        response = self.session.get(url)
+        session = self._get_session()
+        response = session.get(url)
         response.raise_for_status()
         
         move_list = response.json()["results"]
@@ -110,13 +137,16 @@ class DataDownloader:
         
         print(f"Found {total} moves to download")
         
-        for i, move_info in enumerate(move_list, 1):
+        # Thread-safe counter for progress tracking
+        completed_count = 0
+        lock = threading.Lock()
+        
+        def download_and_save_move(move_info):
+            nonlocal completed_count
             try:
                 move_id = move_info["url"].split("/")[-2]
                 move_name = move_info["name"]
                 
-                print(f"Downloading {move_name} ({i}/{total})")
-                
                 # Download detailed move data
                 move_data = self._download_move_detail(move_id)
                 
@@ -125,12 +155,23 @@ class DataDownloader:
                 with open(output_file, 'w', encoding='utf-8') as f:
                     json.dump(move_data, f, indent=2, ensure_ascii=False)
                 
-                # Rate limiting
-                time.sleep(0.1)
+                # Update progress
+                with lock:
+                    completed_count += 1
+                    print(f"Downloaded {move_name} ({completed_count}/{total})")
+                
+                return move_name
                 
             except Exception as e:
-                print(f"Error downloading {move_info['name']}: {e}")
-                continue
+                with lock:
+                    completed_count += 1
+                    print(f"Error downloading {move_info['name']}: {e}")
+                return None
+        
+        with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
+            futures = [executor.submit(download_and_save_move, move_info) for move_info in move_list]
+            for future in as_completed(futures):
+                future.result()  # Wait for completion and handle any exceptions
     
     def download_ability_data(self, limit: Optional[int] = None) -> None:
         """
@@ -146,7 +187,8 @@ class DataDownloader:
         if limit:
             url += f"?limit={limit}"
         
-        response = self.session.get(url)
+        session = self._get_session()
+        response = session.get(url)
         response.raise_for_status()
         
         ability_list = response.json()["results"]
@@ -154,13 +196,16 @@ class DataDownloader:
         
         print(f"Found {total} abilities to download")
         
-        for i, ability_info in enumerate(ability_list, 1):
+        # Thread-safe counter for progress tracking
+        completed_count = 0
+        lock = threading.Lock()
+        
+        def download_and_save_ability(ability_info):
+            nonlocal completed_count
             try:
                 ability_id = ability_info["url"].split("/")[-2]
                 ability_name = ability_info["name"]
                 
-                print(f"Downloading {ability_name} ({i}/{total})")
-                
                 # Download detailed ability data
                 ability_data = self._download_ability_detail(ability_id)
                 
@@ -169,12 +214,23 @@ class DataDownloader:
                 with open(output_file, 'w', encoding='utf-8') as f:
                     json.dump(ability_data, f, indent=2, ensure_ascii=False)
                 
-                # Rate limiting
-                time.sleep(0.1)
+                # Update progress
+                with lock:
+                    completed_count += 1
+                    print(f"Downloaded {ability_name} ({completed_count}/{total})")
+                
+                return ability_name
                 
             except Exception as e:
-                print(f"Error downloading {ability_info['name']}: {e}")
-                continue
+                with lock:
+                    completed_count += 1
+                    print(f"Error downloading {ability_info['name']}: {e}")
+                return None
+        
+        with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
+            futures = [executor.submit(download_and_save_ability, ability_info) for ability_info in ability_list]
+            for future in as_completed(futures):
+                future.result()  # Wait for completion and handle any exceptions
     
     def download_item_data(self, limit: Optional[int] = None) -> None:
         """
@@ -190,7 +246,8 @@ class DataDownloader:
         if limit:
             url += f"?limit={limit}"
         
-        response = self.session.get(url)
+        session = self._get_session()
+        response = session.get(url)
         response.raise_for_status()
         
         item_list = response.json()["results"]
@@ -198,13 +255,16 @@ class DataDownloader:
         
         print(f"Found {total} items to download")
         
-        for i, item_info in enumerate(item_list, 1):
+        # Thread-safe counter for progress tracking
+        completed_count = 0
+        lock = threading.Lock()
+        
+        def download_and_save_item(item_info):
+            nonlocal completed_count
             try:
                 item_id = item_info["url"].split("/")[-2]
                 item_name = item_info["name"]
                 
-                print(f"Downloading {item_name} ({i}/{total})")
-                
                 # Download detailed item data
                 item_data = self._download_item_detail(item_id)
                 
@@ -213,17 +273,29 @@ class DataDownloader:
                 with open(output_file, 'w', encoding='utf-8') as f:
                     json.dump(item_data, f, indent=2, ensure_ascii=False)
                 
-                # Rate limiting
-                time.sleep(0.1)
+                # Update progress
+                with lock:
+                    completed_count += 1
+                    print(f"Downloaded {item_name} ({completed_count}/{total})")
+                
+                return item_name
                 
             except Exception as e:
-                print(f"Error downloading {item_info['name']}: {e}")
-                continue
+                with lock:
+                    completed_count += 1
+                    print(f"Error downloading {item_info['name']}: {e}")
+                return None
+        
+        with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
+            futures = [executor.submit(download_and_save_item, item_info) for item_info in item_list]
+            for future in as_completed(futures):
+                future.result()  # Wait for completion and handle any exceptions
     
     def _download_pokemon_detail(self, pokemon_id: str) -> Dict[str, Any]:
         """Download detailed Pokemon data."""
         url = f"{self.base_url}/pokemon/{pokemon_id}"
-        response = self.session.get(url)
+        session = self._get_session()
+        response = session.get(url)
         response.raise_for_status()
         
         data = response.json()
@@ -263,7 +335,8 @@ class DataDownloader:
     def _download_move_detail(self, move_id: str) -> Dict[str, Any]:
         """Download detailed move data."""
         url = f"{self.base_url}/move/{move_id}"
-        response = self.session.get(url)
+        session = self._get_session()
+        response = session.get(url)
         response.raise_for_status()
         
         data = response.json()
@@ -292,7 +365,8 @@ class DataDownloader:
     def _download_ability_detail(self, ability_id: str) -> Dict[str, Any]:
         """Download detailed ability data."""
         url = f"{self.base_url}/ability/{ability_id}"
-        response = self.session.get(url)
+        session = self._get_session()
+        response = session.get(url)
         response.raise_for_status()
         
         data = response.json()
@@ -315,7 +389,8 @@ class DataDownloader:
     def _download_item_detail(self, item_id: str) -> Dict[str, Any]:
         """Download detailed item data."""
         url = f"{self.base_url}/item/{item_id}"
-        response = self.session.get(url)
+        session = self._get_session()
+        response = session.get(url)
         response.raise_for_status()
         
         data = response.json()
@@ -389,6 +464,7 @@ def main():
     
     parser = argparse.ArgumentParser(description="Download Pokemon data for LocalDex")
     parser.add_argument("--output", default="localdex/data", help="Output directory")
+    parser.add_argument("--max-workers", type=int, default=5, help="Maximum number of concurrent downloads (default: 5)")
     parser.add_argument("--pokemon-limit", type=int, help="Limit number of Pokemon to download (default: all)")
     parser.add_argument("--move-limit", type=int, help="Limit number of moves to download (default: all)")
     parser.add_argument("--ability-limit", type=int, help="Limit number of abilities to download (default: all)")
@@ -400,7 +476,7 @@ def main():
     
     args = parser.parse_args()
     
-    downloader = DataDownloader(args.output)
+    downloader = DataDownloader(args.output, max_workers=args.max_workers)
     
     try:
         if args.pokemon_only:

{localdex-0.1.1a6.dist-info → localdex-0.1.1a10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: localdex
-Version: 0.1.1a6
+Version: 0.1.1a10
 Summary: A local Pokemon data repository/Pokedex with fast offline access
 Home-page: https://github.com/yourusername/localdex
 Author: LocalDex Team

{localdex-0.1.1a6.dist-info → localdex-0.1.1a10.dist-info}/RECORD

@@ -2,7 +2,7 @@ localdex/__init__.py,sha256=rAVSDHi5KVKNiMaKgZZ04_Kl7j2KPCZuWO7OuDU6id8,748
 localdex/cli.py,sha256=79NpeQQYwuDuM6wODUX1AOUppTPULD5T0qgTH5uKd2c,16797
 localdex/core.py,sha256=74ia5z5tQarbWQpbZQPz3L9i3FGEZrCxIy65H90KU6c,20254
 localdex/data_loader.py,sha256=hi9aSTto5Ti-OBGOgrQ-XwD5hmivsUwS1uC4rulhwQI,11366
-localdex/download_data.py,sha256=yOFaS0U5Qq9gFcCzIH-yFA_DGu7Iniq7qRFPy6Ofj1E,15479
+localdex/download_data.py,sha256=ibAHDxL60sV4LVN9isCmf8vvd_aI9IQbyjJpU0FHGUo,18869
 localdex/exceptions.py,sha256=Z02-8Kci6jFDk2nnGdVSHZJMDDWE9vuwuASs4VM3To8,2777
 localdex/data/abilities/adaptability.json,sha256=FGEEZmL80YVcIXHV-h287Wu-UJycM1QEJxiOHK2I4mY,289
 localdex/data/abilities/aerilate.json,sha256=KRZOVH2KGdEQ_3UktFoXoagUOaC8jIPZ6Ti-YGzW44s,301
@@ -4790,8 +4790,8 @@ localdex/models/ability.py,sha256=AQzv3XUHHl4sustMJjPDDjJOjXu2GMLTfcM3-tqQ_1w,30
 localdex/models/item.py,sha256=zXao8F-jBPUGq_YLeGeYeK_dZVI7aZMXtWOPwR3qusY,4677
 localdex/models/move.py,sha256=hfgcWI4ziz5MMvc9ddmkotxzYYdrSUqZZQ72IU5tucs,7629
 localdex/models/pokemon.py,sha256=v5zkxY1NGGR-MczFCZe9fHt6u_BAqAjMiQZlpcLXVGc,5408
-localdex-0.1.1a6.dist-info/METADATA,sha256=IsdXp13Gu_tjAzvAB3RKRmZ_Tj1VEKAiXDJt8H5mPUU,11110
-localdex-0.1.1a6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-localdex-0.1.1a6.dist-info/entry_points.txt,sha256=n5GxSeQo-MRuvrT2wVk7hOzEFFsWf6tkBjkzmGIYJe4,47
-localdex-0.1.1a6.dist-info/top_level.txt,sha256=vtupDMH-IaxVCoEZrmE0QzdTwhaKzngVJbTA1NkR_MY,9
-localdex-0.1.1a6.dist-info/RECORD,,
+localdex-0.1.1a10.dist-info/METADATA,sha256=6vPmzgnk5KrYx0HVTsPCj69yWMM2DeK4-JuQBGcitDc,11111
+localdex-0.1.1a10.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+localdex-0.1.1a10.dist-info/entry_points.txt,sha256=n5GxSeQo-MRuvrT2wVk7hOzEFFsWf6tkBjkzmGIYJe4,47
+localdex-0.1.1a10.dist-info/top_level.txt,sha256=vtupDMH-IaxVCoEZrmE0QzdTwhaKzngVJbTA1NkR_MY,9
+localdex-0.1.1a10.dist-info/RECORD,,