datamule 0.429__cp311-cp311-win_amd64.whl → 0.430__cp311-cp311-win_amd64.whl

datamule/document.py

@@ -17,6 +17,9 @@ class Document:
         self.data = None
         self.content = None
 
+    def _load_content(self):
+        self.content = load_file_content(self.path)
+
     def contains_string(self, pattern):
         """Currently only works for .htm, .html, and .txt files"""
         if self.path.suffix in ['.htm', '.html', '.txt']:

datamule/downloader/premiumdownloader.py

@@ -259,7 +259,8 @@ class PremiumDownloader:
                 keepalive_timeout=60
             )
 
-            async with aiohttp.ClientSession(connector=connector, timeout=aiohttp.ClientTimeout(total=3600)) as session:
+            # timeout should be max 2 hours.
+            async with aiohttp.ClientSession(connector=connector, timeout=aiohttp.ClientTimeout(total=7200)) as session:
                 tasks = [self.download_and_process(session, url, semaphore, decompression_pool, output_dir, processor) for url in urls]
                 await asyncio.gather(*tasks, return_exceptions=True)
 

datamule/parser/document_parsing/helper.py

@@ -62,7 +62,7 @@ def load_file_content(filename):
     elif filename.suffix in ['.html','.htm']:
         return load_html_content(filename)
     else:
-        raise ValueError(f"Unsupported file type: {filename}")
+        raise ValueError(f"Unsupported file type: {filename.suffix}")
 
 def clean_title(title: str) -> str:
     """Clean up section title by removing newlines, periods, and all whitespace, converting to lowercase."""

datamule/portfolio.py

@@ -1,82 +1,95 @@
 from pathlib import Path
 from tqdm import tqdm
-from concurrent.futures import ProcessPoolExecutor
+from concurrent.futures import ThreadPoolExecutor
 from .submission import Submission
 from .downloader.premiumdownloader import PremiumDownloader
 from .downloader.downloader import Downloader
 from .config import Config
+import os
 
 class Portfolio:
-    def create(cls, path):
-        # This method handles the process pool lifecycle
-        with ProcessPoolExecutor() as executor:
-            portfolio = cls(path, executor)
-            return portfolio
-
-    def __init__(self, path, executor=None):
+    def __init__(self, path):
         self.path = Path(path)
-        # check if path exists
+        self.submissions = []
+        self.MAX_WORKERS = os.cpu_count() - 1 
+        
         if self.path.exists():
-            folders = [f for f in self.path.iterdir() if f.is_dir()]
-            print(f"Loading {len(folders)} submissions")
-            
-            if executor is None:
-                # Fall back to sequential loading if no executor
-                self.submissions = [Submission(f) for f in tqdm(folders, desc="Loading submissions")]
-            else:
-                # Use provided executor for parallel loading
-                self.submissions = list(tqdm(
-                    executor.map(Submission, folders),
-                    total=len(folders),
-                    desc="Loading submissions"  
-                ))
-            
-        else:
-            pass
+            self._load_submissions()
+    
+    def _load_submissions(self):
+        folders = [f for f in self.path.iterdir() if f.is_dir()]
+        print(f"Loading {len(folders)} submissions")
+        
+        with ThreadPoolExecutor(max_workers=self.MAX_WORKERS) as executor:
+            self.submissions = list(tqdm(
+                executor.map(Submission, folders),
+                total=len(folders),
+                desc="Loading submissions"
+            ))
+
+    def process_submissions(self, callback):
+        """Process all submissions using a thread pool."""
+        with ThreadPoolExecutor(max_workers=self.MAX_WORKERS) as executor:
+            results = list(tqdm(
+                executor.map(callback, self.submissions),
+                total=len(self.submissions),
+                desc="Processing submissions"
+            ))
+            return results
+
+    def process_documents(self, callback):
+        """Process all documents using a thread pool."""
+        documents = [doc for sub in self.submissions for doc in sub]
+        
+        with ThreadPoolExecutor(max_workers=self.MAX_WORKERS) as executor:
+            results = list(tqdm(
+                executor.map(callback, documents),
+                total=len(documents),
+                desc="Processing documents"
+            ))
+            return results
 
     def download_submissions(self, cik=None, ticker=None, submission_type=None, filing_date=None, provider=None):
         if provider is None:
             config = Config()
             provider = config.get_default_source()
 
-        if provider == 'sec':
-            downloader = Downloader()
-        elif provider == 'datamule':
-            downloader = PremiumDownloader()
-
-        downloader.download_submissions(output_dir=self.path, cik=cik, ticker=ticker, submission_type=submission_type, filing_date=filing_date
-                                        )
+        downloader = PremiumDownloader() if provider == 'datamule' else Downloader()
+        downloader.download_submissions(
+            output_dir=self.path,
+            cik=cik,
+            ticker=ticker,
+            submission_type=submission_type,
+            filing_date=filing_date
+        )
         
         # Reload submissions after download
-        self.__init__(self.path)
-        
+        self._load_submissions()
+
     def __iter__(self):
         return iter(self.submissions)
     
     def document_type(self, document_types):
-        # Convert single document type to list for consistent handling
+        """Filter documents by type(s)."""
         if isinstance(document_types, str):
             document_types = [document_types]
             
         for submission in self.submissions:
             yield from submission.document_type(document_types)
 
-    def contains_string(self, pattern, document_types=None, executor=None):
+    def contains_string(self, pattern, document_types=None):
+        """Search for pattern in documents, with optional type filter."""
         def check_document(document):
             return document if document.contains_string(pattern) else None
         
-        documents = list(self.document_type(document_types) if document_types else (
-            doc for sub in tqdm(self.submissions, desc="Collecting documents") for doc in sub
-        ))
+        # Get documents, filtered by type if specified
+        documents = list(self.document_type(document_types)) if document_types else [
+            doc for sub in self.submissions for doc in sub
+        ]
         
-        if executor:
-            results = list(tqdm(
-                executor.map(check_document, documents),
-                total=len(documents),
-                desc=f"Searching for '{pattern}'"
-            ))
-            yield from (doc for doc in results if doc is not None)
-        else:
-            for document in tqdm(documents, desc=f"Searching for '{pattern}'"):
-                if document.contains_string(pattern):
-                    yield document
+        with ThreadPoolExecutor(max_workers=self.MAX_WORKERS) as executor:
+            results = executor.map(check_document, documents)
+            
+            for doc in tqdm(results, total=len(documents), desc=f"Searching for '{pattern}'"):
+                if doc is not None:
+                    yield doc

{datamule-0.429.dist-info → datamule-0.430.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 0.429
+Version: 0.430
 Summary: Making it easier to use SEC filings.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman
@@ -24,8 +24,8 @@ Requires-Dist: pandas; extra == "dataset-builder"
 Requires-Dist: google-generativeai; extra == "dataset-builder"
 Requires-Dist: psutil; extra == "dataset-builder"
 Provides-Extra: all
-Requires-Dist: google-generativeai; extra == "all"
+Requires-Dist: openai; extra == "all"
 Requires-Dist: pandas; extra == "all"
+Requires-Dist: google-generativeai; extra == "all"
 Requires-Dist: psutil; extra == "all"
-Requires-Dist: openai; extra == "all"
 Requires-Dist: flask; extra == "all"

{datamule-0.429.dist-info → datamule-0.430.dist-info}/RECORD

@@ -1,10 +1,10 @@
 datamule/__init__.py,sha256=ghCMkcNrtQ2dYz9ulnlZ0JSe-aJF0YSVa3da2g0eIWk,2425
 datamule/config.py,sha256=lrXwhbWFAF3eTa6B4OQgvexSYvaFa-EkWofpLn6AKZM,911
-datamule/document.py,sha256=NoJS6Q9f7Z2bZzKkaoxooYHdDwqs-TCw_BkVH3krtIU,5774
+datamule/document.py,sha256=i8F0uPXKsA_PLbNk84phJ6aqHKDaFA_-5z94qH6R5-0,5859
 datamule/helper.py,sha256=8HOjB3Y7svw_zjEY-AY5JKOJ-LrBiuQMPyok3MH6CCg,4716
 datamule/monitor.py,sha256=WVds1HGV_ojYgWmo0b4Dsiv9mzZ85HHnCucH-7XoUw8,9350
 datamule/packageupdater.py,sha256=X73XlXs9bYSPiwtceaSOEq6wSCqKoG8lyhNyuha6aG0,9801
-datamule/portfolio.py,sha256=rXVTtoVoiBgHTuaKUIzul2HY5cZxNeyrTkHwmCnGZMY,3292
+datamule/portfolio.py,sha256=OiFocVGUWKfnxrSORN42GNJhzqG35fybNIaE2S_h49c,3660
 datamule/submission.py,sha256=cfX7fvHQBObff0N1jzikCGTuAUE1bGIyqenLRxch9eg,2865
 datamule/data/company_former_names.csv,sha256=zTBWdV12_JE3aROFOMrFNTHLPW_M4TDruxtl15-XfA0,714528
 datamule/data/company_metadata.csv,sha256=X7uSIwConqC0sz-moIhXIISg6FI7GLGSlvAfDDf8Sd0,3078648
@@ -13,7 +13,7 @@ datamule/data/sec-glossary.csv,sha256=TPjTBVM3kyFd8xHsmihykepvKbuLAAthOfEDjh_H-K
 datamule/data/xbrl_descriptions.csv,sha256=Hg9BOo9zSjR7Khvx0pikILcbmDK_A404dmQtWuESK4s,2631548
 datamule/dataset_builder/dataset_builder.py,sha256=h1JDzLcMKxxMcXcD24EyqjUPp78iLWCovfNLQtpwZi4,10005
 datamule/downloader/downloader.py,sha256=jXt9bTHH0NgV3E54B1xEChNq24FkUvpIX3BvXxVxgVQ,14921
-datamule/downloader/premiumdownloader.py,sha256=jLr9jWDUrOQQXagbpbqc9rc2H_od6tDhyKnmNKABids,14537
+datamule/downloader/premiumdownloader.py,sha256=8th3Oas9l0wc1yTC7TvK0oNNYAF_UKpW3m_TtVc9WTo,14583
 datamule/mulebot/__init__.py,sha256=GM5cTnijSSLO9GXFdsCuz5B1iwGUcxDbpoBQ6zw1Odo,30
 datamule/mulebot/helper.py,sha256=Hzzr2HReHpFe2GfpVU79EXvQFx3oL9UiwkJp--Sd1N4,1114
 datamule/mulebot/mulebot.py,sha256=wN0Tv5fvarXgk1LRCcgPhj8Xgd8uYFn-cMucxTxRsEo,5966
@@ -39,7 +39,7 @@ datamule/parser/document_parsing/basic_13g_parser.py,sha256=H9_MuOgkYVTIGwhj9w_W
 datamule/parser/document_parsing/basic_8k_parser.py,sha256=fzf8q9LOpBMHGWw-sfqUq3pyFZBlw47nLJBQWPhtGGg,2549
 datamule/parser/document_parsing/form_d_parser.py,sha256=NTAfC8W3i2y7aIofXoLlAbY-4F6QVELYfIPIrVErjVY,2105
 datamule/parser/document_parsing/generalized_item_parser.py,sha256=M2bmYivSXe0POyBtDlPMykyyCgG8n1egRpJuZtZTR_g,2694
-datamule/parser/document_parsing/helper.py,sha256=wEhqx70CQ0SyEfmOfSCcNsf2TNTtomZvTbmoOhwG3kk,2607
+datamule/parser/document_parsing/helper.py,sha256=wk4dotrn8Ejbb4OMc7nCxqxK-x_O9ChjM8_Qoh9nOiQ,2614
 datamule/parser/document_parsing/information_table_parser_13fhr.py,sha256=vCIEqxOoBf1lnOKTJAifnPEXihtSPZcwRFW8yWhEZLc,1808
 datamule/parser/document_parsing/insider_trading_parser.py,sha256=IrH5a3qikPjC1GFoYzapfYW_CArbK0FkbJLbWSfsYNQ,7264
 datamule/parser/document_parsing/mappings.py,sha256=dq6EjaxxDHjH-sg62adRwJOf1v736QiLwXavOHs2vy8,5380
@@ -47,8 +47,8 @@ datamule/parser/document_parsing/n_port_p_parser.py,sha256=T6GliMm-TETPsFM-hDKt1
 datamule/parser/document_parsing/sec_parser.py,sha256=YewOdOsi0P25teQuxS5DNEND9ZCyxE2ewK1DoP9mPto,2788
 datamule/parser/document_parsing/sgml_parser.py,sha256=ASpe1SzgPj4qk0VOmmuMiEQeatjcwZzsuO3MvsYCHhc,3410
 datamule/parser/sgml_parsing/sgml_parser_cy.c,sha256=66QwBAmhxkKdhCgCjOkg29umbIgQoK4T5_mmMy3NkkM,841089
-datamule/parser/sgml_parsing/sgml_parser_cy.cp311-win_amd64.pyd,sha256=hxC7wal9q5mM2m0LeL_Zmor4p8OEmvmildMRfNMi9KQ,134144
-datamule-0.429.dist-info/METADATA,sha256=6v54qYPbi034JPyV1c2Qg7124Aq1V19Xy8_ris2YLZc,1037
-datamule-0.429.dist-info/WHEEL,sha256=nkBcd8Ko0v5sEcSagm2-x_RVrb8gBSkTa8VFFZ0Mr1o,101
-datamule-0.429.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
-datamule-0.429.dist-info/RECORD,,
+datamule/parser/sgml_parsing/sgml_parser_cy.cp311-win_amd64.pyd,sha256=ba4PiYLkkt5Zu03EW0rMYSRx7O8OIZl0jhJmQgYnabI,134144
+datamule-0.430.dist-info/METADATA,sha256=Je7_NZ1r2SsMkjhxfVoW4geNldopmZixvucYQE6Y6Pw,1037
+datamule-0.430.dist-info/WHEEL,sha256=aYhuBGe9EmKAtiGSW4PepBJu0RwWKJi5OB2SjRJXvrY,101
+datamule-0.430.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
+datamule-0.430.dist-info/RECORD,,

{datamule-0.429.dist-info → datamule-0.430.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.6.0)
+Generator: setuptools (75.7.0)
 Root-Is-Purelib: false
 Tag: cp311-cp311-win_amd64