datamule 1.5.8__py3-none-any.whl → 1.5.9__py3-none-any.whl

datamule/portfolio.py

@@ -127,7 +127,7 @@ class Portfolio:
             self.accession_numbers = new_accession_numbers
 
     def download_submissions(self, cik=None, ticker=None, submission_type=None, filing_date=None, provider=None,document_type=[],
-                             requests_per_second=5,keep_filtered_metadata=False,standardize_metadata=True, **kwargs):
+                             requests_per_second=5,keep_filtered_metadata=False,standardize_metadata=True,skip_existing=True, **kwargs):
         if provider is None:
             config = Config()
             provider = config.get_default_source()
@@ -135,6 +135,11 @@ class Portfolio:
         # Process CIK and metadata filters
         cik = _process_cik_and_metadata_filters(cik, ticker, **kwargs)
 
+        accession_numbers = self.accession_numbers if hasattr(self, 'accession_numbers') else None
+        skip_accession_numbers = []
+        if skip_existing:
+            skip_accession_numbers = [sub.accession for sub in self]
+
         if provider == 'datamule':
 
             seclibrary_download(
@@ -143,10 +148,11 @@ class Portfolio:
                 api_key=self.api_key,
                 submission_type=submission_type,
                 filing_date=filing_date,
-                accession_numbers=self.accession_numbers if hasattr(self, 'accession_numbers') else None,
+                accession_numbers=accession_numbers,
                 keep_document_types=document_type,
                 keep_filtered_metadata=keep_filtered_metadata,
                 standardize_metadata=standardize_metadata,
+                skip_accession_numbers=skip_accession_numbers
             )
         else:
             sec_download(
@@ -155,10 +161,11 @@ class Portfolio:
                 submission_type=submission_type,
                 filing_date=filing_date,
                 requests_per_second=requests_per_second, 
-                accession_numbers=self.accession_numbers if hasattr(self, 'accession_numbers') else None,
+                accession_numbers=accession_numbers,
                 keep_document_types=document_type,
                 keep_filtered_metadata=keep_filtered_metadata,
                 standardize_metadata=standardize_metadata,
+                skip_accession_numbers=skip_accession_numbers
             )
 
         self.submissions_loaded = False

datamule/sec/submissions/downloader.py

@@ -5,7 +5,8 @@ from tqdm import tqdm
 
 def download(cik=None, submission_type=None, filing_date=None, location=None, name=None, 
              requests_per_second=5, output_dir="filings", accession_numbers=None, 
-             quiet=False, keep_document_types=[],keep_filtered_metadata=False,standardize_metadata=True):
+             quiet=False, keep_document_types=[],keep_filtered_metadata=False,standardize_metadata=True,
+             skip_accession_numbers=[]):
     # Make sure output directory exists
     os.makedirs(output_dir, exist_ok=True)
 
@@ -29,5 +30,6 @@ def download(cik=None, submission_type=None, filing_date=None, location=None, na
         requests_per_second=requests_per_second,
         document_callback=callback_wrapper,
         accession_numbers=accession_numbers,
+        skip_accession_numbers=skip_accession_numbers,
         quiet=quiet
     )

datamule/sec/submissions/streamer.py

@@ -21,7 +21,7 @@ def fix_filing_url(url):
     return url
 
 class Streamer(EFTSQuery):
-    def __init__(self, requests_per_second=5.0, document_callback=None, accession_numbers=None, quiet=False):
+    def __init__(self, requests_per_second=5.0, document_callback=None, accession_numbers=None,skip_accession_numbers=None, quiet=False):
         super().__init__(requests_per_second=requests_per_second, quiet=quiet)
         self.document_callback = document_callback
         self.document_queue = asyncio.Queue()
@@ -32,6 +32,7 @@ class Streamer(EFTSQuery):
         self.documents_processed = 0
         self.total_documents = 0
         self.accession_numbers = accession_numbers
+        self.skip_accession_numbers = skip_accession_numbers
         self.skipped_documents = 0
         
     async def _fetch_worker(self):
@@ -81,6 +82,9 @@ class Streamer(EFTSQuery):
             if self.accession_numbers is not None and accno_w_dash not in self.accession_numbers:
                 return None, None, None
             
+            if self.skip_accession_numbers is not None and accno_w_dash in self.skip_accession_numbers:
+                return None, None, None
+            
             # Construct the URL
             url = f"https://www.sec.gov/Archives/edgar/data/{cik}/{accno_no_dash}/{accno_w_dash}.txt"
             url = fix_filing_url(url)
@@ -218,7 +222,7 @@ class Streamer(EFTSQuery):
         return results
 
 def stream(cik=None, submission_type=None, filing_date=None, location=None, 
-           requests_per_second=5.0, document_callback=None, accession_numbers=None,
+           requests_per_second=5.0, document_callback=None, accession_numbers=None,skip_accession_numbers=[],
            quiet=False, name=None):
     """
     Stream EFTS results and download documents into memory.
@@ -257,6 +261,7 @@ def stream(cik=None, submission_type=None, filing_date=None, location=None,
             requests_per_second=requests_per_second, 
             document_callback=document_callback,
             accession_numbers=accession_numbers,
+            skip_accession_numbers=skip_accession_numbers,
             quiet=quiet
         )
         return await streamer.stream(cik, submission_type, filing_date, location, name)

datamule/sec/xbrl/filter_xbrl.py

@@ -5,8 +5,6 @@ from ..utils import headers
 def fetch_frame(taxonomy, concept, unit, period):
     url = f"https://data.sec.gov/api/xbrl/frames/{taxonomy}/{concept}/{unit}/{period}.json"
     response = requests.get(url, headers=headers)
-    print(url)
-    print(response)
     return response.json()
 
 

datamule/seclibrary/downloader.py

@@ -14,7 +14,6 @@ from queue import Queue, Empty
 from threading import Thread
 from .query import query
 from os import cpu_count
-from ..submission import Submission
 from secsgml import write_sgml_file_to_tar
 
 
@@ -235,7 +234,8 @@ class Downloader:
             processor.stop_workers()
             decompression_pool.shutdown()
 
-    def download(self, submission_type=None, cik=None, filing_date=None, output_dir="downloads", accession_numbers=None, keep_document_types=[], keep_filtered_metadata=False, standardize_metadata=True):
+    def download(self, submission_type=None, cik=None, filing_date=None, output_dir="downloads", accession_numbers=None, keep_document_types=[], keep_filtered_metadata=False, standardize_metadata=True,
+                 skip_accession_numbers=[]):
         """
         Query SEC filings and download/process them.
         
@@ -259,10 +259,18 @@ class Downloader:
             filing_date=filing_date,
             api_key=self.api_key
         )
+
+
         # After querying but before generating URLs
         if accession_numbers:
+            accession_numbers = [str(int(item.replace('-',''))) for item in accession_numbers]
             filings = [filing for filing in filings if filing['accession_number'] in accession_numbers]
         
+
+        if skip_accession_numbers:
+            skip_accession_numbers = [int(item.replace('-','')) for item in skip_accession_numbers]
+            filings = [filing for filing in filings if filing['accession_number'] not in skip_accession_numbers]
+
         # Generate URLs from the query results
         
         print(f"Generating URLs for {len(filings)} filings...")
@@ -355,7 +363,8 @@ class Downloader:
         print(f"Processing speed: {len(urls)/elapsed_time:.2f} files/second")
 
 
-def download(submission_type=None, cik=None, filing_date=None, api_key=None, output_dir="downloads", accession_numbers=None, keep_document_types=[],keep_filtered_metadata=False,standardize_metadata=True):
+def download(submission_type=None, cik=None, filing_date=None, api_key=None, output_dir="downloads", accession_numbers=None, keep_document_types=[],keep_filtered_metadata=False,standardize_metadata=True,
+             skip_accession_numbers=[]):
     """
     Query SEC filings and download/process them.
     
@@ -383,28 +392,6 @@ def download(submission_type=None, cik=None, filing_date=None, api_key=None, out
         accession_numbers=accession_numbers,
         keep_document_types=keep_document_types,
         keep_filtered_metadata=keep_filtered_metadata,
-        standardize_metadata=standardize_metadata
+        standardize_metadata=standardize_metadata,
+        skip_accession_numbers=skip_accession_numbers
     )
-
-def download_files_using_filename(filenames, api_key=None, output_dir="downloads", keep_document_types=[], keep_filtered_metadata=False, standardize_metadata=True):
-    """
-    Download and process SEC filings using specific filenames.
-    
-    Parameters:
-    - filenames: List of specific filenames to download (e.g., ['000091205797006494.sgml', '000100704297000007.sgml.zst'])
-    - api_key: API key for datamule service (optional if DATAMULE_API_KEY env var is set)
-    - output_dir: Directory to save downloaded files
-    - keep_document_types: List of document types to keep (e.g., ['10-K', 'EX-10.1'])
-    - keep_filtered_metadata: Whether to keep metadata for filtered documents
-    - standardize_metadata: Whether to standardize metadata format
-    """
-    downloader = Downloader(api_key=api_key)
-    downloader.QUEUE_SIZE = 1
-    downloader.MAX_CONCURRENT_DOWNLOADS = 1
-    downloader.download_files_using_filename(
-        filenames=filenames,
-        output_dir=output_dir,
-        keep_document_types=keep_document_types,
-        keep_filtered_metadata=keep_filtered_metadata,
-        standardize_metadata=standardize_metadata
-    )

{datamule-1.5.8.dist-info → datamule-1.5.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 1.5.8
+Version: 1.5.9
 Summary: Work with SEC submissions at scale.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

{datamule-1.5.8.dist-info → datamule-1.5.9.dist-info}/RECORD

@@ -3,7 +3,7 @@ datamule/config.py,sha256=Y--CVv7JcgrjJkMOSLrvm2S8B9ost6RMSkGviP-MKtg,883
 datamule/helper.py,sha256=KqhAmTMdvATEh3I-O4xLcAcrHB9zXQERBuwzue7zyQw,3674
 datamule/index.py,sha256=Rrcna9FJV-Oh_K6O2IuUEIDmtay_7UZ4l4jgKCi7A7I,2079
 datamule/package_updater.py,sha256=Z9zaa_y0Z5cknpRn8oPea3gg4kquFHfpfhduKKCZ6NU,958
-datamule/portfolio.py,sha256=Ijx4JFRHSzPoGJRdOTv8c90x79M80LlAXUhUncwYZSo,7755
+datamule/portfolio.py,sha256=eF1eDSwIg-CI8ZmZAHRjCGU0UhuPN4ijxPB0YDT4s2o,8023
 datamule/sheet.py,sha256=TvFqK9eAYuVoJ2uWdAlx5EN6vS9lke-aZf7FqtUiDBc,22304
 datamule/submission.py,sha256=6JIi-ayLL-jENVj6Q4IhmrYlAreJI7xBAHP_NYaDB6k,12918
 datamule/data/listed_filer_metadata.csv,sha256=dT9fQ8AC5P1-Udf_UF0ZkdXJ88jNxJb_tuhi5YYL1rc,2426827
@@ -46,21 +46,21 @@ datamule/sec/utils.py,sha256=JUxwijJiqRMnRJNQzVUamyF5h9ZGc7RnO_zsLOIM73g,2079
 datamule/sec/infrastructure/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/sec/infrastructure/submissions_metadata.py,sha256=f1KarzFSryKm0EV8DCDNsBw5Jv0Tx5aljiGUJkk7DRk,18745
 datamule/sec/submissions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamule/sec/submissions/downloader.py,sha256=tDWn8bsK9XabQo2pBGYSiqTw37MmqM8rEma8Ph7zp-o,1391
+datamule/sec/submissions/downloader.py,sha256=zGS0oJJI8tVF_GnVpZm20MymdYxnjrEjQioSVggw7Ck,1486
 datamule/sec/submissions/eftsquery.py,sha256=mSZon8rlW8dxma7M49ZW5V02Fn-ENOdt9TNO6elBrhE,27983
 datamule/sec/submissions/monitor.py,sha256=ll0nfHzG8FI3bA8zVFrfsfZGnbt5qAD4rRZ4LG2SORY,9567
-datamule/sec/submissions/streamer.py,sha256=EXyWNCD9N6mZmvm9lFSCFodF19zSQ8jfIbWPZNp0K5Y,11253
+datamule/sec/submissions/streamer.py,sha256=Qydj40CmWB_wsPv2dibefRohmCokegG2pR7iZ9C3xLQ,11584
 datamule/sec/submissions/textsearch.py,sha256=MKDXEz_VI_0ljl73_aw2lx4MVzJW5uDt8KxjvJBwPwM,5794
 datamule/sec/xbrl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/sec/xbrl/downloadcompanyfacts.py,sha256=rMWRiCF9ci_gNZMJ9MC2c_PGEd-yEthawQ0CtVwWTjM,3323
-datamule/sec/xbrl/filter_xbrl.py,sha256=g9OT4zrNS0tiUJeBIwbCs_zMisOBkpFnMR3tV4Tr39Q,1316
+datamule/sec/xbrl/filter_xbrl.py,sha256=QiSfm7tsJVLIw2PFqGh8D01qsRe_ZB-mbFhr6KcBa8A,1281
 datamule/sec/xbrl/streamcompanyfacts.py,sha256=Qq88PqW5_j1k3Aqrl0KRmKeF54D6Wbb6H5N2tbvKUzM,3307
 datamule/sec/xbrl/xbrlmonitor.py,sha256=TKFVfSyyUUfUgFQw4WxEVs4g8Nh-2C0tygNIRmTqW3Y,5848
 datamule/seclibrary/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/seclibrary/bq.py,sha256=C8sb_rpXTvchprrFLcbRar4Qi0XWW25tnv1YsHSS5o4,18025
-datamule/seclibrary/downloader.py,sha256=wNRURTGb3eqg12Ltt4578L0WcAm7DmCWg0Rm0Om6Z4U,17959
+datamule/seclibrary/downloader.py,sha256=ylv69VF22IVfrdeCkiGr5mVa2GKrPC9zFiDJU1fiBu8,17262
 datamule/seclibrary/query.py,sha256=qGuursTERRbOGfoDcYcpo4oWkW3PCBW6x1Qf1Puiak4,7352
-datamule-1.5.8.dist-info/METADATA,sha256=kfV8_aDjqzk6OZKmJn4GIffpvTW-SYi55O1qSOEnsGQ,501
-datamule-1.5.8.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-datamule-1.5.8.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
-datamule-1.5.8.dist-info/RECORD,,
+datamule-1.5.9.dist-info/METADATA,sha256=DkoMbTIImVjWfEkqwfe7BBqCpkvBC8CFRRF5v7PKyco,501
+datamule-1.5.9.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+datamule-1.5.9.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
+datamule-1.5.9.dist-info/RECORD,,