datamule 1.0.9__tar.gz → 1.1.1__tar.gz

{datamule-1.0.9 → datamule-1.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 1.0.9
+Version: 1.1.1
 Summary: Making it easier to use SEC filings.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

{datamule-1.0.9 → datamule-1.1.1}/datamule/sec/infrastructure/submissions_metadata.py

@@ -12,7 +12,6 @@ from ..utils import headers
 
 async def download_sec_file(url, target_path):
     """Download submissions.zip from SEC website with progress bar."""
-
     
     async with aiohttp.ClientSession() as session:
         async with session.get(url, headers=headers) as response:
@@ -53,6 +52,9 @@ def extract_metadata(data):
                 for field in ['street1', 'street2', 'city', 'stateOrCountry', 'zipCode', 'stateOrCountryDescription']:
                     result[f"{addr_type}_{field}"] = addr.get(field)
     
+    # Add start_date field (will be populated later)
+    result['start_date'] = ''
+    
     return result
 
 def extract_earliest_filing_date(data):
@@ -78,8 +80,12 @@ def extract_earliest_filing_date(data):
     return earliest_date
 
 def process_former_names(data, cik, current_name):
-    """Process former names into a list of records."""
+    """
+    Process former names into a list of records.
+    Returns former names records and the earliest company date.
+    """
     former_names_records = []
+    earliest_company_date = None
     
     # Process former names if present
     former_names = data.get('formerNames', [])
@@ -98,6 +104,10 @@ def process_former_names(data, cik, current_name):
                 # Clean up date formats (remove time component)
                 if start_date:
                     start_date = start_date.split('T')[0]
+                    # Track earliest company date across all former names
+                    if earliest_company_date is None or start_date < earliest_company_date:
+                        earliest_company_date = start_date
+                        
                 if end_date:
                     end_date = end_date.split('T')[0]
                     # Track latest end date
@@ -114,10 +124,16 @@ def process_former_names(data, cik, current_name):
                 
                 former_names_records.append(record)
     
+    # Find the earliest filing date for the company if no date found in former names
+    if earliest_company_date is None:
+        earliest_company_date = extract_earliest_filing_date(data)
+        if earliest_company_date and 'T' in earliest_company_date:
+            earliest_company_date = earliest_company_date.split('T')[0]
+    
     # For the current name, if we don't have a start date from former names,
-    # we'll try to find the earliest filing date
+    # we'll use the earliest filing date
     if not latest_end_date:
-        latest_end_date = extract_earliest_filing_date(data)
+        latest_end_date = earliest_company_date
     
     # Add current name record with start date as latest end date
     current_record = {
@@ -129,7 +145,8 @@ def process_former_names(data, cik, current_name):
     
     former_names_records.append(current_record)
     
-    return former_names_records
+    # Return both the records and the earliest company date (for metadata)
+    return former_names_records, earliest_company_date
 
 def write_metadata_to_csv(metadata_list, output_path):
     """Write metadata records to CSV and compress with gzip."""
@@ -145,8 +162,8 @@ def write_metadata_to_csv(metadata_list, output_path):
     for metadata in metadata_list:
         fieldnames.update(metadata.keys())
     
-    # Make sure 'name' and 'cik' come first
-    fieldnames = ['name', 'cik'] + [f for f in sorted(fieldnames) if f not in ['name', 'cik']]
+    # Make sure 'name', 'cik', and 'start_date' come first
+    fieldnames = ['name', 'cik', 'start_date'] + [f for f in sorted(fieldnames) if f not in ['name', 'cik', 'start_date']]
     
     # Write directly to gzipped CSV without using StringIO buffer
     with gzip.open(output_path, 'wt', encoding='utf-8', newline='') as gzfile:
@@ -299,7 +316,11 @@ async def extract_and_process_metadata(output_dir, local_zip_path=None, sec_url=
                                 name = metadata.get('name', '')
                                 
                                 # Process former names with the full json_data
-                                former_names_records = process_former_names(json_data, cik, name)
+                                # Now also returning the earliest company date
+                                former_names_records, earliest_company_date = process_former_names(json_data, cik, name)
+                                
+                                # Add the earliest company date to the metadata
+                                metadata['start_date'] = earliest_company_date if earliest_company_date else ''
                                 
                                 # Check if company is listed (has tickers)
                                 tickers = metadata.get('tickers', [])

{datamule-1.0.9 → datamule-1.1.1}/datamule/sec/submissions/eftsquery.py

@@ -6,13 +6,14 @@ from tqdm import tqdm
 from ..utils import RetryException, PreciseRateLimiter, RateMonitor, headers
 
 class EFTSQuery:
-    def __init__(self, requests_per_second=5.0):
+    def __init__(self, requests_per_second=5.0, quiet=False):
         self.base_url = "https://efts.sec.gov/LATEST/search-index"
         self.headers = headers
         self.limiter = PreciseRateLimiter(requests_per_second)
         self.rate_monitor = RateMonitor()
         self.session = None
         self.pbar = None
+        self.quiet = quiet
         self.max_page_size = 100  # EFTS API limit
         self.fetch_queue = asyncio.Queue()
         self.connection_semaphore = asyncio.Semaphore(5)  # Max 5 concurrent connections
@@ -127,6 +128,8 @@ class EFTSQuery:
         return ", ".join(parts)
 
     async def _fetch_json(self, url):
+        if not self.quiet:
+            print(f"Fetching {url}...")
         async with self.connection_semaphore:
             async with self.limiter:
                 try:
@@ -160,18 +163,21 @@ class EFTSQuery:
                             await callback(hits)
                     self.fetch_queue.task_done()
                 except RetryException as e:
-                    print(f"\nRate limited. Sleeping for {e.retry_after} seconds...")
+                    if not self.quiet:
+                        print(f"\nRate limited. Sleeping for {e.retry_after} seconds...")
                     await asyncio.sleep(e.retry_after)
                     # Put back in queue
                     await self.fetch_queue.put((params, from_val, size_val, callback))
                     self.fetch_queue.task_done()
                 except Exception as e:
-                    print(f"\nError fetching {url}: {str(e)}")
+                    if not self.quiet:
+                        print(f"\nError fetching {url}: {str(e)}")
                     self.fetch_queue.task_done()
             except asyncio.CancelledError:
                 break
             except Exception as e:
-                print(f"\nWorker error: {str(e)}")
+                if not self.quiet:
+                    print(f"\nWorker error: {str(e)}")
                 self.fetch_queue.task_done()
 
     def _split_date_range(self, start_date, end_date, num_splits=4):
@@ -322,12 +328,14 @@ class EFTSQuery:
         
         # Skip if no results
         if total_hits == 0:
-            print(f"Skipping negated forms query - no results returned")
+            if not self.quiet:
+                print(f"Skipping negated forms query - no results returned")
             return
             
-        query_desc = self._get_query_description(params)
-        date_range = f"{start_date} to {end_date}"
-        print(f"Planning: Analyzing negated forms query (depth {depth}): {date_range} [{total_hits:,} hits]")
+        if not self.quiet:
+            query_desc = self._get_query_description(params)
+            date_range = f"{start_date} to {end_date}"
+            print(f"Planning: Analyzing negated forms query (depth {depth}): {date_range} [{total_hits:,} hits]")
         
         # If small enough or at max depth, process directly
         if total_hits < self.max_efts_hits or start_date == end_date:
@@ -350,8 +358,9 @@ class EFTSQuery:
             
         total_hits, data = await self._test_query_size(params)
         
-        query_desc = self._get_query_description(params)
-        print(f"Planning: Analyzing {'  '*depth}query: {query_desc} [{total_hits:,} hits]")
+        if not self.quiet:
+            query_desc = self._get_query_description(params)
+            print(f"Planning: Analyzing {'  '*depth}query: {query_desc} [{total_hits:,} hits]")
         
         # If we're at the maximum recursion depth or hits are under limit, process directly
         if depth >= max_depth or total_hits < self.max_efts_hits:
@@ -396,8 +405,9 @@ class EFTSQuery:
 
     async def _start_query_phase(self, callback):
         """Start the query phase after planning is complete"""
-        print("\n--- Starting query phase ---")
-        self.pbar = tqdm(total=self.total_results_to_fetch, desc="Querying documents [Rate: 0/s | 0 MB/s]")
+        if not self.quiet:
+            print("\n--- Starting query phase ---")
+            self.pbar = tqdm(total=self.total_results_to_fetch, desc="Querying documents [Rate: 0/s | 0 MB/s]")
         
         # Queue all pending page requests
         for params, from_val, size_val, callback in self.pending_page_requests:
@@ -425,18 +435,21 @@ class EFTSQuery:
             self.pbar = None
             
             # First check size
-            print("\n--- Starting query planning phase ---")
-            print("Analyzing request and splitting into manageable chunks...")
+            if not self.quiet:
+                print("\n--- Starting query planning phase ---")
+                print("Analyzing request and splitting into manageable chunks...")
             
             total_hits, data = await self._test_query_size(params)
             
             if total_hits == 0:
-                print("No results found for this query.")
+                if not self.quiet:
+                    print("No results found for this query.")
                 return []
                 
             # Get accurate total from aggregation buckets
             self.true_total_docs = self._get_total_from_buckets(data)
-            print(f"Found {self.true_total_docs:,} total documents to retrieve.")
+            if not self.quiet:
+                print(f"Found {self.true_total_docs:,} total documents to retrieve.")
             
             # Start worker tasks
             workers = [asyncio.create_task(self._fetch_worker()) for _ in range(5)]
@@ -458,7 +471,8 @@ class EFTSQuery:
                     negated_forms.append('-0')  # Keep primary documents constraint
                     
                     remaining_docs = self.true_total_docs - self.processed_doc_count
-                    print(f"Planning: Analyzing remaining primary document forms using negation (~{remaining_docs:,} hits)")
+                    if not self.quiet:
+                        print(f"Planning: Analyzing remaining primary document forms using negation (~{remaining_docs:,} hits)")
                     
                     # Process negated forms query with recursive date splitting
                     start_date = params['startdt']
@@ -466,9 +480,9 @@ class EFTSQuery:
                     await self._process_negated_forms_recursive(
                         params, negated_forms, start_date, end_date, 0, collect_hits
                     )
-                else:
+                elif not self.quiet:
                     print("No additional forms to process with negation - not a primary documents query")
-            else:
+            elif not self.quiet:
                 print("No additional forms to process with negation")
             
             # Start the download phase
@@ -488,15 +502,16 @@ class EFTSQuery:
                 self.pbar.close()
                 self.pbar = None
             
-            print(f"\n--- Query complete: {len(all_hits):,} submissions retrieved ---")
+            if not self.quiet:
+                print(f"\n--- Query complete: {len(all_hits):,} submissions retrieved ---")
             return all_hits
 
-def query_efts(cik=None, submission_type=None, filing_date=None, requests_per_second=5.0, callback=None):
+def query_efts(cik=None, submission_type=None, filing_date=None, requests_per_second=5.0, callback=None, quiet=False):
     """
     Convenience function to run a query without managing the async context.
     """
     async def run_query():
-        query = EFTSQuery(requests_per_second=requests_per_second)
+        query = EFTSQuery(requests_per_second=requests_per_second, quiet=quiet)
         return await query.query(cik, submission_type, filing_date, callback)
     
     return asyncio.run(run_query())

{datamule-1.0.9 → datamule-1.1.1}/datamule/sec/submissions/textsearch.py

@@ -9,8 +9,8 @@ class TextSearchEFTSQuery(EFTSQuery):
     """
     Extended EFTSQuery class that adds text search capabilities.
     """
-    def __init__(self, text_query, requests_per_second=5.0):
-        super().__init__(requests_per_second=requests_per_second)
+    def __init__(self, text_query, requests_per_second=5.0, quiet=False):
+        super().__init__(requests_per_second=requests_per_second, quiet=quiet)
         self.text_query = text_query
         
     def _prepare_params(self, cik=None, submission_type=None, filing_date=None):
@@ -46,7 +46,7 @@ async def extract_accession_numbers(hits):
                 accession_numbers.append(acc_no)
     return accession_numbers
 
-def query(text_query, cik=None, submission_type=None, filing_date=None, requests_per_second=5.0):
+def query(text_query, cik=None, submission_type=None, filing_date=None, requests_per_second=5.0, quiet=False):
     """
     Search SEC filings for text and return the full search results.
     
@@ -66,6 +66,8 @@ def query(text_query, cik=None, submission_type=None, filing_date=None, requests
     requests_per_second : float, optional
         Maximum number of requests per second to make to the SEC API.
         Default is 5.0.
+    quiet : bool, optional
+        If True, suppresses all output (progress bars and prints). Default is False.
         
     Returns:
     --------
@@ -73,12 +75,12 @@ def query(text_query, cik=None, submission_type=None, filing_date=None, requests
         Complete search results with all hit data.
     """
     async def run_query():
-        query = TextSearchEFTSQuery(text_query, requests_per_second=requests_per_second)
+        query = TextSearchEFTSQuery(text_query, requests_per_second=requests_per_second, quiet=quiet)
         return await query.query(cik, submission_type, filing_date)
     
     return asyncio.run(run_query())
 
-def filter_text(text_query, cik=None, submission_type=None, filing_date=None, requests_per_second=5.0):
+def filter_text(text_query, cik=None, submission_type=None, filing_date=None, requests_per_second=5.0, quiet=False):
     """
     Search SEC filings for text and return matching accession numbers.
     
@@ -98,6 +100,8 @@ def filter_text(text_query, cik=None, submission_type=None, filing_date=None, re
     requests_per_second : float, optional
         Maximum number of requests per second to make to the SEC API.
         Default is 5.0.
+    quiet : bool, optional
+        If True, suppresses all output (progress bars and prints). Default is False.
         
     Returns:
     --------
@@ -105,7 +109,7 @@ def filter_text(text_query, cik=None, submission_type=None, filing_date=None, re
         List of accession numbers (as strings) for filings that match the text query.
     """
     async def run_query():
-        query_obj = TextSearchEFTSQuery(text_query, requests_per_second=requests_per_second)
+        query_obj = TextSearchEFTSQuery(text_query, requests_per_second=requests_per_second, quiet=quiet)
         
         # Create a collector for accession numbers
         all_acc_nos = []

{datamule-1.0.9 → datamule-1.1.1}/datamule.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 1.0.9
+Version: 1.1.1
 Summary: Making it easier to use SEC filings.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

{datamule-1.0.9 → datamule-1.1.1}/setup.py

@@ -29,7 +29,7 @@ if not file_path.exists():
 setup(
     name="datamule",
     author="John Friedman",
-    version="1.0.9",
+    version="1.1.1",
     description="Making it easier to use SEC filings.",
     packages=find_packages(include=['datamule', 'datamule.*']),
     url="https://github.com/john-friedman/datamule-python",