datamule 1.1.8__py3-none-any.whl → 1.2.1__py3-none-any.whl

datamule/seclibrary/bq.py

@@ -3,9 +3,6 @@ import requests
 import json
 
 def get_information_table(
-    # Required parameters
-    table_type="INFORMATION_TABLE",
-    
     # Optional filtering parameters
     columns=None,
     name_of_issuer=None,
@@ -37,8 +34,6 @@ def get_information_table(
     
     Parameters:
     -----------
-    table_type : str
-        The table to query (default is "INFORMATION_TABLE")
     columns : List[str], optional
         Specific columns to return. If None, all columns are returned.
     
@@ -76,7 +71,7 @@ def get_information_table(
         raise ValueError("No API key found. Please set DATAMULE_API_KEY environment variable or provide api_key parameter")
     
     # 2. Build query parameters
-    params = {'table_type': table_type}
+    params = {'table_type': 'INFORMATION_TABLE'}
     
     # Add columns parameter if provided
     if columns:
@@ -138,7 +133,198 @@ def get_information_table(
                 # Exact match
                 params[api_param_name] = value
     
-    # 3. Make the API request
+    # Call common function to make API request
+    return _make_api_request(params, api_key, print_cost, verbose)
+
+def get_345(
+    # Optional filtering parameters
+    columns=None,
+    is_derivative=None,
+    is_non_derivative=None,
+    security_title=None,
+    transaction_date=None,
+    document_type=None,
+    transaction_code=None,
+    equity_swap_involved=None,
+    transaction_timeliness=None,
+    transaction_shares=None,
+    transaction_price_per_share=None,
+    shares_owned_following_transaction=None,
+    ownership_type=None,
+    deemed_execution_date=None,
+    conversion_or_exercise_price=None,
+    exercise_date=None,
+    expiration_date=None,
+    underlying_security_title=None,
+    underlying_security_shares=None,
+    underlying_security_value=None,
+    accession=None,
+    reporting_owner_cik=None,
+    issuer_cik=None,
+    filing_date=None,
+    
+    # API key handling
+    api_key=None,
+    
+    # Additional options
+    print_cost=True,
+    verbose=False
+):
+    """
+    Query the SEC BigQuery API for Form 345 insider transaction data.
+    
+    Parameters:
+    -----------
+    columns : List[str], optional
+        Specific columns to return. If None, all columns are returned.
+    
+    # Filter parameters
+    is_derivative, security_title, etc. : Various filters that can be:
+        - str/bool: Exact match
+        - List[str]: Match any in list
+        - tuple: (min, max) range for numeric/date fields
+        
+    reporting_owner_cik : str or List[str]
+        CIK(s) of the reporting insider(s). This is matched against an array in BigQuery.
+        Any match within the array will return the record.
+        
+    issuer_cik : str or List[str]
+        CIK(s) of the company/companies
+    
+    api_key : str, optional
+        SEC BigQuery API key. If None, looks for DATAMULE_API_KEY env variable.
+    print_cost : bool
+        Whether to print the query cost information
+    verbose : bool
+        Whether to print additional information about the query
+        
+    Returns:
+    --------
+    List[Dict]
+        A list of dictionaries containing the query results
+        
+    Raises:
+    -------
+    ValueError
+        If API key is missing or invalid
+    Exception
+        For API errors or other issues
+    """
+    
+    # 1. Handle API key
+    if api_key is None:
+        api_key = os.getenv('DATAMULE_API_KEY')
+    
+    if not api_key:
+        raise ValueError("No API key found. Please set DATAMULE_API_KEY environment variable or provide api_key parameter")
+    
+    # 2. Build query parameters
+    params = {'table_type': 'FORM_345_TABLE'}
+    
+    # Add columns parameter if provided
+    if columns:
+        if isinstance(columns, list):
+            params['columns'] = ','.join(columns)
+        else:
+            params['columns'] = columns
+    
+    # Map Python parameter names to API parameter names
+    param_mapping = {
+        'is_derivative': 'isDerivative',
+        'is_non_derivative': 'isNonDerivative',
+        'security_title': 'securityTitle',
+        'transaction_date': 'transactionDate',
+        'document_type': 'documentType',
+        'transaction_code': 'transactionCode',
+        'equity_swap_involved': 'equitySwapInvolved',
+        'transaction_timeliness': 'transactionTimeliness',
+        'transaction_shares': 'transactionShares',
+        'transaction_price_per_share': 'transactionPricePerShare',
+        'shares_owned_following_transaction': 'sharesOwnedFollowingTransaction',
+        'ownership_type': 'ownershipType',
+        'deemed_execution_date': 'deemedExecutionDate',
+        'conversion_or_exercise_price': 'conversionOrExercisePrice',
+        'exercise_date': 'exerciseDate',
+        'expiration_date': 'expirationDate',
+        'underlying_security_title': 'underlyingSecurityTitle',
+        'underlying_security_shares': 'underlyingSecurityShares',
+        'underlying_security_value': 'underlyingSecurityValue',
+        'accession': 'accession',
+        'reporting_owner_cik': 'reportingOwnerCIK',
+        'issuer_cik': 'issuerCIK',
+        'filing_date': 'filingDate'
+    }
+    
+    # Process all possible filter parameters
+    date_params = ['transaction_date', 'filing_date', 'deemed_execution_date', 'exercise_date', 'expiration_date']
+    boolean_params = ['is_derivative', 'is_non_derivative']
+    
+    for param_name, api_param_name in param_mapping.items():
+        value = locals()[param_name]
+        if value is not None:
+            # Handle different filter types
+            if isinstance(value, list):
+                # List filter
+                params[api_param_name] = f"[{','.join(str(v) for v in value)}]"
+            elif isinstance(value, tuple):
+                # Range filter
+                if len(value) == 2:
+                    min_val, max_val = value
+                    # Handle date range specially
+                    if param_name in date_params:
+                        # Dates need to be in quotes within the parentheses
+                        if min_val is None:
+                            min_val = ''
+                        else:
+                            min_val = f"'{min_val}'"
+                        
+                        if max_val is None:
+                            max_val = ''
+                        else:
+                            max_val = f"'{max_val}'"
+                    
+                    range_str = f"({min_val},{max_val})"
+                    params[api_param_name] = range_str
+                else:
+                    raise ValueError(f"Range filter for {param_name} must be a tuple of (min, max)")
+            elif param_name in boolean_params:
+                # Boolean values
+                params[api_param_name] = str(value).lower()
+            else:
+                # Exact match
+                params[api_param_name] = value
+    
+    # Call common function to make API request
+    return _make_api_request(params, api_key, print_cost, verbose)
+
+def _make_api_request(params, api_key, print_cost=True, verbose=False):
+    """
+    Common function to make API requests to the SEC BigQuery API.
+    
+    Parameters:
+    -----------
+    params : dict
+        Query parameters
+    api_key : str
+        API key for authentication
+    print_cost : bool
+        Whether to print cost information
+    verbose : bool
+        Whether to print debugging information
+        
+    Returns:
+    --------
+    List[Dict]
+        Data returned from the API
+        
+    Raises:
+    -------
+    ValueError
+        If API key is invalid
+    Exception
+        For other API errors
+    """
+    # Make the API request
     BASE_URL = "https://sec-bq.jgfriedman99.workers.dev/"
     
     headers = {
@@ -166,7 +352,15 @@ def get_information_table(
         # Extract metadata for cost reporting
         metadata = result.get('metadata', {})
         
-        # 5. Print cost information if requested
+        # Process the data to handle array fields
+        data = result.get('data', [])
+        for row in data:
+            # Check if reportingOwnerCIK is an array that needs processing
+            if 'reportingOwnerCIK' in row and isinstance(row['reportingOwnerCIK'], list):
+                # Transform from [{'v': 'value1'}, {'v': 'value2'}] to comma-separated string
+                row['reportingOwnerCIK'] = ','.join([item['v'] for item in row['reportingOwnerCIK'] if 'v' in item])
+        
+        # Print cost information if requested
         if print_cost and 'billing' in metadata:
             billing = metadata['billing']
             query_info = metadata.get('query_info', {})
@@ -181,11 +375,154 @@ def get_information_table(
             print(f"Cache Hit: {query_info.get('cache_hit', False)}")
             print("==============================\n")
         
-        # 6. Return data
-        return result.get('data', [])
+        # Return data
+        return data
         
     except requests.exceptions.RequestException as e:
-        if response.status_code == 401:
+        if hasattr(e, 'response') and e.response is not None and e.response.status_code == 401:
             raise ValueError("Authentication failed: Invalid API key")
         else:
-            raise Exception(f"Request failed: {str(e)}")
+            raise Exception(f"Request failed: {str(e)}")
+        
+def get_proxy_voting_record(
+    # Optional filtering parameters
+    columns=None,
+    meeting_date=None,
+    isin=None,
+    cusip=None,
+    issuer_name=None,
+    vote_description=None,
+    shares_on_loan=None,
+    shares_voted=None,
+    vote_category=None,
+    vote_record=None,
+    vote_source=None,
+    how_voted=None,
+    figi=None,
+    management_recommendation=None,
+    accession=None,
+    reporting_owner_cik=None,
+    filing_date=None,
+    
+    # API key handling
+    api_key=None,
+    
+    # Additional options
+    print_cost=True,
+    verbose=False
+):
+    """
+    Query the SEC BigQuery API for NPX proxy voting record data.
+    
+    Parameters:
+    -----------
+    columns : List[str], optional
+        Specific columns to return. If None, all columns are returned.
+    
+    # Filter parameters
+    meeting_date, isin, cusip, etc. : Various filters that can be:
+        - str: Exact match
+        - List[str]: Match any in list
+        - tuple: (min, max) range for numeric/date fields
+    
+    shares_on_loan, shares_voted : int/float or tuple
+        Numeric values or (min, max) range
+        
+    filing_date : str or tuple
+        Date string in 'YYYY-MM-DD' format or (start_date, end_date) tuple
+    
+    api_key : str, optional
+        SEC BigQuery API key. If None, looks for DATAMULE_API_KEY env variable.
+    print_cost : bool
+        Whether to print the query cost information
+    verbose : bool
+        Whether to print additional information about the query
+        
+    Returns:
+    --------
+    List[Dict]
+        A list of dictionaries containing the query results
+        
+    Raises:
+    -------
+    ValueError
+        If API key is missing or invalid
+    Exception
+        For API errors or other issues
+    """
+    
+    # 1. Handle API key
+    if api_key is None:
+        api_key = os.getenv('DATAMULE_API_KEY')
+    
+    if not api_key:
+        raise ValueError("No API key found. Please set DATAMULE_API_KEY environment variable or provide api_key parameter")
+    
+    # 2. Build query parameters
+    params = {'table_type': 'NPX_VOTING_TABLE'}
+    
+    # Add columns parameter if provided
+    if columns:
+        if isinstance(columns, list):
+            params['columns'] = ','.join(columns)
+        else:
+            params['columns'] = columns
+    
+    # Map Python parameter names to API parameter names
+    param_mapping = {
+        'meeting_date': 'meetingDate',
+        'isin': 'isin',
+        'cusip': 'cusip',
+        'issuer_name': 'issuerName',
+        'vote_description': 'voteDescription',
+        'shares_on_loan': 'sharesOnLoan',
+        'shares_voted': 'sharesVoted',
+        'vote_category': 'voteCategory',
+        'vote_record': 'voteRecord',
+        'vote_source': 'voteSource',
+        'how_voted': 'howVoted',
+        'figi': 'figi',
+        'management_recommendation': 'managementRecommendation',
+        'accession': 'accession',
+        'reporting_owner_cik': 'reportingOwnerCIK',
+        'filing_date': 'filingDate'
+    }
+    
+    # Process all possible filter parameters
+    date_params = ['meeting_date', 'filing_date']
+    numeric_params = ['shares_on_loan', 'shares_voted']
+    
+    for param_name, api_param_name in param_mapping.items():
+        value = locals()[param_name]
+        if value is not None:
+            # Handle different filter types
+            if isinstance(value, list):
+                # List filter
+                params[api_param_name] = f"[{','.join(str(v) for v in value)}]"
+            elif isinstance(value, tuple):
+                # Range filter
+                if len(value) == 2:
+                    min_val, max_val = value
+                    # Handle date range specially
+                    if param_name in date_params:
+                        # Dates need to be in quotes within the parentheses
+                        if min_val is None:
+                            min_val = ''
+                        else:
+                            min_val = f"'{min_val}'"
+                        
+                        if max_val is None:
+                            max_val = ''
+                        else:
+                            max_val = f"'{max_val}'"
+                    
+                    range_str = f"({min_val},{max_val})"
+                    params[api_param_name] = range_str
+                else:
+                    raise ValueError(f"Range filter for {param_name} must be a tuple of (min, max)")
+            else:
+                # Exact match
+                params[api_param_name] = value
+    
+    # Call common function to make API request
+    return _make_api_request(params, api_key, print_cost, verbose)

datamule/seclibrary/downloader.py

@@ -16,17 +16,35 @@ from threading import Thread
 from secsgml import parse_sgml_submission
 from .query import query
 from os import cpu_count
+from ..submission import Submission
 
 class Downloader:
     def __init__(self, api_key=None):
         self.BASE_URL = "https://library.datamule.xyz/original/nc/"
         self.CHUNK_SIZE = 2 * 1024 * 1024
-        self.MAX_CONCURRENT_DOWNLOADS = 250
+        self.MAX_CONCURRENT_DOWNLOADS = 100
         self.MAX_DECOMPRESSION_WORKERS = cpu_count()
         self.MAX_PROCESSING_WORKERS = cpu_count()
         self.QUEUE_SIZE = 10
         if api_key is not None:
             self._api_key = api_key
+        # Create a shared event loop for async operations
+        self.loop = asyncio.new_event_loop()
+        # Create a thread to run the event loop
+        self.loop_thread = Thread(target=self._run_event_loop, daemon=True)
+        self.loop_thread.start()
+        # Create a queue for async tasks
+        self.async_queue = Queue()
+        
+    def _run_event_loop(self):
+        """Run the event loop in a separate thread"""
+        asyncio.set_event_loop(self.loop)
+        self.loop.run_forever()
+    
+    def _run_coroutine(self, coro):
+        """Run a coroutine in the event loop and return its result"""
+        future = asyncio.run_coroutine_threadsafe(coro, self.loop)
+        return future.result()
 
     @property
     def api_key(self):
@@ -55,7 +73,7 @@ class Downloader:
             print(f"Failed to log error to {error_file}: {str(e)}")
 
     class FileProcessor:
-        def __init__(self, output_dir, max_workers, queue_size, pbar, downloader):
+        def __init__(self, output_dir, max_workers, queue_size, pbar, downloader, keep_document_types=None):
             self.processing_queue = Queue(maxsize=queue_size)
             self.should_stop = False
             self.processing_workers = []
@@ -64,6 +82,7 @@ class Downloader:
             self.batch_size = 50
             self.pbar = pbar
             self.downloader = downloader
+            self.keep_document_types = keep_document_types
 
         def start_processing_workers(self):
             for _ in range(self.max_workers):
@@ -75,7 +94,9 @@ class Downloader:
         def _process_file(self, item):
             filename, content = item
             try:
-                parse_sgml_submission(output_dir=self.output_dir, content=content)
+                submission = Submission(sgml_content=content, keep_document_types=self.keep_document_types)
+                # Use the shared event loop to run save_async
+                self.downloader._run_coroutine(submission.save_async(output_dir=self.output_dir))
                 self.pbar.update(1)
             except Exception as e:
                 accession_dir = os.path.join(self.output_dir, filename.split('.')[0])  
@@ -189,11 +210,11 @@ class Downloader:
             except Exception as e:
                 self._log_error(output_dir, filename, str(e))
 
-    async def process_batch(self, urls, output_dir):
+    async def process_batch(self, urls, output_dir, keep_document_types=None):
         os.makedirs(output_dir, exist_ok=True)
         
         with tqdm(total=len(urls), desc="Processing files") as pbar:
-            processor = self.FileProcessor(output_dir, self.MAX_PROCESSING_WORKERS, self.QUEUE_SIZE, pbar, self)
+            processor = self.FileProcessor(output_dir, self.MAX_PROCESSING_WORKERS, self.QUEUE_SIZE, pbar, self, keep_document_types=keep_document_types)
             processor.start_processing_workers()
 
             semaphore = asyncio.Semaphore(self.MAX_CONCURRENT_DOWNLOADS)
@@ -216,7 +237,7 @@ class Downloader:
             processor.stop_workers()
             decompression_pool.shutdown()
 
-    def download(self, submission_type=None, cik=None, filing_date=None, output_dir="downloads", accession_numbers=None):
+    def download(self, submission_type=None, cik=None, filing_date=None, output_dir="downloads", accession_numbers=None, keep_document_types=None):
         """
         Query SEC filings and download/process them.
         
@@ -225,6 +246,8 @@ class Downloader:
         - cik: Company CIK number(s), string, int, or list
         - filing_date: Filing date(s), string, list, or tuple of (start_date, end_date)
         - output_dir: Directory to save downloaded files
+        - accession_numbers: List of specific accession numbers to download
+        - keep_document_types: List of document types to keep (e.g., ['10-K', 'EX-10.1'])
         """
         if self.api_key is None:
             raise ValueError("No API key found. Please set DATAMULE_API_KEY environment variable or provide api_key in constructor")
@@ -262,15 +285,32 @@ class Downloader:
         start_time = time.time()
         
         # Process the batch asynchronously
-        asyncio.run(self.process_batch(urls, output_dir))
+        asyncio.run(self.process_batch(urls, output_dir, keep_document_types=keep_document_types))
         
         # Calculate and display performance metrics
         elapsed_time = time.time() - start_time
         print(f"\nProcessing completed in {elapsed_time:.2f} seconds")
         print(f"Processing speed: {len(urls)/elapsed_time:.2f} files/second")
+    
+    def __del__(self):
+        """Cleanup when the downloader is garbage collected"""
+        if hasattr(self, 'loop') and self.loop.is_running():
+            self.loop.call_soon_threadsafe(self.loop.stop)
 
 
-def download(submission_type=None, cik=None, filing_date=None, api_key=None, output_dir="downloads", accession_numbers=None):
+def download(submission_type=None, cik=None, filing_date=None, api_key=None, output_dir="downloads", accession_numbers=None, keep_document_types=None):
+    """
+    Query SEC filings and download/process them.
+    
+    Parameters:
+    - submission_type: Filing type(s), string or list (e.g., '10-K', ['10-K', '10-Q'])
+    - cik: Company CIK number(s), string, int, or list
+    - filing_date: Filing date(s), string, list, or tuple of (start_date, end_date)
+    - api_key: API key for datamule service (optional if DATAMULE_API_KEY env var is set)
+    - output_dir: Directory to save downloaded files
+    - accession_numbers: List of specific accession numbers to download
+    - keep_document_types: List of document types to keep (e.g., ['10-K', 'EX-10.1'])
+    """
     if accession_numbers:
         accession_numbers = [int(str(x).replace('-', '')) for x in accession_numbers]
     # check if acc no is empty list
@@ -282,5 +322,6 @@ def download(submission_type=None, cik=None, filing_date=None, api_key=None, out
         cik=cik,
         filing_date=filing_date,
         output_dir=output_dir,
-        accession_numbers=accession_numbers
+        accession_numbers=accession_numbers,
+        keep_document_types=keep_document_types
     )