datamule 1.1.7__py3-none-any.whl → 1.1.8__py3-none-any.whl

datamule/document.py

@@ -11,8 +11,6 @@ class Document:
     def __init__(self, type, content, extension):
         
         self.type = type
-        # we will remove this later #
-        # make sure extension is in lower case
         extension = extension.lower()
         self.content = content
         if extension == '.txt':
@@ -94,6 +92,9 @@ class Document:
 
     # Note: this method will be heavily modified in the future
     def parse(self):
+        # check if we have already parsed the content
+        if self.data:
+            return self.data
         mapping_dict = None
 
         if self.extension == '.xml':
@@ -127,34 +128,243 @@ class Document:
         with open(output_filename, 'w',encoding='utf-8') as f:
             json.dump(self.data, f, indent=2)
 
-    def write_csv(self, output_filename=None, accession_number=None):
+    def to_tabular(self, accession_number=None):
         self.parse()
 
-        with open(output_filename, 'w', newline='') as csvfile:
-            if not self.data:
-                return output_filename
+        if self.type == "INFORMATION TABLE":
+            info_table = self.data['informationTable']['infoTable']
+            if isinstance(info_table, dict):
+                info_table = [info_table]
+
+            flattened = self._flatten_dict(info_table)
 
-            has_document = any('document' in item for item in self.data)
+            # Original field names
+            original_columns = [
+                "nameOfIssuer", "titleOfClass", "cusip", "value", 
+                "shrsOrPrnAmt_sshPrnamt", "shrsOrPrnAmt_sshPrnamtType", 
+                "investmentDiscretion", "votingAuthority_Sole", 
+                "votingAuthority_Shared", "votingAuthority_None", 
+                "reportingOwnerCIK", "putCall", "otherManager", 'figi'
+            ]
+                
+            # Define mapping from original to camelCase field names
+            field_mapping = {
+                "shrsOrPrnAmt_sshPrnamt": "sshPrnamt",
+                "shrsOrPrnAmt_sshPrnamtType": "sshPrnamtType",
+                "votingAuthority_Sole": "votingAuthoritySole",
+                "votingAuthority_Shared": "votingAuthorityShared",
+                "votingAuthority_None": "votingAuthorityNone"
+            }
+                
+            # Create the new expected columns list with mapped field names
+            expected_columns = []
+            for column in original_columns:
+                if column in field_mapping:
+                    expected_columns.append(field_mapping[column])
+                else:
+                    expected_columns.append(column)
+                
+            # Process each item in the flattened data
+            for item in flattened:
+                # Remove newlines from items
+                for key in item:
+                    if isinstance(item[key], str):
+                        item[key] = re.sub(r'\s+', ' ', item[key])
+                    
+                new_item = {}
+                for key, value in item.items():
+                    # Apply the mapping if the key is in our mapping dictionary
+                    if key in field_mapping:
+                        new_item[field_mapping[key]] = value
+                    else:
+                        new_item[key] = value
+                    
+                # Update the original item with the new keys
+                item.clear()
+                item.update(new_item)
+                    
+                # Ensure all expected columns exist
+                for column in expected_columns:
+                    if column not in item:
+                        item[column] = None
+
+                item['accession'] = accession_number
             
-            if has_document and 'document' in self.data:
-                writer = csv.DictWriter(csvfile, ['section', 'text'], quoting=csv.QUOTE_ALL)
-                writer.writeheader()
-                flattened = self._flatten_dict(self.data['document'])
-                for section, text in flattened.items():
-                    writer.writerow({'section': section, 'text': text})
-            else:
-                fieldnames = list(self.data[0].keys())
-                if accession_number:
-                    fieldnames.append('Accession Number')
-                writer = csv.DictWriter(csvfile, fieldnames, quoting=csv.QUOTE_ALL)
-                writer.writeheader()
-                for row in self.data:
-                    if accession_number:
-                        row['Accession Number'] = accession_number
-                    writer.writerow(row)
+            # Add this block to reorder the items to match the expected order
+            ordered_columns = ["nameOfIssuer", "titleOfClass", "cusip", "value", "sshPrnamt", "sshPrnamtType",
+                            "investmentDiscretion", "votingAuthoritySole", "votingAuthorityShared", "votingAuthorityNone",
+                            "reportingOwnerCIK", "putCall", "otherManager", "figi"]
+            if accession_number is not None:
+                ordered_columns.append("accession")
+                
+            ordered_data = []
+            for item in flattened:
+                ordered_item = {column: item.get(column, None) for column in ordered_columns}
+                ordered_data.append(ordered_item)
+                
+            return ordered_data
+        
+        elif self.type in ["3", "4", "5"]:
+            # Master mapping dictionary - includes all possible fields
+            # The order of this dictionary will determine the output column order
+            master_mapping_dict = {
+                # Flag fields (will be set programmatically)
+                "isDerivative": "isDerivative",
+                "isNonDerivative": "isNonDerivative",
+                
+                # Common fields across all types
+                "securityTitle_value": "securityTitle",
+                "transactionDate_value": "transactionDate",
+                "documentType": "documentType",
+                "transactionCoding_transactionFormType": "documentType",
+                "transactionCoding_transactionCode": "transactionCode",
+                "transactionAmounts_transactionAcquiredDisposedCode_value": "transactionCode",
+                "transactionCoding_equitySwapInvolved": "equitySwapInvolved",
+                "transactionTimeliness_value": "transactionTimeliness",
+                "transactionAmounts_transactionShares_value": "transactionShares",
+                "transactionAmounts_transactionPricePerShare_value": "transactionPricePerShare",
+                "postTransactionAmounts_sharesOwnedFollowingTransaction_value": "sharesOwnedFollowingTransaction",
+                "heldFollowingReport": "sharesOwnedFollowingTransaction",  # Form 3
+                "ownershipNature_directOrIndirectOwnership_value": "ownershipType",
+                "ownershipNature_natureOfOwnership_value": "ownershipType",
+                "deemedExecutionDate": "deemedExecutionDate",
+                "deemedExecutionDate_value": "deemedExecutionDate",
+                
+                # Derivative-specific fields
+                "conversionOrExercisePrice_value": "conversionOrExercisePrice",
+                "exerciseDate_value": "exerciseDate",
+                "expirationDate_value": "expirationDate",
+                "underlyingSecurity_underlyingSecurityTitle_value": "underlyingSecurityTitle",
+                "underlyingSecurity_underlyingSecurityShares_value": "underlyingSecurityShares",
+                "underlyingSecurity_underlyingSecurityValue_value": "underlyingSecurityValue",
+                
+                # Footnote fields
+                "transactionPricePerShareFootnote": "transactionPricePerShareFootnote",
+                "transactionAmounts_transactionPricePerShare_footnote": "transactionPricePerShareFootnote",
+                "transactionCodeFootnote": "transactionCodeFootnote",
+                "transactionAmounts_transactionAcquiredDisposedCode_footnote": "transactionCodeFootnote",
+                "transactionCoding_footnote": "transactionCodeFootnote",
+                "natureOfOwnershipFootnote": "natureOfOwnershipFootnote",
+                "ownershipNature_natureOfOwnership_footnote": "natureOfOwnershipFootnote",
+                "sharesOwnedFollowingTransactionFootnote": "sharesOwnedFollowingTransactionFootnote",
+                "postTransactionAmounts_sharesOwnedFollowingTransaction_footnote": "sharesOwnedFollowingTransactionFootnote",
+                "ownershipTypeFootnote": "ownershipTypeFootnote",
+                "ownershipNature_directOrIndirectOwnership_footnote": "ownershipTypeFootnote",
+                "securityTitleFootnote": "securityTitleFootnote",
+                "securityTitle_footnote": "securityTitleFootnote",
+                "transactionSharesFootnote": "transactionSharesFootnote",
+                "transactionAmounts_transactionShares_footnote": "transactionSharesFootnote",
+                "transactionDateFootnote": "transactionDateFootnote",
+                "transactionDate_footnote": "transactionDateFootnote",
+                "conversionOrExercisePriceFootnote": "conversionOrExercisePriceFootnote",
+                "conversionOrExercisePrice_footnote": "conversionOrExercisePriceFootnote",
+                "exerciseDateFootnote": "exerciseDateFootnote",
+                "exerciseDate_footnote": "exerciseDateFootnote",
+                "expirationDateFootnote": "expirationDateFootnote",
+                "expirationDate_footnote": "expirationDateFootnote",
+                "underlyingSecurityTitleFootnote": "underlyingSecurityTitleFootnote",
+                "underlyingSecurity_underlyingSecurityTitle_footnote": "underlyingSecurityTitleFootnote",
+                "underlyingSecuritySharesFootnote": "underlyingSecuritySharesFootnote",
+                "underlyingSecurity_underlyingSecurityShares_footnote": "underlyingSecuritySharesFootnote",
+                "underlyingSecurityValueFootnote": "underlyingSecurityValueFootnote",
+                "underlyingSecurity_underlyingSecurityValue_footnote": "underlyingSecurityValueFootnote"
+            }
+            
+            # Get the unique target column names in order from the mapping dictionary
+            output_columns = []
+            for _, target_key in master_mapping_dict.items():
+                if target_key not in output_columns:
+                    output_columns.append(target_key)
+            
+            # Process function that handles any table type
+            def process_table(table_data, is_derivative):
+                if isinstance(table_data, dict):
+                    table_data = [table_data]
+                
+                flattened = self._flatten_dict(table_data)
+                
+                # Apply mapping to the flattened data and ensure all expected columns are present
+                mapped_data = []
+                for item in flattened:
+                    mapped_item = {}
+                    # First, apply the mapping
+                    for old_key, value in item.items():
+                        target_key = master_mapping_dict.get(old_key, old_key)
+                        mapped_item[target_key] = value
+                    
+                    # Set the derivative/non-derivative flags
+                    mapped_item["isDerivative"] = 1 if is_derivative else 0
+                    mapped_item["isNonDerivative"] = 0 if is_derivative else 1
+                    
+                    # Create a new ordered dictionary with all columns
+                    ordered_item = {}
+                    for column in output_columns:
+                        ordered_item[column] = mapped_item.get(column, None)
+                    
+                    # Add accession_number if available
+                    if accession_number is not None:
+                        ordered_item['accession_number'] = accession_number
+                    
+                    mapped_data.append(ordered_item)
+                
+                return mapped_data
+            
+            # Results container
+            all_results = []
+            
+            # Process non-derivative transactions if they exist
+            if 'nonDerivativeTable' in self.data['ownershipDocument'] and self.data['ownershipDocument']['nonDerivativeTable'] is not None:
+                if 'nonDerivativeTransaction' in self.data['ownershipDocument']['nonDerivativeTable']:
+                    non_deriv_trans = self.data['ownershipDocument']['nonDerivativeTable']['nonDerivativeTransaction']
+                    non_deriv_results = process_table(non_deriv_trans, is_derivative=False)
+                    all_results.extend(non_deriv_results)
+                
+                # Process non-derivative holdings (for Form 3)
+                if 'nonDerivativeHolding' in self.data['ownershipDocument']['nonDerivativeTable']:
+                    non_deriv_hold = self.data['ownershipDocument']['nonDerivativeTable']['nonDerivativeHolding']
+                    non_deriv_hold_results = process_table(non_deriv_hold, is_derivative=False)
+                    all_results.extend(non_deriv_hold_results)
+            
+            # Process derivative transactions if they exist
+            if 'derivativeTable' in self.data['ownershipDocument'] and self.data['ownershipDocument']['derivativeTable'] is not None:
+                if 'derivativeTransaction' in self.data['ownershipDocument']['derivativeTable']:
+                    deriv_trans = self.data['ownershipDocument']['derivativeTable']['derivativeTransaction']
+                    deriv_results = process_table(deriv_trans, is_derivative=True)
+                    all_results.extend(deriv_results)
+                
+                # Process derivative holdings (for Form 3)
+                if 'derivativeHolding' in self.data['ownershipDocument']['derivativeTable']:
+                    deriv_hold = self.data['ownershipDocument']['derivativeTable']['derivativeHolding']
+                    deriv_hold_results = process_table(deriv_hold, is_derivative=True)
+                    all_results.extend(deriv_hold_results)
 
-        return output_filename
-    
+            # check if any rows not in the mapping dict, raise error if so
+            for item in all_results:
+                for key in item.keys():
+                    if key not in master_mapping_dict.values() and key != 'accession_number':
+                        raise ValueError(f"Key '{key}' not found in mapping dictionary")
+
+            
+            return all_results
+        else:
+            raise ValueError("sorry, rejigging conversion to tabular format")
+        
+    def write_csv(self, output_filename, accession_number=None):
+            
+        data = self.to_tabular(accession_number)
+
+        if not data:
+
+            return
+        
+        fieldnames = data[0].keys()
+        
+        with open(output_filename, 'w', newline='') as csvfile:
+            writer = csv.DictWriter(csvfile,fieldnames=fieldnames, quoting=csv.QUOTE_ALL)
+            writer.writeheader()
+            writer.writerows(data)
+
+        
     def _document_to_section_text(self, document_data, parent_key=''):
         items = []
         
@@ -188,7 +398,7 @@ class Document:
     # we'll modify this for every dict
     def _flatten_dict(self, d, parent_key=''):
         items = {}
-        
+
         if isinstance(d, list):
             return [self._flatten_dict(item) for item in d]
                 
@@ -204,8 +414,7 @@ class Document:
    
    # this will all have to be changed. default will be to flatten everything
     def __iter__(self):
-        if not self.data:
-            self.parse()
+        self.parse()
 
         # Let's remove XML iterable for now
 

datamule/portfolio.py

@@ -119,7 +119,7 @@ class Portfolio:
             # First query, just set the accession numbers
             self.accession_numbers = new_accession_numbers
 
-    def download_submissions(self, cik=None, ticker=None, submission_type=None, filing_date=None, provider=None, **kwargs):
+    def download_submissions(self, cik=None, ticker=None, submission_type=None, filing_date=None, provider=None,requests_per_second=5, **kwargs):
         if provider is None:
             config = Config()
             provider = config.get_default_source()
@@ -142,7 +142,7 @@ class Portfolio:
                 cik=cik,
                 submission_type=submission_type,
                 filing_date=filing_date,
-                requests_per_second=5, 
+                requests_per_second=requests_per_second, 
                 accession_numbers=self.accession_numbers if hasattr(self, 'accession_numbers') else None
             )
 
@@ -164,8 +164,6 @@ class Portfolio:
         )
 
         
-        
-        
     def __iter__(self):
         if not self.submissions_loaded:
             self._load_submissions()

datamule/sec/submissions/monitor.py

@@ -20,12 +20,16 @@ async def _process_efts_hits(hits, collected_accession_numbers, data_callback=No
             submission_type = source.get('form')
             ciks = source.get('ciks', [])
             ciks = [str(int(cik)) for cik in ciks]
+
+            filing_date = source.get('file_date')
                 
             # Create standardized filing record
             filing = {
                 'accession_number': accession_number,
                 'submission_type': submission_type,
-                'ciks': ciks
+                'ciks': ciks,
+                'filing_date': filing_date,
+
             }
             
             processed_hits.append(filing)

datamule/seclibrary/bq.py

@@ -0,0 +1,191 @@
+import os
+import requests
+import json
+
+def get_information_table(
+    # Required parameters
+    table_type="INFORMATION_TABLE",
+    
+    # Optional filtering parameters
+    columns=None,
+    name_of_issuer=None,
+    title_of_class=None,
+    cusip=None,
+    value=None,
+    ssh_prnamt=None,
+    ssh_prnamt_type=None,
+    investment_discretion=None,
+    voting_authority_sole=None,
+    voting_authority_shared=None,
+    voting_authority_none=None,
+    reporting_owner_cik=None,
+    put_call=None,
+    other_manager=None,
+    figi=None,
+    accession=None,
+    filing_date=None,
+    
+    # API key handling
+    api_key=None,
+    
+    # Additional options
+    print_cost=True,
+    verbose=False
+):
+    """
+    Query the SEC BigQuery API for 13F-HR information table data.
+    
+    Parameters:
+    -----------
+    table_type : str
+        The table to query (default is "INFORMATION_TABLE")
+    columns : List[str], optional
+        Specific columns to return. If None, all columns are returned.
+    
+    # Filter parameters
+    name_of_issuer, title_of_class, etc. : Various filters that can be:
+        - str: Exact match
+        - List[str]: Match any in list
+        - tuple: (min, max) range for numeric/date fields
+    
+    api_key : str, optional
+        SEC BigQuery API key. If None, looks for DATAMULE_API_KEY env variable.
+    print_cost : bool
+        Whether to print the query cost information
+    verbose : bool
+        Whether to print additional information about the query
+        
+    Returns:
+    --------
+    List[Dict]
+        A list of dictionaries containing the query results
+        
+    Raises:
+    -------
+    ValueError
+        If API key is missing or invalid
+    Exception
+        For API errors or other issues
+    """
+    
+    # 1. Handle API key
+    if api_key is None:
+        api_key = os.getenv('DATAMULE_API_KEY')
+    
+    if not api_key:
+        raise ValueError("No API key found. Please set DATAMULE_API_KEY environment variable or provide api_key parameter")
+    
+    # 2. Build query parameters
+    params = {'table_type': table_type}
+    
+    # Add columns parameter if provided
+    if columns:
+        if isinstance(columns, list):
+            params['columns'] = ','.join(columns)
+        else:
+            params['columns'] = columns
+    
+    # Map Python parameter names to API parameter names
+    param_mapping = {
+        'name_of_issuer': 'nameOfIssuer',
+        'title_of_class': 'titleOfClass',
+        'cusip': 'cusip',
+        'value': 'value',
+        'ssh_prnamt': 'sshPrnamt',
+        'ssh_prnamt_type': 'sshPrnamtType',
+        'investment_discretion': 'investmentDiscretion',
+        'voting_authority_sole': 'votingAuthoritySole',
+        'voting_authority_shared': 'votingAuthorityShared',
+        'voting_authority_none': 'votingAuthorityNone',
+        'reporting_owner_cik': 'reportingOwnerCIK',
+        'put_call': 'putCall',
+        'other_manager': 'otherManager',
+        'figi': 'figi',
+        'accession': 'accession',
+        'filing_date': 'filingDate'
+    }
+    
+    # Process all possible filter parameters
+    for param_name, api_param_name in param_mapping.items():
+        value = locals()[param_name]
+        if value is not None:
+            # Handle different filter types
+            if isinstance(value, list):
+                # List filter
+                params[api_param_name] = f"[{','.join(str(v) for v in value)}]"
+            elif isinstance(value, tuple):
+                # Range filter
+                if len(value) == 2:
+                    min_val, max_val = value
+                    # Handle date range specially
+                    if param_name == 'filing_date':
+                        # Dates need to be in quotes within the parentheses
+                        if min_val is None:
+                            min_val = ''
+                        else:
+                            min_val = f"'{min_val}'"
+                        
+                        if max_val is None:
+                            max_val = ''
+                        else:
+                            max_val = f"'{max_val}'"
+                    
+                    range_str = f"({min_val},{max_val})"
+                    params[api_param_name] = range_str
+                else:
+                    raise ValueError(f"Range filter for {param_name} must be a tuple of (min, max)")
+            else:
+                # Exact match
+                params[api_param_name] = value
+    
+    # 3. Make the API request
+    BASE_URL = "https://sec-bq.jgfriedman99.workers.dev/"
+    
+    headers = {
+        'Authorization': f'Bearer {api_key}',
+        'Accept': 'application/json'
+    }
+    
+    if verbose:
+        print(f"Making request to {BASE_URL} with params: {params}")
+    
+    try:
+        response = requests.get(BASE_URL, params=params, headers=headers)
+        
+        # Check for HTTP errors
+        response.raise_for_status()
+        
+        # Parse response
+        result = response.json()
+        
+        # Check for API-level errors
+        if not result.get('success', False):
+            error_msg = result.get('error', 'Unknown API error')
+            raise Exception(f"API Error: {error_msg}")
+        
+        # Extract metadata for cost reporting
+        metadata = result.get('metadata', {})
+        
+        # 5. Print cost information if requested
+        if print_cost and 'billing' in metadata:
+            billing = metadata['billing']
+            query_info = metadata.get('query_info', {})
+            
+            print("\n=== Query Cost Information ===")
+            print(f"Bytes Processed: {query_info.get('bytes_processed', 0):,} bytes")
+            print(f"Data Processed: {billing.get('tb_processed', 0):.10f} TB")
+            print(f"Cost Rate: ${billing.get('cost_per_tb', 0):.2f}/TB")
+            print(f"Query Cost: ${billing.get('total_charge', 0):.6f}")
+            print(f"Remaining Balance: ${billing.get('remaining_balance', 0):.2f}")
+            print(f"Execution Time: {query_info.get('execution_time_ms', 0)} ms")
+            print(f"Cache Hit: {query_info.get('cache_hit', False)}")
+            print("==============================\n")
+        
+        # 6. Return data
+        return result.get('data', [])
+        
+    except requests.exceptions.RequestException as e:
+        if response.status_code == 401:
+            raise ValueError("Authentication failed: Invalid API key")
+        else:
+            raise Exception(f"Request failed: {str(e)}")

datamule/sheet.py

@@ -1,6 +1,9 @@
 from pathlib import Path
+import csv
+import os
 from .helper import _process_cik_and_metadata_filters, load_package_dataset
 from .sec.xbrl.downloadcompanyfacts import download_company_facts
+from .seclibrary.bq import get_information_table
 
 class Sheet:
     def __init__(self, path):
@@ -26,16 +29,220 @@ class Sheet:
         # Download facts for all CIKs in parallel
         download_company_facts(cik=cik_list, output_dir=self.path)
 
-    def query_345():
-        pass
-    def query_xbrl():
-        pass
+    def get_information_table(
+        self,
+        # Required parameters
+        table_type="INFORMATION_TABLE",
+        
+        # Optional filtering parameters
+        columns=None,
+        name_of_issuer=None,
+        title_of_class=None,
+        cusip=None,
+        value=None,
+        ssh_prnamt=None,
+        ssh_prnamt_type=None,
+        investment_discretion=None,
+        voting_authority_sole=None,
+        voting_authority_shared=None,
+        voting_authority_none=None,
+        reporting_owner_cik=None,
+        put_call=None,
+        other_manager=None,
+        figi=None,
+        accession=None,
+        filing_date=None,
+        
+        # API key handling
+        api_key=None,
+        
+        # Additional options
+        print_cost=True,
+        verbose=False
+    ):
+        """
+        Query the SEC BigQuery API for 13F-HR information table data.
+        
+        Parameters:
+        -----------
+        table_type : str
+            The table to query (default is "INFORMATION_TABLE")
+        columns : List[str], optional
+            Specific columns to return. If None, all columns are returned.
+        
+        # Filter parameters
+        name_of_issuer, title_of_class, etc. : Various filters that can be:
+            - str: Exact match
+            - List[str]: Match any in list
+            - tuple: (min, max) range for numeric/date fields
+        
+        api_key : str, optional
+            SEC BigQuery API key. If None, looks for DATAMULE_API_KEY env variable.
+        print_cost : bool
+            Whether to print the query cost information
+        verbose : bool
+            Whether to print additional information about the query
+            
+        Returns:
+        --------
+        List[Dict]
+            A list of dictionaries containing the query results
+            
+        Raises:
+        -------
+        ValueError
+            If API key is missing or invalid
+        Exception
+            For API errors or other issues
+        """
+
+        return get_information_table(
+            table_type=table_type,
+            columns=columns,
+            name_of_issuer=name_of_issuer,
+            title_of_class=title_of_class,
+            cusip=cusip,
+            value=value,
+            ssh_prnamt=ssh_prnamt,
+            ssh_prnamt_type=ssh_prnamt_type,
+            investment_discretion=investment_discretion,
+            voting_authority_sole=voting_authority_sole,
+            voting_authority_shared=voting_authority_shared,
+            voting_authority_none=voting_authority_none,
+            reporting_owner_cik=reporting_owner_cik,
+            put_call=put_call,
+            other_manager=other_manager,
+            figi=figi,
+            accession=accession,
+            filing_date=filing_date,
+            
+            # API key handling
+            api_key=api_key,
+            
+            # Additional options
+            print_cost=print_cost,
+            verbose=verbose
+        )
 
-    # LIST TUPLE SYNTAX, so e.g. value (0,100) is 0-100, while [0,100] is 0 and 100
-    def get_13fhr(reportingOwnerCIK,nameOfIssuer,titleOfClass,cusip,value,
-                shrsOrPrnAmt_sshPrnamt,shrsOrPrnAmt_sshPrnamtType,investmentDiscretion,otherManager,
-                votingAuthority_Sole,
-                votingAuthority_Shared,
-                votingAuthority_None,
-                filing_date):
-        pass
+    def download_information_table(
+        self,
+        filepath,
+        # Required parameters
+        table_type="INFORMATION_TABLE",
+        
+        # Optional filtering parameters
+        columns=None,
+        name_of_issuer=None,
+        title_of_class=None,
+        cusip=None,
+        value=None,
+        ssh_prnamt=None,
+        ssh_prnamt_type=None,
+        investment_discretion=None,
+        voting_authority_sole=None,
+        voting_authority_shared=None,
+        voting_authority_none=None,
+        reporting_owner_cik=None,
+        put_call=None,
+        other_manager=None,
+        figi=None,
+        accession=None,
+        filing_date=None,
+        
+        # API key handling
+        api_key=None,
+        
+        # Additional options
+        print_cost=True,
+        verbose=False
+    ):
+        """
+        Query the SEC BigQuery API for 13F-HR information table data and save to CSV.
+        
+        Parameters:
+        -----------
+        filepath : str
+            Path where to save the CSV file. If relative, it will be relative to the Sheet's path.
+        
+        table_type : str
+            The table to query (default is "INFORMATION_TABLE")
+        columns : List[str], optional
+            Specific columns to return. If None, all columns are returned.
+        
+        # Filter parameters
+        name_of_issuer, title_of_class, etc. : Various filters that can be:
+            - str: Exact match
+            - List[str]: Match any in list
+            - tuple: (min, max) range for numeric/date fields
+        
+        api_key : str, optional
+            SEC BigQuery API key. If None, looks for DATAMULE_API_KEY env variable.
+        print_cost : bool
+            Whether to print the query cost information
+        verbose : bool
+            Whether to print additional information about the query
+            
+        Returns:
+        --------
+        List[Dict]
+            A list of dictionaries containing the query results
+            
+        Raises:
+        -------
+        ValueError
+            If API key is missing or invalid
+        Exception
+            For API errors or other issues
+        """
+        # Get the data from the API
+        data = self.get_information_table(
+            table_type=table_type,
+            columns=columns,
+            name_of_issuer=name_of_issuer,
+            title_of_class=title_of_class,
+            cusip=cusip,
+            value=value,
+            ssh_prnamt=ssh_prnamt,
+            ssh_prnamt_type=ssh_prnamt_type,
+            investment_discretion=investment_discretion,
+            voting_authority_sole=voting_authority_sole,
+            voting_authority_shared=voting_authority_shared,
+            voting_authority_none=voting_authority_none,
+            reporting_owner_cik=reporting_owner_cik,
+            put_call=put_call,
+            other_manager=other_manager,
+            figi=figi,
+            accession=accession,
+            filing_date=filing_date,
+            api_key=api_key,
+            print_cost=print_cost,
+            verbose=verbose
+        )
+        
+        # If no data returned, nothing to save
+        if not data:
+            if verbose:
+                print("No data returned from API. No file was created.")
+            return data
+        
+        # Resolve filepath - if it's not absolute, make it relative to self.path
+        filepath_obj = Path(filepath)
+        if not filepath_obj.is_absolute():
+            filepath_obj = self.path / filepath_obj
+        
+        # Create directory if it doesn't exist
+        os.makedirs(filepath_obj.parent, exist_ok=True)
+        
+        # Get fieldnames from the first record
+        fieldnames = data[0].keys()
+        
+        # Write to CSV
+        with open(filepath_obj, 'w', newline='') as csvfile:
+            writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
+            writer.writeheader()
+            writer.writerows(data)
+            
+        if verbose:
+            print(f"Saved {len(data)} records to {filepath_obj}")
+            
+        return data

datamule/submission.py

@@ -14,6 +14,7 @@ class Submission:
         if sgml_content is not None:
             self.path = None
             self.metadata, raw_documents = parse_sgml_submission_into_memory(sgml_content)
+            self.documents = []
 
             for idx,doc in enumerate(self.metadata['documents']):
                 type = doc.get('type')
@@ -23,7 +24,7 @@ class Submission:
                     continue
                 filename = doc.get('filename')
                 extension = Path(filename).suffix
-                self.documents = [Document(type=type, content=raw_documents[idx], extension=extension)]
+                self.documents.append(Document(type=type, content=raw_documents[idx], extension=extension))
 
 
         if path is not None:

{datamule-1.1.7.dist-info → datamule-1.1.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 1.1.7
+Version: 1.1.8
 Summary: Making it easier to use SEC filings.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

{datamule-1.1.7.dist-info → datamule-1.1.8.dist-info}/RECORD

@@ -1,11 +1,11 @@
 datamule/__init__.py,sha256=l6YlwT5EeRxPlCtO5Jd4I8l266rSRUJyfFe97cRtSCM,991
 datamule/config.py,sha256=Y--CVv7JcgrjJkMOSLrvm2S8B9ost6RMSkGviP-MKtg,883
-datamule/document.py,sha256=7FBmjWJJfdKrbQ4UH4J8It7W5GEWTFFEUfQdODUrYlQ,10160
+datamule/document.py,sha256=qShyVKHQ1nSCNvSfrhAOMVXprOd1br1rFKLy52S9WnE,22007
 datamule/helper.py,sha256=xgOVnea-lUlQ5I-U0vYUp0VeKPNZehNhqjJvegA3lYE,3342
 datamule/index.py,sha256=0txvbzPcvY1GsdxA-wGdLzAByxSeE_1VyyBp9mZEQRM,2292
-datamule/portfolio.py,sha256=ECevaiF8P6v4mJ7W9IM4hRKNF0GGdQzc1SzBWLnG2qQ,7082
-datamule/sheet.py,sha256=FF0JL8BuAZ7Sd_LY_-sCGJuYlhm3sKgj2jlHUGMjeUQ,1406
-datamule/submission.py,sha256=zWCnucjmfTYcr1Hm9Us-TjGLjWAHuRPtIyaVpLNvs4c,4427
+datamule/portfolio.py,sha256=yWt5gYTjV7rJsLiPUmhc6Vmr3lfvfCR5MSpLQ_6Gdp4,7104
+datamule/sheet.py,sha256=QaArtx7LpT7bwyteelJV67C-lK0RjQbGS3ka7ftdi8w,7978
+datamule/submission.py,sha256=LI7Zr60YbE_tU-v2N09k2dGjfztSgplKZACT3eRUkFE,4463
 datamule/mapping_dicts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/mapping_dicts/txt_mapping_dicts.py,sha256=DQPrGYbAPQxomRUtt4iiMGrwuF7BHc_LeFBQuYBzU9o,6311
 datamule/mapping_dicts/xml_mapping_dicts.py,sha256=Z22yDVwKYonUfM5foQP00dVDE8EHhhMKp0CLqVKV5OI,438
@@ -18,7 +18,7 @@ datamule/sec/rss/monitor.py,sha256=6r4EYaSlGu6VYErlj9zXJsIMLVie1cfacSZU-ESfuBI,1
 datamule/sec/submissions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/sec/submissions/downloader.py,sha256=IB08W8-lQD5Bb0LgzrTN4Xi4HsCw24DybRLHqE1AUrU,3290
 datamule/sec/submissions/eftsquery.py,sha256=mSZon8rlW8dxma7M49ZW5V02Fn-ENOdt9TNO6elBrhE,27983
-datamule/sec/submissions/monitor.py,sha256=F24I9yn1k8ggbCJQ-Vk7go_qJHlpkBzVKFYKDs_CWLs,5287
+datamule/sec/submissions/monitor.py,sha256=Im2kgnUehhTgyY2Vq3uk07n4Vkj4PjII_SsRDi8ehAE,5384
 datamule/sec/submissions/streamer.py,sha256=EXyWNCD9N6mZmvm9lFSCFodF19zSQ8jfIbWPZNp0K5Y,11253
 datamule/sec/submissions/textsearch.py,sha256=-a5yIrrxxtaK10IJeywFmXuJmSndYL9VKm4SC4I9JAs,5808
 datamule/sec/xbrl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -27,9 +27,10 @@ datamule/sec/xbrl/filter_xbrl.py,sha256=g9OT4zrNS0tiUJeBIwbCs_zMisOBkpFnMR3tV4Tr
 datamule/sec/xbrl/streamcompanyfacts.py,sha256=WyJIwuy5mNMXWpx_IkhFzDMe9MOfQ-vNkWl_JzBzFmc,3323
 datamule/sec/xbrl/xbrlmonitor.py,sha256=TKFVfSyyUUfUgFQw4WxEVs4g8Nh-2C0tygNIRmTqW3Y,5848
 datamule/seclibrary/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datamule/seclibrary/bq.py,sha256=C6kafFXWtm-MUjf70H1wTtpwv1Rxpcbk-Kfy8fkBPfo,6469
 datamule/seclibrary/downloader.py,sha256=Zb1TxsIz887tO3MJVP66siYVtNus89ti-g9oZ6VywrM,11500
 datamule/seclibrary/query.py,sha256=qGuursTERRbOGfoDcYcpo4oWkW3PCBW6x1Qf1Puiak4,7352
-datamule-1.1.7.dist-info/METADATA,sha256=gIryya087eiyvgFA5S5vf2s_wKDxaV3ZEAJA7-W4kS8,512
-datamule-1.1.7.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-datamule-1.1.7.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
-datamule-1.1.7.dist-info/RECORD,,
+datamule-1.1.8.dist-info/METADATA,sha256=8HRRMz6l928E5tuHXkPi1_Kf-8nfPSjWQnnfReSxdPM,512
+datamule-1.1.8.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+datamule-1.1.8.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
+datamule-1.1.8.dist-info/RECORD,,