datamule 1.1.6__py3-none-any.whl → 1.1.8__py3-none-any.whl

datamule/document.py

@@ -8,31 +8,32 @@ from .mapping_dicts.xml_mapping_dicts import dict_345
 from selectolax.parser import HTMLParser
 
 class Document:
-    def __init__(self, type, filename):
+    def __init__(self, type, content, extension):
+        
         self.type = type
-        self.path = filename
+        extension = extension.lower()
+        self.content = content
+        if extension == '.txt':
+            self.content = self._preprocess_txt_content()
+        elif extension in ['.htm', '.html']:
+            self.content = self._preprocess_html_content()
 
+        self.extension = extension
+        # this will be filled by parsed
         self.data = None
-        self.content = None
-
 
-    def load_content(self,encoding='utf-8'):
-        with open(self.path, 'r',encoding=encoding) as f:
-            self.content = f.read()
-
-    def _load_text_content(self):
-        with open(self.path) as f:
-            return f.read().translate(str.maketrans({
+    #_load_text_content
+    def _preprocess_txt_content(self):
+            return self.content.read().translate(str.maketrans({
                 '\xa0': ' ', '\u2003': ' ',
                 '\u2018': "'", '\u2019': "'",
                 '\u201c': '"', '\u201d': '"'
             }))
 
     # will deprecate this when we add html2dict
-    def _load_html_content(self):
-        with open(self.path,'rb') as f:
-            parser = HTMLParser(f.read(),detect_encoding=True,decode_errors='ignore')
-        
+    def _preprocess_html_content(self):
+        parser = HTMLParser(self.content,detect_encoding=True,decode_errors='ignore')
+    
         # Remove hidden elements first
         hidden_nodes = parser.css('[style*="display: none"], [style*="display:none"], .hidden, .hide, .d-none')
         for node in hidden_nodes:
@@ -83,36 +84,27 @@ class Document:
             '\u201c': '"', '\u201d': '"'
         }))
 
-    def _load_file_content(self):
-        if self.path.suffix =='.txt':
-            self.content = self._load_text_content()
-        elif self.path.suffix in ['.html','.htm']:
-            self.content =  self._load_html_content()
-        else:
-            raise ValueError(f"Unsupported file type: {self.path.suffix}")
-
-
     def contains_string(self, pattern):
-        """Currently only works for .htm, .html, and .txt files"""
-        if self.path.suffix in ['.htm', '.html', '.txt']:
-            if self.content is None:
-                self.content = self._load_file_content(self.path)
+        """Works for select files"""
+        if self.extension in ['.htm', '.html', '.txt','.xml']:
             return bool(re.search(pattern, self.content))
         return False
 
     # Note: this method will be heavily modified in the future
     def parse(self):
+        # check if we have already parsed the content
+        if self.data:
+            return self.data
         mapping_dict = None
 
-        if self.path.suffix == '.xml':
+        if self.extension == '.xml':
             if self.type in ['3', '4', '5']:
                 mapping_dict = dict_345
 
-            self.load_content()
             self.data = xml2dict(content=self.content, mapping_dict=mapping_dict)
+
         # will deprecate this when we add html2dict
-        elif self.path.suffix in ['.htm', '.html','.txt']:
-            self._load_file_content()
+        elif self.extension in ['.htm', '.html','.txt']:
 
             if self.type == '10-K':
                 mapping_dict = dict_10k
@@ -133,43 +125,246 @@ class Document:
         if not self.data:
             self.parse()
             
-        if output_filename is None:
-            output_filename = f"{self.path.rsplit('.', 1)[0]}.json"
-            
         with open(output_filename, 'w',encoding='utf-8') as f:
             json.dump(self.data, f, indent=2)
 
-    def write_csv(self, output_filename=None, accession_number=None):
+    def to_tabular(self, accession_number=None):
         self.parse()
 
-        if output_filename is None:
-            output_filename = f"{self.path.rsplit('.', 1)[0]}.csv"
+        if self.type == "INFORMATION TABLE":
+            info_table = self.data['informationTable']['infoTable']
+            if isinstance(info_table, dict):
+                info_table = [info_table]
 
-        with open(output_filename, 'w', newline='') as csvfile:
-            if not self.data:
-                return output_filename
+            flattened = self._flatten_dict(info_table)
 
-            has_document = any('document' in item for item in self.data)
+            # Original field names
+            original_columns = [
+                "nameOfIssuer", "titleOfClass", "cusip", "value", 
+                "shrsOrPrnAmt_sshPrnamt", "shrsOrPrnAmt_sshPrnamtType", 
+                "investmentDiscretion", "votingAuthority_Sole", 
+                "votingAuthority_Shared", "votingAuthority_None", 
+                "reportingOwnerCIK", "putCall", "otherManager", 'figi'
+            ]
+                
+            # Define mapping from original to camelCase field names
+            field_mapping = {
+                "shrsOrPrnAmt_sshPrnamt": "sshPrnamt",
+                "shrsOrPrnAmt_sshPrnamtType": "sshPrnamtType",
+                "votingAuthority_Sole": "votingAuthoritySole",
+                "votingAuthority_Shared": "votingAuthorityShared",
+                "votingAuthority_None": "votingAuthorityNone"
+            }
+                
+            # Create the new expected columns list with mapped field names
+            expected_columns = []
+            for column in original_columns:
+                if column in field_mapping:
+                    expected_columns.append(field_mapping[column])
+                else:
+                    expected_columns.append(column)
+                
+            # Process each item in the flattened data
+            for item in flattened:
+                # Remove newlines from items
+                for key in item:
+                    if isinstance(item[key], str):
+                        item[key] = re.sub(r'\s+', ' ', item[key])
+                    
+                new_item = {}
+                for key, value in item.items():
+                    # Apply the mapping if the key is in our mapping dictionary
+                    if key in field_mapping:
+                        new_item[field_mapping[key]] = value
+                    else:
+                        new_item[key] = value
+                    
+                # Update the original item with the new keys
+                item.clear()
+                item.update(new_item)
+                    
+                # Ensure all expected columns exist
+                for column in expected_columns:
+                    if column not in item:
+                        item[column] = None
+
+                item['accession'] = accession_number
             
-            if has_document and 'document' in self.data:
-                writer = csv.DictWriter(csvfile, ['section', 'text'], quoting=csv.QUOTE_ALL)
-                writer.writeheader()
-                flattened = self._flatten_dict(self.data['document'])
-                for section, text in flattened.items():
-                    writer.writerow({'section': section, 'text': text})
-            else:
-                fieldnames = list(self.data[0].keys())
-                if accession_number:
-                    fieldnames.append('Accession Number')
-                writer = csv.DictWriter(csvfile, fieldnames, quoting=csv.QUOTE_ALL)
-                writer.writeheader()
-                for row in self.data:
-                    if accession_number:
-                        row['Accession Number'] = convert_to_dashed_accession(accession_number)
-                    writer.writerow(row)
+            # Add this block to reorder the items to match the expected order
+            ordered_columns = ["nameOfIssuer", "titleOfClass", "cusip", "value", "sshPrnamt", "sshPrnamtType",
+                            "investmentDiscretion", "votingAuthoritySole", "votingAuthorityShared", "votingAuthorityNone",
+                            "reportingOwnerCIK", "putCall", "otherManager", "figi"]
+            if accession_number is not None:
+                ordered_columns.append("accession")
+                
+            ordered_data = []
+            for item in flattened:
+                ordered_item = {column: item.get(column, None) for column in ordered_columns}
+                ordered_data.append(ordered_item)
+                
+            return ordered_data
+        
+        elif self.type in ["3", "4", "5"]:
+            # Master mapping dictionary - includes all possible fields
+            # The order of this dictionary will determine the output column order
+            master_mapping_dict = {
+                # Flag fields (will be set programmatically)
+                "isDerivative": "isDerivative",
+                "isNonDerivative": "isNonDerivative",
+                
+                # Common fields across all types
+                "securityTitle_value": "securityTitle",
+                "transactionDate_value": "transactionDate",
+                "documentType": "documentType",
+                "transactionCoding_transactionFormType": "documentType",
+                "transactionCoding_transactionCode": "transactionCode",
+                "transactionAmounts_transactionAcquiredDisposedCode_value": "transactionCode",
+                "transactionCoding_equitySwapInvolved": "equitySwapInvolved",
+                "transactionTimeliness_value": "transactionTimeliness",
+                "transactionAmounts_transactionShares_value": "transactionShares",
+                "transactionAmounts_transactionPricePerShare_value": "transactionPricePerShare",
+                "postTransactionAmounts_sharesOwnedFollowingTransaction_value": "sharesOwnedFollowingTransaction",
+                "heldFollowingReport": "sharesOwnedFollowingTransaction",  # Form 3
+                "ownershipNature_directOrIndirectOwnership_value": "ownershipType",
+                "ownershipNature_natureOfOwnership_value": "ownershipType",
+                "deemedExecutionDate": "deemedExecutionDate",
+                "deemedExecutionDate_value": "deemedExecutionDate",
+                
+                # Derivative-specific fields
+                "conversionOrExercisePrice_value": "conversionOrExercisePrice",
+                "exerciseDate_value": "exerciseDate",
+                "expirationDate_value": "expirationDate",
+                "underlyingSecurity_underlyingSecurityTitle_value": "underlyingSecurityTitle",
+                "underlyingSecurity_underlyingSecurityShares_value": "underlyingSecurityShares",
+                "underlyingSecurity_underlyingSecurityValue_value": "underlyingSecurityValue",
+                
+                # Footnote fields
+                "transactionPricePerShareFootnote": "transactionPricePerShareFootnote",
+                "transactionAmounts_transactionPricePerShare_footnote": "transactionPricePerShareFootnote",
+                "transactionCodeFootnote": "transactionCodeFootnote",
+                "transactionAmounts_transactionAcquiredDisposedCode_footnote": "transactionCodeFootnote",
+                "transactionCoding_footnote": "transactionCodeFootnote",
+                "natureOfOwnershipFootnote": "natureOfOwnershipFootnote",
+                "ownershipNature_natureOfOwnership_footnote": "natureOfOwnershipFootnote",
+                "sharesOwnedFollowingTransactionFootnote": "sharesOwnedFollowingTransactionFootnote",
+                "postTransactionAmounts_sharesOwnedFollowingTransaction_footnote": "sharesOwnedFollowingTransactionFootnote",
+                "ownershipTypeFootnote": "ownershipTypeFootnote",
+                "ownershipNature_directOrIndirectOwnership_footnote": "ownershipTypeFootnote",
+                "securityTitleFootnote": "securityTitleFootnote",
+                "securityTitle_footnote": "securityTitleFootnote",
+                "transactionSharesFootnote": "transactionSharesFootnote",
+                "transactionAmounts_transactionShares_footnote": "transactionSharesFootnote",
+                "transactionDateFootnote": "transactionDateFootnote",
+                "transactionDate_footnote": "transactionDateFootnote",
+                "conversionOrExercisePriceFootnote": "conversionOrExercisePriceFootnote",
+                "conversionOrExercisePrice_footnote": "conversionOrExercisePriceFootnote",
+                "exerciseDateFootnote": "exerciseDateFootnote",
+                "exerciseDate_footnote": "exerciseDateFootnote",
+                "expirationDateFootnote": "expirationDateFootnote",
+                "expirationDate_footnote": "expirationDateFootnote",
+                "underlyingSecurityTitleFootnote": "underlyingSecurityTitleFootnote",
+                "underlyingSecurity_underlyingSecurityTitle_footnote": "underlyingSecurityTitleFootnote",
+                "underlyingSecuritySharesFootnote": "underlyingSecuritySharesFootnote",
+                "underlyingSecurity_underlyingSecurityShares_footnote": "underlyingSecuritySharesFootnote",
+                "underlyingSecurityValueFootnote": "underlyingSecurityValueFootnote",
+                "underlyingSecurity_underlyingSecurityValue_footnote": "underlyingSecurityValueFootnote"
+            }
+            
+            # Get the unique target column names in order from the mapping dictionary
+            output_columns = []
+            for _, target_key in master_mapping_dict.items():
+                if target_key not in output_columns:
+                    output_columns.append(target_key)
+            
+            # Process function that handles any table type
+            def process_table(table_data, is_derivative):
+                if isinstance(table_data, dict):
+                    table_data = [table_data]
+                
+                flattened = self._flatten_dict(table_data)
+                
+                # Apply mapping to the flattened data and ensure all expected columns are present
+                mapped_data = []
+                for item in flattened:
+                    mapped_item = {}
+                    # First, apply the mapping
+                    for old_key, value in item.items():
+                        target_key = master_mapping_dict.get(old_key, old_key)
+                        mapped_item[target_key] = value
+                    
+                    # Set the derivative/non-derivative flags
+                    mapped_item["isDerivative"] = 1 if is_derivative else 0
+                    mapped_item["isNonDerivative"] = 0 if is_derivative else 1
+                    
+                    # Create a new ordered dictionary with all columns
+                    ordered_item = {}
+                    for column in output_columns:
+                        ordered_item[column] = mapped_item.get(column, None)
+                    
+                    # Add accession_number if available
+                    if accession_number is not None:
+                        ordered_item['accession_number'] = accession_number
+                    
+                    mapped_data.append(ordered_item)
+                
+                return mapped_data
+            
+            # Results container
+            all_results = []
+            
+            # Process non-derivative transactions if they exist
+            if 'nonDerivativeTable' in self.data['ownershipDocument'] and self.data['ownershipDocument']['nonDerivativeTable'] is not None:
+                if 'nonDerivativeTransaction' in self.data['ownershipDocument']['nonDerivativeTable']:
+                    non_deriv_trans = self.data['ownershipDocument']['nonDerivativeTable']['nonDerivativeTransaction']
+                    non_deriv_results = process_table(non_deriv_trans, is_derivative=False)
+                    all_results.extend(non_deriv_results)
+                
+                # Process non-derivative holdings (for Form 3)
+                if 'nonDerivativeHolding' in self.data['ownershipDocument']['nonDerivativeTable']:
+                    non_deriv_hold = self.data['ownershipDocument']['nonDerivativeTable']['nonDerivativeHolding']
+                    non_deriv_hold_results = process_table(non_deriv_hold, is_derivative=False)
+                    all_results.extend(non_deriv_hold_results)
+            
+            # Process derivative transactions if they exist
+            if 'derivativeTable' in self.data['ownershipDocument'] and self.data['ownershipDocument']['derivativeTable'] is not None:
+                if 'derivativeTransaction' in self.data['ownershipDocument']['derivativeTable']:
+                    deriv_trans = self.data['ownershipDocument']['derivativeTable']['derivativeTransaction']
+                    deriv_results = process_table(deriv_trans, is_derivative=True)
+                    all_results.extend(deriv_results)
+                
+                # Process derivative holdings (for Form 3)
+                if 'derivativeHolding' in self.data['ownershipDocument']['derivativeTable']:
+                    deriv_hold = self.data['ownershipDocument']['derivativeTable']['derivativeHolding']
+                    deriv_hold_results = process_table(deriv_hold, is_derivative=True)
+                    all_results.extend(deriv_hold_results)
 
-        return output_filename
-    
+            # check if any rows not in the mapping dict, raise error if so
+            for item in all_results:
+                for key in item.keys():
+                    if key not in master_mapping_dict.values() and key != 'accession_number':
+                        raise ValueError(f"Key '{key}' not found in mapping dictionary")
+
+            
+            return all_results
+        else:
+            raise ValueError("sorry, rejigging conversion to tabular format")
+        
+    def write_csv(self, output_filename, accession_number=None):
+            
+        data = self.to_tabular(accession_number)
+
+        if not data:
+
+            return
+        
+        fieldnames = data[0].keys()
+        
+        with open(output_filename, 'w', newline='') as csvfile:
+            writer = csv.DictWriter(csvfile,fieldnames=fieldnames, quoting=csv.QUOTE_ALL)
+            writer.writeheader()
+            writer.writerows(data)
+
+        
     def _document_to_section_text(self, document_data, parent_key=''):
         items = []
         
@@ -203,7 +398,7 @@ class Document:
     # we'll modify this for every dict
     def _flatten_dict(self, d, parent_key=''):
         items = {}
-        
+
         if isinstance(d, list):
             return [self._flatten_dict(item) for item in d]
                 
@@ -219,13 +414,12 @@ class Document:
    
    # this will all have to be changed. default will be to flatten everything
     def __iter__(self):
-        if not self.data:
-            self.parse()
+        self.parse()
 
         # Let's remove XML iterable for now
 
         # Handle text-based documents
-        if self.path.suffix in ['.txt', '.htm', '.html']:
+        if self.extension in ['.txt', '.htm', '.html']:
             document_data = self.data
             if not document_data:
                 return iter([])
@@ -235,13 +429,13 @@ class Document:
             section_type = None
             
             if self.type in ['10-K', '10-Q']:
-                mapping_dict = txt_mapping_dicts.dict_10k if self.type == '10-K' else txt_mapping_dicts.dict_10q
+                mapping_dict = dict_10k if self.type == '10-K' else dict_10q
             elif self.type == '8-K':
-                mapping_dict = txt_mapping_dicts.dict_8k
+                mapping_dict = dict_8k
             elif self.type == 'SC 13D':
-                mapping_dict = txt_mapping_dicts.dict_13d
+                mapping_dict = dict_13d
             elif self.type == 'SC 13G':
-                mapping_dict = txt_mapping_dicts.dict_13g
+                mapping_dict = dict_13g
             else:
                 return iter([])
                 

datamule/portfolio.py

@@ -119,7 +119,7 @@ class Portfolio:
             # First query, just set the accession numbers
             self.accession_numbers = new_accession_numbers
 
-    def download_submissions(self, cik=None, ticker=None, submission_type=None, filing_date=None, provider=None, **kwargs):
+    def download_submissions(self, cik=None, ticker=None, submission_type=None, filing_date=None, provider=None,requests_per_second=5, **kwargs):
         if provider is None:
             config = Config()
             provider = config.get_default_source()
@@ -142,7 +142,7 @@ class Portfolio:
                 cik=cik,
                 submission_type=submission_type,
                 filing_date=filing_date,
-                requests_per_second=5, # Revisit this later.
+                requests_per_second=requests_per_second, 
                 accession_numbers=self.accession_numbers if hasattr(self, 'accession_numbers') else None
             )
 
@@ -164,8 +164,6 @@ class Portfolio:
         )
 
         
-        
-        
     def __iter__(self):
         if not self.submissions_loaded:
             self._load_submissions()
@@ -179,4 +177,8 @@ class Portfolio:
             document_types = [document_types]
             
         for submission in self.submissions:
-            yield from submission.document_type(document_types)
+            yield from submission.document_type(document_types)
+
+    def keep(self,document_type):
+        for submission in self.__iter__():
+            submission.keep(document_type)

datamule/sec/submissions/downloader.py

@@ -36,7 +36,8 @@ async def download_callback(hit, content, cik, accno, url, output_dir="filings")
         print(f"Error processing {accno}: {e}")
         return None
 
-def download(cik=None, submission_type=None, filing_date=None, requests_per_second=5, output_dir="filings", accession_numbers=None):
+def download(cik=None, submission_type=None, filing_date=None, location=None, name=None, 
+             requests_per_second=5, output_dir="filings", accession_numbers=None, quiet=False):
     """
     Download SEC EDGAR filings and extract their documents.
     
@@ -44,12 +45,25 @@ def download(cik=None, submission_type=None, filing_date=None, requests_per_seco
     - cik: CIK number(s) to query for
     - submission_type: Filing type(s) to query for (default: 10-K)
     - filing_date: Date or date range to query for
+    - location: Location code to filter by (e.g., 'CA' for California)
+    - name: Company name to search for (alternative to providing CIK)
     - requests_per_second: Rate limit for SEC requests
     - output_dir: Directory to save documents
     - accession_numbers: Optional list of accession numbers to filter by
+    - quiet: Whether to suppress progress output
     
     Returns:
     - List of all document paths processed
+    
+    Examples:
+    # Download filings by CIK
+    download(cik="1318605", submission_type="10-K")
+    
+    # Download filings by company name
+    download(name="Tesla", submission_type="10-K")
+    
+    # Download filings with location filter
+    download(name="Apple", location="CA", submission_type="10-K")
     """
         
     # Make sure output directory exists
@@ -62,9 +76,12 @@ def download(cik=None, submission_type=None, filing_date=None, requests_per_seco
     # Call the stream function with our callback
     return stream(
         cik=cik,
+        name=name,
         submission_type=submission_type,
         filing_date=filing_date,
+        location=location,
         requests_per_second=requests_per_second,
         document_callback=callback_wrapper,
-        accession_numbers=accession_numbers
+        accession_numbers=accession_numbers,
+        quiet=quiet
     )