datamule 1.1.7__py3-none-any.whl → 1.2.0__py3-none-any.whl

datamule/document.py

@@ -11,8 +11,6 @@ class Document:
     def __init__(self, type, content, extension):
         
         self.type = type
-        # we will remove this later #
-        # make sure extension is in lower case
         extension = extension.lower()
         self.content = content
         if extension == '.txt':
@@ -94,6 +92,9 @@ class Document:
 
     # Note: this method will be heavily modified in the future
     def parse(self):
+        # check if we have already parsed the content
+        if self.data:
+            return self.data
         mapping_dict = None
 
         if self.extension == '.xml':
@@ -127,34 +128,236 @@ class Document:
         with open(output_filename, 'w',encoding='utf-8') as f:
             json.dump(self.data, f, indent=2)
 
-    def write_csv(self, output_filename=None, accession_number=None):
+    def to_tabular(self, accession_number=None):
+        """
+        Convert the document to a tabular format suitable for CSV output.
+        
+        Args:
+            accession_number: Optional accession number to include in the output
+            
+        Returns:
+            list: List of dictionaries, each representing a row in the tabular output
+        """
         self.parse()
+        
+        # Common function to normalize and process dictionaries
+        def process_records(records, mapping_dict, is_derivative=None):
+            """
+            Process records into a standardized tabular format
+            
+            Args:
+                records: List or single dictionary of records to process
+                mapping_dict: Dictionary mapping source keys to target keys
+                is_derivative: Boolean flag for derivative securities (or None if not applicable)
+                
+            Returns:
+                list: Processed records in tabular format
+            """
+            # Convert single dict to list for uniform processing
+            if isinstance(records, dict):
+                records = [records]
+                
+            # Flatten nested dictionaries
+            flattened = self._flatten_dict(records)
+            
+            # Process each record
+            result = []
+            for item in flattened:
+                # Normalize whitespace in all string values
+                for key in item:
+                    if isinstance(item[key], str):
+                        item[key] = re.sub(r'\s+', ' ', item[key])
+                        
+                # Map keys according to the mapping dictionary
+                mapped_item = {}
+                for old_key, value in item.items():
+                    target_key = mapping_dict.get(old_key, old_key)
+                    mapped_item[target_key] = value
+                    
+                # Set derivative flags if applicable
+                if is_derivative is not None:
+                    mapped_item["isDerivative"] = 1 if is_derivative else 0
+                    mapped_item["isNonDerivative"] = 0 if is_derivative else 1
+                    
+                # Ensure all expected columns exist
+                output_columns = list(dict.fromkeys(mapping_dict.values()))
+                ordered_item = {column: mapped_item.get(column, None) for column in output_columns}
+                
+                # Add accession number if provided
+                if accession_number is not None:
+                    ordered_item['accession'] = accession_number
+                    
+                result.append(ordered_item)
+                
+            return result
+        
+        # Handle different document types
+        if self.type == "INFORMATION TABLE":
+            # Information Table mapping dictionary
+            info_table_mapping = {
+                "nameOfIssuer": "nameOfIssuer", 
+                "titleOfClass": "titleOfClass", 
+                "cusip": "cusip", 
+                "value": "value", 
+                "shrsOrPrnAmt_sshPrnamt": "sshPrnamt", 
+                "shrsOrPrnAmt_sshPrnamtType": "sshPrnamtType", 
+                "investmentDiscretion": "investmentDiscretion", 
+                "votingAuthority_Sole": "votingAuthoritySole", 
+                "votingAuthority_Shared": "votingAuthorityShared", 
+                "votingAuthority_None": "votingAuthorityNone", 
+                "reportingOwnerCIK": "reportingOwnerCIK", 
+                "putCall": "putCall", 
+                "otherManager": "otherManager", 
+                "figi": "figi"
+            }
+            
+            # Process the information table
+            info_table = self.data['informationTable']['infoTable']
+            return process_records(info_table, info_table_mapping)
+            
+        elif self.type == "PROXY VOTING RECORD":
+            # Proxy voting record mapping dictionary
+            proxy_mapping = {
+                'meetingDate': 'meetingDate',
+                'isin': 'isin',  
+                'cusip': 'cusip',
+                'issuerName': 'issuerName',
+                'voteDescription': 'voteDescription',
+                'sharesOnLoan': 'sharesOnLoan',
+                'vote_voteRecord_sharesVoted': 'sharesVoted',
+                'voteCategories_voteCategory_categoryType': 'voteCategory', 
+                'vote_voteRecord': 'voteRecord',
+                'sharesVoted': 'sharesVoted', 
+                'voteSource': 'voteSource', 
+                'vote_voteRecord_howVoted': 'howVoted',
+                'figi': 'figi', 
+                'vote_voteRecord_managementRecommendation': 'managementRecommendation'
+            }
+            
+            # Process proxy voting records if they exist
+            all_results = []
+            if 'proxyVoteTable' in self.data and 'proxyTable' in self.data['proxyVoteTable'] and self.data['proxyVoteTable']['proxyTable'] is not None:
+                proxy_records = self.data['proxyVoteTable']['proxyTable']
+                proxy_results = process_records(proxy_records, proxy_mapping)
+                all_results.extend(proxy_results)
+                
+            return all_results
+            
+        elif self.type in ["3", "4", "5"]:
+            # Forms 3, 4, 5 mapping dictionary
+            form_345_mapping = {
+                # Flag fields (will be set programmatically)
+                "isDerivative": "isDerivative",
+                "isNonDerivative": "isNonDerivative",
+                
+                # Common fields across all types
+                "securityTitle_value": "securityTitle",
+                "transactionDate_value": "transactionDate",
+                "documentType": "documentType",
+                "transactionCoding_transactionFormType": "documentType",
+                "transactionCoding_transactionCode": "transactionCode",
+                "transactionAmounts_transactionAcquiredDisposedCode_value": "transactionCode",
+                "transactionCoding_equitySwapInvolved": "equitySwapInvolved",
+                "transactionTimeliness_value": "transactionTimeliness",
+                "transactionAmounts_transactionShares_value": "transactionShares",
+                "transactionAmounts_transactionPricePerShare_value": "transactionPricePerShare",
+                "postTransactionAmounts_sharesOwnedFollowingTransaction_value": "sharesOwnedFollowingTransaction",
+                "heldFollowingReport": "sharesOwnedFollowingTransaction",  # Form 3
+                "ownershipNature_directOrIndirectOwnership_value": "ownershipType",
+                "ownershipNature_natureOfOwnership_value": "ownershipType",
+                "deemedExecutionDate": "deemedExecutionDate",
+                "deemedExecutionDate_value": "deemedExecutionDate",
+                
+                # Derivative-specific fields
+                "conversionOrExercisePrice_value": "conversionOrExercisePrice",
+                "exerciseDate_value": "exerciseDate",
+                "expirationDate_value": "expirationDate",
+                "underlyingSecurity_underlyingSecurityTitle_value": "underlyingSecurityTitle",
+                "underlyingSecurity_underlyingSecurityShares_value": "underlyingSecurityShares",
+                "underlyingSecurity_underlyingSecurityValue_value": "underlyingSecurityValue",
+                
+                # Footnote fields
+                "transactionPricePerShareFootnote": "transactionPricePerShareFootnote",
+                "transactionAmounts_transactionPricePerShare_footnote": "transactionPricePerShareFootnote",
+                "transactionCodeFootnote": "transactionCodeFootnote",
+                "transactionAmounts_transactionAcquiredDisposedCode_footnote": "transactionCodeFootnote",
+                "transactionCoding_footnote": "transactionCodeFootnote",
+                "natureOfOwnershipFootnote": "natureOfOwnershipFootnote",
+                "ownershipNature_natureOfOwnership_footnote": "natureOfOwnershipFootnote",
+                "sharesOwnedFollowingTransactionFootnote": "sharesOwnedFollowingTransactionFootnote",
+                "postTransactionAmounts_sharesOwnedFollowingTransaction_footnote": "sharesOwnedFollowingTransactionFootnote",
+                "ownershipTypeFootnote": "ownershipTypeFootnote",
+                "ownershipNature_directOrIndirectOwnership_footnote": "ownershipTypeFootnote",
+                "securityTitleFootnote": "securityTitleFootnote",
+                "securityTitle_footnote": "securityTitleFootnote",
+                "transactionSharesFootnote": "transactionSharesFootnote",
+                "transactionAmounts_transactionShares_footnote": "transactionSharesFootnote",
+                "transactionDateFootnote": "transactionDateFootnote",
+                "transactionDate_footnote": "transactionDateFootnote",
+                "conversionOrExercisePriceFootnote": "conversionOrExercisePriceFootnote",
+                "conversionOrExercisePrice_footnote": "conversionOrExercisePriceFootnote",
+                "exerciseDateFootnote": "exerciseDateFootnote",
+                "exerciseDate_footnote": "exerciseDateFootnote",
+                "expirationDateFootnote": "expirationDateFootnote",
+                "expirationDate_footnote": "expirationDateFootnote",
+                "underlyingSecurityTitleFootnote": "underlyingSecurityTitleFootnote",
+                "underlyingSecurity_underlyingSecurityTitle_footnote": "underlyingSecurityTitleFootnote",
+                "underlyingSecuritySharesFootnote": "underlyingSecuritySharesFootnote",
+                "underlyingSecurity_underlyingSecurityShares_footnote": "underlyingSecuritySharesFootnote",
+                "underlyingSecurityValueFootnote": "underlyingSecurityValueFootnote",
+                "underlyingSecurity_underlyingSecurityValue_footnote": "underlyingSecurityValueFootnote"
+            }
+            
+            # Results container
+            all_results = []
+            
+            # Process non-derivative transactions if they exist
+            if 'nonDerivativeTable' in self.data['ownershipDocument'] and self.data['ownershipDocument']['nonDerivativeTable'] is not None:
+                if 'nonDerivativeTransaction' in self.data['ownershipDocument']['nonDerivativeTable']:
+                    non_deriv_trans = self.data['ownershipDocument']['nonDerivativeTable']['nonDerivativeTransaction']
+                    non_deriv_results = process_records(non_deriv_trans, form_345_mapping, is_derivative=False)
+                    all_results.extend(non_deriv_results)
+                
+                # Process non-derivative holdings (for Form 3)
+                if 'nonDerivativeHolding' in self.data['ownershipDocument']['nonDerivativeTable']:
+                    non_deriv_hold = self.data['ownershipDocument']['nonDerivativeTable']['nonDerivativeHolding']
+                    non_deriv_hold_results = process_records(non_deriv_hold, form_345_mapping, is_derivative=False)
+                    all_results.extend(non_deriv_hold_results)
+            
+            # Process derivative transactions if they exist
+            if 'derivativeTable' in self.data['ownershipDocument'] and self.data['ownershipDocument']['derivativeTable'] is not None:
+                if 'derivativeTransaction' in self.data['ownershipDocument']['derivativeTable']:
+                    deriv_trans = self.data['ownershipDocument']['derivativeTable']['derivativeTransaction']
+                    deriv_results = process_records(deriv_trans, form_345_mapping, is_derivative=True)
+                    all_results.extend(deriv_results)
+                
+                # Process derivative holdings (for Form 3)
+                if 'derivativeHolding' in self.data['ownershipDocument']['derivativeTable']:
+                    deriv_hold = self.data['ownershipDocument']['derivativeTable']['derivativeHolding']
+                    deriv_hold_results = process_records(deriv_hold, form_345_mapping, is_derivative=True)
+                    all_results.extend(deriv_hold_results)
+            
+            return all_results
+        
+        else:
+            raise ValueError(f"Document type '{self.type}' is not supported for tabular conversion")
+    
+    def write_csv(self, output_filename, accession_number=None):
+            
+        data = self.to_tabular(accession_number)
 
-        with open(output_filename, 'w', newline='') as csvfile:
-            if not self.data:
-                return output_filename
+        if not data:
 
-            has_document = any('document' in item for item in self.data)
-            
-            if has_document and 'document' in self.data:
-                writer = csv.DictWriter(csvfile, ['section', 'text'], quoting=csv.QUOTE_ALL)
-                writer.writeheader()
-                flattened = self._flatten_dict(self.data['document'])
-                for section, text in flattened.items():
-                    writer.writerow({'section': section, 'text': text})
-            else:
-                fieldnames = list(self.data[0].keys())
-                if accession_number:
-                    fieldnames.append('Accession Number')
-                writer = csv.DictWriter(csvfile, fieldnames, quoting=csv.QUOTE_ALL)
-                writer.writeheader()
-                for row in self.data:
-                    if accession_number:
-                        row['Accession Number'] = accession_number
-                    writer.writerow(row)
+            return
+        
+        fieldnames = data[0].keys()
+        
+        with open(output_filename, 'w', newline='') as csvfile:
+            writer = csv.DictWriter(csvfile,fieldnames=fieldnames, quoting=csv.QUOTE_ALL)
+            writer.writeheader()
+            writer.writerows(data)
 
-        return output_filename
-    
+        
     def _document_to_section_text(self, document_data, parent_key=''):
         items = []
         
@@ -188,7 +391,7 @@ class Document:
     # we'll modify this for every dict
     def _flatten_dict(self, d, parent_key=''):
         items = {}
-        
+
         if isinstance(d, list):
             return [self._flatten_dict(item) for item in d]
                 
@@ -204,8 +407,7 @@ class Document:
    
    # this will all have to be changed. default will be to flatten everything
     def __iter__(self):
-        if not self.data:
-            self.parse()
+        self.parse()
 
         # Let's remove XML iterable for now
 

datamule/portfolio.py

@@ -119,7 +119,7 @@ class Portfolio:
             # First query, just set the accession numbers
             self.accession_numbers = new_accession_numbers
 
-    def download_submissions(self, cik=None, ticker=None, submission_type=None, filing_date=None, provider=None, **kwargs):
+    def download_submissions(self, cik=None, ticker=None, submission_type=None, filing_date=None, provider=None,requests_per_second=5, **kwargs):
         if provider is None:
             config = Config()
             provider = config.get_default_source()
@@ -142,7 +142,7 @@ class Portfolio:
                 cik=cik,
                 submission_type=submission_type,
                 filing_date=filing_date,
-                requests_per_second=5, 
+                requests_per_second=requests_per_second, 
                 accession_numbers=self.accession_numbers if hasattr(self, 'accession_numbers') else None
             )
 
@@ -164,8 +164,6 @@ class Portfolio:
         )
 
         
-        
-        
     def __iter__(self):
         if not self.submissions_loaded:
             self._load_submissions()

datamule/sec/submissions/monitor.py

@@ -20,12 +20,16 @@ async def _process_efts_hits(hits, collected_accession_numbers, data_callback=No
             submission_type = source.get('form')
             ciks = source.get('ciks', [])
             ciks = [str(int(cik)) for cik in ciks]
+
+            filing_date = source.get('file_date')
                 
             # Create standardized filing record
             filing = {
                 'accession_number': accession_number,
                 'submission_type': submission_type,
-                'ciks': ciks
+                'ciks': ciks,
+                'filing_date': filing_date,
+
             }
             
             processed_hits.append(filing)