datamule 1.2.7__tar.gz → 1.2.9__tar.gz

{datamule-1.2.7 → datamule-1.2.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 1.2.7
+Version: 1.2.9
 Summary: Making it easier to use SEC filings.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

datamule-1.2.9/datamule/document/mappings/d.py

@@ -0,0 +1,125 @@
+issuer_list_d_dict = {
+    'issuer_issuerAddress_street1': 'issuerStreet1',
+    'issuer_cik': 'issuerCik', 
+    'issuer_issuerAddress_stateOrCountryDescription': 'issuerStateOrCountryDescription',
+    'issuer_issuerAddress_zipCode': 'issuerZipCode',
+    'issuer_issuerPhoneNumber': 'issuerPhoneNumber',
+    'issuer_yearOfInc_value': 'yearOfIncValue',
+    'issuer_issuerAddress_stateOrCountry': 'issuerStateOrCountry',
+    'issuer_jurisdictionOfInc': 'jurisdictionOfInc',
+    'issuer_entityType': 'entityType',
+    'issuer_issuerAddress_street2': 'issuerStreet2',
+    'issuer_entityName': 'entityName',
+    'accession': 'accession',
+    'issuer_edgarPreviousNameList_value': 'edgarPreviousNameListValue',
+    'issuer_entityTypeOtherDesc': 'entityTypeOtherDesc',
+    'issuer_yearOfInc_yetToBeFormed': 'yearOfIncYetToBeFormed',
+    'issuer_yearOfInc_withinFiveYears': 'yearOfIncWithinFiveYears',
+    'issuer_issuerPreviousNameList_value': 'issuerPreviousNameListValue',
+    'issuer_issuerAddress_city': 'issuerCity'
+}
+
+metadata_d_dict = {
+"testOrLive" : "testOrLive",
+"schemaVersion" : "schemaVersion",
+"accession" : "accession",
+"submissionType" : "submissionType",
+}
+offering_data_d_dict = {
+    'salesCompensationList_recipient_foreignSolicitation': 'foreignSolicitation',
+    'typeOfFiling_dateOfFirstSale_yetToOccur': 'dateOfFirstSaleYetToOccur',
+    'industryGroup_investmentFundInfo_is40Act': 'is40Act',
+    'salesCommissionsFindersFees_findersFees_dollarAmount': 'findersFeesAmount',
+    'offeringSalesAmounts_totalRemaining': 'totalRemaining',
+    'issuerSize_aggregateNetAssetValueRange': 'aggregateNetAssetValueRange',
+    'typesOfSecuritiesOffered_isSecurityToBeAcquiredType': 'isSecurityToBeAcquiredType',
+    'salesCompensationList_recipient_recipientAddress_stateOrCountryDescription': 'recipientStateOrCountryDescription',
+    'typesOfSecuritiesOffered_isEquityType': 'isEquityType',
+    'investors_totalNumberAlreadyInvested': 'totalNumberAlreadyInvested',
+    'minimumInvestmentAccepted': 'minimumInvestmentAccepted',
+    'salesCompensationList_recipient_associatedBDName': 'associatedBDName',
+    'salesCompensationList_recipient_statesOfSolicitationList_state': 'statesOfSolicitationState',
+    'businessCombinationTransaction_isBusinessCombinationTransaction': 'isBusinessCombinationTransaction',
+    'useOfProceeds_grossProceedsUsed_isEstimate': 'grossProceedsUsedIsEstimate',
+    'federalExemptionsExclusions_item': 'federalExemptionsExclusionsItem',
+    'useOfProceeds_grossProceedsUsed_dollarAmount': 'grossProceedsUsedAmount',
+    'industryGroup_industryGroupType': 'industryGroupType',
+    'signatureBlock_signature_nameOfSigner': 'nameOfSigner',
+    'signatureBlock_signature_signatureDate': 'signatureDate',
+    'salesCommissionsFindersFees_salesCommissions_isEstimate': 'salesCommissionsIsEstimate',
+    'typesOfSecuritiesOffered_isOtherType': 'isOtherType',
+    'salesCompensationList_recipient_associatedBDCRDNumber': 'associatedBDCRDNumber',
+    'salesCompensationList_recipient_recipientAddress_stateOrCountry': 'recipientStateOrCountry',
+    'typesOfSecuritiesOffered_descriptionOfOtherType': 'descriptionOfOtherType',
+    'salesCommissionsFindersFees_salesCommissions_dollarAmount': 'salesCommissionsAmount',
+    'useOfProceeds_clarificationOfResponse': 'useOfProceedsClarification',
+    'accession': 'accession',
+    'typesOfSecuritiesOffered_isPooledInvestmentFundType': 'isPooledInvestmentFundType',
+    'salesCompensationList_recipient_statesOfSolicitationList_value': 'statesOfSolicitationValue',
+    'signatureBlock_signature_signatureName': 'signatureName',
+    'typeOfFiling_newOrAmendment_isAmendment': 'isAmendment',
+    'issuerSize_revenueRange': 'revenueRange',
+    'salesCommissionsFindersFees_clarificationOfResponse': 'salesCommissionsFindersFeesClarification',
+    'salesCompensationList_recipient_recipientAddress_zipCode': 'recipientZipCode',
+    'salesCompensationList_recipient_recipientAddress_city': 'recipientCity',
+    'typesOfSecuritiesOffered_isOptionToAcquireType': 'isOptionToAcquireType',
+    'businessCombinationTransaction_clarificationOfResponse': 'businessCombinationClarification',
+    'typesOfSecuritiesOffered_isTenantInCommonType': 'isTenantInCommonType',
+    'salesCompensationList_recipient_statesOfSolicitationList_description': 'statesOfSolicitationDescription',
+    'offeringSalesAmounts_totalOfferingAmount': 'totalOfferingAmount',
+    'investors_numberNonAccreditedInvestors': 'numberNonAccreditedInvestors',
+    'signatureBlock_authorizedRepresentative': 'authorizedRepresentative',
+    'signatureBlock_signature_issuerName': 'issuerName',
+    'salesCompensationList_recipient_recipientAddress_street2': 'recipientStreet2',
+    'typesOfSecuritiesOffered_isDebtType': 'isDebtType',
+    'salesCompensationList_recipient_recipientAddress_street1': 'recipientStreet1',
+    'signatureBlock_signature_signatureTitle': 'signatureTitle',
+    'industryGroup_investmentFundInfo_investmentFundType': 'investmentFundType',
+    'salesCommissionsFindersFees_findersFees_isEstimate': 'findersFeesIsEstimate',
+    'typeOfFiling_dateOfFirstSale_value': 'dateOfFirstSaleValue',
+    'offeringSalesAmounts_totalAmountSold': 'totalAmountSold',
+    'offeringSalesAmounts_clarificationOfResponse': 'offeringSalesAmountsClarification',
+    'investors_hasNonAccreditedInvestors': 'hasNonAccreditedInvestors',
+    'salesCompensationList_recipient_recipientCRDNumber': 'recipientCRDNumber',
+    'typesOfSecuritiesOffered_isMineralPropertyType': 'isMineralPropertyType',
+    'salesCompensationList_recipient_recipientName': 'recipientName',
+    'durationOfOffering_moreThanOneYear': 'moreThanOneYear'
+}
+primary_issuer_d_dict = {
+    'yearOfInc_withinFiveYears': 'yearOfIncWithinFiveYears',
+    'entityTypeOtherDesc': 'entityTypeOtherDesc',
+    'jurisdictionOfInc': 'jurisdictionOfInc',
+    'issuerAddress_street1': 'issuerStreet1',
+    'issuerAddress_zipCode': 'issuerZipCode',
+    'issuerPreviousNameList_previousName': 'issuerPreviousName',
+    'entityType': 'entityType',
+    'issuerPreviousNameList_value': 'issuerPreviousNameListValue',
+    'issuerPhoneNumber': 'issuerPhoneNumber',
+    'yearOfInc_value': 'yearOfIncValue',
+    'yearOfInc_yetToBeFormed': 'yearOfIncYetToBeFormed',
+    'edgarPreviousNameList_previousName': 'edgarPreviousName',
+    'edgarPreviousNameList_value': 'edgarPreviousNameListValue',
+    'issuerAddress_stateOrCountry': 'issuerStateOrCountry',
+    'entityName': 'entityName',
+    'accession': 'accession',
+    'issuerAddress_street2': 'issuerStreet2',
+    'issuerAddress_city': 'issuerCity',
+    'issuerAddress_stateOrCountryDescription': 'issuerStateOrCountryDescription',
+    'cik': 'cik',
+    'yearOfInc_overFiveYears': 'yearOfIncOverFiveYears'
+}
+
+related_persons_d_dict = {
+    'relatedPersonInfo_relatedPersonAddress_stateOrCountry': 'relatedPersonStateOrCountry',
+    'relatedPersonInfo_relatedPersonRelationshipList_relationship': 'relatedPersonRelationship',
+    'relatedPersonInfo_relationshipClarification': 'relationshipClarification',
+    'relatedPersonInfo_relatedPersonName_lastName': 'relatedPersonLastName',
+    'accession': 'accession',
+    'relatedPersonInfo_relatedPersonName_middleName': 'relatedPersonMiddleName',
+    'relatedPersonInfo_relatedPersonAddress_zipCode': 'relatedPersonZipCode',
+    'relatedPersonInfo_relatedPersonAddress_city': 'relatedPersonCity',
+    'relatedPersonInfo_relatedPersonAddress_street1': 'relatedPersonStreet1',
+    'relatedPersonInfo_relatedPersonAddress_stateOrCountryDescription': 'relatedPersonStateOrCountryDescription',
+    'relatedPersonInfo_relatedPersonName_firstName': 'relatedPersonFirstName',
+    'relatedPersonInfo_relatedPersonAddress_street2': 'relatedPersonStreet2'
+}

{datamule-1.2.7 → datamule-1.2.9}/datamule/document/processing.py

@@ -20,10 +20,13 @@ def process_tabular_data(self):
     elif self.type in ["25-NSE", "25-NSE/A"]:
         tables = process_25nse(self.data, self.accession)
     # complete mark:
-    elif self.type in ["N-PX","N-PX/A"]:
-        tables = process_npx(self.data, self.accession)
     elif self.type in ["EX-102"]:
         tables = process_ex102_abs(self.data, self.accession)
+    elif self.type in ["D","D/A"]:
+        tables = process_d(self.data, self.accession)
+    elif self.type in ["N-PX","N-PX/A"]:
+        tables = process_npx(self.data, self.accession)
+
 
     elif self.type in ["SBSEF","SBSEF/A","SBSEF-V","SBSEF-W"]:
         tables = process_sbsef(self.data, self.accession)
@@ -49,8 +52,7 @@ def process_tabular_data(self):
     #     tables = process_c(self.data, self.accession)
     elif self.type in ["CFPORTAL","CFPORTAL/A","CFPORTAL-W"]:
         tables = process_cfportal(self.data, self.accession)
-    # elif self.type in ["D","D/A"]:
-    #     tables = process_d(self.data, self.accession)
+
     # elif self.type in ["MA","MA-A","MA/A","MA-I","MA-I/A","MA-W"]:
     #     tables = process_ma(self.data, self.accession)
     # elif self.type in ["N-CEN","N-CEN/A"]:
@@ -102,6 +104,67 @@ def _flatten_dict(d, parent_key=''):
                 
     return items
 
+# flattens in a different way
+def flatten_dict_to_rows(d, parent_key='', sep='_'):
+
+    if isinstance(d, list):
+        # If input is a list, flatten each item and return all rows
+        all_rows = []
+        for item in d:
+            all_rows.extend(flatten_dict_to_rows(item, parent_key, sep))
+        return all_rows
+    
+    if not isinstance(d, dict):
+        # If input is a primitive value, return single row
+        return [{parent_key: d}] if parent_key else []
+    
+    # Input is a dictionary
+    rows = [{}]
+    
+    for k, v in d.items():
+        new_key = f"{parent_key}{sep}{k}" if parent_key else k
+        
+        if isinstance(v, dict):
+            # Recursively flatten nested dictionaries
+            nested_rows = flatten_dict_to_rows(v, new_key, sep)
+            # Cross-product with existing rows
+            new_rows = []
+            for row in rows:
+                for nested_row in nested_rows:
+                    combined_row = row.copy()
+                    combined_row.update(nested_row)
+                    new_rows.append(combined_row)
+            rows = new_rows
+            
+        elif isinstance(v, list):
+            # Handle lists - create multiple rows
+            if not v:  # Empty list
+                for row in rows:
+                    row[new_key] = ''
+            else:
+                new_rows = []
+                for row in rows:
+                    for list_item in v:
+                        new_row = row.copy()
+                        if isinstance(list_item, dict):
+                            # Recursively flatten dict items in list
+                            nested_rows = flatten_dict_to_rows(list_item, new_key, sep)
+                            for nested_row in nested_rows:
+                                combined_row = new_row.copy()
+                                combined_row.update(nested_row)
+                                new_rows.append(combined_row)
+                        else:
+                            # Primitive value in list
+                            new_row[new_key] = list_item
+                            new_rows.append(new_row)
+                rows = new_rows
+        else:
+            # Handle primitive values
+            for row in rows:
+                row[new_key] = v
+    
+    return rows
+
 def process_ownership(data, accession):
     tables = []
     if 'ownershipDocument' not in data:
@@ -353,33 +416,41 @@ def process_cfportal(data, accession):
     
     return tables
 
-# def process_d(data, accession):
-#     tables = []
-#     primary_issuer = safe_get(data, ['edgarSubmission', 'primaryIssuer'])
-#     if primary_issuer:
-#         metadata = Table(_flatten_dict(primary_issuer), 'metadata_d', accession)
-        
-#         metadata_columns = ['schemaVersion', 'submissionType', 'testOrLive', 'returnCopy', 'contactData', 'notificationAddressList']
-#         for col in metadata_columns:
-#             col_data = safe_get(data, ['edgarSubmission', col])
-#             if col_data:
-#                 metadata.add_column(col, col_data)
-        
-#         tables.append(metadata)
-    
-#     issuer_list = safe_get(data, ['edgarSubmission', 'issuerList'])
-#     if issuer_list:
-#         tables.append(Table(_flatten_dict(issuer_list), 'primary_issuer_d', accession))
-    
-#     offering_data = safe_get(data, ['edgarSubmission', 'offeringData'])
-#     if offering_data:
-#         tables.append(Table(_flatten_dict(offering_data), 'offering_data_d', accession))
-    
-#     related_persons_list = safe_get(data, ['edgarSubmission', 'relatedPersonsList'])
-#     if related_persons_list:
-#         tables.append(Table(_flatten_dict(related_persons_list), 'related_persons_list_d', accession))
+def process_d(data, accession):
+    tables = []
+    groups = [('contactData', 'contact_data_d'),
+                ('notificationAddressList', 'notification_address_list_d'),
+                ('primaryIssuer', 'primary_issuer_d'),
+                ('issuerList', 'issuer_list_d'),
+                ('relatedPersonsList', 'related_persons_list_d'),
+                ('offeringData', 'offering_data_d'),
+    ]
+    for group,table_type in groups:
+        if group == 'relatedPersonList':
+            group_data = data['edgarSubmission'].pop('relatedPersonInfo', None)
+            data['edgarSubmission'].pop(group, None)
+        elif group == 'issuerList':
+            group_data = data['edgarSubmission'].pop('issuerList', None)
+        else:
+            group_data = data['edgarSubmission'].pop(group, None)
+            
+        if group_data:
+            # Special handling ONLY for relatedPersonsList
+            if group in ['relatedPersonsList', 'issuerList','offeringData']:
+                # Use the new flatten_dict_to_rows ONLY for this key
+                flattened_rows = flatten_dict_to_rows(group_data)
+                if flattened_rows:
+                    tables.append(Table(flattened_rows, table_type, accession))
+            else:
+                # Everything else remains EXACTLY the same
+                tables.append(Table(_flatten_dict(group_data), table_type, accession))
+
+
+
+    metadata_table = Table(_flatten_dict(data['edgarSubmission']), 'metadata_d', accession)
+    tables.append(metadata_table)
     
-#     return tables
+    return tables
 
 # def process_nmfp(data, accession):
 #     tables = []

{datamule-1.2.7 → datamule-1.2.9}/datamule/document/table.py

@@ -20,6 +20,7 @@ from .mappings.twentyfourf2nt import *
 from .mappings.information_table import *
 from .mappings.submission_metadata import *
 from .mappings.ex102_abs import *
+from .mappings.d import *
 
 from pathlib import Path
 import csv
@@ -201,6 +202,17 @@ class Table():
         elif self.type == 'signature_schedule_13':
             mapping_dict = signature_schedule_13_dict
 
+        # D 
+        elif self.type == 'issuer_list_d':
+            mapping_dict = issuer_list_d_dict
+        elif self.type == 'metadata_d':
+            mapping_dict = metadata_d_dict
+        elif self.type == 'offering_data_d':
+            mapping_dict = offering_data_d_dict
+        elif self.type == 'primary_issuer_d':
+            mapping_dict = primary_issuer_d_dict
+        elif self.type == 'related_persons_list_d':
+            mapping_dict = related_persons_d_dict
         # SDR
         elif self.type == 'sdr':
             mapping_dict = sdr_dict

{datamule-1.2.7 → datamule-1.2.9}/datamule.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 1.2.7
+Version: 1.2.9
 Summary: Making it easier to use SEC filings.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

{datamule-1.2.7 → datamule-1.2.9}/datamule.egg-info/SOURCES.txt

@@ -19,6 +19,7 @@ datamule/document/table.py
 datamule/document/mappings/__init__.py
 datamule/document/mappings/atsn.py
 datamule/document/mappings/cfportal.py
+datamule/document/mappings/d.py
 datamule/document/mappings/ex102_abs.py
 datamule/document/mappings/ex99a_sdr.py
 datamule/document/mappings/ex99c_sdr.py

{datamule-1.2.7 → datamule-1.2.9}/setup.py

@@ -30,7 +30,7 @@ if not file_path.exists():
 setup(
     name="datamule",
     author="John Friedman",
-    version="1.2.7",
+    version="1.2.9",
     description="Making it easier to use SEC filings.",
     packages=find_packages(include=['datamule', 'datamule.*']),
     url="https://github.com/john-friedman/datamule-python",