datamule 1.2.6__py3-none-any.whl → 1.2.8__py3-none-any.whl

datamule/document/document.py

@@ -141,7 +141,7 @@ class Document:
         with open(output_filename, 'w',encoding='utf-8') as f:
             json.dump(self.data, f, indent=2)
 
-    def to_tabular(self):
+    def tables(self):
         if self.type == 'submission_metadata':
             return process_tabular_data(self)
         elif self.extension != '.xml':
@@ -155,7 +155,7 @@ class Document:
         output_folder = Path(output_folder)
         output_folder.mkdir(exist_ok=True)
             
-        tables = self.to_tabular()
+        tables = self.tables()
 
         if not tables:
             return

datamule/document/mappings/d.py

@@ -0,0 +1,127 @@
+issuer_list_d_dict = {
+    'issuer_issuerAddress_street1': 'issuerStreet1',
+    'issuer_cik': 'issuerCik', 
+    'issuer_issuerAddress_stateOrCountryDescription': 'issuerStateOrCountryDescription',
+    'issuer_issuerAddress_zipCode': 'issuerZipCode',
+    'issuer_issuerPhoneNumber': 'issuerPhoneNumber',
+    'issuer_yearOfInc_value': 'yearOfIncValue',
+    'issuer_issuerAddress_stateOrCountry': 'issuerStateOrCountry',
+    'issuer_jurisdictionOfInc': 'jurisdictionOfInc',
+    'issuer_entityType': 'entityType',
+    'issuer_issuerAddress_street2': 'issuerStreet2',
+    'issuer_entityName': 'entityName',
+    'accession': 'accessionNumber',
+    'issuer_edgarPreviousNameList_value': 'edgarPreviousNameListValue',
+    'issuer_entityTypeOtherDesc': 'entityTypeOtherDesc',
+    'issuer_yearOfInc_yetToBeFormed': 'yearOfIncYetToBeFormed',
+    'issuer_yearOfInc_withinFiveYears': 'yearOfIncWithinFiveYears',
+    'issuer_issuerPreviousNameList_value': 'issuerPreviousNameListValue',
+    'issuer_issuerAddress_city': 'issuerCity'
+}
+
+metadata_d_dict = {
+"testOrLive" : "testOrLive",
+"schemaVersion" : "schemaVersion",
+"accession" : "accession",
+"submissionType" : "submissionType",
+}
+offering_data_d_dict = {
+    'salesCompensationList_recipient': 'salesCompensationRecipient',
+    'useOfProceeds_clarificationOfResponse': 'useOfProceedsClarification',
+    'industryGroup_investmentFundInfo_investmentFundType': 'investmentFundType',
+    'typeOfFiling_newOrAmendment_isAmendment': 'isAmendment',
+    'salesCompensationList_recipient_recipientAddress_stateOrCountryDescription': 'recipientStateOrCountryDescription',
+    'signatureBlock_signature_signatureName': 'signatureName',
+    'federalExemptionsExclusions_item': 'federalExemptionsItem',
+    'salesCommissionsFindersFees_findersFees_isEstimate': 'findersFeesIsEstimate',
+    'salesCommissionsFindersFees_salesCommissions_dollarAmount': 'salesCommissionsDollarAmount',
+    'salesCompensationList_recipient_recipientName': 'recipientName',
+    'salesCommissionsFindersFees_salesCommissions_isEstimate': 'salesCommissionsIsEstimate',
+    'offeringSalesAmounts_totalRemaining': 'totalRemaining',
+    'salesCommissionsFindersFees_clarificationOfResponse': 'salesCommissionsClarification',
+    'salesCompensationList_recipient_statesOfSolicitationList_value': 'recipientStatesOfSolicitationValue',
+    'investors_numberNonAccreditedInvestors': 'numberNonAccreditedInvestors',
+    'typesOfSecuritiesOffered_isTenantInCommonType': 'isTenantInCommonType',
+    'businessCombinationTransaction_isBusinessCombinationTransaction': 'isBusinessCombinationTransaction',
+    'typesOfSecuritiesOffered_isSecurityToBeAcquiredType': 'isSecurityToBeAcquiredType',
+    'issuerSize_aggregateNetAssetValueRange': 'aggregateNetAssetValueRange',
+    'typesOfSecuritiesOffered_isPooledInvestmentFundType': 'isPooledInvestmentFundType',
+    'offeringSalesAmounts_clarificationOfResponse': 'offeringSalesAmountsClarification',
+    'signatureBlock_signature_nameOfSigner': 'nameOfSigner',
+    'industryGroup_investmentFundInfo_is40Act': 'is40Act',
+    'salesCompensationList_recipient_recipientAddress_city': 'recipientCity',
+    'typeOfFiling_dateOfFirstSale_yetToOccur': 'dateOfFirstSaleYetToOccur',
+    'signatureBlock_signature': 'signature',
+    'salesCompensationList_recipient_foreignSolicitation': 'recipientForeignSolicitation',
+    'businessCombinationTransaction_clarificationOfResponse': 'businessCombinationClarification',
+    'salesCompensationList_recipient_associatedBDName': 'recipientAssociatedBDName',
+    'salesCompensationList_recipient_statesOfSolicitationList_state': 'recipientStatesOfSolicitationState',
+    'typeOfFiling_dateOfFirstSale_value': 'dateOfFirstSaleValue',
+    'signatureBlock_signature_signatureTitle': 'signatureTitle',
+    'signatureBlock_signature_issuerName': 'signatureIssuerName',
+    'durationOfOffering_moreThanOneYear': 'durationOfferingMoreThanOneYear',
+    'offeringSalesAmounts_totalAmountSold': 'totalAmountSold',
+    'signatureBlock_signature_signatureDate': 'signatureDate',
+    'issuerSize_revenueRange': 'revenueRange',
+    'typesOfSecuritiesOffered_isOptionToAcquireType': 'isOptionToAcquireType',
+    'signatureBlock_authorizedRepresentative': 'authorizedRepresentative',
+    'salesCompensationList_recipient_recipientAddress_street2': 'recipientStreet2',
+    'useOfProceeds_grossProceedsUsed_isEstimate': 'grossProceedsUsedIsEstimate',
+    'salesCommissionsFindersFees_findersFees_dollarAmount': 'findersFeesDollarAmount',
+    'typesOfSecuritiesOffered_isEquityType': 'isEquityType',
+    'typesOfSecuritiesOffered_descriptionOfOtherType': 'descriptionOfOtherType',
+    'salesCompensationList_recipient_recipientAddress_street1': 'recipientStreet1',
+    'minimumInvestmentAccepted': 'minimumInvestmentAccepted',
+    'typesOfSecuritiesOffered_isOtherType': 'isOtherType',
+    'salesCompensationList_recipient_recipientCRDNumber': 'recipientCRDNumber',
+    'typesOfSecuritiesOffered_isDebtType': 'isDebtType',
+    'accession': 'accessionNumber',
+    'investors_totalNumberAlreadyInvested': 'totalNumberAlreadyInvested',
+    'typesOfSecuritiesOffered_isMineralPropertyType': 'isMineralPropertyType',
+    'industryGroup_industryGroupType': 'industryGroupType',
+    'salesCompensationList_recipient_statesOfSolicitationList_description': 'recipientStatesOfSolicitationDescription',
+    'salesCompensationList_recipient_associatedBDCRDNumber': 'recipientAssociatedBDCRDNumber',
+    'useOfProceeds_grossProceedsUsed_dollarAmount': 'grossProceedsUsedDollarAmount',
+    'investors_hasNonAccreditedInvestors': 'hasNonAccreditedInvestors',
+    'salesCompensationList_recipient_recipientAddress_zipCode': 'recipientZipCode',
+    'offeringSalesAmounts_totalOfferingAmount': 'totalOfferingAmount',
+    'salesCompensationList_recipient_recipientAddress_stateOrCountry': 'recipientStateOrCountry'
+}
+primary_issuer_d_dict = {
+    'yearOfInc_withinFiveYears': 'yearOfIncWithinFiveYears',
+    'entityTypeOtherDesc': 'entityTypeOtherDesc',
+    'jurisdictionOfInc': 'jurisdictionOfInc',
+    'issuerAddress_street1': 'issuerStreet1',
+    'issuerAddress_zipCode': 'issuerZipCode',
+    'issuerPreviousNameList_previousName': 'issuerPreviousName',
+    'entityType': 'entityType',
+    'issuerPreviousNameList_value': 'issuerPreviousNameListValue',
+    'issuerPhoneNumber': 'issuerPhoneNumber',
+    'yearOfInc_value': 'yearOfIncValue',
+    'yearOfInc_yetToBeFormed': 'yearOfIncYetToBeFormed',
+    'edgarPreviousNameList_previousName': 'edgarPreviousName',
+    'edgarPreviousNameList_value': 'edgarPreviousNameListValue',
+    'issuerAddress_stateOrCountry': 'issuerStateOrCountry',
+    'entityName': 'entityName',
+    'accession': 'accessionNumber',
+    'issuerAddress_street2': 'issuerStreet2',
+    'issuerAddress_city': 'issuerCity',
+    'issuerAddress_stateOrCountryDescription': 'issuerStateOrCountryDescription',
+    'cik': 'cik',
+    'yearOfInc_overFiveYears': 'yearOfIncOverFiveYears'
+}
+
+related_persons_d_dict = {
+    'relatedPersonInfo_relatedPersonAddress_stateOrCountry': 'relatedPersonStateOrCountry',
+    'relatedPersonInfo_relatedPersonRelationshipList_relationship': 'relatedPersonRelationship',
+    'relatedPersonInfo_relationshipClarification': 'relationshipClarification',
+    'relatedPersonInfo_relatedPersonName_lastName': 'relatedPersonLastName',
+    'accession': 'accessionNumber',
+    'relatedPersonInfo_relatedPersonName_middleName': 'relatedPersonMiddleName',
+    'relatedPersonInfo_relatedPersonAddress_zipCode': 'relatedPersonZipCode',
+    'relatedPersonInfo_relatedPersonAddress_city': 'relatedPersonCity',
+    'relatedPersonInfo_relatedPersonAddress_street1': 'relatedPersonStreet1',
+    'relatedPersonInfo_relatedPersonAddress_stateOrCountryDescription': 'relatedPersonStateOrCountryDescription',
+    'relatedPersonInfo_relatedPersonName_firstName': 'relatedPersonFirstName',
+    'relatedPersonInfo_relatedPersonAddress_street2': 'relatedPersonStreet2'
+}

datamule/document/mappings/ex102_abs.py

@@ -0,0 +1,63 @@
+# Assets dictionary mapping
+assets_dict_ex102_abs = {
+    'assetNumber': 'assetNumber',
+    'DefeasedStatusCode': 'DefeasedStatusCode',
+    'defeasanceOptionStartDate': 'defeasanceOptionStartDate',
+    'mostRecentDebtServiceCoverageNetOperatingIncomePercentage': 'mostRecentDebtServiceCoverageNetOperatingIncomePercentage',
+    'mostRecentDebtServiceAmount': 'mostRecentDebtServiceAmount',
+    'debtServiceCoverageSecuritizationCode': 'debtServiceCoverageSecuritizationCode',
+    'debtServiceCoverageNetOperatingIncomeSecuritizationPercentage': 'debtServiceCoverageNetOperatingIncomeSecuritizationPercentage',
+    'valuationSecuritizationDate': 'valuationSecuritizationDate',
+    'physicalOccupancySecuritizationPercentage': 'physicalOccupancySecuritizationPercentage',
+    'revenueSecuritizationAmount': 'revenueSecuritizationAmount',
+    'valuationSourceSecuritizationCode': 'valuationSourceSecuritizationCode',
+    'financialsSecuritizationDate': 'financialsSecuritizationDate',
+    'mostRecentNetCashFlowAmount': 'mostRecentNetCashFlowAmount',
+    'operatingExpensesAmount': 'operatingExpensesAmount',
+    'operatingExpensesSecuritizationAmount': 'operatingExpensesSecuritizationAmount',
+    'netOperatingIncomeNetCashFlowSecuritizationCode': 'netOperatingIncomeNetCashFlowSecuritizationCode',
+    'mostRecentValuationSourceCode': 'mostRecentValuationSourceCode',
+    'mostRecentDebtServiceCoverageNetCashFlowpercentage': 'mostRecentDebtServiceCoverageNetCashFlowpercentage',
+    'debtServiceCoverageNetCashFlowSecuritizationPercentage': 'debtServiceCoverageNetCashFlowSecuritizationPercentage',
+    'mostRecentAnnualLeaseRolloverReviewDate': 'mostRecentAnnualLeaseRolloverReviewDate',
+    'mostRecentRevenueAmount': 'mostRecentRevenueAmount',
+    'mostRecentPhysicalOccupancyPercentage': 'mostRecentPhysicalOccupancyPercentage',
+    'mostRecentNetOperatingIncomeAmount': 'mostRecentNetOperatingIncomeAmount',
+    'netOperatingIncomeSecuritizationAmount': 'netOperatingIncomeSecuritizationAmount',
+    'netOperatingIncomeNetCashFlowCode': 'netOperatingIncomeNetCashFlowCode',
+    'mostRecentFinancialsStartDate': 'mostRecentFinancialsStartDate',
+    'mostRecentFinancialsEndDate': 'mostRecentFinancialsEndDate',
+    'accession': 'accession',
+    'valuationSecuritizationAmount': 'valuationSecuritizationAmount',
+    'mostRecentValuationDate': 'mostRecentValuationDate',
+    'mostRecentValuationAmount': 'mostRecentValuationAmount',
+    'mostRecentDebtServiceCoverageCode': 'mostRecentDebtServiceCoverageCode',
+    'netCashFlowFlowSecuritizationAmount': 'netCashFlowFlowSecuritizationAmount'
+}
+
+# Properties dictionary mapping
+properties_dict_ex102_abs = {
+    'unitsBedsRoomsNumber': 'unitsBedsRoomsNumber',
+    'propertyCounty': 'propertyCounty',
+    'squareFeetLargestTenantNumber': 'squareFeetLargestTenantNumber',
+    'netRentableSquareFeetNumber': 'netRentableSquareFeetNumber',
+    'leaseExpirationThirdLargestTenantDate': 'leaseExpirationThirdLargestTenantDate',
+    'leaseExpirationLargestTenantDate': 'leaseExpirationLargestTenantDate',
+    'propertyZip': 'propertyZip',
+    'squareFeetThirdLargestTenantNumber': 'squareFeetThirdLargestTenantNumber',
+    'propertyStatusCode': 'propertyStatusCode',
+    'propertyState': 'propertyState',
+    'yearBuiltNumber': 'yearBuiltNumber',
+    'propertyCity': 'propertyCity',
+    'propertyName': 'propertyName',
+    'propertyAddress': 'propertyAddress',
+    'yearLastRenovated': 'yearLastRenovated',
+    'leaseExpirationSecondLargestTenantDate': 'leaseExpirationSecondLargestTenantDate',
+    'thirdLargestTenant': 'thirdLargestTenant',
+    'unitsBedsRoomsSecuritizationNumber': 'unitsBedsRoomsSecuritizationNumber',
+    'propertyTypeCode': 'propertyTypeCode',
+    'largestTenant': 'largestTenant',
+    'squareFeetSecondLargestTenantNumber': 'squareFeetSecondLargestTenantNumber',
+    'netRentableSquareFeetSecuritizationNumber': 'netRentableSquareFeetSecuritizationNumber',
+    'secondLargestTenant': 'secondLargestTenant'
+}

datamule/document/processing.py

@@ -20,9 +20,14 @@ def process_tabular_data(self):
     elif self.type in ["25-NSE", "25-NSE/A"]:
         tables = process_25nse(self.data, self.accession)
     # complete mark:
+    elif self.type in ["EX-102"]:
+        tables = process_ex102_abs(self.data, self.accession)
+    elif self.type in ["D","D/A"]:
+        tables = process_d(self.data, self.accession)
     elif self.type in ["N-PX","N-PX/A"]:
         tables = process_npx(self.data, self.accession)
 
+
     elif self.type in ["SBSEF","SBSEF/A","SBSEF-V","SBSEF-W"]:
         tables = process_sbsef(self.data, self.accession)
     elif self.type in ["SDR","SDR/A","SDR-W","SDR-A"]:
@@ -47,8 +52,7 @@ def process_tabular_data(self):
     #     tables = process_c(self.data, self.accession)
     elif self.type in ["CFPORTAL","CFPORTAL/A","CFPORTAL-W"]:
         tables = process_cfportal(self.data, self.accession)
-    # elif self.type in ["D","D/A"]:
-    #     tables = process_d(self.data, self.accession)
+
     # elif self.type in ["MA","MA-A","MA/A","MA-I","MA-I/A","MA-W"]:
     #     tables = process_ma(self.data, self.accession)
     # elif self.type in ["N-CEN","N-CEN/A"]:
@@ -70,8 +74,7 @@ def process_tabular_data(self):
         tables = process_reg_a(self.data, self.accession)
     # elif self.type in ["SBSE","SBSE/A","SBSE-A","SBSE-A/A","SBSE-BD","SBSE-BD/A","SBSE-C","SBSE-W","SBSE-CCO-RPT","SBSE-CCO-RPT/A"]:
     #     tables = process_sbs(self.data, self.accession)
-    # elif self.type in ["EX-102"]:
-    #     tables = process_ex102_abs(self.data, self.accession)
+
     elif self.type == "PROXY VOTING RECORD":
         tables = process_proxy_voting_record(self.data, self.accession)
     elif self.type == 'submission_metadata':
@@ -101,6 +104,67 @@ def _flatten_dict(d, parent_key=''):
                 
     return items
 
+# flattens in a different way
+def flatten_dict_to_rows(d, parent_key='', sep='_'):
+
+    if isinstance(d, list):
+        # If input is a list, flatten each item and return all rows
+        all_rows = []
+        for item in d:
+            all_rows.extend(flatten_dict_to_rows(item, parent_key, sep))
+        return all_rows
+    
+    if not isinstance(d, dict):
+        # If input is a primitive value, return single row
+        return [{parent_key: d}] if parent_key else []
+    
+    # Input is a dictionary
+    rows = [{}]
+    
+    for k, v in d.items():
+        new_key = f"{parent_key}{sep}{k}" if parent_key else k
+        
+        if isinstance(v, dict):
+            # Recursively flatten nested dictionaries
+            nested_rows = flatten_dict_to_rows(v, new_key, sep)
+            # Cross-product with existing rows
+            new_rows = []
+            for row in rows:
+                for nested_row in nested_rows:
+                    combined_row = row.copy()
+                    combined_row.update(nested_row)
+                    new_rows.append(combined_row)
+            rows = new_rows
+            
+        elif isinstance(v, list):
+            # Handle lists - create multiple rows
+            if not v:  # Empty list
+                for row in rows:
+                    row[new_key] = ''
+            else:
+                new_rows = []
+                for row in rows:
+                    for list_item in v:
+                        new_row = row.copy()
+                        if isinstance(list_item, dict):
+                            # Recursively flatten dict items in list
+                            nested_rows = flatten_dict_to_rows(list_item, new_key, sep)
+                            for nested_row in nested_rows:
+                                combined_row = new_row.copy()
+                                combined_row.update(nested_row)
+                                new_rows.append(combined_row)
+                        else:
+                            # Primitive value in list
+                            new_row[new_key] = list_item
+                            new_rows.append(new_row)
+                rows = new_rows
+        else:
+            # Handle primitive values
+            for row in rows:
+                row[new_key] = v
+    
+    return rows
+
 def process_ownership(data, accession):
     tables = []
     if 'ownershipDocument' not in data:
@@ -352,33 +416,33 @@ def process_cfportal(data, accession):
     
     return tables
 
-# def process_d(data, accession):
-#     tables = []
-#     primary_issuer = safe_get(data, ['edgarSubmission', 'primaryIssuer'])
-#     if primary_issuer:
-#         metadata = Table(_flatten_dict(primary_issuer), 'metadata_d', accession)
-        
-#         metadata_columns = ['schemaVersion', 'submissionType', 'testOrLive', 'returnCopy', 'contactData', 'notificationAddressList']
-#         for col in metadata_columns:
-#             col_data = safe_get(data, ['edgarSubmission', col])
-#             if col_data:
-#                 metadata.add_column(col, col_data)
-        
-#         tables.append(metadata)
-    
-#     issuer_list = safe_get(data, ['edgarSubmission', 'issuerList'])
-#     if issuer_list:
-#         tables.append(Table(_flatten_dict(issuer_list), 'primary_issuer_d', accession))
-    
-#     offering_data = safe_get(data, ['edgarSubmission', 'offeringData'])
-#     if offering_data:
-#         tables.append(Table(_flatten_dict(offering_data), 'offering_data_d', accession))
-    
-#     related_persons_list = safe_get(data, ['edgarSubmission', 'relatedPersonsList'])
-#     if related_persons_list:
-#         tables.append(Table(_flatten_dict(related_persons_list), 'related_persons_list_d', accession))
+def process_d(data, accession):
+    tables = []
+    groups = ['contactData', 'notificationAddressList', 'primaryIssuer', 'issuerList', 'relatedPersonsList', 'offeringData']
+    for group in groups:
+        if group == 'relatedPersonList':
+            group_data = data['edgarSubmission'].pop('relatedPersonInfo', None)
+            data['edgarSubmission'].pop(group, None)
+        elif group == 'issuerList':
+            group_data = data['edgarSubmission'].pop('issuerList', None)
+        else:
+            group_data = data['edgarSubmission'].pop(group, None)
+            
+        if group_data:
+            # Special handling ONLY for relatedPersonsList
+            if group in ['relatedPersonsList', 'issuerList']:
+                # Use the new flatten_dict_to_rows ONLY for this key
+                flattened_rows = flatten_dict_to_rows(group_data)
+                if flattened_rows:
+                    tables.append(Table(flattened_rows, f'{group}_d', accession))
+            else:
+                # Everything else remains EXACTLY the same
+                tables.append(Table(_flatten_dict(group_data), f'{group}_d', accession))
+
+    metadata_table = Table(_flatten_dict(data['edgarSubmission']), 'metadata_d', accession)
+    tables.append(metadata_table)
     
-#     return tables
+    return tables
 
 # def process_nmfp(data, accession):
 #     tables = []
@@ -589,13 +653,39 @@ def process_reg_a(data, accession):
     
 #     return tables
 
-# def process_ex102_abs(data, accession):
-#     tables = []
-#     asset_data = safe_get(data, ['assetData'])
-#     if asset_data:
-#         tables.append(Table(_flatten_dict(asset_data), 'abs', accession))
-#     raise NotImplementedError("Need to implement the rest of the ABS processing")
-#     return tables
+def process_ex102_abs(data, accession):
+    tables = []
+    data = safe_get(data, ['assetData', 'assets'])
+    
+    # Create assets list: all items without their 'property' field
+    assets = [{k: v for k, v in item.items() if k != 'property'} for item in data]
+
+    # Create properties list in a more vectorized way
+    properties = []
+    
+    # Handle dictionary properties
+    properties.extend([
+        item['property'] | {'assetNumber': item['assetNumber']}
+        for item in data
+        if 'property' in item and isinstance(item['property'], dict)
+    ])
+    
+    # Handle list properties - flatten in one operation
+    properties.extend([
+        prop | {'assetNumber': item['assetNumber']}
+        for item in data
+        if 'property' in item and isinstance(item['property'], list)
+        for prop in item['property']
+        if isinstance(prop, dict)
+    ])
+    
+    if assets:
+        tables.append(Table(_flatten_dict(assets), 'assets_ex102_absee', accession))
+    
+    if properties:
+        tables.append(Table(_flatten_dict(properties), 'properties_ex102_absee', accession))
+    
+    return tables
 
 # def process_ma(data, accession):
 #     tables = []

datamule/document/table.py

@@ -19,6 +19,11 @@ from .mappings.twentyfivense import *
 from .mappings.twentyfourf2nt import *
 from .mappings.information_table import *
 from .mappings.submission_metadata import *
+from .mappings.ex102_abs import *
+from .mappings.d import *
+
+from pathlib import Path
+import csv
 # need to check if mappings correctly create new columns
 class Table():
     def __init__(self, data, type,accession):
@@ -27,11 +32,18 @@ class Table():
         self.type = type
         self.data = data
         self.accession = accession
-        self.columns = self.determine_columns()
+        self.columns = self.determine_columns_complete()
+
+    def determine_columns_complete(self):
+        if not self.data:
+            return []
+        return list(set().union(*(row.keys() for row in self.data)))
+
 
     def determine_columns(self):
         if len(self.data) == 0:
             return []
+        
         return self.data[0].keys()
 
     def add_column(self,column_name,value):
@@ -190,6 +202,17 @@ class Table():
         elif self.type == 'signature_schedule_13':
             mapping_dict = signature_schedule_13_dict
 
+        # D 
+        elif self.type == 'issuerList_d':
+            mapping_dict = issuer_list_d_dict
+        elif self.type == 'metadata_d':
+            mapping_dict = metadata_d_dict
+        elif self.type == 'offeringData_d':
+            mapping_dict = offering_data_d_dict
+        elif self.type == 'primaryIssuer_d':
+            mapping_dict = primary_issuer_d_dict
+        elif self.type == 'relatedPersonsList_d':
+            mapping_dict = related_persons_d_dict
         # SDR
         elif self.type == 'sdr':
             mapping_dict = sdr_dict
@@ -227,7 +250,11 @@ class Table():
             mapping_dict = item_9_24f2nt_dict
         elif self.type == 'signature_info_schedule_a':
             mapping_dict = signature_24f2nt_dict
-
+        # ABS
+        elif self.type == 'assets_ex102_absee':
+            mapping_dict = assets_dict_ex102_abs
+        elif self.type =='properties_ex102_absee':
+            mapping_dict = properties_dict_ex102_abs
         # submission metadata
         elif self.type == 'document_submission_metadata':
             mapping_dict = document_submission_metadata_dict
@@ -250,9 +277,6 @@ class Table():
             for old_key, new_key in mapping_dict.items():
                 if old_key in row:
                     ordered_row[new_key] = row.pop(old_key)
-                else:
-                    # if the old key is not present, set the new key to None
-                    ordered_row[new_key] = None
             
             # Then add any remaining keys that weren't in the mapping
             for key, value in row.items():
@@ -262,4 +286,30 @@ class Table():
             row.clear()
             row.update(ordered_row)
 
-        self.determine_columns()
+        # Update the columns after mapping
+        columns = set(self.columns)
+        # remove the old columns that are now in the mapping
+        columns.difference_update(mapping_dict.keys())
+        # add the new columns from the mapping
+        columns.update(mapping_dict.values())
+        # add the accession column to the columns
+        columns.add('accession')
+
+        self.columns = list(columns)
+
+    def write_csv(self, output_file):
+        output_file = Path(output_file)
+        fieldnames = self.columns
+        
+        # Check if the file already exists
+        if output_file.exists():
+            # Append to existing file without writing header
+            with open(output_file, 'a', newline='') as csvfile:
+                writer = csv.DictWriter(csvfile, fieldnames=fieldnames, quoting=csv.QUOTE_ALL)
+                writer.writerows(self.data)
+        else:
+            # Create new file with header
+            with open(output_file, 'w', newline='') as csvfile:
+                writer = csv.DictWriter(csvfile, fieldnames=fieldnames, quoting=csv.QUOTE_ALL)
+                writer.writeheader()
+                writer.writerows(self.data)

{datamule-1.2.6.dist-info → datamule-1.2.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 1.2.6
+Version: 1.2.8
 Summary: Making it easier to use SEC filings.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

{datamule-1.2.6.dist-info → datamule-1.2.8.dist-info}/RECORD

@@ -7,12 +7,14 @@ datamule/portfolio.py,sha256=8fiK-vfZM5-NJSvOEsDR2YDb-2njjzFk6l7BiRyrzOM,7168
 datamule/sheet.py,sha256=TvFqK9eAYuVoJ2uWdAlx5EN6vS9lke-aZf7FqtUiDBc,22304
 datamule/submission.py,sha256=Yh5nG3ioumhl6z30wJdIEmKjDDNSuo0r2xycZSIaeIg,11035
 datamule/document/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamule/document/document.py,sha256=CQqlHCVwp0TfOjE173jJSh9O-zpYuo4Ixr85vXCJA1E,9977
-datamule/document/processing.py,sha256=Wn2Dpe6vXqVDm1qgIc-nR273wKBQEtstxZbugvuPZdI,28526
-datamule/document/table.py,sha256=TiOQyFp8CLWt-hHHX8BCjKuaBgeb-uO23ekD8QoUsqM,10993
+datamule/document/document.py,sha256=menUFoeWwiY0rJnBkQiqY4NWnO0J17-qs8jFvO_1jiY,9969
+datamule/document/processing.py,sha256=eWrLxBXmoCoKyUN1gY57ikl9m0KRdACzahJT9PnWQcA,31668
+datamule/document/table.py,sha256=jC2itnolWG9DpCjw0x6Ma5yVJiHvQT3Gd_5yl53L5a4,12939
 datamule/document/mappings/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/document/mappings/atsn.py,sha256=qkZGNIhyPC3VTTOjQ8-FSCQIhUy4XeSycUGLShxNVCo,17743
 datamule/document/mappings/cfportal.py,sha256=bR9d6DDY0kJ_HGx_hND2y1PNNkZjemYZ2KdyFAcv760,25257
+datamule/document/mappings/d.py,sha256=nRpfxkHIf8wYBMmY84biU1Smci9fF9pFYCG_iUddGtU,7928
+datamule/document/mappings/ex102_abs.py,sha256=FdGKvteRh_HsYgILF-8o4R6aSsjYwcaLpJxzdru4FTE,3976
 datamule/document/mappings/ex99a_sdr.py,sha256=PNdj9I0ZhNicPObLelNmjp33EgTwzvukqkBDnwxarE0,19
 datamule/document/mappings/ex99c_sdr.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/document/mappings/ex99g_sdr.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -54,7 +56,7 @@ datamule/seclibrary/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSu
 datamule/seclibrary/bq.py,sha256=C8sb_rpXTvchprrFLcbRar4Qi0XWW25tnv1YsHSS5o4,18025
 datamule/seclibrary/downloader.py,sha256=PIgz_7ASUTZOHcUZGcD1SmLaGSbq7xe7EiJT0Z7HU4M,13653
 datamule/seclibrary/query.py,sha256=qGuursTERRbOGfoDcYcpo4oWkW3PCBW6x1Qf1Puiak4,7352
-datamule-1.2.6.dist-info/METADATA,sha256=WwrQVyxcEIzhkzy_0WQhkoXHt_nXbmzarCMa2pYl2mw,490
-datamule-1.2.6.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-datamule-1.2.6.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
-datamule-1.2.6.dist-info/RECORD,,
+datamule-1.2.8.dist-info/METADATA,sha256=mQvixJX2sn9NJbH3LkZ1H3IeRIwHwCgBjgWVVXjOt-Q,490
+datamule-1.2.8.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+datamule-1.2.8.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
+datamule-1.2.8.dist-info/RECORD,,