datamule 1.2.5__py3-none-any.whl → 1.2.9__py3-none-any.whl

datamule/__init__.py

@@ -8,6 +8,7 @@ from .index import Index
 from .package_updater import PackageUpdater
 
 
+
 # Keep the notebook environment setup
 def _is_notebook_env():
     """Check if the code is running in a Jupyter or Colab environment."""

datamule/document/document.py

@@ -118,10 +118,11 @@ class Document:
         # will deprecate this when we add html2dict
         elif self.extension in ['.htm', '.html','.txt']:
 
-            if self.type == '10-K':
-                mapping_dict = dict_10k
-            elif self.type == '10-Q':
+
+            if self.type == '10-Q':
                 mapping_dict = dict_10q
+            elif self.type == '10-K':
+                mapping_dict = dict_10k
             elif self.type == '8-K':
                 mapping_dict = dict_8k
             elif self.type == 'SC 13D':
@@ -140,18 +141,21 @@ class Document:
         with open(output_filename, 'w',encoding='utf-8') as f:
             json.dump(self.data, f, indent=2)
 
-    def to_tabular(self):
-        if self.extension != '.xml':
+    def tables(self):
+        if self.type == 'submission_metadata':
+            return process_tabular_data(self)
+        elif self.extension != '.xml':
             return []
-        self.parse()
-        return process_tabular_data(self)
+        else:
+            self.parse()
+            return process_tabular_data(self)
 
 
     def write_csv(self, output_folder):
         output_folder = Path(output_folder)
         output_folder.mkdir(exist_ok=True)
             
-        tables = self.to_tabular()
+        tables = self.tables()
 
         if not tables:
             return

datamule/document/mappings/d.py

@@ -0,0 +1,125 @@
+issuer_list_d_dict = {
+    'issuer_issuerAddress_street1': 'issuerStreet1',
+    'issuer_cik': 'issuerCik', 
+    'issuer_issuerAddress_stateOrCountryDescription': 'issuerStateOrCountryDescription',
+    'issuer_issuerAddress_zipCode': 'issuerZipCode',
+    'issuer_issuerPhoneNumber': 'issuerPhoneNumber',
+    'issuer_yearOfInc_value': 'yearOfIncValue',
+    'issuer_issuerAddress_stateOrCountry': 'issuerStateOrCountry',
+    'issuer_jurisdictionOfInc': 'jurisdictionOfInc',
+    'issuer_entityType': 'entityType',
+    'issuer_issuerAddress_street2': 'issuerStreet2',
+    'issuer_entityName': 'entityName',
+    'accession': 'accession',
+    'issuer_edgarPreviousNameList_value': 'edgarPreviousNameListValue',
+    'issuer_entityTypeOtherDesc': 'entityTypeOtherDesc',
+    'issuer_yearOfInc_yetToBeFormed': 'yearOfIncYetToBeFormed',
+    'issuer_yearOfInc_withinFiveYears': 'yearOfIncWithinFiveYears',
+    'issuer_issuerPreviousNameList_value': 'issuerPreviousNameListValue',
+    'issuer_issuerAddress_city': 'issuerCity'
+}
+
+metadata_d_dict = {
+"testOrLive" : "testOrLive",
+"schemaVersion" : "schemaVersion",
+"accession" : "accession",
+"submissionType" : "submissionType",
+}
+offering_data_d_dict = {
+    'salesCompensationList_recipient_foreignSolicitation': 'foreignSolicitation',
+    'typeOfFiling_dateOfFirstSale_yetToOccur': 'dateOfFirstSaleYetToOccur',
+    'industryGroup_investmentFundInfo_is40Act': 'is40Act',
+    'salesCommissionsFindersFees_findersFees_dollarAmount': 'findersFeesAmount',
+    'offeringSalesAmounts_totalRemaining': 'totalRemaining',
+    'issuerSize_aggregateNetAssetValueRange': 'aggregateNetAssetValueRange',
+    'typesOfSecuritiesOffered_isSecurityToBeAcquiredType': 'isSecurityToBeAcquiredType',
+    'salesCompensationList_recipient_recipientAddress_stateOrCountryDescription': 'recipientStateOrCountryDescription',
+    'typesOfSecuritiesOffered_isEquityType': 'isEquityType',
+    'investors_totalNumberAlreadyInvested': 'totalNumberAlreadyInvested',
+    'minimumInvestmentAccepted': 'minimumInvestmentAccepted',
+    'salesCompensationList_recipient_associatedBDName': 'associatedBDName',
+    'salesCompensationList_recipient_statesOfSolicitationList_state': 'statesOfSolicitationState',
+    'businessCombinationTransaction_isBusinessCombinationTransaction': 'isBusinessCombinationTransaction',
+    'useOfProceeds_grossProceedsUsed_isEstimate': 'grossProceedsUsedIsEstimate',
+    'federalExemptionsExclusions_item': 'federalExemptionsExclusionsItem',
+    'useOfProceeds_grossProceedsUsed_dollarAmount': 'grossProceedsUsedAmount',
+    'industryGroup_industryGroupType': 'industryGroupType',
+    'signatureBlock_signature_nameOfSigner': 'nameOfSigner',
+    'signatureBlock_signature_signatureDate': 'signatureDate',
+    'salesCommissionsFindersFees_salesCommissions_isEstimate': 'salesCommissionsIsEstimate',
+    'typesOfSecuritiesOffered_isOtherType': 'isOtherType',
+    'salesCompensationList_recipient_associatedBDCRDNumber': 'associatedBDCRDNumber',
+    'salesCompensationList_recipient_recipientAddress_stateOrCountry': 'recipientStateOrCountry',
+    'typesOfSecuritiesOffered_descriptionOfOtherType': 'descriptionOfOtherType',
+    'salesCommissionsFindersFees_salesCommissions_dollarAmount': 'salesCommissionsAmount',
+    'useOfProceeds_clarificationOfResponse': 'useOfProceedsClarification',
+    'accession': 'accession',
+    'typesOfSecuritiesOffered_isPooledInvestmentFundType': 'isPooledInvestmentFundType',
+    'salesCompensationList_recipient_statesOfSolicitationList_value': 'statesOfSolicitationValue',
+    'signatureBlock_signature_signatureName': 'signatureName',
+    'typeOfFiling_newOrAmendment_isAmendment': 'isAmendment',
+    'issuerSize_revenueRange': 'revenueRange',
+    'salesCommissionsFindersFees_clarificationOfResponse': 'salesCommissionsFindersFeesClarification',
+    'salesCompensationList_recipient_recipientAddress_zipCode': 'recipientZipCode',
+    'salesCompensationList_recipient_recipientAddress_city': 'recipientCity',
+    'typesOfSecuritiesOffered_isOptionToAcquireType': 'isOptionToAcquireType',
+    'businessCombinationTransaction_clarificationOfResponse': 'businessCombinationClarification',
+    'typesOfSecuritiesOffered_isTenantInCommonType': 'isTenantInCommonType',
+    'salesCompensationList_recipient_statesOfSolicitationList_description': 'statesOfSolicitationDescription',
+    'offeringSalesAmounts_totalOfferingAmount': 'totalOfferingAmount',
+    'investors_numberNonAccreditedInvestors': 'numberNonAccreditedInvestors',
+    'signatureBlock_authorizedRepresentative': 'authorizedRepresentative',
+    'signatureBlock_signature_issuerName': 'issuerName',
+    'salesCompensationList_recipient_recipientAddress_street2': 'recipientStreet2',
+    'typesOfSecuritiesOffered_isDebtType': 'isDebtType',
+    'salesCompensationList_recipient_recipientAddress_street1': 'recipientStreet1',
+    'signatureBlock_signature_signatureTitle': 'signatureTitle',
+    'industryGroup_investmentFundInfo_investmentFundType': 'investmentFundType',
+    'salesCommissionsFindersFees_findersFees_isEstimate': 'findersFeesIsEstimate',
+    'typeOfFiling_dateOfFirstSale_value': 'dateOfFirstSaleValue',
+    'offeringSalesAmounts_totalAmountSold': 'totalAmountSold',
+    'offeringSalesAmounts_clarificationOfResponse': 'offeringSalesAmountsClarification',
+    'investors_hasNonAccreditedInvestors': 'hasNonAccreditedInvestors',
+    'salesCompensationList_recipient_recipientCRDNumber': 'recipientCRDNumber',
+    'typesOfSecuritiesOffered_isMineralPropertyType': 'isMineralPropertyType',
+    'salesCompensationList_recipient_recipientName': 'recipientName',
+    'durationOfOffering_moreThanOneYear': 'moreThanOneYear'
+}
+primary_issuer_d_dict = {
+    'yearOfInc_withinFiveYears': 'yearOfIncWithinFiveYears',
+    'entityTypeOtherDesc': 'entityTypeOtherDesc',
+    'jurisdictionOfInc': 'jurisdictionOfInc',
+    'issuerAddress_street1': 'issuerStreet1',
+    'issuerAddress_zipCode': 'issuerZipCode',
+    'issuerPreviousNameList_previousName': 'issuerPreviousName',
+    'entityType': 'entityType',
+    'issuerPreviousNameList_value': 'issuerPreviousNameListValue',
+    'issuerPhoneNumber': 'issuerPhoneNumber',
+    'yearOfInc_value': 'yearOfIncValue',
+    'yearOfInc_yetToBeFormed': 'yearOfIncYetToBeFormed',
+    'edgarPreviousNameList_previousName': 'edgarPreviousName',
+    'edgarPreviousNameList_value': 'edgarPreviousNameListValue',
+    'issuerAddress_stateOrCountry': 'issuerStateOrCountry',
+    'entityName': 'entityName',
+    'accession': 'accession',
+    'issuerAddress_street2': 'issuerStreet2',
+    'issuerAddress_city': 'issuerCity',
+    'issuerAddress_stateOrCountryDescription': 'issuerStateOrCountryDescription',
+    'cik': 'cik',
+    'yearOfInc_overFiveYears': 'yearOfIncOverFiveYears'
+}
+
+related_persons_d_dict = {
+    'relatedPersonInfo_relatedPersonAddress_stateOrCountry': 'relatedPersonStateOrCountry',
+    'relatedPersonInfo_relatedPersonRelationshipList_relationship': 'relatedPersonRelationship',
+    'relatedPersonInfo_relationshipClarification': 'relationshipClarification',
+    'relatedPersonInfo_relatedPersonName_lastName': 'relatedPersonLastName',
+    'accession': 'accession',
+    'relatedPersonInfo_relatedPersonName_middleName': 'relatedPersonMiddleName',
+    'relatedPersonInfo_relatedPersonAddress_zipCode': 'relatedPersonZipCode',
+    'relatedPersonInfo_relatedPersonAddress_city': 'relatedPersonCity',
+    'relatedPersonInfo_relatedPersonAddress_street1': 'relatedPersonStreet1',
+    'relatedPersonInfo_relatedPersonAddress_stateOrCountryDescription': 'relatedPersonStateOrCountryDescription',
+    'relatedPersonInfo_relatedPersonName_firstName': 'relatedPersonFirstName',
+    'relatedPersonInfo_relatedPersonAddress_street2': 'relatedPersonStreet2'
+}

datamule/document/mappings/ex102_abs.py

@@ -0,0 +1,63 @@
+# Assets dictionary mapping
+assets_dict_ex102_abs = {
+    'assetNumber': 'assetNumber',
+    'DefeasedStatusCode': 'DefeasedStatusCode',
+    'defeasanceOptionStartDate': 'defeasanceOptionStartDate',
+    'mostRecentDebtServiceCoverageNetOperatingIncomePercentage': 'mostRecentDebtServiceCoverageNetOperatingIncomePercentage',
+    'mostRecentDebtServiceAmount': 'mostRecentDebtServiceAmount',
+    'debtServiceCoverageSecuritizationCode': 'debtServiceCoverageSecuritizationCode',
+    'debtServiceCoverageNetOperatingIncomeSecuritizationPercentage': 'debtServiceCoverageNetOperatingIncomeSecuritizationPercentage',
+    'valuationSecuritizationDate': 'valuationSecuritizationDate',
+    'physicalOccupancySecuritizationPercentage': 'physicalOccupancySecuritizationPercentage',
+    'revenueSecuritizationAmount': 'revenueSecuritizationAmount',
+    'valuationSourceSecuritizationCode': 'valuationSourceSecuritizationCode',
+    'financialsSecuritizationDate': 'financialsSecuritizationDate',
+    'mostRecentNetCashFlowAmount': 'mostRecentNetCashFlowAmount',
+    'operatingExpensesAmount': 'operatingExpensesAmount',
+    'operatingExpensesSecuritizationAmount': 'operatingExpensesSecuritizationAmount',
+    'netOperatingIncomeNetCashFlowSecuritizationCode': 'netOperatingIncomeNetCashFlowSecuritizationCode',
+    'mostRecentValuationSourceCode': 'mostRecentValuationSourceCode',
+    'mostRecentDebtServiceCoverageNetCashFlowpercentage': 'mostRecentDebtServiceCoverageNetCashFlowpercentage',
+    'debtServiceCoverageNetCashFlowSecuritizationPercentage': 'debtServiceCoverageNetCashFlowSecuritizationPercentage',
+    'mostRecentAnnualLeaseRolloverReviewDate': 'mostRecentAnnualLeaseRolloverReviewDate',
+    'mostRecentRevenueAmount': 'mostRecentRevenueAmount',
+    'mostRecentPhysicalOccupancyPercentage': 'mostRecentPhysicalOccupancyPercentage',
+    'mostRecentNetOperatingIncomeAmount': 'mostRecentNetOperatingIncomeAmount',
+    'netOperatingIncomeSecuritizationAmount': 'netOperatingIncomeSecuritizationAmount',
+    'netOperatingIncomeNetCashFlowCode': 'netOperatingIncomeNetCashFlowCode',
+    'mostRecentFinancialsStartDate': 'mostRecentFinancialsStartDate',
+    'mostRecentFinancialsEndDate': 'mostRecentFinancialsEndDate',
+    'accession': 'accession',
+    'valuationSecuritizationAmount': 'valuationSecuritizationAmount',
+    'mostRecentValuationDate': 'mostRecentValuationDate',
+    'mostRecentValuationAmount': 'mostRecentValuationAmount',
+    'mostRecentDebtServiceCoverageCode': 'mostRecentDebtServiceCoverageCode',
+    'netCashFlowFlowSecuritizationAmount': 'netCashFlowFlowSecuritizationAmount'
+}
+
+# Properties dictionary mapping
+properties_dict_ex102_abs = {
+    'unitsBedsRoomsNumber': 'unitsBedsRoomsNumber',
+    'propertyCounty': 'propertyCounty',
+    'squareFeetLargestTenantNumber': 'squareFeetLargestTenantNumber',
+    'netRentableSquareFeetNumber': 'netRentableSquareFeetNumber',
+    'leaseExpirationThirdLargestTenantDate': 'leaseExpirationThirdLargestTenantDate',
+    'leaseExpirationLargestTenantDate': 'leaseExpirationLargestTenantDate',
+    'propertyZip': 'propertyZip',
+    'squareFeetThirdLargestTenantNumber': 'squareFeetThirdLargestTenantNumber',
+    'propertyStatusCode': 'propertyStatusCode',
+    'propertyState': 'propertyState',
+    'yearBuiltNumber': 'yearBuiltNumber',
+    'propertyCity': 'propertyCity',
+    'propertyName': 'propertyName',
+    'propertyAddress': 'propertyAddress',
+    'yearLastRenovated': 'yearLastRenovated',
+    'leaseExpirationSecondLargestTenantDate': 'leaseExpirationSecondLargestTenantDate',
+    'thirdLargestTenant': 'thirdLargestTenant',
+    'unitsBedsRoomsSecuritizationNumber': 'unitsBedsRoomsSecuritizationNumber',
+    'propertyTypeCode': 'propertyTypeCode',
+    'largestTenant': 'largestTenant',
+    'squareFeetSecondLargestTenantNumber': 'squareFeetSecondLargestTenantNumber',
+    'netRentableSquareFeetSecuritizationNumber': 'netRentableSquareFeetSecuritizationNumber',
+    'secondLargestTenant': 'secondLargestTenant'
+}

datamule/document/mappings/information_table.py

@@ -1,3 +1,4 @@
+# Ready for mass testing
 
 # Information Table (13F-HR Securities) mapping
 information_table_dict = {

datamule/document/mappings/ownership.py

@@ -1,4 +1,4 @@
-# Mapping dictionaries for SEC filing table types based on actual field occurrences
+# Ready for mass testing
 
 # Non-derivative transaction ownership mapping
 non_derivative_transaction_ownership_dict = {

datamule/document/mappings/proxy_voting_record.py

@@ -1 +1,17 @@
-proxy_voting_record_dict = {}
+proxy_voting_record_dict = {
+    'meetingDate': 'meetingDate',
+    'accession': 'accessionNumber',
+    'vote_voteRecord_managementRecommendation': 'managementRecommendation',
+    'sharesVoted': 'sharesVoted',  # Top-level sharesVoted
+    'vote_voteRecord_howVoted': 'howVoted',
+    'sharesOnLoan': 'sharesOnLoan',
+    'cusip': 'cusip',
+    'issuerName': 'issuerName',
+    'voteCategories_voteCategory_categoryType': 'categoryType',
+    'voteDescription': 'voteDescription',
+    'voteManager_otherManagers_otherManager': 'otherManager',
+    'vote_voteRecord_sharesVoted': 'recordSharesVoted',  # To distinguish from top-level sharesVoted
+    'isin': 'isin',
+    'voteSource': 'voteSource',
+    'voteSeries': 'voteSeries'
+}

datamule/document/mappings/submission_metadata.py

@@ -0,0 +1,9 @@
+# Note: submission_metadata is my designation, not SEC for the header of the Submission tag
+
+document_submission_metadata_dict = {
+    'accession':'accession',
+    'type':'type',
+    'sequence' : 'sequence',
+    'filename' : 'filename', 
+    'description':'description'
+}

datamule/document/mappings/thirteenfhr.py

@@ -1,5 +1,72 @@
+# Ready for mass testing
 
 # 13F-HR (Institutional Investment Manager Holdings) mapping
-thirteenfhr_dict = {
-
-}
+thirteenfhr_dict =  {
+    # Cover Page Mapping
+    'formData_coverPage_reportCalendarOrQuarter': 'reportCalendarOrQuarter',
+    'formData_coverPage_filingManager_name': 'filingManagerName',
+    'formData_coverPage_filingManager_address_street1': 'filingManagerStreet1',
+    'formData_coverPage_filingManager_address_street2': 'filingManagerStreet2',
+    'formData_coverPage_filingManager_address_city': 'filingManagerCity',
+    'formData_coverPage_filingManager_address_stateOrCountry': 'filingManagerStateOrCountry',
+    'formData_coverPage_filingManager_address_zipCode': 'filingManagerZipCode',
+    'formData_coverPage_crdNumber': 'crdNumber',
+    'formData_coverPage_secFileNumber': 'secFileNumber',
+    'formData_coverPage_form13FFileNumber': 'form13FFileNumber',
+    'formData_coverPage_reportType': 'reportType',
+    'formData_coverPage_isAmendment': 'isAmendment',
+    'formData_coverPage_amendmentNo': 'amendmentNo',
+    'formData_coverPage_amendmentInfo_amendmentType': 'amendmentType',
+    'formData_coverPage_amendmentInfo_confDeniedExpired': 'confDeniedExpired',
+    'formData_coverPage_additionalInformation': 'additionalInformation',
+    'formData_coverPage_provideInfoForInstruction5': 'provideInfoForInstruction5',
+    
+    # Other Managers Info Mapping
+    'formData_coverPage_otherManagersInfo_otherManager': 'otherManager',
+    'formData_coverPage_otherManagersInfo_otherManager_cik': 'otherManagerCik',
+    'formData_coverPage_otherManagersInfo_otherManager_name': 'otherManagerName',
+    'formData_coverPage_otherManagersInfo_otherManager_crdNumber': 'otherManagerCrdNumber',
+    'formData_coverPage_otherManagersInfo_otherManager_secFileNumber': 'otherManagerSecFileNumber',
+    'formData_coverPage_otherManagersInfo_otherManager_form13FFileNumber': 'otherManagerForm13FFileNumber',
+    
+    # Summary Page Mapping
+    'formData_summaryPage_isConfidentialOmitted': 'isConfidentialOmitted',
+    'formData_summaryPage_otherIncludedManagersCount': 'otherIncludedManagersCount',
+    'formData_summaryPage_tableEntryTotal': 'tableEntryTotal',
+    'formData_summaryPage_tableValueTotal': 'tableValueTotal',
+    
+    # Other Managers 2 Info Mapping
+    'formData_summaryPage_otherManagers2Info_otherManager2': 'otherManager2',
+    'formData_summaryPage_otherManagers2Info_otherManager2_sequenceNumber': 'otherManager2SequenceNumber',
+    'formData_summaryPage_otherManagers2Info_otherManager2_otherManager_cik': 'otherManager2Cik',
+    'formData_summaryPage_otherManagers2Info_otherManager2_otherManager_name': 'otherManager2Name',
+    'formData_summaryPage_otherManagers2Info_otherManager2_otherManager_crdNumber': 'otherManager2CrdNumber',
+    'formData_summaryPage_otherManagers2Info_otherManager2_otherManager_secFileNumber': 'otherManager2SecFileNumber',
+    'formData_summaryPage_otherManagers2Info_otherManager2_otherManager_form13FFileNumber': 'otherManager2Form13FFileNumber',
+    
+    # Signature Block Mapping
+    'formData_signatureBlock_name': 'signatureName',
+    'formData_signatureBlock_title': 'signatureTitle',
+    'formData_signatureBlock_phone': 'signaturePhone',
+    'formData_signatureBlock_signature': 'signature',
+    'formData_signatureBlock_city': 'signatureCity',
+    'formData_signatureBlock_stateOrCountry': 'signatureStateOrCountry',
+    'formData_signatureBlock_signatureDate': 'signatureDate',
+    
+    # Header Data Mapping
+    'headerData_filerInfo_periodOfReport': 'periodOfReport',
+    'headerData_filerInfo_filer_fileNumber': 'filerFileNumber',
+    'headerData_filerInfo_filer_credentials_cik': 'filerCik',
+    'headerData_filerInfo_filer_credentials_ccc': 'filerCcc',
+    'headerData_filerInfo_flags_confirmingCopyFlag': 'confirmingCopyFlag',
+    'headerData_filerInfo_flags_returnCopyFlag': 'returnCopyFlag',
+    'headerData_filerInfo_flags_overrideInternetFlag': 'overrideInternetFlag',
+    'headerData_filerInfo_denovoRequest': 'denovoRequest',
+    'headerData_filerInfo_liveTestFlag': 'liveTestFlag',
+    'headerData_submissionType': 'submissionType',
+    
+    # Schema and Metadata Mapping
+    'schemaLocation': 'schemaLocation',
+    'schemaVersion': 'schemaVersion',
+    'accession': 'accessionNumber'
+}

datamule/document/mappings/twentyfivense.py

@@ -1,3 +1,4 @@
+# Ready for mass testing
 # 25-NSE mapping
 twentyfive_nse_dict = {
     'descriptionClassSecurity': 'securityDescription',

datamule/document/processing.py

@@ -17,6 +17,17 @@ def process_tabular_data(self):
         tables = process_13fhr(self.data, self.accession)
     elif self.type in ["INFORMATION TABLE"]:
         tables = process_information_table(self.data, self.accession)
+    elif self.type in ["25-NSE", "25-NSE/A"]:
+        tables = process_25nse(self.data, self.accession)
+    # complete mark:
+    elif self.type in ["EX-102"]:
+        tables = process_ex102_abs(self.data, self.accession)
+    elif self.type in ["D","D/A"]:
+        tables = process_d(self.data, self.accession)
+    elif self.type in ["N-PX","N-PX/A"]:
+        tables = process_npx(self.data, self.accession)
+
+
     elif self.type in ["SBSEF","SBSEF/A","SBSEF-V","SBSEF-W"]:
         tables = process_sbsef(self.data, self.accession)
     elif self.type in ["SDR","SDR/A","SDR-W","SDR-A"]:
@@ -33,8 +44,7 @@ def process_tabular_data(self):
         tables = process_144(self.data, self.accession)
     elif self.type in ["24F-2NT", "24F-2NT/A"]:
         tables = process_24f2nt(self.data, self.accession)
-    elif self.type in ["25-NSE", "25-NSE/A"]:
-        tables = process_25nse(self.data, self.accession)
+
     elif self.type in ["ATS-N", "ATS-N/A"]:
         tables = process_ats(self.data, self.accession)
     # elif self.type in ["C","C-W","C-U","C-U-W","C/A","C/A-W",
@@ -42,8 +52,7 @@ def process_tabular_data(self):
     #     tables = process_c(self.data, self.accession)
     elif self.type in ["CFPORTAL","CFPORTAL/A","CFPORTAL-W"]:
         tables = process_cfportal(self.data, self.accession)
-    # elif self.type in ["D","D/A"]:
-    #     tables = process_d(self.data, self.accession)
+
     # elif self.type in ["MA","MA-A","MA/A","MA-I","MA-I/A","MA-W"]:
     #     tables = process_ma(self.data, self.accession)
     # elif self.type in ["N-CEN","N-CEN/A"]:
@@ -53,8 +62,7 @@ def process_tabular_data(self):
     #     tables = process_nmfp(self.data, self.accession)
     # elif self.type in ["NPORT-P","NPORT-P/A"]:
     #     tables = process_nportp(self.data, self.accession)
-    elif self.type in ["N-PX","N-PX/A"]:
-        tables = process_npx(self.data, self.accession)
+
     # elif self.type in ["TA-1","TA-1/A","TA-W","TA-2","TA-2/A"]:
     #     tables = process_ta(self.data, self.accession)
     elif self.type in ["X-17A-5","X-17A-5/A"]:
@@ -66,10 +74,11 @@ def process_tabular_data(self):
         tables = process_reg_a(self.data, self.accession)
     # elif self.type in ["SBSE","SBSE/A","SBSE-A","SBSE-A/A","SBSE-BD","SBSE-BD/A","SBSE-C","SBSE-W","SBSE-CCO-RPT","SBSE-CCO-RPT/A"]:
     #     tables = process_sbs(self.data, self.accession)
-    # elif self.type in ["EX-102"]:
-    #     tables = process_ex102_abs(self.data, self.accession)
+
     elif self.type == "PROXY VOTING RECORD":
         tables = process_proxy_voting_record(self.data, self.accession)
+    elif self.type == 'submission_metadata':
+        tables = process_submission_metadata(self.content, self.accession)
     else:
         warn(f"Processing for {self.type} is not implemented yet.")
         return []
@@ -95,6 +104,67 @@ def _flatten_dict(d, parent_key=''):
                 
     return items
 
+# flattens in a different way
+def flatten_dict_to_rows(d, parent_key='', sep='_'):
+
+    if isinstance(d, list):
+        # If input is a list, flatten each item and return all rows
+        all_rows = []
+        for item in d:
+            all_rows.extend(flatten_dict_to_rows(item, parent_key, sep))
+        return all_rows
+    
+    if not isinstance(d, dict):
+        # If input is a primitive value, return single row
+        return [{parent_key: d}] if parent_key else []
+    
+    # Input is a dictionary
+    rows = [{}]
+    
+    for k, v in d.items():
+        new_key = f"{parent_key}{sep}{k}" if parent_key else k
+        
+        if isinstance(v, dict):
+            # Recursively flatten nested dictionaries
+            nested_rows = flatten_dict_to_rows(v, new_key, sep)
+            # Cross-product with existing rows
+            new_rows = []
+            for row in rows:
+                for nested_row in nested_rows:
+                    combined_row = row.copy()
+                    combined_row.update(nested_row)
+                    new_rows.append(combined_row)
+            rows = new_rows
+            
+        elif isinstance(v, list):
+            # Handle lists - create multiple rows
+            if not v:  # Empty list
+                for row in rows:
+                    row[new_key] = ''
+            else:
+                new_rows = []
+                for row in rows:
+                    for list_item in v:
+                        new_row = row.copy()
+                        if isinstance(list_item, dict):
+                            # Recursively flatten dict items in list
+                            nested_rows = flatten_dict_to_rows(list_item, new_key, sep)
+                            for nested_row in nested_rows:
+                                combined_row = new_row.copy()
+                                combined_row.update(nested_row)
+                                new_rows.append(combined_row)
+                        else:
+                            # Primitive value in list
+                            new_row[new_key] = list_item
+                            new_rows.append(new_row)
+                rows = new_rows
+        else:
+            # Handle primitive values
+            for row in rows:
+                row[new_key] = v
+    
+    return rows
+
 def process_ownership(data, accession):
     tables = []
     if 'ownershipDocument' not in data:
@@ -346,33 +416,41 @@ def process_cfportal(data, accession):
     
     return tables
 
-# def process_d(data, accession):
-#     tables = []
-#     primary_issuer = safe_get(data, ['edgarSubmission', 'primaryIssuer'])
-#     if primary_issuer:
-#         metadata = Table(_flatten_dict(primary_issuer), 'metadata_d', accession)
-        
-#         metadata_columns = ['schemaVersion', 'submissionType', 'testOrLive', 'returnCopy', 'contactData', 'notificationAddressList']
-#         for col in metadata_columns:
-#             col_data = safe_get(data, ['edgarSubmission', col])
-#             if col_data:
-#                 metadata.add_column(col, col_data)
-        
-#         tables.append(metadata)
-    
-#     issuer_list = safe_get(data, ['edgarSubmission', 'issuerList'])
-#     if issuer_list:
-#         tables.append(Table(_flatten_dict(issuer_list), 'primary_issuer_d', accession))
-    
-#     offering_data = safe_get(data, ['edgarSubmission', 'offeringData'])
-#     if offering_data:
-#         tables.append(Table(_flatten_dict(offering_data), 'offering_data_d', accession))
-    
-#     related_persons_list = safe_get(data, ['edgarSubmission', 'relatedPersonsList'])
-#     if related_persons_list:
-#         tables.append(Table(_flatten_dict(related_persons_list), 'related_persons_list_d', accession))
+def process_d(data, accession):
+    tables = []
+    groups = [('contactData', 'contact_data_d'),
+                ('notificationAddressList', 'notification_address_list_d'),
+                ('primaryIssuer', 'primary_issuer_d'),
+                ('issuerList', 'issuer_list_d'),
+                ('relatedPersonsList', 'related_persons_list_d'),
+                ('offeringData', 'offering_data_d'),
+    ]
+    for group,table_type in groups:
+        if group == 'relatedPersonList':
+            group_data = data['edgarSubmission'].pop('relatedPersonInfo', None)
+            data['edgarSubmission'].pop(group, None)
+        elif group == 'issuerList':
+            group_data = data['edgarSubmission'].pop('issuerList', None)
+        else:
+            group_data = data['edgarSubmission'].pop(group, None)
+            
+        if group_data:
+            # Special handling ONLY for relatedPersonsList
+            if group in ['relatedPersonsList', 'issuerList','offeringData']:
+                # Use the new flatten_dict_to_rows ONLY for this key
+                flattened_rows = flatten_dict_to_rows(group_data)
+                if flattened_rows:
+                    tables.append(Table(flattened_rows, table_type, accession))
+            else:
+                # Everything else remains EXACTLY the same
+                tables.append(Table(_flatten_dict(group_data), table_type, accession))
+
+
+
+    metadata_table = Table(_flatten_dict(data['edgarSubmission']), 'metadata_d', accession)
+    tables.append(metadata_table)
     
-#     return tables
+    return tables
 
 # def process_nmfp(data, accession):
 #     tables = []
@@ -583,13 +661,39 @@ def process_reg_a(data, accession):
     
 #     return tables
 
-# def process_ex102_abs(data, accession):
-#     tables = []
-#     asset_data = safe_get(data, ['assetData'])
-#     if asset_data:
-#         tables.append(Table(_flatten_dict(asset_data), 'abs', accession))
-#     raise NotImplementedError("Need to implement the rest of the ABS processing")
-#     return tables
+def process_ex102_abs(data, accession):
+    tables = []
+    data = safe_get(data, ['assetData', 'assets'])
+    
+    # Create assets list: all items without their 'property' field
+    assets = [{k: v for k, v in item.items() if k != 'property'} for item in data]
+
+    # Create properties list in a more vectorized way
+    properties = []
+    
+    # Handle dictionary properties
+    properties.extend([
+        item['property'] | {'assetNumber': item['assetNumber']}
+        for item in data
+        if 'property' in item and isinstance(item['property'], dict)
+    ])
+    
+    # Handle list properties - flatten in one operation
+    properties.extend([
+        prop | {'assetNumber': item['assetNumber']}
+        for item in data
+        if 'property' in item and isinstance(item['property'], list)
+        for prop in item['property']
+        if isinstance(prop, dict)
+    ])
+    
+    if assets:
+        tables.append(Table(_flatten_dict(assets), 'assets_ex102_absee', accession))
+    
+    if properties:
+        tables.append(Table(_flatten_dict(properties), 'properties_ex102_absee', accession))
+    
+    return tables
 
 # def process_ma(data, accession):
 #     tables = []
@@ -601,4 +705,28 @@ def process_reg_a(data, accession):
 #     raise NotImplementedError("Need to implement the rest of the MA processing")
 
 # def process_ncen(data, accession):
-#     raise NotImplementedError("Need to implement the N-CEN processing")
+#     raise NotImplementedError("Need to implement the N-CEN processing")
+
+# WIP
+# Note: going to pause this for now, as I don't have a great way of putting this in a csv.
+def process_submission_metadata(data,accession):
+    tables = []
+    document_data = safe_get(data, ['documents'])
+    if document_data:
+        tables.append(Table(_flatten_dict(document_data), 'document_submission_metadata', accession))
+
+    reporting_owner_data = safe_get(data,['reporting-owner'])
+    if reporting_owner_data:
+        tables.append(Table(_flatten_dict(reporting_owner_data), 'reporting_owner_submission_metadata', accession))
+
+    issuer_data = safe_get(data,['issuer'])
+    if issuer_data:
+        tables.append(Table(_flatten_dict(issuer_data), 'issuer_submission_metadata', accession))
+        
+    # # construct metadata
+    # accession-number date-of-filing-date-change, depositor-cik effectiveness-date
+
+    # # other tables
+    # depositor, securitizer
+        
+    return tables