datamule 1.2.5__tar.gz → 1.2.7__tar.gz

{datamule-1.2.5 → datamule-1.2.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 1.2.5
+Version: 1.2.7
 Summary: Making it easier to use SEC filings.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman
@@ -10,7 +10,6 @@ Requires-Dist: tqdm
 Requires-Dist: requests
 Requires-Dist: nest_asyncio
 Requires-Dist: aiofiles
-Requires-Dist: polars
 Requires-Dist: setuptools
 Requires-Dist: selectolax
 Requires-Dist: pytz

{datamule-1.2.5 → datamule-1.2.7}/datamule/__init__.py

@@ -8,6 +8,7 @@ from .index import Index
 from .package_updater import PackageUpdater
 
 
+
 # Keep the notebook environment setup
 def _is_notebook_env():
     """Check if the code is running in a Jupyter or Colab environment."""

{datamule-1.2.5 → datamule-1.2.7}/datamule/document/document.py

@@ -118,10 +118,11 @@ class Document:
         # will deprecate this when we add html2dict
         elif self.extension in ['.htm', '.html','.txt']:
 
-            if self.type == '10-K':
-                mapping_dict = dict_10k
-            elif self.type == '10-Q':
+
+            if self.type == '10-Q':
                 mapping_dict = dict_10q
+            elif self.type == '10-K':
+                mapping_dict = dict_10k
             elif self.type == '8-K':
                 mapping_dict = dict_8k
             elif self.type == 'SC 13D':
@@ -140,18 +141,21 @@ class Document:
         with open(output_filename, 'w',encoding='utf-8') as f:
             json.dump(self.data, f, indent=2)
 
-    def to_tabular(self):
-        if self.extension != '.xml':
+    def tables(self):
+        if self.type == 'submission_metadata':
+            return process_tabular_data(self)
+        elif self.extension != '.xml':
             return []
-        self.parse()
-        return process_tabular_data(self)
+        else:
+            self.parse()
+            return process_tabular_data(self)
 
 
     def write_csv(self, output_folder):
         output_folder = Path(output_folder)
         output_folder.mkdir(exist_ok=True)
             
-        tables = self.to_tabular()
+        tables = self.tables()
 
         if not tables:
             return

datamule-1.2.7/datamule/document/mappings/ex102_abs.py

@@ -0,0 +1,63 @@
+# Assets dictionary mapping
+assets_dict_ex102_abs = {
+    'assetNumber': 'assetNumber',
+    'DefeasedStatusCode': 'DefeasedStatusCode',
+    'defeasanceOptionStartDate': 'defeasanceOptionStartDate',
+    'mostRecentDebtServiceCoverageNetOperatingIncomePercentage': 'mostRecentDebtServiceCoverageNetOperatingIncomePercentage',
+    'mostRecentDebtServiceAmount': 'mostRecentDebtServiceAmount',
+    'debtServiceCoverageSecuritizationCode': 'debtServiceCoverageSecuritizationCode',
+    'debtServiceCoverageNetOperatingIncomeSecuritizationPercentage': 'debtServiceCoverageNetOperatingIncomeSecuritizationPercentage',
+    'valuationSecuritizationDate': 'valuationSecuritizationDate',
+    'physicalOccupancySecuritizationPercentage': 'physicalOccupancySecuritizationPercentage',
+    'revenueSecuritizationAmount': 'revenueSecuritizationAmount',
+    'valuationSourceSecuritizationCode': 'valuationSourceSecuritizationCode',
+    'financialsSecuritizationDate': 'financialsSecuritizationDate',
+    'mostRecentNetCashFlowAmount': 'mostRecentNetCashFlowAmount',
+    'operatingExpensesAmount': 'operatingExpensesAmount',
+    'operatingExpensesSecuritizationAmount': 'operatingExpensesSecuritizationAmount',
+    'netOperatingIncomeNetCashFlowSecuritizationCode': 'netOperatingIncomeNetCashFlowSecuritizationCode',
+    'mostRecentValuationSourceCode': 'mostRecentValuationSourceCode',
+    'mostRecentDebtServiceCoverageNetCashFlowpercentage': 'mostRecentDebtServiceCoverageNetCashFlowpercentage',
+    'debtServiceCoverageNetCashFlowSecuritizationPercentage': 'debtServiceCoverageNetCashFlowSecuritizationPercentage',
+    'mostRecentAnnualLeaseRolloverReviewDate': 'mostRecentAnnualLeaseRolloverReviewDate',
+    'mostRecentRevenueAmount': 'mostRecentRevenueAmount',
+    'mostRecentPhysicalOccupancyPercentage': 'mostRecentPhysicalOccupancyPercentage',
+    'mostRecentNetOperatingIncomeAmount': 'mostRecentNetOperatingIncomeAmount',
+    'netOperatingIncomeSecuritizationAmount': 'netOperatingIncomeSecuritizationAmount',
+    'netOperatingIncomeNetCashFlowCode': 'netOperatingIncomeNetCashFlowCode',
+    'mostRecentFinancialsStartDate': 'mostRecentFinancialsStartDate',
+    'mostRecentFinancialsEndDate': 'mostRecentFinancialsEndDate',
+    'accession': 'accession',
+    'valuationSecuritizationAmount': 'valuationSecuritizationAmount',
+    'mostRecentValuationDate': 'mostRecentValuationDate',
+    'mostRecentValuationAmount': 'mostRecentValuationAmount',
+    'mostRecentDebtServiceCoverageCode': 'mostRecentDebtServiceCoverageCode',
+    'netCashFlowFlowSecuritizationAmount': 'netCashFlowFlowSecuritizationAmount'
+}
+
+# Properties dictionary mapping
+properties_dict_ex102_abs = {
+    'unitsBedsRoomsNumber': 'unitsBedsRoomsNumber',
+    'propertyCounty': 'propertyCounty',
+    'squareFeetLargestTenantNumber': 'squareFeetLargestTenantNumber',
+    'netRentableSquareFeetNumber': 'netRentableSquareFeetNumber',
+    'leaseExpirationThirdLargestTenantDate': 'leaseExpirationThirdLargestTenantDate',
+    'leaseExpirationLargestTenantDate': 'leaseExpirationLargestTenantDate',
+    'propertyZip': 'propertyZip',
+    'squareFeetThirdLargestTenantNumber': 'squareFeetThirdLargestTenantNumber',
+    'propertyStatusCode': 'propertyStatusCode',
+    'propertyState': 'propertyState',
+    'yearBuiltNumber': 'yearBuiltNumber',
+    'propertyCity': 'propertyCity',
+    'propertyName': 'propertyName',
+    'propertyAddress': 'propertyAddress',
+    'yearLastRenovated': 'yearLastRenovated',
+    'leaseExpirationSecondLargestTenantDate': 'leaseExpirationSecondLargestTenantDate',
+    'thirdLargestTenant': 'thirdLargestTenant',
+    'unitsBedsRoomsSecuritizationNumber': 'unitsBedsRoomsSecuritizationNumber',
+    'propertyTypeCode': 'propertyTypeCode',
+    'largestTenant': 'largestTenant',
+    'squareFeetSecondLargestTenantNumber': 'squareFeetSecondLargestTenantNumber',
+    'netRentableSquareFeetSecuritizationNumber': 'netRentableSquareFeetSecuritizationNumber',
+    'secondLargestTenant': 'secondLargestTenant'
+}

{datamule-1.2.5 → datamule-1.2.7}/datamule/document/mappings/information_table.py

@@ -1,3 +1,4 @@
+# Ready for mass testing
 
 # Information Table (13F-HR Securities) mapping
 information_table_dict = {

{datamule-1.2.5 → datamule-1.2.7}/datamule/document/mappings/ownership.py

@@ -1,4 +1,4 @@
-# Mapping dictionaries for SEC filing table types based on actual field occurrences
+# Ready for mass testing
 
 # Non-derivative transaction ownership mapping
 non_derivative_transaction_ownership_dict = {

datamule-1.2.7/datamule/document/mappings/proxy_voting_record.py

@@ -0,0 +1,17 @@
+proxy_voting_record_dict = {
+    'meetingDate': 'meetingDate',
+    'accession': 'accessionNumber',
+    'vote_voteRecord_managementRecommendation': 'managementRecommendation',
+    'sharesVoted': 'sharesVoted',  # Top-level sharesVoted
+    'vote_voteRecord_howVoted': 'howVoted',
+    'sharesOnLoan': 'sharesOnLoan',
+    'cusip': 'cusip',
+    'issuerName': 'issuerName',
+    'voteCategories_voteCategory_categoryType': 'categoryType',
+    'voteDescription': 'voteDescription',
+    'voteManager_otherManagers_otherManager': 'otherManager',
+    'vote_voteRecord_sharesVoted': 'recordSharesVoted',  # To distinguish from top-level sharesVoted
+    'isin': 'isin',
+    'voteSource': 'voteSource',
+    'voteSeries': 'voteSeries'
+}

datamule-1.2.7/datamule/document/mappings/submission_metadata.py

@@ -0,0 +1,9 @@
+# Note: submission_metadata is my designation, not SEC for the header of the Submission tag
+
+document_submission_metadata_dict = {
+    'accession':'accession',
+    'type':'type',
+    'sequence' : 'sequence',
+    'filename' : 'filename', 
+    'description':'description'
+}

datamule-1.2.7/datamule/document/mappings/thirteenfhr.py

@@ -0,0 +1,72 @@
+# Ready for mass testing
+
+# 13F-HR (Institutional Investment Manager Holdings) mapping
+thirteenfhr_dict =  {
+    # Cover Page Mapping
+    'formData_coverPage_reportCalendarOrQuarter': 'reportCalendarOrQuarter',
+    'formData_coverPage_filingManager_name': 'filingManagerName',
+    'formData_coverPage_filingManager_address_street1': 'filingManagerStreet1',
+    'formData_coverPage_filingManager_address_street2': 'filingManagerStreet2',
+    'formData_coverPage_filingManager_address_city': 'filingManagerCity',
+    'formData_coverPage_filingManager_address_stateOrCountry': 'filingManagerStateOrCountry',
+    'formData_coverPage_filingManager_address_zipCode': 'filingManagerZipCode',
+    'formData_coverPage_crdNumber': 'crdNumber',
+    'formData_coverPage_secFileNumber': 'secFileNumber',
+    'formData_coverPage_form13FFileNumber': 'form13FFileNumber',
+    'formData_coverPage_reportType': 'reportType',
+    'formData_coverPage_isAmendment': 'isAmendment',
+    'formData_coverPage_amendmentNo': 'amendmentNo',
+    'formData_coverPage_amendmentInfo_amendmentType': 'amendmentType',
+    'formData_coverPage_amendmentInfo_confDeniedExpired': 'confDeniedExpired',
+    'formData_coverPage_additionalInformation': 'additionalInformation',
+    'formData_coverPage_provideInfoForInstruction5': 'provideInfoForInstruction5',
+    
+    # Other Managers Info Mapping
+    'formData_coverPage_otherManagersInfo_otherManager': 'otherManager',
+    'formData_coverPage_otherManagersInfo_otherManager_cik': 'otherManagerCik',
+    'formData_coverPage_otherManagersInfo_otherManager_name': 'otherManagerName',
+    'formData_coverPage_otherManagersInfo_otherManager_crdNumber': 'otherManagerCrdNumber',
+    'formData_coverPage_otherManagersInfo_otherManager_secFileNumber': 'otherManagerSecFileNumber',
+    'formData_coverPage_otherManagersInfo_otherManager_form13FFileNumber': 'otherManagerForm13FFileNumber',
+    
+    # Summary Page Mapping
+    'formData_summaryPage_isConfidentialOmitted': 'isConfidentialOmitted',
+    'formData_summaryPage_otherIncludedManagersCount': 'otherIncludedManagersCount',
+    'formData_summaryPage_tableEntryTotal': 'tableEntryTotal',
+    'formData_summaryPage_tableValueTotal': 'tableValueTotal',
+    
+    # Other Managers 2 Info Mapping
+    'formData_summaryPage_otherManagers2Info_otherManager2': 'otherManager2',
+    'formData_summaryPage_otherManagers2Info_otherManager2_sequenceNumber': 'otherManager2SequenceNumber',
+    'formData_summaryPage_otherManagers2Info_otherManager2_otherManager_cik': 'otherManager2Cik',
+    'formData_summaryPage_otherManagers2Info_otherManager2_otherManager_name': 'otherManager2Name',
+    'formData_summaryPage_otherManagers2Info_otherManager2_otherManager_crdNumber': 'otherManager2CrdNumber',
+    'formData_summaryPage_otherManagers2Info_otherManager2_otherManager_secFileNumber': 'otherManager2SecFileNumber',
+    'formData_summaryPage_otherManagers2Info_otherManager2_otherManager_form13FFileNumber': 'otherManager2Form13FFileNumber',
+    
+    # Signature Block Mapping
+    'formData_signatureBlock_name': 'signatureName',
+    'formData_signatureBlock_title': 'signatureTitle',
+    'formData_signatureBlock_phone': 'signaturePhone',
+    'formData_signatureBlock_signature': 'signature',
+    'formData_signatureBlock_city': 'signatureCity',
+    'formData_signatureBlock_stateOrCountry': 'signatureStateOrCountry',
+    'formData_signatureBlock_signatureDate': 'signatureDate',
+    
+    # Header Data Mapping
+    'headerData_filerInfo_periodOfReport': 'periodOfReport',
+    'headerData_filerInfo_filer_fileNumber': 'filerFileNumber',
+    'headerData_filerInfo_filer_credentials_cik': 'filerCik',
+    'headerData_filerInfo_filer_credentials_ccc': 'filerCcc',
+    'headerData_filerInfo_flags_confirmingCopyFlag': 'confirmingCopyFlag',
+    'headerData_filerInfo_flags_returnCopyFlag': 'returnCopyFlag',
+    'headerData_filerInfo_flags_overrideInternetFlag': 'overrideInternetFlag',
+    'headerData_filerInfo_denovoRequest': 'denovoRequest',
+    'headerData_filerInfo_liveTestFlag': 'liveTestFlag',
+    'headerData_submissionType': 'submissionType',
+    
+    # Schema and Metadata Mapping
+    'schemaLocation': 'schemaLocation',
+    'schemaVersion': 'schemaVersion',
+    'accession': 'accessionNumber'
+}

{datamule-1.2.5 → datamule-1.2.7}/datamule/document/mappings/twentyfivense.py

@@ -1,3 +1,4 @@
+# Ready for mass testing
 # 25-NSE mapping
 twentyfive_nse_dict = {
     'descriptionClassSecurity': 'securityDescription',

{datamule-1.2.5 → datamule-1.2.7}/datamule/document/processing.py

@@ -17,6 +17,14 @@ def process_tabular_data(self):
         tables = process_13fhr(self.data, self.accession)
     elif self.type in ["INFORMATION TABLE"]:
         tables = process_information_table(self.data, self.accession)
+    elif self.type in ["25-NSE", "25-NSE/A"]:
+        tables = process_25nse(self.data, self.accession)
+    # complete mark:
+    elif self.type in ["N-PX","N-PX/A"]:
+        tables = process_npx(self.data, self.accession)
+    elif self.type in ["EX-102"]:
+        tables = process_ex102_abs(self.data, self.accession)
+
     elif self.type in ["SBSEF","SBSEF/A","SBSEF-V","SBSEF-W"]:
         tables = process_sbsef(self.data, self.accession)
     elif self.type in ["SDR","SDR/A","SDR-W","SDR-A"]:
@@ -33,8 +41,7 @@ def process_tabular_data(self):
         tables = process_144(self.data, self.accession)
     elif self.type in ["24F-2NT", "24F-2NT/A"]:
         tables = process_24f2nt(self.data, self.accession)
-    elif self.type in ["25-NSE", "25-NSE/A"]:
-        tables = process_25nse(self.data, self.accession)
+
     elif self.type in ["ATS-N", "ATS-N/A"]:
         tables = process_ats(self.data, self.accession)
     # elif self.type in ["C","C-W","C-U","C-U-W","C/A","C/A-W",
@@ -53,8 +60,7 @@ def process_tabular_data(self):
     #     tables = process_nmfp(self.data, self.accession)
     # elif self.type in ["NPORT-P","NPORT-P/A"]:
     #     tables = process_nportp(self.data, self.accession)
-    elif self.type in ["N-PX","N-PX/A"]:
-        tables = process_npx(self.data, self.accession)
+
     # elif self.type in ["TA-1","TA-1/A","TA-W","TA-2","TA-2/A"]:
     #     tables = process_ta(self.data, self.accession)
     elif self.type in ["X-17A-5","X-17A-5/A"]:
@@ -66,10 +72,11 @@ def process_tabular_data(self):
         tables = process_reg_a(self.data, self.accession)
     # elif self.type in ["SBSE","SBSE/A","SBSE-A","SBSE-A/A","SBSE-BD","SBSE-BD/A","SBSE-C","SBSE-W","SBSE-CCO-RPT","SBSE-CCO-RPT/A"]:
     #     tables = process_sbs(self.data, self.accession)
-    # elif self.type in ["EX-102"]:
-    #     tables = process_ex102_abs(self.data, self.accession)
+
     elif self.type == "PROXY VOTING RECORD":
         tables = process_proxy_voting_record(self.data, self.accession)
+    elif self.type == 'submission_metadata':
+        tables = process_submission_metadata(self.content, self.accession)
     else:
         warn(f"Processing for {self.type} is not implemented yet.")
         return []
@@ -583,13 +590,39 @@ def process_reg_a(data, accession):
     
 #     return tables
 
-# def process_ex102_abs(data, accession):
-#     tables = []
-#     asset_data = safe_get(data, ['assetData'])
-#     if asset_data:
-#         tables.append(Table(_flatten_dict(asset_data), 'abs', accession))
-#     raise NotImplementedError("Need to implement the rest of the ABS processing")
-#     return tables
+def process_ex102_abs(data, accession):
+    tables = []
+    data = safe_get(data, ['assetData', 'assets'])
+    
+    # Create assets list: all items without their 'property' field
+    assets = [{k: v for k, v in item.items() if k != 'property'} for item in data]
+
+    # Create properties list in a more vectorized way
+    properties = []
+    
+    # Handle dictionary properties
+    properties.extend([
+        item['property'] | {'assetNumber': item['assetNumber']}
+        for item in data
+        if 'property' in item and isinstance(item['property'], dict)
+    ])
+    
+    # Handle list properties - flatten in one operation
+    properties.extend([
+        prop | {'assetNumber': item['assetNumber']}
+        for item in data
+        if 'property' in item and isinstance(item['property'], list)
+        for prop in item['property']
+        if isinstance(prop, dict)
+    ])
+    
+    if assets:
+        tables.append(Table(_flatten_dict(assets), 'assets_ex102_absee', accession))
+    
+    if properties:
+        tables.append(Table(_flatten_dict(properties), 'properties_ex102_absee', accession))
+    
+    return tables
 
 # def process_ma(data, accession):
 #     tables = []
@@ -601,4 +634,28 @@ def process_reg_a(data, accession):
 #     raise NotImplementedError("Need to implement the rest of the MA processing")
 
 # def process_ncen(data, accession):
-#     raise NotImplementedError("Need to implement the N-CEN processing")
+#     raise NotImplementedError("Need to implement the N-CEN processing")
+
+# WIP
+# Note: going to pause this for now, as I don't have a great way of putting this in a csv.
+def process_submission_metadata(data,accession):
+    tables = []
+    document_data = safe_get(data, ['documents'])
+    if document_data:
+        tables.append(Table(_flatten_dict(document_data), 'document_submission_metadata', accession))
+
+    reporting_owner_data = safe_get(data,['reporting-owner'])
+    if reporting_owner_data:
+        tables.append(Table(_flatten_dict(reporting_owner_data), 'reporting_owner_submission_metadata', accession))
+
+    issuer_data = safe_get(data,['issuer'])
+    if issuer_data:
+        tables.append(Table(_flatten_dict(issuer_data), 'issuer_submission_metadata', accession))
+        
+    # # construct metadata
+    # accession-number date-of-filing-date-change, depositor-cik effectiveness-date
+
+    # # other tables
+    # depositor, securitizer
+        
+    return tables

{datamule-1.2.5 → datamule-1.2.7}/datamule/document/table.py

@@ -18,7 +18,11 @@ from .mappings.thirteenfhr import *
 from .mappings.twentyfivense import *
 from .mappings.twentyfourf2nt import *
 from .mappings.information_table import *
+from .mappings.submission_metadata import *
+from .mappings.ex102_abs import *
 
+from pathlib import Path
+import csv
 # need to check if mappings correctly create new columns
 class Table():
     def __init__(self, data, type,accession):
@@ -27,11 +31,18 @@ class Table():
         self.type = type
         self.data = data
         self.accession = accession
-        self.columns = self.determine_columns()
+        self.columns = self.determine_columns_complete()
+
+    def determine_columns_complete(self):
+        if not self.data:
+            return []
+        return list(set().union(*(row.keys() for row in self.data)))
+
 
     def determine_columns(self):
         if len(self.data) == 0:
             return []
+        
         return self.data[0].keys()
 
     def add_column(self,column_name,value):
@@ -227,6 +238,15 @@ class Table():
             mapping_dict = item_9_24f2nt_dict
         elif self.type == 'signature_info_schedule_a':
             mapping_dict = signature_24f2nt_dict
+        # ABS
+        elif self.type == 'assets_ex102_absee':
+            mapping_dict = assets_dict_ex102_abs
+        elif self.type =='properties_ex102_absee':
+            mapping_dict = properties_dict_ex102_abs
+        # submission metadata
+        elif self.type == 'document_submission_metadata':
+            mapping_dict = document_submission_metadata_dict
+        
 
         else:
             mapping_dict = {}
@@ -245,9 +265,6 @@ class Table():
             for old_key, new_key in mapping_dict.items():
                 if old_key in row:
                     ordered_row[new_key] = row.pop(old_key)
-                else:
-                    # if the old key is not present, set the new key to None
-                    ordered_row[new_key] = None
             
             # Then add any remaining keys that weren't in the mapping
             for key, value in row.items():
@@ -257,4 +274,30 @@ class Table():
             row.clear()
             row.update(ordered_row)
 
-        self.determine_columns()
+        # Update the columns after mapping
+        columns = set(self.columns)
+        # remove the old columns that are now in the mapping
+        columns.difference_update(mapping_dict.keys())
+        # add the new columns from the mapping
+        columns.update(mapping_dict.values())
+        # add the accession column to the columns
+        columns.add('accession')
+
+        self.columns = list(columns)
+
+    def write_csv(self, output_file):
+        output_file = Path(output_file)
+        fieldnames = self.columns
+        
+        # Check if the file already exists
+        if output_file.exists():
+            # Append to existing file without writing header
+            with open(output_file, 'a', newline='') as csvfile:
+                writer = csv.DictWriter(csvfile, fieldnames=fieldnames, quoting=csv.QUOTE_ALL)
+                writer.writerows(self.data)
+        else:
+            # Create new file with header
+            with open(output_file, 'w', newline='') as csvfile:
+                writer = csv.DictWriter(csvfile, fieldnames=fieldnames, quoting=csv.QUOTE_ALL)
+                writer.writeheader()
+                writer.writerows(self.data)

{datamule-1.2.5 → datamule-1.2.7}/datamule/helper.py

@@ -79,7 +79,16 @@ def _process_cik_and_metadata_filters(cik=None, ticker=None, **kwargs):
 
         # Convert ticker to CIK if provided
         if ticker is not None:
-            cik = get_cik_from_dataset('listed_filer_metadata', 'ticker', ticker)
+            if isinstance(ticker, str):
+                ticker = [ticker]
+
+            ciks_from_ticker = []
+            for t in ticker:
+                ciks = get_cik_from_dataset('listed_filer_metadata', 'ticker', t)
+                if ciks:
+                    ciks_from_ticker.extend(ciks)
+
+            cik = ciks
 
         # Normalize CIK format
         if cik is not None:

{datamule-1.2.5 → datamule-1.2.7}/datamule/index.py

@@ -1,16 +1,16 @@
-from pathlib import Path
+
 from .sec.submissions.textsearch import query
-from .helper import _process_cik_and_metadata_filters, load_package_dataset
+from .helper import _process_cik_and_metadata_filters
+from pathlib import Path
 
 class Index:
-    def __init__(self, path=None):
-        self.path = Path(path) if path else None
+    def __init__(self):
+        pass
         
     def search_submissions(
         self,
         text_query,
-        start_date=None,
-        end_date=None,
+        filing_date=None,
         submission_type=None,
         cik=None,
         ticker=None,
@@ -47,16 +47,14 @@ class Index:
         # Execute the search query
         results = query(
             f'{text_query}',
-            filing_date=(start_date, end_date),
+            filing_date=filing_date,
             requests_per_second=requests_per_second,
             quiet=quiet,
             submission_type=submission_type,
             **kwargs
         )
         
-        # Save results to path if specified
-        if self.path:
-            self._save_results(results, text_query)
+
             
         return results
     

{datamule-1.2.5 → datamule-1.2.7}/datamule/portfolio.py

@@ -9,22 +9,28 @@ import os
 from .helper import _process_cik_and_metadata_filters
 from .seclibrary.downloader import download as seclibrary_download
 from .sec.xbrl.filter_xbrl import filter_xbrl
-from .sec.submissions.monitor import monitor
-from .sec.xbrl.xbrlmonitor import XBRLMonitor
+from .sec.submissions.monitor import Monitor
+#from .sec.xbrl.xbrlmonitor import XBRLMonitor
 
 
 class Portfolio:
     def __init__(self, path):
         self.path = Path(path)
+        self.api_key = None
         self.submissions = []
         self.submissions_loaded = False
         self.MAX_WORKERS = os.cpu_count() - 1 
+
+        self.monitor = Monitor()
         
         if self.path.exists():
             self._load_submissions()
             self.submissions_loaded = True
         else:
             self.path.mkdir(parents=True, exist_ok=True)
+
+    def set_api_key(self, api_key):
+        self.api_key = api_key
     
     def _load_submissions(self):
         folders = [f for f in self.path.iterdir() if f.is_dir()]
@@ -132,6 +138,7 @@ class Portfolio:
             seclibrary_download(
                 output_dir=self.path,
                 cik=cik,
+                api_key=self.api_key,
                 submission_type=submission_type,
                 filing_date=filing_date,
                 accession_numbers=self.accession_numbers if hasattr(self, 'accession_numbers') else None,
@@ -149,20 +156,18 @@ class Portfolio:
             )
 
         self.submissions_loaded = False
-    def monitor_submissions(self,data_callback=None, poll_callback=None, submission_type=None, cik=None, 
-           polling_interval=200, requests_per_second=5, quiet=False, start_date=None, ticker=None, **kwargs):
+    def monitor_submissions(self, data_callback=None, interval_callback=None,
+                            polling_interval=1000, quiet=True, start_date=None,
+                            validation_interval=600000):
         
-        cik = _process_cik_and_metadata_filters(cik, ticker, **kwargs)
 
-        monitor(
+        self.monitor.monitor_submissions(
             data_callback=data_callback,
-            poll_callback=poll_callback,
-            cik=cik,
-            submission_type=submission_type,
+            interval_callback=interval_callback,
             polling_interval=polling_interval,
-            requests_per_second=requests_per_second,
             quiet=quiet,
-            start_date=start_date
+            start_date=start_date,
+            validation_interval=validation_interval
         )