datamule 1.0.3__py3-none-any.whl → 1.0.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. datamule/__init__.py +2 -13
  2. datamule/document.py +8 -9
  3. datamule/helper.py +85 -105
  4. datamule/portfolio.py +105 -29
  5. datamule/submission.py +0 -38
  6. {datamule-1.0.3.dist-info → datamule-1.0.7.dist-info}/METADATA +2 -8
  7. datamule-1.0.7.dist-info/RECORD +10 -0
  8. datamule/book/__init__.py +0 -0
  9. datamule/book/book.py +0 -34
  10. datamule/book/eftsquery.py +0 -127
  11. datamule/book/xbrl_retriever.py +0 -88
  12. datamule/data/company_former_names.csv +0 -8148
  13. datamule/data/company_metadata.csv +0 -10049
  14. datamule/data/company_tickers.csv +0 -9999
  15. datamule/data/sec-glossary.csv +0 -728
  16. datamule/data/xbrl_descriptions.csv +0 -10024
  17. datamule/downloader/downloader.py +0 -374
  18. datamule/downloader/premiumdownloader.py +0 -335
  19. datamule/mapping_dicts/txt_mapping_dicts.py +0 -234
  20. datamule/mapping_dicts/xml_mapping_dicts.py +0 -19
  21. datamule/monitor.py +0 -283
  22. datamule/mulebot/__init__.py +0 -1
  23. datamule/mulebot/helper.py +0 -35
  24. datamule/mulebot/mulebot.py +0 -130
  25. datamule/mulebot/mulebot_server/__init__.py +0 -1
  26. datamule/mulebot/mulebot_server/server.py +0 -87
  27. datamule/mulebot/mulebot_server/static/css/minimalist.css +0 -174
  28. datamule/mulebot/mulebot_server/static/scripts/artifacts.js +0 -68
  29. datamule/mulebot/mulebot_server/static/scripts/chat.js +0 -92
  30. datamule/mulebot/mulebot_server/static/scripts/filingArtifacts.js +0 -56
  31. datamule/mulebot/mulebot_server/static/scripts/listArtifacts.js +0 -15
  32. datamule/mulebot/mulebot_server/static/scripts/main.js +0 -57
  33. datamule/mulebot/mulebot_server/static/scripts/prefilledPrompt.js +0 -27
  34. datamule/mulebot/mulebot_server/static/scripts/suggestions.js +0 -47
  35. datamule/mulebot/mulebot_server/static/scripts/tableArtifacts.js +0 -129
  36. datamule/mulebot/mulebot_server/static/scripts/utils.js +0 -28
  37. datamule/mulebot/mulebot_server/templates/chat-minimalist.html +0 -91
  38. datamule/mulebot/search.py +0 -52
  39. datamule/mulebot/tools.py +0 -82
  40. datamule/packageupdater.py +0 -207
  41. datamule-1.0.3.dist-info/RECORD +0 -43
  42. {datamule-1.0.3.dist-info → datamule-1.0.7.dist-info}/WHEEL +0 -0
  43. {datamule-1.0.3.dist-info → datamule-1.0.7.dist-info}/top_level.txt +0 -0
datamule/mulebot/mulebot_server/static/scripts/utils.js DELETED
@@ -1,28 +0,0 @@
1
- // utils.js
2
- export function renderMetadata(artifactData) {
3
- const metadataHtml = `
4
- <div class="card mb-3">
5
- <div class="card-body">
6
- <h5 class="card-title">Metadata</h5>
7
- <p><strong>Fact:</strong> ${artifactData.fact}</p>
8
- ${artifactData.cik ? `<p><strong>CIK:</strong> ${artifactData.cik}</p>` : ''}
9
- ${artifactData.category ? `<p><strong>Category:</strong> ${artifactData.category}</p>` : ''}
10
- ${artifactData.label ? `<p><strong>Label:</strong> ${artifactData.label}</p>` : ''}
11
- ${artifactData.description ? `<p><strong>Description:</strong> ${artifactData.description}</p>` : ''}
12
- ${artifactData.unit ? `<p><strong>Unit:</strong> ${artifactData.unit}</p>` : ''}
13
- </div>
14
- </div>
15
- `;
16
- document.getElementById('metadata-content').innerHTML = metadataHtml;
17
- }
18
-
19
- export function downloadCSV(table) {
20
- let csv = '';
21
- const headers = Object.keys(table.table[0]);
22
- csv += headers.join(',') + '\n';
23
- table.table.forEach(row => {
24
- csv += Object.values(row).join(',') + '\n';
25
- });
26
- const blob = new Blob([csv], { type: 'text/csv;charset=utf-8;' });
27
- saveAs(blob, `${table.fact}.csv`);
28
- }
datamule/mulebot/mulebot_server/templates/chat-minimalist.html DELETED
@@ -1,91 +0,0 @@
1
- <!DOCTYPE html>
2
- <html lang="en">
3
-
4
- <head>
5
- <meta charset="UTF-8">
6
- <meta name="viewport" content="width=device-width, initial-scale=1.0">
7
- <title>Mulebot</title>
8
- <link href="https://cdnjs.cloudflare.com/ajax/libs/bootstrap/5.1.3/css/bootstrap.min.css" rel="stylesheet">
9
- <script src="https://cdnjs.cloudflare.com/ajax/libs/FileSaver.js/2.0.5/FileSaver.min.js"></script>
10
- <script src="https://cdnjs.cloudflare.com/ajax/libs/jszip/3.7.1/jszip.min.js"></script>
11
- <link rel="stylesheet" href="{{ url_for('static', filename='css/minimalist.css') }}">
12
- </head>
13
-
14
- <body>
15
- <div class="container mt-5">
16
- <div class="d-flex justify-content-between align-items-center mb-4">
17
- <h1>Mulebot</h1>
18
- <div>
19
- <a href="https://github.com/john-friedman/datamule-python" target="_blank" class="social-btn">
20
- <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor"
21
- class="bi bi-github" viewBox="0 0 16 16">
22
- <path
23
- d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.012 8.012 0 0 0 16 8c0-4.42-3.58-8-8-8z" />
24
- </svg>
25
- GitHub
26
- </a>
27
- <a href="https://datamule.xyz" target="_blank" class="social-btn">
28
- <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor"
29
- viewBox="0 0 16 16">
30
- <path
31
- d="M0 8a8 8 0 1 1 16 0A8 8 0 0 1 0 8zm7.5-6.923c-.67.204-1.335.82-1.887 1.855-.143.268-.276.56-.395.872.705.157 1.472.257 2.282.287V1.077zM4.249 3.539c.142-.384.304-.744.481-1.078a6.7 6.7 0 0 1 .597-.933A7.01 7.01 0 0 0 3.051 3.05c.362.184.763.349 1.198.49zM3.509 7.5c.036-1.07.188-2.087.436-3.008a9.124 9.124 0 0 1-1.565-.667A6.964 6.964 0 0 0 1.018 7.5h2.49zm1.4-2.741a12.344 12.344 0 0 0-.4 2.741H7.5V5.091c-.91-.03-1.783-.145-2.591-.332zM8.5 5.09V7.5h2.99a12.342 12.342 0 0 0-.399-2.741c-.808.187-1.681.301-2.591.332zM4.51 8.5c.035.987.176 1.914.399 2.741A13.612 13.612 0 0 1 7.5 10.91V8.5H4.51zm3.99 0v2.409c.91.03 1.783.145 2.591.332.223-.827.364-1.754.4-2.741H8.5zm-3.282 3.696c.12.312.252.604.395.872.552 1.035 1.218 1.65 1.887 1.855V11.91c-.81.03-1.577.13-2.282.287zm.11 2.276a6.696 6.696 0 0 1-.598-.933 8.853 8.853 0 0 1-.481-1.079 8.38 8.38 0 0 0-1.198.49 7.01 7.01 0 0 0 2.276 1.522zm-1.383-2.964A13.36 13.36 0 0 1 3.508 8.5h-2.49a6.963 6.963 0 0 0 1.362 3.675c.47-.258.995-.482 1.565-.667zm6.728 2.964a7.009 7.009 0 0 0 2.275-1.521 8.376 8.376 0 0 0-1.197-.49 8.853 8.853 0 0 1-.481 1.078 6.688 6.688 0 0 1-.597.933zM8.5 11.909v3.014c.67-.204 1.335-.82 1.887-1.855.143-.268.276-.56.395-.872A12.63 12.63 0 0 0 8.5 11.91zm3.555-.401c.57.185 1.095.409 1.565.667A6.963 6.963 0 0 0 14.982 8.5h-2.49a13.36 13.36 0 0 1-.437 3.008zM14.982 7.5a6.963 6.963 0 0 0-1.362-3.675c-.47.258-.995.482-1.565.667.248.92.4 1.938.437 3.008h2.49zM11.27 2.461c.177.334.339.694.482 1.078a8.368 8.368 0 0 0 1.196-.49 7.01 7.01 0 0 0-2.275-1.52c.218.283.418.597.597.932zm-.488 1.343a7.765 7.765 0 0 0-.395-.872C9.835 1.897 9.17 1.282 8.5 1.077V4.09c.81-.03 1.577-.13 2.282-.287z" />
32
- </svg>
33
- DataMule Website
34
- </a>
35
- </div>
36
- </div>
37
- <div class="row">
38
- <div class="col-md-6">
39
- <div class="card mb-3">
40
- <div class="card-body">
41
- <div id="chat-outer-container">
42
- <div id="chat-container" class="mb-3"></div>
43
- <div id="thinking-indicator" class="thinking-indicator">
44
- <span>Bot is thinking</span><span class="dot-animation">...</span>
45
- </div>
46
- </div>
47
- <form id="chat-form">
48
- <div class="input-group">
49
- <input type="text" id="user-input" class="form-control"
50
- placeholder="Type your message...">
51
- <button type="submit" class="btn btn-primary">Send</button>
52
- </div>
53
- </form>
54
- </div>
55
- </div>
56
- </div>
57
- <div class="col-md-6">
58
- <button id="toggle-artifacts" class="btn btn-secondary mb-3">Show Artifacts</button>
59
- <div id="artifact-container" class="card mb-3" style="display: none;">
60
- <div class="card-body">
61
- <div id="artifact-content"></div>
62
- </div>
63
- </div>
64
- <div class="suggestion-box">
65
- <h5 class="mb-2">Suggested Commands:</h5>
66
- <ul class="list-unstyled">
67
- <li class="suggestion-item">Get all company facts for Tesla</li>
68
- <li class="suggestion-item">Get Ford's CIK</li>
69
- <li class="suggestion-item">Get all 10-K urls for META</li>
70
- <li class="suggestion-item">Get the management discussion and analysis section from
71
- https://www.sec.gov/Archives/edgar/data/1318605/000095017022000796/tsla-20211231.htm</li>
72
- </ul>
73
- </div>
74
- </div>
75
- </div>
76
- </div>
77
- <script src="https://cdnjs.cloudflare.com/ajax/libs/bootstrap/5.1.3/js/bootstrap.bundle.min.js"></script>
78
-
79
- <!-- Application scripts -->
80
- <script type="module" src="{{ url_for('static', filename='scripts/utils.js') }}"></script>
81
- <script type="module" src="{{ url_for('static', filename='scripts/chat.js') }}"></script>
82
- <script type="module" src="{{ url_for('static', filename='scripts/listArtifacts.js') }}"></script>
83
- <script type="module" src="{{ url_for('static', filename='scripts/filingArtifacts.js') }}"></script>
84
- <script type="module" src="{{ url_for('static', filename='scripts/tableArtifacts.js') }}"></script>
85
- <script type="module" src="{{ url_for('static', filename='scripts/artifacts.js') }}"></script>
86
- <script type="module" src="{{ url_for('static', filename='scripts/suggestions.js') }}"></script>
87
- <script type="module" src="{{ url_for('static', filename='scripts/prefilledPrompt.js') }}"></script>
88
- <script type="module" src="{{ url_for('static', filename='scripts/main.js') }}"></script>
89
- </body>
90
-
91
- </html>
datamule/mulebot/search.py DELETED
@@ -1,52 +0,0 @@
1
- import difflib
2
- from typing import Dict, List, Any
3
-
4
- def search_filing(query: str, nested_dict: Dict[str, Any], max_matches: int = 20, score_cutoff: float = 0.6) -> List[Dict[str, Any]]:
5
- max_matches = min(max_matches, 20)
6
- query = query.lower() # Convert query to lowercase
7
-
8
- def flatten_dict(d: Dict[str, Any], parent_path: List[str] = None) -> List[Dict[str, Any]]:
9
- parent_path = parent_path or []
10
- items = []
11
-
12
- if isinstance(d, dict):
13
- for k, v in d.items():
14
- new_path = parent_path + [k]
15
- if k == 'title' and isinstance(v, str):
16
- items.append({'path': new_path, 'title': v, 'title_lower': v.lower()})
17
- items.extend(flatten_dict(v, new_path))
18
- elif isinstance(d, list):
19
- for i, item in enumerate(d):
20
- new_path = parent_path + [str(i)]
21
- items.extend(flatten_dict(item, new_path))
22
-
23
- return items
24
-
25
- flat_list = flatten_dict(nested_dict)
26
- all_titles_lower = [item['title_lower'] for item in flat_list]
27
-
28
- matches = difflib.get_close_matches(query, all_titles_lower, n=max_matches, cutoff=score_cutoff)
29
-
30
- results = []
31
- for match in matches:
32
- similarity = difflib.SequenceMatcher(None, query, match).ratio()
33
- for item in flat_list:
34
- if item['title_lower'] == match:
35
- # Navigate to the correct nested dictionary
36
- d = nested_dict
37
- for key in item['path'][:-1]: # Exclude the last 'title' key
38
- if key.isdigit():
39
- d = d[int(key)]
40
- else:
41
- d = d[key]
42
- results.append({
43
- 'path': '.'.join(item['path'][:-1]), # Exclude the last 'title' key
44
- 'content': d,
45
- 'similarity': similarity
46
- })
47
- break
48
-
49
- results.sort(key=lambda x: x['similarity'], reverse=True)
50
-
51
-
52
- return [item['content'] for item in results[:max_matches]]
datamule/mulebot/tools.py DELETED
@@ -1,82 +0,0 @@
1
-
2
- get_company_concept_tool = {
3
- "type": "function",
4
- "function": {
5
- "name": "get_company_concept",
6
- "description": "ONLY use this when explicitly asked to get company XBRL concepts or facts for a given ticker",
7
- "parameters": {
8
- "type": "object",
9
- "properties": {
10
- "ticker": {"type": "string", "description": "The ticker of the company to get facts for"}
11
- },
12
- "required": ["ticker"]
13
- }
14
- }
15
- }
16
-
17
- identifier_to_cik_tool = {
18
- "type": "function",
19
- "function": {
20
- "name": "identifier_to_cik",
21
- "description": "ONLY use this when explicitly asked to convert a company's ticker to a CIK.",
22
- "parameters": {
23
- "type": "object",
24
- "properties": {
25
- "ticker": {"type": "string", "description": "The ticker to convert to a CIK"},
26
- },
27
- "required": ["ticker"]
28
- }
29
- }
30
- }
31
-
32
- get_filing_urls_tool = {
33
- "type": "function",
34
- "function": {
35
- "name": "get_filing_urls",
36
- "description": "ONLY use this when explicitly asked to get URLs of filings for a given company or multiple",
37
- "parameters": {
38
- "type": "object",
39
- "properties": {
40
- "ticker": {"type": "string", "description": "Ticker symbol of the company. Can be a single ticker or a list of tickers."},
41
- "form": {"type": "string", "description": "Form type to get (e.g., '10-K', '10-Q')"},
42
- "date": {"type": "string", "description": "Date of the filing, can be a single date, a range, or a list of dates. Format: 'YYYY-MM-DD'. If range use a tuple of two dates."},
43
- },
44
- "required": ["ticker"]
45
- }
46
- }
47
- }
48
-
49
- find_filing_section_by_title_tool = {
50
- "type": "function",
51
- "function": {
52
- "name": "find_filing_section_by_title",
53
- "description": "ONLY use this when explicitly given a filing URL and told to find a specific section",
54
- "parameters": {
55
- "type": "object",
56
- "properties": {
57
- "url": {"type": "string", "description": "URL of the filing to parse"},
58
- "title": {"type": "string", "description": "The section title to search for in the filing"}
59
- },
60
- "required": ["url","title"]
61
- }
62
- }
63
- }
64
-
65
- return_title_tool = {
66
- "type": "function",
67
- "function": {
68
- "name": "return_title",
69
- "description": "use this to select a title",
70
- "parameters": {
71
- "type": "object",
72
- "properties": {
73
- "title": {"type": "string", "description": "The title to return"}
74
- },
75
- "required": ["title"]
76
- }
77
- }
78
- }
79
-
80
-
81
-
82
- tools = [get_company_concept_tool, identifier_to_cik_tool, get_filing_urls_tool, find_filing_section_by_title_tool]
datamule/packageupdater.py DELETED
@@ -1,207 +0,0 @@
1
- import asyncio
2
- import aiohttp
3
- import json
4
- import csv
5
- import os
6
- from pkg_resources import resource_filename
7
- from .helper import headers
8
- from .downloader.downloader import PreciseRateLimiter, RateMonitor
9
-
10
- class PackageUpdater:
11
- def __init__(self):
12
- self.limiter = PreciseRateLimiter(5) # 5 requests per second
13
- self.rate_monitor = RateMonitor()
14
- self.headers = headers
15
-
16
- async def _fetch_json(self, session, url):
17
- """Fetch JSON with rate limiting and monitoring."""
18
- async with self.limiter:
19
- try:
20
- async with session.get(url) as response:
21
- response.raise_for_status()
22
- content = await response.read()
23
- await self.rate_monitor.add_request(len(content))
24
- return await response.json()
25
- except Exception as e:
26
- print(f"Error fetching {url}: {str(e)}")
27
- return None
28
-
29
- async def _update_company_tickers(self):
30
- """Update company tickers data files."""
31
- url = 'https://www.sec.gov/files/company_tickers.json'
32
-
33
- # Define file paths
34
- json_file = resource_filename('datamule', 'data/company_tickers.json')
35
- csv_file = resource_filename('datamule', 'data/company_tickers.csv')
36
-
37
- # Define temporary file paths
38
- temp_json_file = json_file + '.temp'
39
- temp_csv_file = csv_file + '.temp'
40
-
41
- async with aiohttp.ClientSession(headers=self.headers) as session:
42
- try:
43
- data = await self._fetch_json(session, url)
44
- if not data:
45
- raise Exception("Failed to fetch company tickers data")
46
-
47
- # Save the raw JSON file
48
- with open(temp_json_file, 'w') as f:
49
- json.dump(data, f, indent=4)
50
-
51
- # Convert to CSV
52
- with open(temp_csv_file, 'w', newline='') as csvfile:
53
- fieldnames = ['cik', 'ticker', 'title']
54
- writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
55
- writer.writeheader()
56
- for _, company in data.items():
57
- writer.writerow({
58
- 'cik': str(company['cik_str']).zfill(10),
59
- 'ticker': company['ticker'],
60
- 'title': company['title']
61
- })
62
-
63
- # Replace original files
64
- for src, dst in [(temp_json_file, json_file), (temp_csv_file, csv_file)]:
65
- if os.path.exists(dst):
66
- os.remove(dst)
67
- os.rename(src, dst)
68
-
69
- print(f"Company tickers successfully updated")
70
- return True
71
-
72
- except Exception as e:
73
- print(f"Error updating company tickers: {str(e)}")
74
- return False
75
-
76
- finally:
77
- # Clean up temp files
78
- for temp_file in [temp_json_file, temp_csv_file]:
79
- if os.path.exists(temp_file):
80
- try:
81
- os.remove(temp_file)
82
- except Exception as e:
83
- print(f"Warning: Could not remove {temp_file}: {str(e)}")
84
-
85
- async def _process_metadata_batch(self, session, companies, metadata_writer, former_names_writer):
86
- """Process a batch of companies for metadata updates."""
87
- metadata_fields = [
88
- 'cik', 'name', 'entityType', 'sic', 'sicDescription', 'ownerOrg',
89
- 'insiderTransactionForOwnerExists', 'insiderTransactionForIssuerExists',
90
- 'tickers', 'exchanges', 'ein', 'description', 'website', 'investorWebsite',
91
- 'category', 'fiscalYearEnd', 'stateOfIncorporation', 'stateOfIncorporationDescription',
92
- 'phone', 'flags', 'mailing_street1', 'mailing_street2', 'mailing_city',
93
- 'mailing_stateOrCountry', 'mailing_zipCode', 'mailing_stateOrCountryDescription',
94
- 'business_street1', 'business_street2', 'business_city', 'business_stateOrCountry',
95
- 'business_zipCode', 'business_stateOrCountryDescription'
96
- ]
97
-
98
- tasks = []
99
- for company in companies:
100
- cik = company['cik']
101
- url = f'https://data.sec.gov/submissions/CIK{str(cik).zfill(10)}.json'
102
- tasks.append(self._fetch_json(session, url))
103
-
104
- results = await asyncio.gather(*tasks, return_exceptions=True)
105
-
106
- for company, result in zip(companies, results):
107
- if isinstance(result, Exception) or not result:
108
- print(f"Error processing CIK {company['cik']}: {str(result) if isinstance(result, Exception) else 'No data'}")
109
- continue
110
-
111
- try:
112
- metadata = {field: result.get(field, '') for field in metadata_fields if field not in ['tickers', 'exchanges']}
113
- metadata['cik'] = company['cik']
114
- metadata['tickers'] = ','.join(result.get('tickers', []))
115
- metadata['exchanges'] = ','.join(result.get('exchanges', []))
116
-
117
- # Add address information
118
- for address_type in ['mailing', 'business']:
119
- address = result.get('addresses', {}).get(address_type, {})
120
- for key, value in address.items():
121
- metadata[f'{address_type}_{key}'] = value if value is not None else ''
122
-
123
- metadata_writer.writerow(metadata)
124
-
125
- for former_name in result.get('formerNames', []):
126
- former_names_writer.writerow({
127
- 'cik': company['cik'],
128
- 'former_name': former_name['name'],
129
- 'from_date': former_name['from'],
130
- 'to_date': former_name['to']
131
- })
132
-
133
- except Exception as e:
134
- print(f"Error processing metadata for CIK {company['cik']}: {str(e)}")
135
-
136
- async def _update_company_metadata(self):
137
- """Update company metadata and former names files."""
138
- metadata_file = resource_filename('datamule', 'data/company_metadata.csv')
139
- former_names_file = resource_filename('datamule', 'data/company_former_names.csv')
140
-
141
- temp_metadata_file = metadata_file + '.temp'
142
- temp_former_names_file = former_names_file + '.temp'
143
-
144
- # Load current company tickers
145
- with open(resource_filename('datamule', 'data/company_tickers.csv'), 'r') as f:
146
- company_tickers = list(csv.DictReader(f))
147
-
148
- metadata_fields = ['cik', 'name', 'entityType', 'sic', 'sicDescription', 'ownerOrg',
149
- 'insiderTransactionForOwnerExists', 'insiderTransactionForIssuerExists',
150
- 'tickers', 'exchanges', 'ein', 'description', 'website', 'investorWebsite',
151
- 'category', 'fiscalYearEnd', 'stateOfIncorporation', 'stateOfIncorporationDescription',
152
- 'phone', 'flags', 'mailing_street1', 'mailing_street2', 'mailing_city',
153
- 'mailing_stateOrCountry', 'mailing_zipCode', 'mailing_stateOrCountryDescription',
154
- 'business_street1', 'business_street2', 'business_city', 'business_stateOrCountry',
155
- 'business_zipCode', 'business_stateOrCountryDescription']
156
-
157
- former_names_fields = ['cik', 'former_name', 'from_date', 'to_date']
158
-
159
- try:
160
- async with aiohttp.ClientSession(headers=self.headers) as session:
161
- with open(temp_metadata_file, 'w', newline='') as mf, \
162
- open(temp_former_names_file, 'w', newline='') as fnf:
163
-
164
- metadata_writer = csv.DictWriter(mf, fieldnames=metadata_fields)
165
- metadata_writer.writeheader()
166
-
167
- former_names_writer = csv.DictWriter(fnf, fieldnames=former_names_fields)
168
- former_names_writer.writeheader()
169
-
170
- # Process in batches of 10 companies
171
- batch_size = 10
172
- for i in range(0, len(company_tickers), batch_size):
173
- batch = company_tickers[i:i + batch_size]
174
- await self._process_metadata_batch(
175
- session, batch, metadata_writer, former_names_writer
176
- )
177
-
178
- # Replace original files
179
- for src, dst in [(temp_metadata_file, metadata_file),
180
- (temp_former_names_file, former_names_file)]:
181
- if os.path.exists(dst):
182
- os.remove(dst)
183
- os.rename(src, dst)
184
-
185
- print("Company metadata successfully updated")
186
- return True
187
-
188
- except Exception as e:
189
- print(f"Error updating company metadata: {str(e)}")
190
- return False
191
-
192
- finally:
193
- # Clean up temp files
194
- for temp_file in [temp_metadata_file, temp_former_names_file]:
195
- if os.path.exists(temp_file):
196
- try:
197
- os.remove(temp_file)
198
- except Exception as e:
199
- print(f"Warning: Could not remove {temp_file}: {str(e)}")
200
-
201
- def update_company_tickers(self):
202
- """Update company tickers data files."""
203
- return asyncio.run(self._update_company_tickers())
204
-
205
- def update_company_metadata(self):
206
- """Update company metadata and former names files."""
207
- return asyncio.run(self._update_company_metadata())
datamule-1.0.3.dist-info/RECORD DELETED
@@ -1,43 +0,0 @@
1
- datamule/__init__.py,sha256=IDVK3i5i5DxLlQJ_71aYkloGNi528JOUx8hU6bDzLXM,1255
2
- datamule/config.py,sha256=Y--CVv7JcgrjJkMOSLrvm2S8B9ost6RMSkGviP-MKtg,883
3
- datamule/document.py,sha256=-XT3nkRb1oCOken7n3AUCI8itQNfuKVN86cuD3rys3E,10912
4
- datamule/helper.py,sha256=tr3AQWus9dHNZFKpLSglWjcb8zmm5qDXjOWACMhvMxQ,4594
5
- datamule/monitor.py,sha256=NgzOB_RCLAZDrowYagPWIM4PGNngd1lA3K9Qplkk3Ys,11325
6
- datamule/packageupdater.py,sha256=vEGqlTj6FudIeVHBVJltPh2eBDEqMG9HYmnyrRVKeSU,9595
7
- datamule/portfolio.py,sha256=U_QRNk_CbMmi3nJ0VBIwc9SVEGq6kA8LCZHBj9nOGXs,4032
8
- datamule/submission.py,sha256=uioIYJbsoe-87nRPyzlo-LZ8Hp7HG7A4KPGSnw86PKY,2790
9
- datamule/book/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
10
- datamule/book/book.py,sha256=ns7wVk_gLX5KhAghnSTxKcWIB8bKg7pwsM3es5GfCyM,1422
11
- datamule/book/eftsquery.py,sha256=MW-rMb3L-S1sgAR_Y-Ca3zGa1W9hclTzH52twUK87oM,4596
12
- datamule/book/xbrl_retriever.py,sha256=qLzrskqrQ8guEMFudlUCWFvRqAgSthhql5FXh_E3eCo,2910
13
- datamule/data/company_former_names.csv,sha256=HE9cAv-_QKFX6jT-_-D0rHmaDyQuAzL4MJwank5O1U8,706380
14
- datamule/data/company_metadata.csv,sha256=yPovrCVjYwLWTU_hBUFJymp8iNO0NBYuq_QwOkRLoN8,3068599
15
- datamule/data/company_tickers.csv,sha256=GW6lOP54RiGJCx-d9N5jEBy7tGVgU3zI-5xHJXrZfSI,400363
16
- datamule/data/sec-glossary.csv,sha256=-cN7GjiadLw5C1sv4zSeCnfeZZDYeSgJl-0ydarMAo0,251209
17
- datamule/data/xbrl_descriptions.csv,sha256=SQ9wUURNqG424rnTiZtopsxV2q-PvU4NMj52LqgDsvg,2621524
18
- datamule/downloader/downloader.py,sha256=vnMsw0oWqRa84scu6ZcywxbJxsIn38vLV0tybakx3jQ,15217
19
- datamule/downloader/premiumdownloader.py,sha256=YhGFwkYqjLkdc5ex2YKM-L7nBAPm5MMCdTwVVP0JO78,14314
20
- datamule/mapping_dicts/txt_mapping_dicts.py,sha256=DQPrGYbAPQxomRUtt4iiMGrwuF7BHc_LeFBQuYBzU9o,6311
21
- datamule/mapping_dicts/xml_mapping_dicts.py,sha256=Z22yDVwKYonUfM5foQP00dVDE8EHhhMKp0CLqVKV5OI,438
22
- datamule/mulebot/__init__.py,sha256=YvZXV6xQ0iP-oGD8rloufjdwJL6D46P3NNr0CY9PQCA,29
23
- datamule/mulebot/helper.py,sha256=olztOwltfELZ-IERM2bRNLBavD04kfB6ueWTisJAleA,1080
24
- datamule/mulebot/mulebot.py,sha256=XbtgvXBSFu9OaaLW_k1KDgHVTNQGV8_0ZwNMFad-pPU,5837
25
- datamule/mulebot/search.py,sha256=mwvbB6Fex5dEQkfxkCL53ne5pXdVno-5KlZ5vZyGnJQ,2073
26
- datamule/mulebot/tools.py,sha256=ctnGc2HItR-Roi-QXkc7GEaAOEYQiFRtfmdmIxNxYXk,2940
27
- datamule/mulebot/mulebot_server/__init__.py,sha256=x1QhXys7BWxi2g9_ZHUYA6S6rL3VL2718x4rYtGaaIg,33
28
- datamule/mulebot/mulebot_server/server.py,sha256=M7kU4aZUoi8X8DUKZNckLewBiI637Krbeap31qD2jt8,3547
29
- datamule/mulebot/mulebot_server/static/css/minimalist.css,sha256=Tz1tz8oF_esbfCvLTJBmTfb-5MIiqjfhU_4A4nto1mo,2974
30
- datamule/mulebot/mulebot_server/static/scripts/artifacts.js,sha256=WUAoI3LtEBEt3x-Ri0gwd6YT0JtGNwDZ_b8tuhWWSsg,2258
31
- datamule/mulebot/mulebot_server/static/scripts/chat.js,sha256=q8vV_KtzuNCXCfXqavM1HROIkYHItOAmaR8P1OjSqa0,3108
32
- datamule/mulebot/mulebot_server/static/scripts/filingArtifacts.js,sha256=wxeIM2RzF6Zh_9ivnYuNyTzIgIcEz0-zX8gTCvyACJo,2034
33
- datamule/mulebot/mulebot_server/static/scripts/listArtifacts.js,sha256=DZFLe-45mmzWvJPO1be5Ivfqx0BInrXfduQ1IhbHWzk,429
34
- datamule/mulebot/mulebot_server/static/scripts/main.js,sha256=NEIVih1WJeQ-qo5k8hnmgFHd7N839Mr6hJur856oXVQ,1882
35
- datamule/mulebot/mulebot_server/static/scripts/prefilledPrompt.js,sha256=mGhAXQnjnSxYqVqg1mE5g_ev0-aDhh849xunQtRchnY,1093
36
- datamule/mulebot/mulebot_server/static/scripts/suggestions.js,sha256=TCyz8OYuXeIG9qNRgwU2fhz18YNXpy4Bl9mk66lXefo,1795
37
- datamule/mulebot/mulebot_server/static/scripts/tableArtifacts.js,sha256=UtkUpLvELNI4Ibpb7VstgVA9Tk-8jbkxXhmXsgufFa4,4437
38
- datamule/mulebot/mulebot_server/static/scripts/utils.js,sha256=oGPMtyT9dvuqHqrfZj33t4vLZiF8UJrMXB1hpPXRNu4,1255
39
- datamule/mulebot/mulebot_server/templates/chat-minimalist.html,sha256=MsTbgpnLD0JCQiKKP3XeeNJRNsRqKsRa1j_XXW7nBKw,6975
40
- datamule-1.0.3.dist-info/METADATA,sha256=8PZAcyMcoQTNaV21b9N09t8cd4Uw0Kxm6aImKXlSsCo,732
41
- datamule-1.0.3.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
42
- datamule-1.0.3.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
43
- datamule-1.0.3.dist-info/RECORD,,