datamule 1.0.3__py3-none-any.whl → 1.0.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- datamule/__init__.py +2 -13
- datamule/document.py +0 -1
- datamule/helper.py +85 -105
- datamule/portfolio.py +105 -29
- datamule/submission.py +0 -38
- {datamule-1.0.3.dist-info → datamule-1.0.6.dist-info}/METADATA +2 -8
- datamule-1.0.6.dist-info/RECORD +10 -0
- datamule/book/__init__.py +0 -0
- datamule/book/book.py +0 -34
- datamule/book/eftsquery.py +0 -127
- datamule/book/xbrl_retriever.py +0 -88
- datamule/data/company_former_names.csv +0 -8148
- datamule/data/company_metadata.csv +0 -10049
- datamule/data/company_tickers.csv +0 -9999
- datamule/data/sec-glossary.csv +0 -728
- datamule/data/xbrl_descriptions.csv +0 -10024
- datamule/downloader/downloader.py +0 -374
- datamule/downloader/premiumdownloader.py +0 -335
- datamule/mapping_dicts/txt_mapping_dicts.py +0 -234
- datamule/mapping_dicts/xml_mapping_dicts.py +0 -19
- datamule/monitor.py +0 -283
- datamule/mulebot/__init__.py +0 -1
- datamule/mulebot/helper.py +0 -35
- datamule/mulebot/mulebot.py +0 -130
- datamule/mulebot/mulebot_server/__init__.py +0 -1
- datamule/mulebot/mulebot_server/server.py +0 -87
- datamule/mulebot/mulebot_server/static/css/minimalist.css +0 -174
- datamule/mulebot/mulebot_server/static/scripts/artifacts.js +0 -68
- datamule/mulebot/mulebot_server/static/scripts/chat.js +0 -92
- datamule/mulebot/mulebot_server/static/scripts/filingArtifacts.js +0 -56
- datamule/mulebot/mulebot_server/static/scripts/listArtifacts.js +0 -15
- datamule/mulebot/mulebot_server/static/scripts/main.js +0 -57
- datamule/mulebot/mulebot_server/static/scripts/prefilledPrompt.js +0 -27
- datamule/mulebot/mulebot_server/static/scripts/suggestions.js +0 -47
- datamule/mulebot/mulebot_server/static/scripts/tableArtifacts.js +0 -129
- datamule/mulebot/mulebot_server/static/scripts/utils.js +0 -28
- datamule/mulebot/mulebot_server/templates/chat-minimalist.html +0 -91
- datamule/mulebot/search.py +0 -52
- datamule/mulebot/tools.py +0 -82
- datamule/packageupdater.py +0 -207
- datamule-1.0.3.dist-info/RECORD +0 -43
- {datamule-1.0.3.dist-info → datamule-1.0.6.dist-info}/WHEEL +0 -0
- {datamule-1.0.3.dist-info → datamule-1.0.6.dist-info}/top_level.txt +0 -0
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
// utils.js
|
|
2
|
-
export function renderMetadata(artifactData) {
|
|
3
|
-
const metadataHtml = `
|
|
4
|
-
<div class="card mb-3">
|
|
5
|
-
<div class="card-body">
|
|
6
|
-
<h5 class="card-title">Metadata</h5>
|
|
7
|
-
<p><strong>Fact:</strong> ${artifactData.fact}</p>
|
|
8
|
-
${artifactData.cik ? `<p><strong>CIK:</strong> ${artifactData.cik}</p>` : ''}
|
|
9
|
-
${artifactData.category ? `<p><strong>Category:</strong> ${artifactData.category}</p>` : ''}
|
|
10
|
-
${artifactData.label ? `<p><strong>Label:</strong> ${artifactData.label}</p>` : ''}
|
|
11
|
-
${artifactData.description ? `<p><strong>Description:</strong> ${artifactData.description}</p>` : ''}
|
|
12
|
-
${artifactData.unit ? `<p><strong>Unit:</strong> ${artifactData.unit}</p>` : ''}
|
|
13
|
-
</div>
|
|
14
|
-
</div>
|
|
15
|
-
`;
|
|
16
|
-
document.getElementById('metadata-content').innerHTML = metadataHtml;
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
export function downloadCSV(table) {
|
|
20
|
-
let csv = '';
|
|
21
|
-
const headers = Object.keys(table.table[0]);
|
|
22
|
-
csv += headers.join(',') + '\n';
|
|
23
|
-
table.table.forEach(row => {
|
|
24
|
-
csv += Object.values(row).join(',') + '\n';
|
|
25
|
-
});
|
|
26
|
-
const blob = new Blob([csv], { type: 'text/csv;charset=utf-8;' });
|
|
27
|
-
saveAs(blob, `${table.fact}.csv`);
|
|
28
|
-
}
|
|
@@ -1,91 +0,0 @@
|
|
|
1
|
-
<!DOCTYPE html>
|
|
2
|
-
<html lang="en">
|
|
3
|
-
|
|
4
|
-
<head>
|
|
5
|
-
<meta charset="UTF-8">
|
|
6
|
-
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
7
|
-
<title>Mulebot</title>
|
|
8
|
-
<link href="https://cdnjs.cloudflare.com/ajax/libs/bootstrap/5.1.3/css/bootstrap.min.css" rel="stylesheet">
|
|
9
|
-
<script src="https://cdnjs.cloudflare.com/ajax/libs/FileSaver.js/2.0.5/FileSaver.min.js"></script>
|
|
10
|
-
<script src="https://cdnjs.cloudflare.com/ajax/libs/jszip/3.7.1/jszip.min.js"></script>
|
|
11
|
-
<link rel="stylesheet" href="{{ url_for('static', filename='css/minimalist.css') }}">
|
|
12
|
-
</head>
|
|
13
|
-
|
|
14
|
-
<body>
|
|
15
|
-
<div class="container mt-5">
|
|
16
|
-
<div class="d-flex justify-content-between align-items-center mb-4">
|
|
17
|
-
<h1>Mulebot</h1>
|
|
18
|
-
<div>
|
|
19
|
-
<a href="https://github.com/john-friedman/datamule-python" target="_blank" class="social-btn">
|
|
20
|
-
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor"
|
|
21
|
-
class="bi bi-github" viewBox="0 0 16 16">
|
|
22
|
-
<path
|
|
23
|
-
d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.012 8.012 0 0 0 16 8c0-4.42-3.58-8-8-8z" />
|
|
24
|
-
</svg>
|
|
25
|
-
GitHub
|
|
26
|
-
</a>
|
|
27
|
-
<a href="https://datamule.xyz" target="_blank" class="social-btn">
|
|
28
|
-
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor"
|
|
29
|
-
viewBox="0 0 16 16">
|
|
30
|
-
<path
|
|
31
|
-
d="M0 8a8 8 0 1 1 16 0A8 8 0 0 1 0 8zm7.5-6.923c-.67.204-1.335.82-1.887 1.855-.143.268-.276.56-.395.872.705.157 1.472.257 2.282.287V1.077zM4.249 3.539c.142-.384.304-.744.481-1.078a6.7 6.7 0 0 1 .597-.933A7.01 7.01 0 0 0 3.051 3.05c.362.184.763.349 1.198.49zM3.509 7.5c.036-1.07.188-2.087.436-3.008a9.124 9.124 0 0 1-1.565-.667A6.964 6.964 0 0 0 1.018 7.5h2.49zm1.4-2.741a12.344 12.344 0 0 0-.4 2.741H7.5V5.091c-.91-.03-1.783-.145-2.591-.332zM8.5 5.09V7.5h2.99a12.342 12.342 0 0 0-.399-2.741c-.808.187-1.681.301-2.591.332zM4.51 8.5c.035.987.176 1.914.399 2.741A13.612 13.612 0 0 1 7.5 10.91V8.5H4.51zm3.99 0v2.409c.91.03 1.783.145 2.591.332.223-.827.364-1.754.4-2.741H8.5zm-3.282 3.696c.12.312.252.604.395.872.552 1.035 1.218 1.65 1.887 1.855V11.91c-.81.03-1.577.13-2.282.287zm.11 2.276a6.696 6.696 0 0 1-.598-.933 8.853 8.853 0 0 1-.481-1.079 8.38 8.38 0 0 0-1.198.49 7.01 7.01 0 0 0 2.276 1.522zm-1.383-2.964A13.36 13.36 0 0 1 3.508 8.5h-2.49a6.963 6.963 0 0 0 1.362 3.675c.47-.258.995-.482 1.565-.667zm6.728 2.964a7.009 7.009 0 0 0 2.275-1.521 8.376 8.376 0 0 0-1.197-.49 8.853 8.853 0 0 1-.481 1.078 6.688 6.688 0 0 1-.597.933zM8.5 11.909v3.014c.67-.204 1.335-.82 1.887-1.855.143-.268.276-.56.395-.872A12.63 12.63 0 0 0 8.5 11.91zm3.555-.401c.57.185 1.095.409 1.565.667A6.963 6.963 0 0 0 14.982 8.5h-2.49a13.36 13.36 0 0 1-.437 3.008zM14.982 7.5a6.963 6.963 0 0 0-1.362-3.675c-.47.258-.995.482-1.565.667.248.92.4 1.938.437 3.008h2.49zM11.27 2.461c.177.334.339.694.482 1.078a8.368 8.368 0 0 0 1.196-.49 7.01 7.01 0 0 0-2.275-1.52c.218.283.418.597.597.932zm-.488 1.343a7.765 7.765 0 0 0-.395-.872C9.835 1.897 9.17 1.282 8.5 1.077V4.09c.81-.03 1.577-.13 2.282-.287z" />
|
|
32
|
-
</svg>
|
|
33
|
-
DataMule Website
|
|
34
|
-
</a>
|
|
35
|
-
</div>
|
|
36
|
-
</div>
|
|
37
|
-
<div class="row">
|
|
38
|
-
<div class="col-md-6">
|
|
39
|
-
<div class="card mb-3">
|
|
40
|
-
<div class="card-body">
|
|
41
|
-
<div id="chat-outer-container">
|
|
42
|
-
<div id="chat-container" class="mb-3"></div>
|
|
43
|
-
<div id="thinking-indicator" class="thinking-indicator">
|
|
44
|
-
<span>Bot is thinking</span><span class="dot-animation">...</span>
|
|
45
|
-
</div>
|
|
46
|
-
</div>
|
|
47
|
-
<form id="chat-form">
|
|
48
|
-
<div class="input-group">
|
|
49
|
-
<input type="text" id="user-input" class="form-control"
|
|
50
|
-
placeholder="Type your message...">
|
|
51
|
-
<button type="submit" class="btn btn-primary">Send</button>
|
|
52
|
-
</div>
|
|
53
|
-
</form>
|
|
54
|
-
</div>
|
|
55
|
-
</div>
|
|
56
|
-
</div>
|
|
57
|
-
<div class="col-md-6">
|
|
58
|
-
<button id="toggle-artifacts" class="btn btn-secondary mb-3">Show Artifacts</button>
|
|
59
|
-
<div id="artifact-container" class="card mb-3" style="display: none;">
|
|
60
|
-
<div class="card-body">
|
|
61
|
-
<div id="artifact-content"></div>
|
|
62
|
-
</div>
|
|
63
|
-
</div>
|
|
64
|
-
<div class="suggestion-box">
|
|
65
|
-
<h5 class="mb-2">Suggested Commands:</h5>
|
|
66
|
-
<ul class="list-unstyled">
|
|
67
|
-
<li class="suggestion-item">Get all company facts for Tesla</li>
|
|
68
|
-
<li class="suggestion-item">Get Ford's CIK</li>
|
|
69
|
-
<li class="suggestion-item">Get all 10-K urls for META</li>
|
|
70
|
-
<li class="suggestion-item">Get the management discussion and analysis section from
|
|
71
|
-
https://www.sec.gov/Archives/edgar/data/1318605/000095017022000796/tsla-20211231.htm</li>
|
|
72
|
-
</ul>
|
|
73
|
-
</div>
|
|
74
|
-
</div>
|
|
75
|
-
</div>
|
|
76
|
-
</div>
|
|
77
|
-
<script src="https://cdnjs.cloudflare.com/ajax/libs/bootstrap/5.1.3/js/bootstrap.bundle.min.js"></script>
|
|
78
|
-
|
|
79
|
-
<!-- Application scripts -->
|
|
80
|
-
<script type="module" src="{{ url_for('static', filename='scripts/utils.js') }}"></script>
|
|
81
|
-
<script type="module" src="{{ url_for('static', filename='scripts/chat.js') }}"></script>
|
|
82
|
-
<script type="module" src="{{ url_for('static', filename='scripts/listArtifacts.js') }}"></script>
|
|
83
|
-
<script type="module" src="{{ url_for('static', filename='scripts/filingArtifacts.js') }}"></script>
|
|
84
|
-
<script type="module" src="{{ url_for('static', filename='scripts/tableArtifacts.js') }}"></script>
|
|
85
|
-
<script type="module" src="{{ url_for('static', filename='scripts/artifacts.js') }}"></script>
|
|
86
|
-
<script type="module" src="{{ url_for('static', filename='scripts/suggestions.js') }}"></script>
|
|
87
|
-
<script type="module" src="{{ url_for('static', filename='scripts/prefilledPrompt.js') }}"></script>
|
|
88
|
-
<script type="module" src="{{ url_for('static', filename='scripts/main.js') }}"></script>
|
|
89
|
-
</body>
|
|
90
|
-
|
|
91
|
-
</html>
|
datamule/mulebot/search.py
DELETED
|
@@ -1,52 +0,0 @@
|
|
|
1
|
-
import difflib
|
|
2
|
-
from typing import Dict, List, Any
|
|
3
|
-
|
|
4
|
-
def search_filing(query: str, nested_dict: Dict[str, Any], max_matches: int = 20, score_cutoff: float = 0.6) -> List[Dict[str, Any]]:
|
|
5
|
-
max_matches = min(max_matches, 20)
|
|
6
|
-
query = query.lower() # Convert query to lowercase
|
|
7
|
-
|
|
8
|
-
def flatten_dict(d: Dict[str, Any], parent_path: List[str] = None) -> List[Dict[str, Any]]:
|
|
9
|
-
parent_path = parent_path or []
|
|
10
|
-
items = []
|
|
11
|
-
|
|
12
|
-
if isinstance(d, dict):
|
|
13
|
-
for k, v in d.items():
|
|
14
|
-
new_path = parent_path + [k]
|
|
15
|
-
if k == 'title' and isinstance(v, str):
|
|
16
|
-
items.append({'path': new_path, 'title': v, 'title_lower': v.lower()})
|
|
17
|
-
items.extend(flatten_dict(v, new_path))
|
|
18
|
-
elif isinstance(d, list):
|
|
19
|
-
for i, item in enumerate(d):
|
|
20
|
-
new_path = parent_path + [str(i)]
|
|
21
|
-
items.extend(flatten_dict(item, new_path))
|
|
22
|
-
|
|
23
|
-
return items
|
|
24
|
-
|
|
25
|
-
flat_list = flatten_dict(nested_dict)
|
|
26
|
-
all_titles_lower = [item['title_lower'] for item in flat_list]
|
|
27
|
-
|
|
28
|
-
matches = difflib.get_close_matches(query, all_titles_lower, n=max_matches, cutoff=score_cutoff)
|
|
29
|
-
|
|
30
|
-
results = []
|
|
31
|
-
for match in matches:
|
|
32
|
-
similarity = difflib.SequenceMatcher(None, query, match).ratio()
|
|
33
|
-
for item in flat_list:
|
|
34
|
-
if item['title_lower'] == match:
|
|
35
|
-
# Navigate to the correct nested dictionary
|
|
36
|
-
d = nested_dict
|
|
37
|
-
for key in item['path'][:-1]: # Exclude the last 'title' key
|
|
38
|
-
if key.isdigit():
|
|
39
|
-
d = d[int(key)]
|
|
40
|
-
else:
|
|
41
|
-
d = d[key]
|
|
42
|
-
results.append({
|
|
43
|
-
'path': '.'.join(item['path'][:-1]), # Exclude the last 'title' key
|
|
44
|
-
'content': d,
|
|
45
|
-
'similarity': similarity
|
|
46
|
-
})
|
|
47
|
-
break
|
|
48
|
-
|
|
49
|
-
results.sort(key=lambda x: x['similarity'], reverse=True)
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
return [item['content'] for item in results[:max_matches]]
|
datamule/mulebot/tools.py
DELETED
|
@@ -1,82 +0,0 @@
|
|
|
1
|
-
|
|
2
|
-
get_company_concept_tool = {
|
|
3
|
-
"type": "function",
|
|
4
|
-
"function": {
|
|
5
|
-
"name": "get_company_concept",
|
|
6
|
-
"description": "ONLY use this when explicitly asked to get company XBRL concepts or facts for a given ticker",
|
|
7
|
-
"parameters": {
|
|
8
|
-
"type": "object",
|
|
9
|
-
"properties": {
|
|
10
|
-
"ticker": {"type": "string", "description": "The ticker of the company to get facts for"}
|
|
11
|
-
},
|
|
12
|
-
"required": ["ticker"]
|
|
13
|
-
}
|
|
14
|
-
}
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
identifier_to_cik_tool = {
|
|
18
|
-
"type": "function",
|
|
19
|
-
"function": {
|
|
20
|
-
"name": "identifier_to_cik",
|
|
21
|
-
"description": "ONLY use this when explicitly asked to convert a company's ticker to a CIK.",
|
|
22
|
-
"parameters": {
|
|
23
|
-
"type": "object",
|
|
24
|
-
"properties": {
|
|
25
|
-
"ticker": {"type": "string", "description": "The ticker to convert to a CIK"},
|
|
26
|
-
},
|
|
27
|
-
"required": ["ticker"]
|
|
28
|
-
}
|
|
29
|
-
}
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
get_filing_urls_tool = {
|
|
33
|
-
"type": "function",
|
|
34
|
-
"function": {
|
|
35
|
-
"name": "get_filing_urls",
|
|
36
|
-
"description": "ONLY use this when explicitly asked to get URLs of filings for a given company or multiple",
|
|
37
|
-
"parameters": {
|
|
38
|
-
"type": "object",
|
|
39
|
-
"properties": {
|
|
40
|
-
"ticker": {"type": "string", "description": "Ticker symbol of the company. Can be a single ticker or a list of tickers."},
|
|
41
|
-
"form": {"type": "string", "description": "Form type to get (e.g., '10-K', '10-Q')"},
|
|
42
|
-
"date": {"type": "string", "description": "Date of the filing, can be a single date, a range, or a list of dates. Format: 'YYYY-MM-DD'. If range use a tuple of two dates."},
|
|
43
|
-
},
|
|
44
|
-
"required": ["ticker"]
|
|
45
|
-
}
|
|
46
|
-
}
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
find_filing_section_by_title_tool = {
|
|
50
|
-
"type": "function",
|
|
51
|
-
"function": {
|
|
52
|
-
"name": "find_filing_section_by_title",
|
|
53
|
-
"description": "ONLY use this when explicitly given a filing URL and told to find a specific section",
|
|
54
|
-
"parameters": {
|
|
55
|
-
"type": "object",
|
|
56
|
-
"properties": {
|
|
57
|
-
"url": {"type": "string", "description": "URL of the filing to parse"},
|
|
58
|
-
"title": {"type": "string", "description": "The section title to search for in the filing"}
|
|
59
|
-
},
|
|
60
|
-
"required": ["url","title"]
|
|
61
|
-
}
|
|
62
|
-
}
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
return_title_tool = {
|
|
66
|
-
"type": "function",
|
|
67
|
-
"function": {
|
|
68
|
-
"name": "return_title",
|
|
69
|
-
"description": "use this to select a title",
|
|
70
|
-
"parameters": {
|
|
71
|
-
"type": "object",
|
|
72
|
-
"properties": {
|
|
73
|
-
"title": {"type": "string", "description": "The title to return"}
|
|
74
|
-
},
|
|
75
|
-
"required": ["title"]
|
|
76
|
-
}
|
|
77
|
-
}
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
tools = [get_company_concept_tool, identifier_to_cik_tool, get_filing_urls_tool, find_filing_section_by_title_tool]
|
datamule/packageupdater.py
DELETED
|
@@ -1,207 +0,0 @@
|
|
|
1
|
-
import asyncio
|
|
2
|
-
import aiohttp
|
|
3
|
-
import json
|
|
4
|
-
import csv
|
|
5
|
-
import os
|
|
6
|
-
from pkg_resources import resource_filename
|
|
7
|
-
from .helper import headers
|
|
8
|
-
from .downloader.downloader import PreciseRateLimiter, RateMonitor
|
|
9
|
-
|
|
10
|
-
class PackageUpdater:
|
|
11
|
-
def __init__(self):
|
|
12
|
-
self.limiter = PreciseRateLimiter(5) # 5 requests per second
|
|
13
|
-
self.rate_monitor = RateMonitor()
|
|
14
|
-
self.headers = headers
|
|
15
|
-
|
|
16
|
-
async def _fetch_json(self, session, url):
|
|
17
|
-
"""Fetch JSON with rate limiting and monitoring."""
|
|
18
|
-
async with self.limiter:
|
|
19
|
-
try:
|
|
20
|
-
async with session.get(url) as response:
|
|
21
|
-
response.raise_for_status()
|
|
22
|
-
content = await response.read()
|
|
23
|
-
await self.rate_monitor.add_request(len(content))
|
|
24
|
-
return await response.json()
|
|
25
|
-
except Exception as e:
|
|
26
|
-
print(f"Error fetching {url}: {str(e)}")
|
|
27
|
-
return None
|
|
28
|
-
|
|
29
|
-
async def _update_company_tickers(self):
|
|
30
|
-
"""Update company tickers data files."""
|
|
31
|
-
url = 'https://www.sec.gov/files/company_tickers.json'
|
|
32
|
-
|
|
33
|
-
# Define file paths
|
|
34
|
-
json_file = resource_filename('datamule', 'data/company_tickers.json')
|
|
35
|
-
csv_file = resource_filename('datamule', 'data/company_tickers.csv')
|
|
36
|
-
|
|
37
|
-
# Define temporary file paths
|
|
38
|
-
temp_json_file = json_file + '.temp'
|
|
39
|
-
temp_csv_file = csv_file + '.temp'
|
|
40
|
-
|
|
41
|
-
async with aiohttp.ClientSession(headers=self.headers) as session:
|
|
42
|
-
try:
|
|
43
|
-
data = await self._fetch_json(session, url)
|
|
44
|
-
if not data:
|
|
45
|
-
raise Exception("Failed to fetch company tickers data")
|
|
46
|
-
|
|
47
|
-
# Save the raw JSON file
|
|
48
|
-
with open(temp_json_file, 'w') as f:
|
|
49
|
-
json.dump(data, f, indent=4)
|
|
50
|
-
|
|
51
|
-
# Convert to CSV
|
|
52
|
-
with open(temp_csv_file, 'w', newline='') as csvfile:
|
|
53
|
-
fieldnames = ['cik', 'ticker', 'title']
|
|
54
|
-
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
|
|
55
|
-
writer.writeheader()
|
|
56
|
-
for _, company in data.items():
|
|
57
|
-
writer.writerow({
|
|
58
|
-
'cik': str(company['cik_str']).zfill(10),
|
|
59
|
-
'ticker': company['ticker'],
|
|
60
|
-
'title': company['title']
|
|
61
|
-
})
|
|
62
|
-
|
|
63
|
-
# Replace original files
|
|
64
|
-
for src, dst in [(temp_json_file, json_file), (temp_csv_file, csv_file)]:
|
|
65
|
-
if os.path.exists(dst):
|
|
66
|
-
os.remove(dst)
|
|
67
|
-
os.rename(src, dst)
|
|
68
|
-
|
|
69
|
-
print(f"Company tickers successfully updated")
|
|
70
|
-
return True
|
|
71
|
-
|
|
72
|
-
except Exception as e:
|
|
73
|
-
print(f"Error updating company tickers: {str(e)}")
|
|
74
|
-
return False
|
|
75
|
-
|
|
76
|
-
finally:
|
|
77
|
-
# Clean up temp files
|
|
78
|
-
for temp_file in [temp_json_file, temp_csv_file]:
|
|
79
|
-
if os.path.exists(temp_file):
|
|
80
|
-
try:
|
|
81
|
-
os.remove(temp_file)
|
|
82
|
-
except Exception as e:
|
|
83
|
-
print(f"Warning: Could not remove {temp_file}: {str(e)}")
|
|
84
|
-
|
|
85
|
-
async def _process_metadata_batch(self, session, companies, metadata_writer, former_names_writer):
|
|
86
|
-
"""Process a batch of companies for metadata updates."""
|
|
87
|
-
metadata_fields = [
|
|
88
|
-
'cik', 'name', 'entityType', 'sic', 'sicDescription', 'ownerOrg',
|
|
89
|
-
'insiderTransactionForOwnerExists', 'insiderTransactionForIssuerExists',
|
|
90
|
-
'tickers', 'exchanges', 'ein', 'description', 'website', 'investorWebsite',
|
|
91
|
-
'category', 'fiscalYearEnd', 'stateOfIncorporation', 'stateOfIncorporationDescription',
|
|
92
|
-
'phone', 'flags', 'mailing_street1', 'mailing_street2', 'mailing_city',
|
|
93
|
-
'mailing_stateOrCountry', 'mailing_zipCode', 'mailing_stateOrCountryDescription',
|
|
94
|
-
'business_street1', 'business_street2', 'business_city', 'business_stateOrCountry',
|
|
95
|
-
'business_zipCode', 'business_stateOrCountryDescription'
|
|
96
|
-
]
|
|
97
|
-
|
|
98
|
-
tasks = []
|
|
99
|
-
for company in companies:
|
|
100
|
-
cik = company['cik']
|
|
101
|
-
url = f'https://data.sec.gov/submissions/CIK{str(cik).zfill(10)}.json'
|
|
102
|
-
tasks.append(self._fetch_json(session, url))
|
|
103
|
-
|
|
104
|
-
results = await asyncio.gather(*tasks, return_exceptions=True)
|
|
105
|
-
|
|
106
|
-
for company, result in zip(companies, results):
|
|
107
|
-
if isinstance(result, Exception) or not result:
|
|
108
|
-
print(f"Error processing CIK {company['cik']}: {str(result) if isinstance(result, Exception) else 'No data'}")
|
|
109
|
-
continue
|
|
110
|
-
|
|
111
|
-
try:
|
|
112
|
-
metadata = {field: result.get(field, '') for field in metadata_fields if field not in ['tickers', 'exchanges']}
|
|
113
|
-
metadata['cik'] = company['cik']
|
|
114
|
-
metadata['tickers'] = ','.join(result.get('tickers', []))
|
|
115
|
-
metadata['exchanges'] = ','.join(result.get('exchanges', []))
|
|
116
|
-
|
|
117
|
-
# Add address information
|
|
118
|
-
for address_type in ['mailing', 'business']:
|
|
119
|
-
address = result.get('addresses', {}).get(address_type, {})
|
|
120
|
-
for key, value in address.items():
|
|
121
|
-
metadata[f'{address_type}_{key}'] = value if value is not None else ''
|
|
122
|
-
|
|
123
|
-
metadata_writer.writerow(metadata)
|
|
124
|
-
|
|
125
|
-
for former_name in result.get('formerNames', []):
|
|
126
|
-
former_names_writer.writerow({
|
|
127
|
-
'cik': company['cik'],
|
|
128
|
-
'former_name': former_name['name'],
|
|
129
|
-
'from_date': former_name['from'],
|
|
130
|
-
'to_date': former_name['to']
|
|
131
|
-
})
|
|
132
|
-
|
|
133
|
-
except Exception as e:
|
|
134
|
-
print(f"Error processing metadata for CIK {company['cik']}: {str(e)}")
|
|
135
|
-
|
|
136
|
-
async def _update_company_metadata(self):
|
|
137
|
-
"""Update company metadata and former names files."""
|
|
138
|
-
metadata_file = resource_filename('datamule', 'data/company_metadata.csv')
|
|
139
|
-
former_names_file = resource_filename('datamule', 'data/company_former_names.csv')
|
|
140
|
-
|
|
141
|
-
temp_metadata_file = metadata_file + '.temp'
|
|
142
|
-
temp_former_names_file = former_names_file + '.temp'
|
|
143
|
-
|
|
144
|
-
# Load current company tickers
|
|
145
|
-
with open(resource_filename('datamule', 'data/company_tickers.csv'), 'r') as f:
|
|
146
|
-
company_tickers = list(csv.DictReader(f))
|
|
147
|
-
|
|
148
|
-
metadata_fields = ['cik', 'name', 'entityType', 'sic', 'sicDescription', 'ownerOrg',
|
|
149
|
-
'insiderTransactionForOwnerExists', 'insiderTransactionForIssuerExists',
|
|
150
|
-
'tickers', 'exchanges', 'ein', 'description', 'website', 'investorWebsite',
|
|
151
|
-
'category', 'fiscalYearEnd', 'stateOfIncorporation', 'stateOfIncorporationDescription',
|
|
152
|
-
'phone', 'flags', 'mailing_street1', 'mailing_street2', 'mailing_city',
|
|
153
|
-
'mailing_stateOrCountry', 'mailing_zipCode', 'mailing_stateOrCountryDescription',
|
|
154
|
-
'business_street1', 'business_street2', 'business_city', 'business_stateOrCountry',
|
|
155
|
-
'business_zipCode', 'business_stateOrCountryDescription']
|
|
156
|
-
|
|
157
|
-
former_names_fields = ['cik', 'former_name', 'from_date', 'to_date']
|
|
158
|
-
|
|
159
|
-
try:
|
|
160
|
-
async with aiohttp.ClientSession(headers=self.headers) as session:
|
|
161
|
-
with open(temp_metadata_file, 'w', newline='') as mf, \
|
|
162
|
-
open(temp_former_names_file, 'w', newline='') as fnf:
|
|
163
|
-
|
|
164
|
-
metadata_writer = csv.DictWriter(mf, fieldnames=metadata_fields)
|
|
165
|
-
metadata_writer.writeheader()
|
|
166
|
-
|
|
167
|
-
former_names_writer = csv.DictWriter(fnf, fieldnames=former_names_fields)
|
|
168
|
-
former_names_writer.writeheader()
|
|
169
|
-
|
|
170
|
-
# Process in batches of 10 companies
|
|
171
|
-
batch_size = 10
|
|
172
|
-
for i in range(0, len(company_tickers), batch_size):
|
|
173
|
-
batch = company_tickers[i:i + batch_size]
|
|
174
|
-
await self._process_metadata_batch(
|
|
175
|
-
session, batch, metadata_writer, former_names_writer
|
|
176
|
-
)
|
|
177
|
-
|
|
178
|
-
# Replace original files
|
|
179
|
-
for src, dst in [(temp_metadata_file, metadata_file),
|
|
180
|
-
(temp_former_names_file, former_names_file)]:
|
|
181
|
-
if os.path.exists(dst):
|
|
182
|
-
os.remove(dst)
|
|
183
|
-
os.rename(src, dst)
|
|
184
|
-
|
|
185
|
-
print("Company metadata successfully updated")
|
|
186
|
-
return True
|
|
187
|
-
|
|
188
|
-
except Exception as e:
|
|
189
|
-
print(f"Error updating company metadata: {str(e)}")
|
|
190
|
-
return False
|
|
191
|
-
|
|
192
|
-
finally:
|
|
193
|
-
# Clean up temp files
|
|
194
|
-
for temp_file in [temp_metadata_file, temp_former_names_file]:
|
|
195
|
-
if os.path.exists(temp_file):
|
|
196
|
-
try:
|
|
197
|
-
os.remove(temp_file)
|
|
198
|
-
except Exception as e:
|
|
199
|
-
print(f"Warning: Could not remove {temp_file}: {str(e)}")
|
|
200
|
-
|
|
201
|
-
def update_company_tickers(self):
|
|
202
|
-
"""Update company tickers data files."""
|
|
203
|
-
return asyncio.run(self._update_company_tickers())
|
|
204
|
-
|
|
205
|
-
def update_company_metadata(self):
|
|
206
|
-
"""Update company metadata and former names files."""
|
|
207
|
-
return asyncio.run(self._update_company_metadata())
|
datamule-1.0.3.dist-info/RECORD
DELETED
|
@@ -1,43 +0,0 @@
|
|
|
1
|
-
datamule/__init__.py,sha256=IDVK3i5i5DxLlQJ_71aYkloGNi528JOUx8hU6bDzLXM,1255
|
|
2
|
-
datamule/config.py,sha256=Y--CVv7JcgrjJkMOSLrvm2S8B9ost6RMSkGviP-MKtg,883
|
|
3
|
-
datamule/document.py,sha256=-XT3nkRb1oCOken7n3AUCI8itQNfuKVN86cuD3rys3E,10912
|
|
4
|
-
datamule/helper.py,sha256=tr3AQWus9dHNZFKpLSglWjcb8zmm5qDXjOWACMhvMxQ,4594
|
|
5
|
-
datamule/monitor.py,sha256=NgzOB_RCLAZDrowYagPWIM4PGNngd1lA3K9Qplkk3Ys,11325
|
|
6
|
-
datamule/packageupdater.py,sha256=vEGqlTj6FudIeVHBVJltPh2eBDEqMG9HYmnyrRVKeSU,9595
|
|
7
|
-
datamule/portfolio.py,sha256=U_QRNk_CbMmi3nJ0VBIwc9SVEGq6kA8LCZHBj9nOGXs,4032
|
|
8
|
-
datamule/submission.py,sha256=uioIYJbsoe-87nRPyzlo-LZ8Hp7HG7A4KPGSnw86PKY,2790
|
|
9
|
-
datamule/book/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
10
|
-
datamule/book/book.py,sha256=ns7wVk_gLX5KhAghnSTxKcWIB8bKg7pwsM3es5GfCyM,1422
|
|
11
|
-
datamule/book/eftsquery.py,sha256=MW-rMb3L-S1sgAR_Y-Ca3zGa1W9hclTzH52twUK87oM,4596
|
|
12
|
-
datamule/book/xbrl_retriever.py,sha256=qLzrskqrQ8guEMFudlUCWFvRqAgSthhql5FXh_E3eCo,2910
|
|
13
|
-
datamule/data/company_former_names.csv,sha256=HE9cAv-_QKFX6jT-_-D0rHmaDyQuAzL4MJwank5O1U8,706380
|
|
14
|
-
datamule/data/company_metadata.csv,sha256=yPovrCVjYwLWTU_hBUFJymp8iNO0NBYuq_QwOkRLoN8,3068599
|
|
15
|
-
datamule/data/company_tickers.csv,sha256=GW6lOP54RiGJCx-d9N5jEBy7tGVgU3zI-5xHJXrZfSI,400363
|
|
16
|
-
datamule/data/sec-glossary.csv,sha256=-cN7GjiadLw5C1sv4zSeCnfeZZDYeSgJl-0ydarMAo0,251209
|
|
17
|
-
datamule/data/xbrl_descriptions.csv,sha256=SQ9wUURNqG424rnTiZtopsxV2q-PvU4NMj52LqgDsvg,2621524
|
|
18
|
-
datamule/downloader/downloader.py,sha256=vnMsw0oWqRa84scu6ZcywxbJxsIn38vLV0tybakx3jQ,15217
|
|
19
|
-
datamule/downloader/premiumdownloader.py,sha256=YhGFwkYqjLkdc5ex2YKM-L7nBAPm5MMCdTwVVP0JO78,14314
|
|
20
|
-
datamule/mapping_dicts/txt_mapping_dicts.py,sha256=DQPrGYbAPQxomRUtt4iiMGrwuF7BHc_LeFBQuYBzU9o,6311
|
|
21
|
-
datamule/mapping_dicts/xml_mapping_dicts.py,sha256=Z22yDVwKYonUfM5foQP00dVDE8EHhhMKp0CLqVKV5OI,438
|
|
22
|
-
datamule/mulebot/__init__.py,sha256=YvZXV6xQ0iP-oGD8rloufjdwJL6D46P3NNr0CY9PQCA,29
|
|
23
|
-
datamule/mulebot/helper.py,sha256=olztOwltfELZ-IERM2bRNLBavD04kfB6ueWTisJAleA,1080
|
|
24
|
-
datamule/mulebot/mulebot.py,sha256=XbtgvXBSFu9OaaLW_k1KDgHVTNQGV8_0ZwNMFad-pPU,5837
|
|
25
|
-
datamule/mulebot/search.py,sha256=mwvbB6Fex5dEQkfxkCL53ne5pXdVno-5KlZ5vZyGnJQ,2073
|
|
26
|
-
datamule/mulebot/tools.py,sha256=ctnGc2HItR-Roi-QXkc7GEaAOEYQiFRtfmdmIxNxYXk,2940
|
|
27
|
-
datamule/mulebot/mulebot_server/__init__.py,sha256=x1QhXys7BWxi2g9_ZHUYA6S6rL3VL2718x4rYtGaaIg,33
|
|
28
|
-
datamule/mulebot/mulebot_server/server.py,sha256=M7kU4aZUoi8X8DUKZNckLewBiI637Krbeap31qD2jt8,3547
|
|
29
|
-
datamule/mulebot/mulebot_server/static/css/minimalist.css,sha256=Tz1tz8oF_esbfCvLTJBmTfb-5MIiqjfhU_4A4nto1mo,2974
|
|
30
|
-
datamule/mulebot/mulebot_server/static/scripts/artifacts.js,sha256=WUAoI3LtEBEt3x-Ri0gwd6YT0JtGNwDZ_b8tuhWWSsg,2258
|
|
31
|
-
datamule/mulebot/mulebot_server/static/scripts/chat.js,sha256=q8vV_KtzuNCXCfXqavM1HROIkYHItOAmaR8P1OjSqa0,3108
|
|
32
|
-
datamule/mulebot/mulebot_server/static/scripts/filingArtifacts.js,sha256=wxeIM2RzF6Zh_9ivnYuNyTzIgIcEz0-zX8gTCvyACJo,2034
|
|
33
|
-
datamule/mulebot/mulebot_server/static/scripts/listArtifacts.js,sha256=DZFLe-45mmzWvJPO1be5Ivfqx0BInrXfduQ1IhbHWzk,429
|
|
34
|
-
datamule/mulebot/mulebot_server/static/scripts/main.js,sha256=NEIVih1WJeQ-qo5k8hnmgFHd7N839Mr6hJur856oXVQ,1882
|
|
35
|
-
datamule/mulebot/mulebot_server/static/scripts/prefilledPrompt.js,sha256=mGhAXQnjnSxYqVqg1mE5g_ev0-aDhh849xunQtRchnY,1093
|
|
36
|
-
datamule/mulebot/mulebot_server/static/scripts/suggestions.js,sha256=TCyz8OYuXeIG9qNRgwU2fhz18YNXpy4Bl9mk66lXefo,1795
|
|
37
|
-
datamule/mulebot/mulebot_server/static/scripts/tableArtifacts.js,sha256=UtkUpLvELNI4Ibpb7VstgVA9Tk-8jbkxXhmXsgufFa4,4437
|
|
38
|
-
datamule/mulebot/mulebot_server/static/scripts/utils.js,sha256=oGPMtyT9dvuqHqrfZj33t4vLZiF8UJrMXB1hpPXRNu4,1255
|
|
39
|
-
datamule/mulebot/mulebot_server/templates/chat-minimalist.html,sha256=MsTbgpnLD0JCQiKKP3XeeNJRNsRqKsRa1j_XXW7nBKw,6975
|
|
40
|
-
datamule-1.0.3.dist-info/METADATA,sha256=8PZAcyMcoQTNaV21b9N09t8cd4Uw0Kxm6aImKXlSsCo,732
|
|
41
|
-
datamule-1.0.3.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
|
|
42
|
-
datamule-1.0.3.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
|
|
43
|
-
datamule-1.0.3.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|