epstein-files 1.0.16__py3-none-any.whl → 1.1.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- epstein_files/__init__.py +26 -17
- epstein_files/documents/communication.py +10 -14
- epstein_files/documents/document.py +5 -1
- epstein_files/documents/email.py +164 -78
- epstein_files/documents/imessage/text_message.py +42 -25
- epstein_files/documents/messenger_log.py +31 -12
- epstein_files/documents/other_file.py +13 -12
- epstein_files/epstein_files.py +19 -80
- epstein_files/util/constant/common_words.py +3 -3
- epstein_files/util/constant/html.py +13 -6
- epstein_files/util/constant/names.py +10 -7
- epstein_files/util/constant/output_files.py +3 -0
- epstein_files/util/constant/strings.py +6 -2
- epstein_files/util/constant/urls.py +1 -1
- epstein_files/util/constants.py +18 -22
- epstein_files/util/env.py +46 -36
- epstein_files/util/file_helper.py +1 -2
- epstein_files/util/highlighted_group.py +1007 -187
- epstein_files/util/logging.py +8 -1
- epstein_files/util/output.py +166 -51
- epstein_files/util/rich.py +55 -79
- epstein_files/util/timer.py +1 -1
- epstein_files/util/word_count.py +3 -4
- {epstein_files-1.0.16.dist-info → epstein_files-1.1.2.dist-info}/METADATA +1 -1
- epstein_files-1.1.2.dist-info/RECORD +33 -0
- epstein_files-1.0.16.dist-info/RECORD +0 -33
- {epstein_files-1.0.16.dist-info → epstein_files-1.1.2.dist-info}/LICENSE +0 -0
- {epstein_files-1.0.16.dist-info → epstein_files-1.1.2.dist-info}/WHEEL +0 -0
- {epstein_files-1.0.16.dist-info → epstein_files-1.1.2.dist-info}/entry_points.txt +0 -0
epstein_files/util/logging.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
from os import environ
|
|
3
|
+
from sys import exit
|
|
3
4
|
|
|
4
5
|
import datefinder
|
|
5
6
|
import rich_argparse_plus
|
|
@@ -39,7 +40,7 @@ class LogHighlighter(ReprHighlighter):
|
|
|
39
40
|
|
|
40
41
|
log_console = Console(color_system='256', theme=Theme(LOG_THEME))
|
|
41
42
|
log_handler = RichHandler(console=log_console, highlighter=LogHighlighter())
|
|
42
|
-
logging.basicConfig(level="NOTSET", format="%(message)s", datefmt="
|
|
43
|
+
logging.basicConfig(level="NOTSET", format="%(message)s", datefmt=" ", handlers=[log_handler])
|
|
43
44
|
logger = logging.getLogger("rich")
|
|
44
45
|
|
|
45
46
|
|
|
@@ -58,3 +59,9 @@ if env_log_level_str:
|
|
|
58
59
|
|
|
59
60
|
logger.warning(f"Setting log level to {env_log_level} based on {LOG_LEVEL_ENV_VAR} env var...")
|
|
60
61
|
logger.setLevel(env_log_level)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def exit_with_error(msg: str) -> None:
|
|
65
|
+
print('')
|
|
66
|
+
logger.error(msg + '\n')
|
|
67
|
+
exit(1)
|
epstein_files/util/output.py
CHANGED
|
@@ -2,7 +2,8 @@ import json
|
|
|
2
2
|
|
|
3
3
|
from rich.padding import Padding
|
|
4
4
|
|
|
5
|
-
from epstein_files.documents.
|
|
5
|
+
from epstein_files.documents.document import Document
|
|
6
|
+
from epstein_files.documents.email import JUNK_EMAILERS, KRASSNER_RECIPIENTS, Email
|
|
6
7
|
from epstein_files.documents.messenger_log import MessengerLog
|
|
7
8
|
from epstein_files.documents.other_file import FIRST_FEW_LINES, OtherFile
|
|
8
9
|
from epstein_files.epstein_files import EpsteinFiles, count_by_month
|
|
@@ -10,7 +11,8 @@ from epstein_files.util.constant import output_files
|
|
|
10
11
|
from epstein_files.util.constant.html import *
|
|
11
12
|
from epstein_files.util.constant.names import *
|
|
12
13
|
from epstein_files.util.constant.output_files import JSON_FILES_JSON_PATH, JSON_METADATA_PATH
|
|
13
|
-
from epstein_files.util.
|
|
14
|
+
from epstein_files.util.constant.strings import TIMESTAMP_DIM, TIMESTAMP_STYLE
|
|
15
|
+
from epstein_files.util.data import dict_sets_to_lists, sort_dict
|
|
14
16
|
from epstein_files.util.env import args
|
|
15
17
|
from epstein_files.util.file_helper import log_file_write
|
|
16
18
|
from epstein_files.util.logging import logger
|
|
@@ -27,59 +29,49 @@ DEFAULT_EMAILERS = [
|
|
|
27
29
|
AL_SECKEL,
|
|
28
30
|
DANIEL_SIAD,
|
|
29
31
|
JEAN_LUC_BRUNEL,
|
|
30
|
-
STEVEN_HOFFENBERG,
|
|
31
32
|
RENATA_BOLOTOVA,
|
|
33
|
+
STEVEN_HOFFENBERG,
|
|
32
34
|
MASHA_DROKOVA,
|
|
33
35
|
EHUD_BARAK,
|
|
34
36
|
MARTIN_NOWAK,
|
|
35
37
|
STEVE_BANNON,
|
|
38
|
+
TYLER_SHEARS,
|
|
36
39
|
JIDE_ZEITLIN,
|
|
40
|
+
CHRISTINA_GALBRAITH,
|
|
37
41
|
DAVID_STERN,
|
|
38
42
|
MOHAMED_WAHEED_HASSAN,
|
|
39
43
|
JENNIFER_JACQUET,
|
|
40
|
-
TYLER_SHEARS,
|
|
41
|
-
CHRISTINA_GALBRAITH,
|
|
42
44
|
ZUBAIR_KHAN,
|
|
43
45
|
None,
|
|
44
46
|
]
|
|
45
47
|
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
ARIANE_DE_ROTHSCHILD,
|
|
48
|
+
INVALID_FOR_EPSTEIN_WEB = JUNK_EMAILERS + KRASSNER_RECIPIENTS + [
|
|
49
|
+
'ACT for America',
|
|
50
|
+
'BS Stern',
|
|
51
|
+
INTELLIGENCE_SQUARED,
|
|
52
|
+
UNKNOWN,
|
|
52
53
|
]
|
|
53
54
|
|
|
54
|
-
if len(set(DEFAULT_EMAILERS).intersection(set(DEFAULT_EMAILER_TABLES))) > 0:
|
|
55
|
-
raise RuntimeError(f"Some names appear in both DEFAULT_EMAILERS and DEFAULT_EMAILER_TABLES")
|
|
56
|
-
|
|
57
55
|
|
|
58
56
|
def print_emails_section(epstein_files: EpsteinFiles) -> list[Email]:
|
|
59
57
|
"""Returns emails that were printed (may contain dupes if printed for both author and recipient)."""
|
|
60
58
|
print_section_header(('Selections from ' if not args.all_emails else '') + 'His Emails')
|
|
61
|
-
print_other_page_link(epstein_files)
|
|
62
59
|
emailers_to_print: list[str | None]
|
|
63
|
-
emailer_tables: list[str | None] = []
|
|
64
60
|
already_printed_emails: list[Email] = []
|
|
65
61
|
num_emails_printed_since_last_color_key = 0
|
|
66
62
|
|
|
67
63
|
if args.names:
|
|
68
64
|
emailers_to_print = args.names
|
|
69
65
|
else:
|
|
70
|
-
print_centered(Padding(epstein_files.table_of_emailers(), (2, 0)))
|
|
71
|
-
|
|
72
66
|
if args.all_emails:
|
|
73
67
|
emailers_to_print = sorted(epstein_files.all_emailers(), key=lambda e: epstein_files.earliest_email_at(e))
|
|
74
|
-
console.print('Email conversations are sorted chronologically based on time of the first email.')
|
|
75
|
-
print_numbered_list_of_emailers(emailers_to_print, epstein_files)
|
|
76
68
|
else:
|
|
77
69
|
emailers_to_print = DEFAULT_EMAILERS
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
70
|
+
|
|
71
|
+
print_other_page_link(epstein_files)
|
|
72
|
+
console.line(2)
|
|
73
|
+
console.print(table_of_selected_emailers(emailers_to_print, epstein_files))
|
|
74
|
+
console.print(Padding(_all_emailers_table(epstein_files), (2, 0)))
|
|
83
75
|
|
|
84
76
|
for author in emailers_to_print:
|
|
85
77
|
author_emails = epstein_files.print_emails_for(author)
|
|
@@ -91,12 +83,6 @@ def print_emails_section(epstein_files: EpsteinFiles) -> list[Email]:
|
|
|
91
83
|
print_color_key()
|
|
92
84
|
num_emails_printed_since_last_color_key = 0
|
|
93
85
|
|
|
94
|
-
if emailer_tables:
|
|
95
|
-
print_author_header(f"Email Tables for {len(emailer_tables)} Other People", 'white')
|
|
96
|
-
|
|
97
|
-
for name in DEFAULT_EMAILER_TABLES:
|
|
98
|
-
epstein_files.print_emails_table_for(name)
|
|
99
|
-
|
|
100
86
|
if not args.names:
|
|
101
87
|
epstein_files.print_email_device_info()
|
|
102
88
|
|
|
@@ -138,35 +124,106 @@ def print_json_stats(epstein_files: EpsteinFiles) -> None:
|
|
|
138
124
|
|
|
139
125
|
def print_other_files_section(files: list[OtherFile], epstein_files: EpsteinFiles) -> None:
|
|
140
126
|
"""Returns the OtherFile objects that were interesting enough to print."""
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
print_section_header(f"{FIRST_FEW_LINES} of {len(files)} {
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
console.line(1)
|
|
148
|
-
else:
|
|
149
|
-
print_other_page_link(epstein_files)
|
|
150
|
-
console.line(2)
|
|
151
|
-
|
|
152
|
-
for table in [category_table, other_files_preview_table]:
|
|
153
|
-
table.title = f"{header_pfx}{table.title}"
|
|
154
|
-
|
|
155
|
-
print_centered(category_table)
|
|
156
|
-
console.line(2)
|
|
127
|
+
title_pfx = '' if args.all_other_files else 'Selected '
|
|
128
|
+
category_table = OtherFile.count_by_category_table(files, title_pfx=title_pfx)
|
|
129
|
+
other_files_preview_table = OtherFile.files_preview_table(files, title_pfx=title_pfx)
|
|
130
|
+
print_section_header(f"{FIRST_FEW_LINES} of {len(files)} {title_pfx}Files That Are Neither Emails Nor Text Messages")
|
|
131
|
+
print_other_page_link(epstein_files)
|
|
132
|
+
print_centered(Padding(category_table, (2, 0)))
|
|
157
133
|
console.print(other_files_preview_table)
|
|
158
134
|
|
|
159
135
|
|
|
160
|
-
def print_text_messages_section(
|
|
136
|
+
def print_text_messages_section(imessage_logs: list[MessengerLog]) -> None:
|
|
161
137
|
"""Print summary table and stats for text messages."""
|
|
138
|
+
if not imessage_logs:
|
|
139
|
+
logger.warning(f"No MessengerLog objects to output...")
|
|
140
|
+
return
|
|
141
|
+
|
|
162
142
|
print_section_header('All of His Text Messages')
|
|
163
|
-
print_centered("(conversations are sorted chronologically based on timestamp of first message)
|
|
143
|
+
print_centered("(conversations are sorted chronologically based on timestamp of first message in the log file)", style='dim')
|
|
144
|
+
console.line(2)
|
|
164
145
|
|
|
165
|
-
|
|
146
|
+
if not args.names:
|
|
147
|
+
print_centered(MessengerLog.summary_table(imessage_logs))
|
|
148
|
+
console.line(2)
|
|
149
|
+
|
|
150
|
+
for log_file in imessage_logs:
|
|
166
151
|
console.print(Padding(log_file))
|
|
167
152
|
console.line(2)
|
|
168
153
|
|
|
169
|
-
|
|
154
|
+
|
|
155
|
+
def table_of_selected_emailers(_list: list[str | None], epstein_files: EpsteinFiles) -> Table:
|
|
156
|
+
"""Add the first emailed_at timestamp for each emailer if 'epstein_files' provided."""
|
|
157
|
+
header_pfx = '' if args.all_emails else 'Selected '
|
|
158
|
+
table = build_table(f'{header_pfx}Email Conversations Grouped by Counterparty Will Appear in this Order')
|
|
159
|
+
table.add_column('Start Date')
|
|
160
|
+
table.add_column('Name', max_width=25, no_wrap=True)
|
|
161
|
+
table.add_column('Category', justify='center', style='dim italic')
|
|
162
|
+
table.add_column('Num', justify='right', style='wheat4')
|
|
163
|
+
table.add_column('Info', style='white italic')
|
|
164
|
+
current_year = 1990
|
|
165
|
+
current_year_month = current_year * 12
|
|
166
|
+
grey_idx = 0
|
|
167
|
+
|
|
168
|
+
for i, name in enumerate(_list):
|
|
169
|
+
earliest_email_date = (epstein_files.earliest_email_at(name) or FALLBACK_TIMESTAMP).date()
|
|
170
|
+
year_months = (earliest_email_date.year * 12) + earliest_email_date.month
|
|
171
|
+
|
|
172
|
+
# Color year rollovers more brightly
|
|
173
|
+
if current_year != earliest_email_date.year:
|
|
174
|
+
grey_idx = 0
|
|
175
|
+
elif current_year_month != year_months:
|
|
176
|
+
grey_idx = ((current_year_month - 1) % 12) + 1
|
|
177
|
+
|
|
178
|
+
current_year_month = year_months
|
|
179
|
+
current_year = earliest_email_date.year
|
|
180
|
+
category = get_category_for_name(name)
|
|
181
|
+
info = get_info_for_name(name)
|
|
182
|
+
|
|
183
|
+
if category and category.plain == 'paula_heil_fisher': # TODO: hacky
|
|
184
|
+
category = None
|
|
185
|
+
elif category and info:
|
|
186
|
+
info = info.removeprefix(f"{category.plain}, ")
|
|
187
|
+
elif not name:
|
|
188
|
+
info = Text('(emails whose author or recipient could not be determined)', style='medium_purple4')
|
|
189
|
+
|
|
190
|
+
table.add_row(
|
|
191
|
+
Text(str(earliest_email_date), style=f"grey{GREY_NUMBERS[grey_idx]}"),
|
|
192
|
+
Text(name or UNKNOWN, style=get_style_for_name(name or UNKNOWN, default_style='dim')),
|
|
193
|
+
category,
|
|
194
|
+
f"{len(epstein_files.emails_for(name)):,}",
|
|
195
|
+
info or '',
|
|
196
|
+
)
|
|
197
|
+
|
|
198
|
+
return table
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def write_complete_emails_timeline(epstein_files: EpsteinFiles) -> None:
|
|
202
|
+
"""Print a table of all emails in chronological order."""
|
|
203
|
+
emails = [email for email in epstein_files.non_duplicate_emails() if not email.is_junk_mail()]
|
|
204
|
+
table = build_table(f'All {len(emails):,} Non-Junk Emails in Chronological Order', highlight=True)
|
|
205
|
+
table.add_column('ID', style=TIMESTAMP_DIM)
|
|
206
|
+
table.add_column('Sent At', style='dim')
|
|
207
|
+
table.add_column('Author', max_width=20)
|
|
208
|
+
table.add_column('Recipients', max_width=22)
|
|
209
|
+
table.add_column('Length', justify='right', style='wheat4')
|
|
210
|
+
table.add_column('Subject')
|
|
211
|
+
|
|
212
|
+
for email in Document.sort_by_timestamp(emails):
|
|
213
|
+
if email.is_junk_mail():
|
|
214
|
+
continue
|
|
215
|
+
|
|
216
|
+
table.add_row(
|
|
217
|
+
email.epstein_media_link(link_txt=email.source_file_id()),
|
|
218
|
+
email.timestamp_without_seconds(),
|
|
219
|
+
email.author_txt(),
|
|
220
|
+
email.recipients_txt(max_full_names=1),
|
|
221
|
+
f"{email.length()}",
|
|
222
|
+
email.subject(),
|
|
223
|
+
)
|
|
224
|
+
|
|
225
|
+
console.line(2)
|
|
226
|
+
console.print(table)
|
|
170
227
|
|
|
171
228
|
|
|
172
229
|
def write_json_metadata(epstein_files: EpsteinFiles) -> None:
|
|
@@ -202,6 +259,64 @@ def write_urls() -> None:
|
|
|
202
259
|
logger.warning(f"Wrote {len(url_vars)} URL variables to '{URLS_ENV}'\n")
|
|
203
260
|
|
|
204
261
|
|
|
262
|
+
def _all_emailers_table(epstein_files: EpsteinFiles) -> Table:
|
|
263
|
+
attributed_emails = [e for e in epstein_files.non_duplicate_emails() if e.author]
|
|
264
|
+
footer = f"(identified {len(epstein_files.email_author_counts)} authors of {len(attributed_emails):,}"
|
|
265
|
+
footer = f"{footer} out of {len(epstein_files.non_duplicate_emails()):,} emails)"
|
|
266
|
+
counts_table = build_table("All of the Email Counterparties Who Appear in the Files", caption=footer)
|
|
267
|
+
|
|
268
|
+
add_cols_to_table(counts_table, [
|
|
269
|
+
'Name',
|
|
270
|
+
{'name': 'Count', 'justify': 'right', 'style': 'bold bright_white'},
|
|
271
|
+
{'name': 'Sent', 'justify': 'right', 'style': 'gray74'},
|
|
272
|
+
{'name': 'Recv', 'justify': 'right', 'style': 'gray74'},
|
|
273
|
+
{'name': 'First', 'style': TIMESTAMP_STYLE},
|
|
274
|
+
{'name': 'Last', 'style': LAST_TIMESTAMP_STYLE},
|
|
275
|
+
{'name': 'Days', 'justify': 'right', 'style': 'dim'},
|
|
276
|
+
JMAIL,
|
|
277
|
+
EPSTEIN_MEDIA,
|
|
278
|
+
EPSTEIN_WEB,
|
|
279
|
+
'Twitter',
|
|
280
|
+
])
|
|
281
|
+
|
|
282
|
+
emailer_counts = {
|
|
283
|
+
emailer: epstein_files.email_author_counts[emailer] + epstein_files.email_recipient_counts[emailer]
|
|
284
|
+
for emailer in epstein_files.all_emailers(True)
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
for name, count in sort_dict(emailer_counts):
|
|
288
|
+
style = get_style_for_name(name, default_style=DEFAULT_NAME_STYLE)
|
|
289
|
+
emails = epstein_files.emails_for(name)
|
|
290
|
+
|
|
291
|
+
counts_table.add_row(
|
|
292
|
+
Text.from_markup(link_markup(epsteinify_name_url(name or UNKNOWN), name or UNKNOWN, style)),
|
|
293
|
+
f"{count:,}",
|
|
294
|
+
str(epstein_files.email_author_counts[name]),
|
|
295
|
+
str(epstein_files.email_recipient_counts[name]),
|
|
296
|
+
emails[0].date_str(),
|
|
297
|
+
emails[-1].date_str(),
|
|
298
|
+
f"{epstein_files.email_conversation_length_in_days(name)}",
|
|
299
|
+
link_text_obj(search_jmail_url(name), JMAIL) if name else '',
|
|
300
|
+
link_text_obj(epstein_media_person_url(name), EPSTEIN_MEDIA) if _is_ok_for_epstein_web(name) else '',
|
|
301
|
+
link_text_obj(epstein_web_person_url(name), EPSTEIN_WEB) if _is_ok_for_epstein_web(name) else '',
|
|
302
|
+
link_text_obj(search_twitter_url(name), 'search X') if name else '',
|
|
303
|
+
)
|
|
304
|
+
|
|
305
|
+
return counts_table
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def _is_ok_for_epstein_web(name: str | None) -> bool:
|
|
309
|
+
"""Return True if it's likely that EpsteinWeb has a page for this name."""
|
|
310
|
+
if name is None or ' ' not in name:
|
|
311
|
+
return False
|
|
312
|
+
elif '@' in name or '/' in name or '??' in name:
|
|
313
|
+
return False
|
|
314
|
+
elif name in INVALID_FOR_EPSTEIN_WEB:
|
|
315
|
+
return False
|
|
316
|
+
|
|
317
|
+
return True
|
|
318
|
+
|
|
319
|
+
|
|
205
320
|
def _verify_all_emails_were_printed(epstein_files: EpsteinFiles, already_printed_emails: list[Email]) -> None:
|
|
206
321
|
"""Log warnings if some emails were never printed."""
|
|
207
322
|
email_ids_that_were_printed = set([email.file_id for email in already_printed_emails])
|
epstein_files/util/rich.py
CHANGED
|
@@ -14,32 +14,33 @@ from rich.theme import Theme
|
|
|
14
14
|
|
|
15
15
|
from epstein_files.util.constant.html import CONSOLE_HTML_FORMAT, HTML_TERMINAL_THEME, PAGE_TITLE
|
|
16
16
|
from epstein_files.util.constant.names import UNKNOWN
|
|
17
|
-
from epstein_files.util.constant.
|
|
18
|
-
from epstein_files.util.constant.strings import DEFAULT, EMAIL, NA, QUESTION_MARKS, TEXT_MESSAGE, SiteType
|
|
17
|
+
from epstein_files.util.constant.strings import DEFAULT, EMAIL, NA, QUESTION_MARKS, TEXT_MESSAGE
|
|
19
18
|
from epstein_files.util.constant.urls import *
|
|
20
19
|
from epstein_files.util.constants import FALLBACK_TIMESTAMP, HEADER_ABBREVIATIONS
|
|
21
20
|
from epstein_files.util.data import json_safe
|
|
22
21
|
from epstein_files.util.env import args
|
|
23
22
|
from epstein_files.util.file_helper import log_file_write
|
|
24
|
-
from epstein_files.util.highlighted_group import ALL_HIGHLIGHTS, HIGHLIGHTED_NAMES, EpsteinHighlighter
|
|
23
|
+
from epstein_files.util.highlighted_group import (ALL_HIGHLIGHTS, HIGHLIGHTED_NAMES, EpsteinHighlighter,
|
|
24
|
+
get_category_for_name, get_info_for_name, get_style_for_name)
|
|
25
25
|
from epstein_files.util.logging import logger
|
|
26
26
|
|
|
27
27
|
TITLE_WIDTH = 50
|
|
28
|
+
MIN_AUTHOR_PANEL_WIDTH = 80
|
|
28
29
|
NUM_COLOR_KEY_COLS = 4
|
|
29
30
|
NA_TXT = Text(NA, style='dim')
|
|
30
|
-
QUESTION_MARK_TXT = Text(QUESTION_MARKS, style='dim')
|
|
31
31
|
GREY_NUMBERS = [58, 39, 39, 35, 30, 27, 23, 23, 19, 19, 15, 15, 15]
|
|
32
32
|
|
|
33
33
|
DEFAULT_NAME_STYLE = 'gray46'
|
|
34
34
|
INFO_STYLE = 'white dim italic'
|
|
35
|
-
KEY_STYLE='honeydew2 bold'
|
|
36
|
-
LAST_TIMESTAMP_STYLE='wheat4'
|
|
35
|
+
KEY_STYLE = 'honeydew2 bold'
|
|
36
|
+
LAST_TIMESTAMP_STYLE = 'wheat4'
|
|
37
|
+
OTHER_PAGE_MSG_STYLE = 'gray78 dim'
|
|
37
38
|
SECTION_HEADER_STYLE = 'bold white on blue3'
|
|
38
39
|
SOCIAL_MEDIA_LINK_STYLE = 'pale_turquoise4'
|
|
39
40
|
SUBSTACK_POST_LINK_STYLE = 'bright_cyan'
|
|
40
41
|
SYMBOL_STYLE = 'grey70'
|
|
41
42
|
TABLE_BORDER_STYLE = 'grey46'
|
|
42
|
-
TABLE_TITLE_STYLE = f"
|
|
43
|
+
TABLE_TITLE_STYLE = f"gray54 italic"
|
|
43
44
|
TITLE_STYLE = 'black on bright_white bold'
|
|
44
45
|
|
|
45
46
|
AUX_SITE_LINK_STYLE = 'dark_orange3'
|
|
@@ -47,6 +48,7 @@ OTHER_SITE_LINK_STYLE = 'dark_goldenrod'
|
|
|
47
48
|
|
|
48
49
|
DEFAULT_TABLE_KWARGS = {
|
|
49
50
|
'border_style': TABLE_BORDER_STYLE,
|
|
51
|
+
'caption_style': 'navajo_white3 dim italic',
|
|
50
52
|
'header_style': "bold",
|
|
51
53
|
'title_style': TABLE_TITLE_STYLE,
|
|
52
54
|
}
|
|
@@ -83,15 +85,21 @@ highlighter = CONSOLE_ARGS['highlighter']
|
|
|
83
85
|
def add_cols_to_table(table: Table, col_names: list[str | dict]) -> None:
|
|
84
86
|
"""Left most col will be left justified, rest are center justified."""
|
|
85
87
|
for i, col in enumerate(col_names):
|
|
88
|
+
justify='left' if i == 0 else 'center'
|
|
89
|
+
|
|
86
90
|
if isinstance(col, dict):
|
|
87
91
|
col_name = col['name']
|
|
88
92
|
kwargs = col
|
|
89
93
|
del kwargs['name']
|
|
94
|
+
|
|
95
|
+
if 'justify' in col:
|
|
96
|
+
justify = col['justify']
|
|
97
|
+
del col['justify']
|
|
90
98
|
else:
|
|
91
99
|
col_name = col
|
|
92
100
|
kwargs = {}
|
|
93
101
|
|
|
94
|
-
table.add_column(col_name, justify=
|
|
102
|
+
table.add_column(col_name, justify=justify, **kwargs)
|
|
95
103
|
|
|
96
104
|
|
|
97
105
|
def build_highlighter(pattern: str) -> EpsteinHighlighter:
|
|
@@ -143,11 +151,11 @@ def parenthesize(msg: str | Text, style: str = '') -> Text:
|
|
|
143
151
|
return Text('(', style=style).append(txt).append(')')
|
|
144
152
|
|
|
145
153
|
|
|
146
|
-
def
|
|
147
|
-
|
|
148
|
-
color = color or
|
|
149
|
-
|
|
150
|
-
panel = Panel(
|
|
154
|
+
def print_author_panel(msg: str, color: str | None, footer: str | None = None) -> None:
|
|
155
|
+
"""Print a panel with the name of an emailer and a few tidbits of information about them."""
|
|
156
|
+
color = 'white' if (not color or color == DEFAULT) else color
|
|
157
|
+
width = max(MIN_AUTHOR_PANEL_WIDTH, len(msg) + 4)
|
|
158
|
+
panel = Panel(Text(msg, justify='center'), width=width, style=f"black on {color} bold")
|
|
151
159
|
console.print('\n', Align.center(panel))
|
|
152
160
|
|
|
153
161
|
if footer:
|
|
@@ -180,26 +188,27 @@ def print_color_key() -> None:
|
|
|
180
188
|
print_centered(vertically_pad(color_table))
|
|
181
189
|
|
|
182
190
|
|
|
183
|
-
def
|
|
184
|
-
|
|
191
|
+
def print_title_page_header(epstein_files: 'EpsteinFiles') -> None:
|
|
192
|
+
print_page_title(width=TITLE_WIDTH)
|
|
193
|
+
site_type = EMAIL if (args.all_emails or args.email_timeline) else TEXT_MESSAGE
|
|
194
|
+
title = f"This is the " + ('chronological ' if args.email_timeline else '') + f"Epstein {site_type.title()}s Page"
|
|
195
|
+
print_starred_header(title, num_spaces=9 if args.all_emails else 6, num_stars=14)
|
|
196
|
+
print_centered(f"These documents come from the Nov. 2025 House Oversight Committee release.\n", style='gray74')
|
|
197
|
+
other_site_msg = "another page with" + (' all of' if other_site_type() == EMAIL else '')
|
|
198
|
+
other_site_msg += f" Epstein's {other_site_type()}s also generated by this code"
|
|
199
|
+
|
|
200
|
+
links = [
|
|
201
|
+
Text.from_markup(link_markup(other_site_url(), other_site_msg, f"{OTHER_SITE_LINK_STYLE} bold")),
|
|
202
|
+
link_text_obj(WORD_COUNT_URL, 'most frequently used words in the emails and texts', AUX_SITE_LINK_STYLE),
|
|
203
|
+
link_text_obj(JSON_METADATA_URL, 'author attribution explanations', AUX_SITE_LINK_STYLE),
|
|
204
|
+
link_text_obj(WORD_COUNT_URL, "epstein's json files", AUX_SITE_LINK_STYLE),
|
|
205
|
+
]
|
|
185
206
|
|
|
186
|
-
|
|
187
|
-
|
|
207
|
+
for link in links:
|
|
208
|
+
print_centered(parenthesize(link))
|
|
188
209
|
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
site_type = EMAIL if args.all_emails else TEXT_MESSAGE
|
|
192
|
-
print_starred_header(f"This is the Epstein {site_type.title()}s site", num_spaces=4, num_stars=14)
|
|
193
|
-
other_site_msg = "another site with" + (' all of' if other_site_type() == EMAIL else '')
|
|
194
|
-
other_site_msg += f" Epstein's {other_site_type}s also generated by this code"
|
|
195
|
-
other_site_link_markup = link_markup(other_site_url(), other_site_msg, OTHER_SITE_LINK_STYLE)
|
|
196
|
-
print_centered(parenthesize(Text.from_markup(other_site_link_markup)), style='bold')
|
|
197
|
-
word_count_link = link_text_obj(WORD_COUNT_URL, 'most frequently used words in the emails and texts', AUX_SITE_LINK_STYLE)
|
|
198
|
-
print_centered(parenthesize(word_count_link))
|
|
199
|
-
metadata_link = link_text_obj(JSON_METADATA_URL, 'author attribution explanations', AUX_SITE_LINK_STYLE)
|
|
200
|
-
print_centered(parenthesize(metadata_link))
|
|
201
|
-
json_link = link_text_obj(WORD_COUNT_URL, "epstein's json files", AUX_SITE_LINK_STYLE)
|
|
202
|
-
print_centered(parenthesize(json_link))
|
|
210
|
+
|
|
211
|
+
def print_title_page_tables(epstein_files: 'EpsteinFiles') -> None:
|
|
203
212
|
_print_external_links()
|
|
204
213
|
console.line()
|
|
205
214
|
_print_abbreviations_table()
|
|
@@ -212,8 +221,6 @@ def print_header(epstein_files: 'EpsteinFiles') -> None:
|
|
|
212
221
|
|
|
213
222
|
|
|
214
223
|
def print_json(label: str, obj: object, skip_falsey: bool = False) -> None:
|
|
215
|
-
print(obj)
|
|
216
|
-
|
|
217
224
|
if isinstance(obj, dict):
|
|
218
225
|
if skip_falsey:
|
|
219
226
|
obj = {k: v for k, v in obj.items() if v}
|
|
@@ -226,64 +233,33 @@ def print_json(label: str, obj: object, skip_falsey: bool = False) -> None:
|
|
|
226
233
|
console.line()
|
|
227
234
|
|
|
228
235
|
|
|
229
|
-
def print_numbered_list_of_emailers(_list: list[str | None], epstein_files = None) -> None:
|
|
230
|
-
"""Add the first emailed_at timestamp for each emailer if 'epstein_files' provided."""
|
|
231
|
-
current_year = 1990
|
|
232
|
-
current_year_month = current_year * 12
|
|
233
|
-
grey_idx = 0
|
|
234
|
-
console.line()
|
|
235
|
-
|
|
236
|
-
for i, name in enumerate(_list):
|
|
237
|
-
indent = ' ' if i < 9 else (' ' if i < 99 else ' ')
|
|
238
|
-
txt = Text((indent) + F" {i + 1}. ", style=DEFAULT_NAME_STYLE)
|
|
239
|
-
|
|
240
|
-
if epstein_files:
|
|
241
|
-
earliest_email_date = (epstein_files.earliest_email_at(name) or FALLBACK_TIMESTAMP).date()
|
|
242
|
-
year_months = (earliest_email_date.year * 12) + earliest_email_date.month
|
|
243
|
-
|
|
244
|
-
# Color year rollovers more brightly
|
|
245
|
-
if current_year != earliest_email_date.year:
|
|
246
|
-
grey_idx = 0
|
|
247
|
-
elif current_year_month != year_months:
|
|
248
|
-
grey_idx = ((current_year_month - 1) % 12) + 1
|
|
249
|
-
|
|
250
|
-
current_year_month = year_months
|
|
251
|
-
current_year = earliest_email_date.year
|
|
252
|
-
txt.append(escape(f"[{earliest_email_date}] "), style=f"grey{GREY_NUMBERS[grey_idx]}")
|
|
253
|
-
|
|
254
|
-
txt.append(highlighter(name or UNKNOWN))
|
|
255
|
-
|
|
256
|
-
if epstein_files:
|
|
257
|
-
num_days_in_converation = epstein_files.email_conversation_length_in_days(name)
|
|
258
|
-
msg = f" ({len(epstein_files.emails_for(name))} emails over {num_days_in_converation:,} days)"
|
|
259
|
-
txt.append(msg, style=f'dim italic')
|
|
260
|
-
|
|
261
|
-
console.print(txt)
|
|
262
|
-
|
|
263
|
-
console.line()
|
|
264
|
-
|
|
265
|
-
|
|
266
236
|
def print_other_page_link(epstein_files: 'EpsteinFiles') -> None:
|
|
267
237
|
markup_msg = link_markup(other_site_url(), 'the other page', style='light_slate_grey bold')
|
|
268
238
|
|
|
269
239
|
if other_site_type() == EMAIL:
|
|
270
|
-
txt = Text.from_markup(markup_msg).append(f' is uncurated and has all {len(epstein_files.
|
|
271
|
-
txt.append(f"
|
|
240
|
+
txt = Text.from_markup(markup_msg).append(f' is uncurated and has all {len(epstein_files.emails):,} emails')
|
|
241
|
+
txt.append(f" and {len(epstein_files.other_files)} unclassifiable files")
|
|
272
242
|
else:
|
|
273
|
-
txt = Text.from_markup(markup_msg).append(f' displays
|
|
243
|
+
txt = Text.from_markup(markup_msg).append(f' displays a limited collection of emails and')
|
|
274
244
|
txt.append(" unclassifiable files of particular interest")
|
|
275
245
|
|
|
276
|
-
print_centered(parenthesize(txt), style=
|
|
246
|
+
print_centered(parenthesize(txt), style=OTHER_PAGE_MSG_STYLE)
|
|
247
|
+
chrono_emails_markup = link_text_obj(CHRONOLOGICAL_EMAILS_URL, 'a page', style='light_slate_grey bold')
|
|
248
|
+
chrono_emails_txt = Text(f"there's also ").append(chrono_emails_markup)
|
|
249
|
+
chrono_emails_txt.append(' with a table of all the emails in chronological order')
|
|
250
|
+
print_centered(parenthesize(chrono_emails_txt), style=OTHER_PAGE_MSG_STYLE)
|
|
277
251
|
|
|
278
252
|
|
|
279
253
|
def print_page_title(expand: bool = True, width: int | None = None) -> None:
|
|
254
|
+
warning = f"This page was generated by {link_markup('https://pypi.org/project/rich/', 'rich')}."
|
|
255
|
+
print_centered(f"{warning} It is not optimized for mobile.", style='dim')
|
|
280
256
|
title_panel = Panel(Text(PAGE_TITLE, justify='center'), expand=expand, style=TITLE_STYLE, width=width)
|
|
281
|
-
|
|
257
|
+
print_centered(vertically_pad(title_panel))
|
|
282
258
|
_print_social_media_links()
|
|
283
259
|
console.line(2)
|
|
284
260
|
|
|
285
261
|
|
|
286
|
-
def
|
|
262
|
+
def print_subtitle_panel(msg: str, style: str = 'black on white', padding: tuple | None = None, centered: bool = False) -> None:
|
|
287
263
|
_padding: list[int] = list(padding or [0, 0, 0, 0])
|
|
288
264
|
_padding[2] += 1 # Bottom pad
|
|
289
265
|
actual_padding: tuple[int, int, int, int] = tuple(_padding)
|
|
@@ -298,7 +274,7 @@ def print_panel(msg: str, style: str = 'black on white', padding: tuple | None =
|
|
|
298
274
|
def print_section_header(msg: str, style: str = SECTION_HEADER_STYLE, is_centered: bool = False) -> None:
|
|
299
275
|
panel = Panel(Text(msg, justify='center'), expand=True, padding=(1, 1), style=style)
|
|
300
276
|
panel = Align.center(panel) if is_centered else panel
|
|
301
|
-
console.print(Padding(panel, (3,
|
|
277
|
+
console.print(Padding(panel, (3, 0, 1, 0)))
|
|
302
278
|
|
|
303
279
|
|
|
304
280
|
def print_starred_header(msg: str, num_stars: int = 7, num_spaces: int = 2, style: str = TITLE_STYLE) -> None:
|
|
@@ -383,5 +359,5 @@ def _print_social_media_links() -> None:
|
|
|
383
359
|
print_centered(join_texts(social_links, join=' / '))#, encloser='()'))#, encloser='‹›'))
|
|
384
360
|
|
|
385
361
|
|
|
386
|
-
|
|
387
|
-
|
|
362
|
+
if args.colors_only:
|
|
363
|
+
print_json('THEME_STYLES', THEME_STYLES)
|
epstein_files/util/timer.py
CHANGED
|
@@ -11,7 +11,7 @@ class Timer:
|
|
|
11
11
|
decimals: int = 2
|
|
12
12
|
|
|
13
13
|
def print_at_checkpoint(self, msg: str) -> None:
|
|
14
|
-
logger.warning(f"{msg} in {self.seconds_since_checkpoint_str()}")
|
|
14
|
+
logger.warning(f"{msg} in {self.seconds_since_checkpoint_str()}...")
|
|
15
15
|
self.checkpoint_at = time.perf_counter()
|
|
16
16
|
|
|
17
17
|
def seconds_since_checkpoint_str(self) -> str:
|
epstein_files/util/word_count.py
CHANGED
|
@@ -17,7 +17,7 @@ from epstein_files.util.data import ALL_NAMES, flatten, sort_dict
|
|
|
17
17
|
from epstein_files.util.env import args
|
|
18
18
|
from epstein_files.util.logging import logger
|
|
19
19
|
from epstein_files.util.rich import (console, highlighter, print_centered, print_color_key, print_page_title,
|
|
20
|
-
|
|
20
|
+
print_subtitle_panel, print_starred_header, write_html)
|
|
21
21
|
from epstein_files.util.search_result import MatchedLine, SearchResult
|
|
22
22
|
from epstein_files.util.timer import Timer
|
|
23
23
|
|
|
@@ -196,7 +196,6 @@ def write_word_counts_html() -> None:
|
|
|
196
196
|
epstein_files = EpsteinFiles.get_files(timer)
|
|
197
197
|
email_subjects: set[str] = set()
|
|
198
198
|
word_count = WordCount()
|
|
199
|
-
|
|
200
199
|
# Remove dupes, junk mail, and fwded articles from emails
|
|
201
200
|
emails = [e for e in epstein_files.non_duplicate_emails() if not (e.is_junk_mail() or e.is_fwded_article())]
|
|
202
201
|
|
|
@@ -225,7 +224,7 @@ def write_word_counts_html() -> None:
|
|
|
225
224
|
for i, msg in enumerate(imessage_log.messages):
|
|
226
225
|
if args.names and msg.author not in args.names:
|
|
227
226
|
continue
|
|
228
|
-
elif HTML_REGEX.search(
|
|
227
|
+
elif HTML_REGEX.search(msg.text):
|
|
229
228
|
continue
|
|
230
229
|
|
|
231
230
|
for word in msg.text.split():
|
|
@@ -239,7 +238,7 @@ def write_word_counts_html() -> None:
|
|
|
239
238
|
console.line()
|
|
240
239
|
console.print(word_count)
|
|
241
240
|
console.line(2)
|
|
242
|
-
|
|
241
|
+
print_subtitle_panel(f"{len(COMMON_WORDS_LIST):,} Excluded Words", centered=True)
|
|
243
242
|
console.print(', '.join(COMMON_WORDS_LIST), highlight=False)
|
|
244
243
|
write_html(WORD_COUNT_HTML_PATH)
|
|
245
244
|
timer.print_at_checkpoint(f"Finished counting words")
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: epstein-files
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.1.2
|
|
4
4
|
Summary: Tools for working with the Jeffrey Epstein documents released in November 2025.
|
|
5
5
|
Home-page: https://michelcrypt4d4mus.github.io/epstein_text_messages/
|
|
6
6
|
License: GPL-3.0-or-later
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
epstein_files/__init__.py,sha256=dPSYqe6CZyXFJviCsrVLPy-uxJxG5xS_W9ZDjUih5f0,5484
|
|
2
|
+
epstein_files/documents/communication.py,sha256=4xcmCg4108D4Rln4tiXbm5pRBRfBGpMxcCORUCMnT6k,1908
|
|
3
|
+
epstein_files/documents/document.py,sha256=eQ0IgOUZiKz-KFgUwkWariQ5HqyM2VY3ZCJC4qRsnDg,17401
|
|
4
|
+
epstein_files/documents/email.py,sha256=b5klJTrqTydaSupC5fJkV64nDxulkWE1DAvXF8FaLCY,43369
|
|
5
|
+
epstein_files/documents/emails/email_header.py,sha256=wkPfSLbmzkAeQwvhf0bAeFDLPbQT-EeG0v8vNNLYktM,7502
|
|
6
|
+
epstein_files/documents/imessage/text_message.py,sha256=w_U2bNIKtH7rMSNP4Q0BoTDrQZ6HE2IUSFjy6rBxrgY,3348
|
|
7
|
+
epstein_files/documents/json_file.py,sha256=WcZW5NNqA67rHTdopbOGtup00muNaLlvrNgKb-K4zO8,1504
|
|
8
|
+
epstein_files/documents/messenger_log.py,sha256=pAHH8FntEyQCwoVFI3B5utSqS5LKhQrj5UD1hl3pnbg,7419
|
|
9
|
+
epstein_files/documents/other_file.py,sha256=TeEzsfGN_mTFZPhfyt9ihxK9oTCYwI8sRLplTsgpOMY,9893
|
|
10
|
+
epstein_files/epstein_files.py,sha256=K6hgDlwHFNbhdIQcc5RJN_-g_xLhNLQ1yelpzCZrZBw,15210
|
|
11
|
+
epstein_files/util/constant/common_words.py,sha256=C1JERPnOGHV2UMC71aEue1i9QTQog-RfT3IzdcYQOYQ,3702
|
|
12
|
+
epstein_files/util/constant/html.py,sha256=MFooFV8KfFBCm9hL1u6A3hi_u37i7lL6UKAYoKQj3PI,1505
|
|
13
|
+
epstein_files/util/constant/names.py,sha256=yHZ46IcrzOOEr6mdTllxaG_YvLrdhmJlWCwIGv4FaiU,10738
|
|
14
|
+
epstein_files/util/constant/output_files.py,sha256=et1y3AzkxKqK0k-wDhMEsQFMfoXrUpoJCn6nQONurkU,1911
|
|
15
|
+
epstein_files/util/constant/strings.py,sha256=2TLP_TWgErsXb9lF8k0lZVAAQ8QAc5ytKQi3PD9CAzY,1961
|
|
16
|
+
epstein_files/util/constant/urls.py,sha256=VqgqxC2IbX90yw9kfGTwAwc7VVo46TCgDVrkPy3adV4,5127
|
|
17
|
+
epstein_files/util/constants.py,sha256=ZllrMB7Imlp4ihviGOMxB5qU_hnbS7rP3PnjS0fVbAI,112742
|
|
18
|
+
epstein_files/util/data.py,sha256=JccGFZGiCGm7XtwpQTocIjGYOr6hTUpEPwHhjyW9Xnc,3164
|
|
19
|
+
epstein_files/util/doc_cfg.py,sha256=aBIm0hyxf-aeMsb8ZUNiQFVsPFimjVUIkrVdDrg1iQU,9105
|
|
20
|
+
epstein_files/util/env.py,sha256=MbS-wD0iMJyg6u-adxVGvTlXY4-ubzIjG20ovz1EMHU,6147
|
|
21
|
+
epstein_files/util/file_helper.py,sha256=PGPqXmt4Oz4bE45ybvaCZfI0w_PGKirTsrv7xw86gmY,2903
|
|
22
|
+
epstein_files/util/highlighted_group.py,sha256=Pt34bfLjkdl7VBEogeJoofXhJC9fTPHYCgb4NNDlPpI,52422
|
|
23
|
+
epstein_files/util/logging.py,sha256=F45YqEKAiIb0rDZnOB7XuaY-dOkOKrsfSzO1VVqY508,2097
|
|
24
|
+
epstein_files/util/output.py,sha256=IXBiCQYy6Z2BUGDCLeQUfJ1NUzTItjvgg-k0i4KpOzA,13751
|
|
25
|
+
epstein_files/util/rich.py,sha256=0hyzML8yCSF6FfjQIt7aJyEqG8unQhDOQzfd7e6FTAI,14599
|
|
26
|
+
epstein_files/util/search_result.py,sha256=1fxe0KPBQXBk4dLfu6m0QXIzYfZCzvaSkWqvghJGzxY,567
|
|
27
|
+
epstein_files/util/timer.py,sha256=QqqXAQofKPWkDngNwG0mOqRn7nHcAR-BGQjqAwZfXoE,840
|
|
28
|
+
epstein_files/util/word_count.py,sha256=J6aZkodXwowf09GykLgJuqwSRzrMjvefgKiM8S-T9LA,9234
|
|
29
|
+
epstein_files-1.1.2.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
|
|
30
|
+
epstein_files-1.1.2.dist-info/METADATA,sha256=clg2flS9u3aeaYxXqlwvnemMyDKBHuuMvUgiJj8d4qs,5866
|
|
31
|
+
epstein_files-1.1.2.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
|
|
32
|
+
epstein_files-1.1.2.dist-info/entry_points.txt,sha256=5qYgwAXpxegeAicD_rzda_trDRnUC51F5UVDpcZ7j6Q,240
|
|
33
|
+
epstein_files-1.1.2.dist-info/RECORD,,
|