epstein-files 1.4.1__py3-none-any.whl → 1.5.0__py3-none-any.whl

epstein_files/documents/doj_file.py

@@ -0,0 +1,242 @@
+import re
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import ClassVar, Self
+
+from rich.console import Console, ConsoleOptions, RenderableType, RenderResult
+from rich.padding import Padding
+from rich.panel import Panel
+from rich.text import Text
+
+from epstein_files.documents.document import INFO_INDENT, Document
+from epstein_files.documents.email import Email
+from epstein_files.documents.emails.email_header import FIELDS_COLON_PATTERN
+from epstein_files.documents.other_file import Metadata, OtherFile
+from epstein_files.util.constant.names import RENATA_BOLOTOVA
+from epstein_files.util.constants import FALLBACK_TIMESTAMP
+from epstein_files.util.data import without_falsey
+from epstein_files.util.layout.left_bar_panel import LeftBarPanel
+from epstein_files.util.logging import logger
+from epstein_files.util.rich import RAINBOW, INFO_STYLE, SKIPPED_FILE_MSG_PADDING, highlighter, link_text_obj
+
+CHECK_LINK_FOR_DETAILS = 'not shown here, check original PDF for details'
+IMAGE_PANEL_REGEX = re.compile(r"\n╭─* Page \d+, Image \d+.*?╯\n", re.DOTALL)
+IGNORE_LINE_REGEX = re.compile(r"^(\d+\n?|[\s+❑]{2,})$")
+MIN_VALID_LENGTH = 10
+
+OTHER_DOC_URLS = {
+    '245-22.pdf': 'https://www.justice.gov/multimedia/Court%20Records/Government%20of%20the%20United%20States%20Virgin%20Islands%20v.%20JPMorgan%20Chase%20Bank,%20N.A.,%20No.%20122-cv-10904%20(S.D.N.Y.%202022)/245-22.pdf'
+}
+
+# DojFile specific repair
+OCR_REPAIRS: dict[str | re.Pattern, str] = {
+    re.compile(fr"({FIELDS_COLON_PATTERN}.*\n)\nSubject:", re.MULTILINE): r'\1Subject:',
+}
+
+BAD_DOJ_FILE_IDS = [
+    'EFTA00008511',
+    'EFTA00008503',
+    'EFTA00002512',
+    'EFTA00008501',
+    'EFTA00008500',
+    'EFTA00008514',
+    'EFTA00001940',
+    'EFTA00008410',
+    'EFTA00008411',
+    'EFTA00008519',
+    'EFTA00008493',
+    'EFTA00008527',
+    'EFTA00008473',
+    'EFTA00001846',
+    'EFTA00000052',
+    'EFTA00008445',
+    'EFTA00008480',
+    'EFTA00001124',
+    'EFTA00002509',
+    'EFTA00008497',
+    'EFTA00001031',
+    'EFTA00005495',
+    'EFTA00002830',
+    'EFTA00001937',
+    'EFTA00008496',
+    'EFTA00008441',
+    'EFTA00008415',
+    'EFTA00000675',
+    'EFTA00002538',
+    'EFTA00000672',
+    'EFTA00002814',
+    'EFTA00002812',
+    'EFTA00002543',
+    'EFTA00002813',
+    'EFTA00002523',
+    'EFTA00002079',
+    'EFTA00002805',
+    'EFTA00001840',
+    'EFTA00001114',
+    'EFTA00002812',
+    'EFTA00002543',
+    'EFTA00002786',
+    'EFTA00001271',
+    'EFTA00002523',
+    'EFTA00001979',
+    'EFTA00002110',
+    'EFTA00008504',
+    'EFTA00001368',
+    'EFTA00000134',
+    'EFTA00000471',
+    'EFTA00001848',
+    'EFTA00008506',
+]
+
+PHONE_BILL_IDS = {
+    'EFTA00006770': 'covering 2006-02-01 to 2006-06-16',
+    'EFTA00006870': 'covering 2006-02-09 to 2006-07',
+    'EFTA00006970': 'covering 2006-04-15 to 2006-07-16',
+    # 'EFTA00007070':  # TODO: not a messy phone bill, short, has additional info at end
+}
+
+INTERESTING_DOJ_FILES = {
+    'EFTA02640711': 'Jabor Y home address (HBJ)',
+    'EFTA00039689': 'Dilorio emails to SEC about Signature Bank, Hapoalim, Bioptix / RIOT, Honig, etc.',
+    'EFTA00039025': 'Investigation and Review of the Federal Bureau of Prisons Custody, Care, and Supervision of Jeffrey Epstein',
+    'EFTA02296929': f"{RENATA_BOLOTOVA} appears to know Epstein's final girlfriend",
+    'EFTA01273102': f"payment from Epstein to {RENATA_BOLOTOVA}'s father's account at Sberbank",
+}
+
+NO_IMAGE_SUFFIX = """
+╭──── Page 1, Image 1 ─────╮
+│ (no text found in image) │
+╰──────────────────────────╯
+""".strip()
+
+
+@dataclass
+class DojFile(OtherFile):
+    """
+    Class for the files released by DOJ on 2026-01-30 with `EFTA000` prefix.
+    """
+    _border_style: str | None = None
+
+    border_style_rainbow_idx: ClassVar[int] = 0  # ClassVar to help change color as we print, no impact beyond fancier output
+    max_timestamp: ClassVar[datetime] = datetime(2025, 1, 29) # Overloaded in DojFile
+
+    @property
+    def border_style(self) -> str:
+        """Use a rainbow to make sure each printed object has different color for those before and after."""
+        if self._border_style is None:
+            self._border_style = RAINBOW[int(self.border_style_rainbow_idx % len(RAINBOW))]
+            type(self).border_style_rainbow_idx += 1
+
+        return self._border_style
+
+    @property
+    def info(self) -> list[Text]:
+        """Overloads superclass to adjust formatting."""
+        return [Text(' ').append(sentence) for sentence in super().info]
+
+    @property
+    def is_bad_ocr(self) -> bool:
+        return self.file_id in BAD_DOJ_FILE_IDS
+
+    @property
+    def is_empty(self) -> bool:
+        """Overloads superclass method."""
+        return len(self.text.strip().removesuffix(NO_IMAGE_SUFFIX)) < MIN_VALID_LENGTH
+
+    @property
+    def prettified_text(self) -> Text:
+        """Returns the string we want to print as the body of the document."""
+        style = ''
+
+        if self.file_id in PHONE_BILL_IDS:
+            pages = self.text.split('MetroPCS')
+            text = f"{pages[0]}\n\n(Redacted phone bill {PHONE_BILL_IDS[self.file_id]} {CHECK_LINK_FOR_DETAILS})"
+        elif self.config and self.config.replace_text_with:
+            if len(self.config.replace_text_with) < 300:
+                style = INFO_STYLE
+                text = f'(Text of {self.config.replace_text_with} {CHECK_LINK_FOR_DETAILS})'
+            else:
+                text = self.config.replace_text_with
+        else:
+            text = self.text
+
+        return Text(text, style)
+
+    @property
+    def timestamp_sort_key(self) -> tuple[datetime, str, int]:
+        """Overloads parent method."""
+        dupe_idx = 0
+        # TODO: Years of 2001 are often garbage pared from '1.6' etc.
+        sort_timestamp = self.timestamp or FALLBACK_TIMESTAMP
+        sort_timestamp = FALLBACK_TIMESTAMP if sort_timestamp.year <= 2001 else sort_timestamp
+        return (sort_timestamp, self.file_id, dupe_idx)
+
+    def __post_init__(self):
+        super().__post_init__()
+
+        if self.file_id in PHONE_BILL_IDS:
+            self.strip_image_ocr_panels()
+
+    def doj_link(self) -> Text:
+        """Link to this file on the DOJ site."""
+        return link_text_obj(self.external_url, self.url_slug)
+
+    def external_links_txt(self, _style: str = '', include_alt_links: bool = True) -> Text:
+        """Overrides super() method to apply self.border_style."""
+        return super().external_links_txt(self.border_style, include_alt_links=include_alt_links)
+
+    def image_with_no_text_msg(self) -> RenderableType:
+        """One line of linked text to show if this file doesn't seem to have any OCR text."""
+        return Padding(
+            Text('').append(self.doj_link()).append(f" is a single image with no text..."),
+            (0, 0, 0, 1)
+        )
+
+    def printable_document(self) -> Self | Email:
+        """Return a copy of this `DojFile` with simplified text if file ID is in `REPLACEMENT_TEXT`."""
+        if Document.is_email(self):
+            try:
+                return Email(self.file_path, text=self.text)  # Pass text= to avoid reprocessing
+            except Exception as e:
+                self.warn(f"Error creating Email object, trying full reload of text...")
+                return Email(self.file_path)
+        else:
+            return self
+
+    def strip_image_ocr_panels(self) -> None:
+        """Removes the ╭--- Page 5, Image 1 ---- panels from the text."""
+        new_text, num_replaced = IMAGE_PANEL_REGEX.subn('', self.text)
+        self.warn(f"Stripped {num_replaced} image panels.")
+        self._set_computed_fields(text=new_text)
+
+    def _repair(self) -> None:
+        """Overloads superclass method."""
+        new_text = self.repair_ocr_text(OCR_REPAIRS, self.text)
+        self._set_computed_fields(text=new_text)
+        self._remove_number_only_lines()
+
+    def _remove_number_only_lines(self) -> None:
+        """Remove number only lines (which happen a lot in legal doc OCR) if there are more than a certain amount of them."""
+        non_number_lines = [line for line in self.lines if not IGNORE_LINE_REGEX.match(line)]
+        number_only_line_count = len(self.lines) - len(non_number_lines)
+
+        if number_only_line_count > 20:
+            self.warn(f"Reduced line count from {len(self.lines)} to {len(non_number_lines)}")
+            self._set_computed_fields(lines=non_number_lines)
+
+    def __rich_console__(self, console: Console, options: ConsoleOptions) -> RenderResult:
+        doc = self.printable_document()
+
+        # Emails handle their own formatting
+        if isinstance(doc, Email):
+            yield doc
+        else:
+            yield (info_panel := self.file_info_panel())
+            border_style = info_panel.renderables[0].border_style
+            panel_args = [self.prettified_text, border_style]
+
+            if self.panel_title_timestamp:
+                panel_args.append(Text(f"[{self.panel_title_timestamp}]", style='dim italic ' + border_style))
+
+            table = LeftBarPanel.build(*panel_args)
+            yield Padding(table, (0, 0, 1, 1))

epstein_files/documents/doj_files/full_text.py

@@ -0,0 +1,166 @@
+EFTA00009622_TEXT = """
+[redacted] s GJ Test.          7/19/2006
+
+[redacted] dob          16 yrs
+
+[redacted] - Loxahatchee
+
+2005 [redacted]          9th grade
+
+[redacted] was outside          asked to date [redacted]
+
+1/10/19 [redacted]
+
+~~talked in depth~~
+
+met for first time the night before went to JE
+11:30 ish 10
+do you want to make $200
+
+met my friend Jeff  bighouse
+
+but you have to give massage 45 min + that's all you have to do get $200
+
+[redacted] told [redacted] + [redacted] went to Bathroom
+to argue
+came back out [redacted] called Jeff on the phone.
+1 assistant answered the pho...
+
+Page 2
+Jeff asst answered phone - [redacted] had someone to give massage Lady asked how old I was
+she told the lady 18 + went to [redacted]
+
+Its $200     didn't care what
+[redacted] - Lady trying to verify 18 Yes
+Graduated from [redacted]     Known [redacted] for a long time
+& the next morning
+
+[redacted] + another girl     - going to Mall he [redacted] didn't care
+
+[redacted] asked dad for $     yeh sure fine ok
+
+House in Palm Beach
+all got out of car     gate at back door     security guard the saw us     the [redacted] asked us where...
+Page 3
+
+[redacted] said Jeffrey
+Went to kitchen     waited 15 min
+
+JE + asst     walked in back door
+
+shook hands     met both JE asst.
+(A) JE     chose gonna go first
+
+JE said how about you
+[redacted]     upstairs     "asst lady"
+
+walked me up there stripping out all this picture
+
+She pulled out massage table me     front of couch     blanket over table     opened drawer     whole bunch of lotions     + pick some out
+She told me keep bra + panties on + get undress
+
+No [redacted]     told her I thought a massage keep your clothing
+
+Page 4
+
+stay in the room
+I came in the room     HE + shook hands
+Came back in the room
+
+JE:     old     50ish
+
+initially     guy golfy     khaki pants pullover
+
+t-shirt + jeans     thong     bra
+
+hesitant     thinking what's going on. I didn't care cause I wanted $200
+
+He laid on massage table     He said grab any 3 lotions
+
+Started to give him
+
+What M.S.     did you go to told     him she was 16 Not     do you like to do     small talk [redacted]
+
+Sat on table     15-20 minute conversation     another house in NY
+
+Page 5
+
+He asked if I wanted to make an extra $300
+Straddle him
+
+massage over -
+but if you're not comfortable giving a massage can I     [redacted]     on you     ~~anything~~
+[redacted]     said ok
+JE left + came ba
+
+[redacted]     kind of like a [redacted]
+grabbed that + layed down back on table     ~~old~~ ~~old~~ hands
+
+Give me a     massage on chest use the [redacted]     10 minutes or so ( after     a couple ~~of~~ minutes ( on [redacted]
+(A) [redacted] said if he did that he would have to pay her extra money if he >
+Page 6
+
+[redacted]     not she would have to didn't     tell her     ~~get~~ ~~over~~ do her clothes at He [redacted]     you get paid
+
+[redacted]
+
+Yes     Saw [redacted]
+
+He opened wallet     got the   $200
+
+He told her to write Name + #
+
+after that left the room
+
+I didn't see him [redacted]
+
+I was scared to tell
+
+[redacted]     her body not his body
+Went down by herself    got 1st  No didn't do anything  but [redacted]  $200 ~~too~~
+
+Page 7
+I figure it was probably because she brought me
+
+What happened     I got $300 I was excited
+
+What did you have to do with him
+If you do this you get that
+Laughing ~~in~~     Thanks for fully me I was going to ~~much~~ ~~go~~ ~~many~~ Oh sorry.
+Didn't plan it
+
+If you ever need money
+Oh yeh     - everybody
+
+I did not ever go back to house
+
+at school my best friend I told her if you ever want to make 300.-o massage Told [redacted]     [redacted]
+
+Page 8
+[redacted]     telling everybody I confronted her     altercation get into a fight [redacted]
+
+Spreading rumors about me Let me see in your purse you don't have to lie Stefand I worked at [redacted]
+
+I didn't think it was her business and     why would tell on myself
+
+You don't have to lie I knew what happen
+Everything came out
+
+[redacted]     then I didn't want to tell them what had happened,
+
+I [redacted]
+
+[redacted] Family Setty a lot problems
+
+No     the only [redacted]     sometimes
+Page 9
+
+[redacted]     Have you ever da[ting] [redacted] Survey -     Yeh
+[redacted]     not anymore
+14 birthday     my dad
+I did [redacted]     lots of time
+
+$250,000     its a joke
+tall skinny     dark hair
+Page 10
+""".strip()