pycaption 2.3.2__tar.gz → 2.3.4__tar.gz

{pycaption-2.3.2/pycaption.egg-info → pycaption-2.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pycaption
-Version: 2.3.2
+Version: 2.3.4
 Summary: Closed caption converter
 Author: Joe Norton
 Author-email: joey@nortoncrew.com

{pycaption-2.3.2 → pycaption-2.3.4}/pycaption/base.py

@@ -5,12 +5,15 @@ CaptionSet -> CaptionList -> Caption -> CaptionNode.  Also provides the
 CaptionConverter orchestrator and base classes for readers/writers.
 """
 
+import logging
 import os
 from datetime import timedelta
 from numbers import Number
 
 from .exceptions import CaptionReadError, CaptionReadTimingError, InvalidInputError
 
+logger = logging.getLogger(__name__)
+
 # `und` a special identifier for an undetermined language according to ISO 639-2
 DEFAULT_LANGUAGE_CODE = os.getenv("PYCAPTION_DEFAULT_LANG", "und")
 
@@ -64,6 +67,46 @@ class BaseReader:
     def __init__(self, *args, **kwargs):
         pass
 
+    @staticmethod
+    def _decode_content(content):
+        """Decode bytes to str (UTF-8 with BOM handling).
+
+        :param content: str or bytes input.
+        :returns: decoded str with BOM stripped.
+        :raises InvalidInputError: if content is not str/bytes, is empty
+            bytes, or is not valid UTF-8.
+        """
+        if isinstance(content, bytes):
+            if not content:
+                raise InvalidInputError("The content is empty.")
+            try:
+                content = content.decode("utf-8-sig")
+            except UnicodeDecodeError as e:
+                raise InvalidInputError(f"Content is not valid UTF-8: {e}") from e
+        elif isinstance(content, str):
+            if content.startswith(""):
+                content = content[1:]
+        else:
+            raise InvalidInputError(
+                "The content must be a unicode string or UTF-8 bytes."
+            )
+        content = BaseReader._repair_double_encoding(content)
+        return content
+
+    @staticmethod
+    def _repair_double_encoding(text):
+        """Fix double-encoded UTF-8 (bytes misread as CP-1252)."""
+        try:
+            repaired = text.encode("cp1252").decode("utf-8")
+        except (UnicodeEncodeError, UnicodeDecodeError):
+            return text
+        if repaired != text:
+            logger.warning(
+                "Detected and repaired double-encoded UTF-8 in caption content"
+            )
+            return repaired
+        return text
+
     def detect(self, content):
         """Return True if content appears to be in this reader's format.
 
@@ -369,7 +412,11 @@ class CaptionSet:
     """
 
     def __init__(
-        self, captions, styles=None, layout_info=None, regions=None,
+        self,
+        captions,
+        styles=None,
+        layout_info=None,
+        regions=None,
         visual_alignment_default=None,
     ):
         """

{pycaption-2.3.2 → pycaption-2.3.4}/pycaption/dfxp/reader.py

@@ -21,7 +21,6 @@ from ..exceptions import (
     CaptionReadNoCaptions,
     CaptionReadSyntaxError,
     CaptionReadTimingError,
-    InvalidInputError,
 )
 from ..geometry import (
     Alignment,
@@ -108,9 +107,10 @@ class DFXPReader(BaseReader):
     def detect(self, content):
         """Return True if content looks like a DFXP/TTML document.
 
-        :type content: str
+        :type content: str or bytes
         :rtype: bool
         """
+        content = self._decode_content(content)
         lowered = content.lower()
         return bool(re.search(r"<tt[\s>]", lowered)) and "</tt>" in lowered
 
@@ -122,8 +122,7 @@ class DFXPReader(BaseReader):
         :raises InvalidInputError: if content is not a string
         :raises CaptionReadNoCaptions: if no captions are found
         """
-        if not isinstance(content, str):
-            raise InvalidInputError("The content is not a unicode string.")
+        content = self._decode_content(content)
 
         dfxp_document = LayoutAwareDFXPParser(
             content, read_invalid_positioning=self.read_invalid_positioning

{pycaption-2.3.2 → pycaption-2.3.4}/pycaption/microdvd.py

@@ -20,7 +20,6 @@ from .exceptions import (
     CaptionReadNoCaptions,
     CaptionReadSyntaxError,
     CaptionReadTimingError,
-    InvalidInputError,
 )
 from .geometry import HorizontalAlignmentEnum
 
@@ -30,6 +29,7 @@ class MicroDVDReader(BaseReader):
 
     def detect(self, content):
         """Return True if content starts with MicroDVD frame markers."""
+        content = self._decode_content(content)
         return re.match(r"{\d+}{\d+}", content) is not None
 
     def read(self, content, lang=DEFAULT_LANGUAGE_CODE):
@@ -39,8 +39,7 @@ class MicroDVDReader(BaseReader):
         :param lang: Language code to assign.
         :rtype: CaptionSet
         """
-        if not isinstance(content, str):
-            raise InvalidInputError("The content is not a unicode string.")
+        content = self._decode_content(content)
 
         lines = content.splitlines()
         captions = CaptionList()

{pycaption-2.3.2 → pycaption-2.3.4}/pycaption/sami/reader.py

@@ -10,7 +10,6 @@ from ..base import BaseReader, Caption, CaptionList, CaptionNode, CaptionSet
 from ..exceptions import (
     CaptionReadNoCaptions,
     CaptionReadTimingError,
-    InvalidInputError,
 )
 from ..geometry import Alignment, HorizontalAlignmentEnum, Layout, Padding, Size
 from .parser import SAMIParser
@@ -32,6 +31,7 @@ class SAMIReader(BaseReader):
 
     def detect(self, content):
         """Return True if content looks like a SAMI document."""
+        content = self._decode_content(content)
         return "<sami" in content.lower()
 
     def read(self, content):
@@ -42,8 +42,7 @@ class SAMIReader(BaseReader):
         :raises InvalidInputError: if content is not a string
         :raises CaptionReadNoCaptions: if no captions are found
         """
-        if not isinstance(content, str):
-            raise InvalidInputError("The content is not a unicode string.")
+        content = self._decode_content(content)
 
         content, doc_styles, doc_langs = SAMIParser().feed(content)
         sami_soup = BeautifulSoup(content, features="lxml")

{pycaption-2.3.2 → pycaption-2.3.4}/pycaption/scc/reader.py

@@ -87,7 +87,6 @@ from pycaption.exceptions import (
     CaptionLineLengthError,
     CaptionReadNoCaptions,
     CaptionReadTimingError,
-    InvalidInputError,
 )
 
 from .constants import (
@@ -148,10 +147,11 @@ class SCCReader(BaseReader):
     def detect(self, content):
         """Checks whether the given content is a proper SCC file
 
-        :type content: str
+        :type content: str or bytes
 
         :rtype: bool
         """
+        content = self._decode_content(content)
         lines = content.splitlines()
         if lines[0] == HEADER:
             return True
@@ -177,8 +177,7 @@ class SCCReader(BaseReader):
 
         :rtype: CaptionSet
         """
-        if not isinstance(content, str):
-            raise InvalidInputError("The content is not a unicode string.")
+        content = self._decode_content(content)
 
         self.simulate_roll_up = simulate_roll_up
         self.time_translator.offset = offset * 1000000

{pycaption-2.3.2 → pycaption-2.3.4}/pycaption/srt.py

@@ -6,7 +6,7 @@ from .base import (
     BaseReader, BaseWriter, Caption, CaptionList, CaptionNode, CaptionSet,
     merge_caption_list,
 )
-from .exceptions import CaptionReadNoCaptions, InvalidInputError
+from .exceptions import CaptionReadNoCaptions
 from .geometry import HorizontalAlignmentEnum
 
 
@@ -19,6 +19,7 @@ class SRTReader(BaseReader):
         Checks that the first line is a sequence number and the second
         contains an arrow ('-->').
         """
+        content = self._decode_content(content)
         lines = content.splitlines()
         if lines[0].isdigit() and "-->" in lines[1]:
             return True
@@ -34,8 +35,7 @@ class SRTReader(BaseReader):
         :raises InvalidInputError: if content is not a string.
         :raises CaptionReadNoCaptions: if no captions are found.
         """
-        if not isinstance(content, str):
-            raise InvalidInputError("The content is not a unicode string.")
+        content = self._decode_content(content)
 
         lines = content.splitlines()
         start_line = 0

{pycaption-2.3.2 → pycaption-2.3.4}/pycaption/webvtt/reader.py

@@ -15,7 +15,6 @@ from ..exceptions import (
     CaptionReadNoCaptions,
     CaptionReadSyntaxError,
     CaptionReadWarning,
-    InvalidInputError,
 )
 from ..geometry import (
     Alignment,
@@ -126,8 +125,7 @@ class WebVTTReader(BaseReader):
         optionally followed by a space or tab and header metadata.
         Handles BOM-prefixed content.
         """
-        if content.startswith(""):
-            content = content[1:]
+        content = self._decode_content(content)
         first_line = content.splitlines()[0] if content.strip() else ""
         return (
             first_line == "WEBVTT"
@@ -147,20 +145,7 @@ class WebVTTReader(BaseReader):
         :raises InvalidInputError: If content is not a string.
         :raises CaptionReadNoCaptions: If no cues are found.
         """
-        if isinstance(content, bytes):
-            if not content:
-                raise InvalidInputError("The content is not a unicode string.")
-            try:
-                content = content.decode("utf-8")
-            except UnicodeDecodeError as e:
-                raise InvalidInputError(
-                    f"WebVTT content is not valid UTF-8: {e}"
-                ) from e
-        elif not isinstance(content, str):
-            raise InvalidInputError("The content is not a unicode string.")
-
-        if content.startswith(""):
-            content = content[1:]
+        content = self._decode_content(content)
 
         # str.splitlines() handles CR, LF, CRLF (W3C WebVTT §3 RULE-FMT-005)
         lines = content.splitlines()

{pycaption-2.3.2 → pycaption-2.3.4/pycaption.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pycaption
-Version: 2.3.2
+Version: 2.3.4
 Summary: Closed caption converter
 Author: Joe Norton
 Author-email: joey@nortoncrew.com

{pycaption-2.3.2 → pycaption-2.3.4}/pycaption.egg-info/SOURCES.txt

@@ -41,9 +41,11 @@ tests/__init__.py
 tests/conftest.py
 tests/mixins.py
 tests/test_base.py
+tests/test_bytes_input.py
 tests/test_dfxp.py
 tests/test_dfxp_conversion.py
 tests/test_dfxp_extras.py
+tests/test_double_encoding.py
 tests/test_functions.py
 tests/test_geometry.py
 tests/test_microdvd.py

{pycaption-2.3.2 → pycaption-2.3.4}/setup.py

@@ -20,7 +20,7 @@ transcript_dependencies = ["nltk==3.10.0"]
 
 setup(
     name="pycaption",
-    version="2.3.2",
+    version="2.3.4",
     description="Closed caption converter",
     long_description=open(README_PATH).read(),
     author="Joe Norton",

{pycaption-2.3.2 → pycaption-2.3.4}/tests/mixins.py

@@ -17,10 +17,20 @@ class ReaderTestingMixIn:
     def assert_negative_answer_for_detection(self, different_sample):
         assert self.reader.detect(different_sample) is False
 
-    def test_reader_only_supports_unicode_input(self):
+    def test_reader_rejects_non_string_non_bytes_input(self):
+        with pytest.raises(InvalidInputError) as exc_info:
+            self.reader.read(123)
+        assert "must be a unicode string or UTF-8 bytes" in exc_info.value.args[0]
+
+    def test_reader_rejects_empty_bytes(self):
         with pytest.raises(InvalidInputError) as exc_info:
             self.reader.read(b"")
-        assert exc_info.value.args[0] == "The content is not a unicode string."
+        assert exc_info.value.args[0] == "The content is empty."
+
+    def test_reader_rejects_invalid_utf8_bytes(self):
+        with pytest.raises(InvalidInputError) as exc_info:
+            self.reader.read(b"\xff\xfe invalid utf8")
+        assert "not valid UTF-8" in exc_info.value.args[0]
 
 
 class WebVTTTestingMixIn:

pycaption-2.3.4/tests/test_bytes_input.py

@@ -0,0 +1,161 @@
+"""Tests for bytes input support across all readers.
+
+All readers should accept bytes (raw file content) and decode as UTF-8
+internally, preventing the double-encoding gibberish that occurs when
+callers decode with the wrong system encoding (e.g., cp1252).
+"""
+
+from pycaption import (
+    DFXPReader,
+    MicroDVDReader,
+    SAMIReader,
+    SCCReader,
+    SRTReader,
+    WebVTTReader,
+)
+
+
+class TestSRTReaderBytes:
+    def setup_class(self):
+        self.reader = SRTReader()
+
+    def test_read_bytes_produces_same_result_as_str(self, sample_srt):
+        from_str = self.reader.read(sample_srt)
+        from_bytes = self.reader.read(sample_srt.encode("utf-8"))
+        str_captions = from_str.get_captions("en-US")
+        bytes_captions = from_bytes.get_captions("en-US")
+        assert len(str_captions) == len(bytes_captions)
+        for s, b in zip(str_captions, bytes_captions):
+            assert s.get_text() == b.get_text()
+            assert s.start == b.start
+            assert s.end == b.end
+
+    def test_read_bytes_preserves_music_notes(self, sample_srt):
+        captions = self.reader.read(sample_srt.encode("utf-8"))
+        texts = [c.get_text() for c in captions.get_captions("en-US")]
+        assert any("♪" in t for t in texts)
+
+    def test_read_bytes_with_bom(self, sample_srt):
+        content_with_bom = b"\xef\xbb\xbf" + sample_srt.encode("utf-8")
+        captions = self.reader.read(content_with_bom)
+        assert len(captions.get_captions("en-US")) == 7
+
+    def test_detect_bytes(self, sample_srt):
+        assert self.reader.detect(sample_srt.encode("utf-8")) is True
+
+
+class TestWebVTTReaderBytes:
+    def setup_class(self):
+        self.reader = WebVTTReader()
+
+    def test_read_bytes_produces_same_result_as_str(self, sample_webvtt):
+        from_str = self.reader.read(sample_webvtt)
+        from_bytes = self.reader.read(sample_webvtt.encode("utf-8"))
+        str_captions = from_str.get_captions("en-US")
+        bytes_captions = from_bytes.get_captions("en-US")
+        assert len(str_captions) == len(bytes_captions)
+        for s, b in zip(str_captions, bytes_captions):
+            assert s.get_text() == b.get_text()
+
+    def test_read_bytes_with_bom(self, sample_webvtt):
+        content_with_bom = b"\xef\xbb\xbf" + sample_webvtt.encode("utf-8")
+        captions = self.reader.read(content_with_bom)
+        assert len(captions.get_captions("en-US")) > 0
+
+    def test_detect_bytes(self, sample_webvtt):
+        assert self.reader.detect(sample_webvtt.encode("utf-8")) is True
+
+
+class TestDFXPReaderBytes:
+    def setup_class(self):
+        self.reader = DFXPReader()
+
+    def test_read_bytes_produces_same_result_as_str(self, sample_dfxp):
+        from_str = self.reader.read(sample_dfxp)
+        from_bytes = self.reader.read(sample_dfxp.encode("utf-8"))
+        for lang in from_str.get_languages():
+            str_captions = from_str.get_captions(lang)
+            bytes_captions = from_bytes.get_captions(lang)
+            assert len(str_captions) == len(bytes_captions)
+            for s, b in zip(str_captions, bytes_captions):
+                assert s.get_text() == b.get_text()
+
+    def test_read_bytes_with_bom(self, sample_dfxp):
+        content_with_bom = b"\xef\xbb\xbf" + sample_dfxp.encode("utf-8")
+        captions = self.reader.read(content_with_bom)
+        assert not captions.is_empty()
+
+    def test_detect_bytes(self, sample_dfxp):
+        assert self.reader.detect(sample_dfxp.encode("utf-8")) is True
+
+
+class TestSAMIReaderBytes:
+    def setup_class(self):
+        self.reader = SAMIReader()
+
+    def test_read_bytes_produces_same_result_as_str(self, sample_sami):
+        from_str = self.reader.read(sample_sami)
+        from_bytes = self.reader.read(sample_sami.encode("utf-8"))
+        for lang in from_str.get_languages():
+            str_captions = from_str.get_captions(lang)
+            bytes_captions = from_bytes.get_captions(lang)
+            assert len(str_captions) == len(bytes_captions)
+            for s, b in zip(str_captions, bytes_captions):
+                assert s.get_text() == b.get_text()
+
+    def test_read_bytes_preserves_music_notes(self, sample_sami):
+        captions = self.reader.read(sample_sami.encode("utf-8"))
+        langs = list(captions.get_languages())
+        texts = [c.get_text() for c in captions.get_captions(langs[0])]
+        assert any("♪" in t for t in texts)
+
+    def test_read_bytes_with_bom(self, sample_sami):
+        content_with_bom = b"\xef\xbb\xbf" + sample_sami.encode("utf-8")
+        captions = self.reader.read(content_with_bom)
+        assert not captions.is_empty()
+
+    def test_detect_bytes(self, sample_sami):
+        assert self.reader.detect(sample_sami.encode("utf-8")) is True
+
+
+class TestSCCReaderBytes:
+    def test_read_bytes_produces_same_result_as_str(self, sample_scc_pop_on):
+        from_str = SCCReader().read(sample_scc_pop_on)
+        from_bytes = SCCReader().read(sample_scc_pop_on.encode("utf-8"))
+        str_captions = from_str.get_captions("en-US")
+        bytes_captions = from_bytes.get_captions("en-US")
+        assert len(str_captions) == len(bytes_captions)
+        for s, b in zip(str_captions, bytes_captions):
+            assert s.get_text() == b.get_text()
+
+    def test_read_bytes_with_bom(self, sample_scc_pop_on):
+        content_with_bom = b"\xef\xbb\xbf" + sample_scc_pop_on.encode("utf-8")
+        captions = SCCReader().read(content_with_bom)
+        assert len(captions.get_captions("en-US")) > 0
+
+    def test_detect_bytes(self, sample_scc_pop_on):
+        assert SCCReader().detect(sample_scc_pop_on.encode("utf-8")) is True
+
+
+class TestMicroDVDReaderBytes:
+    def setup_class(self):
+        self.reader = MicroDVDReader()
+
+    def test_read_bytes_produces_same_result_as_str(self, sample_microdvd):
+        from_str = self.reader.read(sample_microdvd)
+        from_bytes = self.reader.read(sample_microdvd.encode("utf-8"))
+        str_captions = from_str.get_captions("und")
+        bytes_captions = from_bytes.get_captions("und")
+        assert len(str_captions) == len(bytes_captions)
+        for s, b in zip(str_captions, bytes_captions):
+            assert s.get_text() == b.get_text()
+
+    def test_read_bytes_with_bom(self, sample_microdvd):
+        content_with_bom = b"\xef\xbb\xbf" + sample_microdvd.encode("utf-8")
+        captions = self.reader.read(content_with_bom)
+        assert not captions.is_empty()
+
+    def test_detect_bytes(self, sample_microdvd):
+        assert self.reader.detect(sample_microdvd.encode("utf-8")) is True
+
+

pycaption-2.3.4/tests/test_double_encoding.py

@@ -0,0 +1,89 @@
+import logging
+
+import pytest
+
+from pycaption import SAMIReader, SRTReader
+from pycaption.base import BaseReader
+from pycaption.dfxp import DFXPReader
+
+
+def _double_encode(text):
+    """Simulate double-encoding: UTF-8 bytes misread as CP-1252, re-encoded."""
+    return text.encode("utf-8").decode("cp1252")
+
+
+ORIGINAL_CHARS = ["♪", "—", "’", "é"]
+
+
+class TestRepairDoubleEncoding:
+    @pytest.mark.parametrize("original", ORIGINAL_CHARS)
+    def test_fixes_double_encoded(self, original):
+        garbled = _double_encode(original)
+        assert BaseReader._repair_double_encoding(garbled) == original
+
+    def test_leaves_clean_utf8_alone(self):
+        clean = "♪ This is — perfectly fine é text"
+        assert BaseReader._repair_double_encoding(clean) == clean
+
+    def test_logs_warning_on_repair(self, caplog):
+        garbled = _double_encode("♪")
+        with caplog.at_level(logging.WARNING, logger="pycaption.base"):
+            BaseReader._repair_double_encoding(garbled)
+        assert "double-encoded" in caplog.text.lower()
+
+    def test_no_warning_for_clean_input(self, caplog):
+        with caplog.at_level(logging.WARNING, logger="pycaption.base"):
+            BaseReader._repair_double_encoding("♪ Music ♪")
+        assert caplog.text == ""
+
+
+class TestDoubleEncodingEndToEnd:
+    def test_srt_reader(self):
+        garbled_note = _double_encode("♪")
+        content = (
+            "1\n"
+            "00:00:01,000 --> 00:00:02,000\n"
+            f"{garbled_note} Music {garbled_note}\n"
+        )
+        captions = SRTReader().read(content)
+        nodes = captions.get_captions("en-US")[0].nodes
+        text = "".join(n.content for n in nodes)
+        assert "♪" in text
+        assert garbled_note not in text
+
+    def test_dfxp_reader(self):
+        garbled = _double_encode("élève")
+        content = (
+            '<?xml version="1.0" encoding="UTF-8"?>\n'
+            '<tt xml:lang="en" xmlns="http://www.w3.org/ns/ttml">\n'
+            "  <body><div>\n"
+            f'    <p begin="00:00:01.000" end="00:00:02.000">'
+            f"{garbled}</p>\n"
+            "  </div></body>\n"
+            "</tt>\n"
+        )
+        captions = DFXPReader().read(content)
+        nodes = captions.get_captions("en")[0].nodes
+        text = "".join(n.content for n in nodes)
+        assert "élève" in text
+        assert garbled not in text
+
+    def test_sami_reader(self):
+        garbled_dash = _double_encode("—")
+        content = (
+            "<SAMI>\n"
+            "<Body>\n"
+            "  <Sync Start=1000>\n"
+            f"    <P Class=enCC>{garbled_dash} Hello</P>\n"
+            "  </Sync>\n"
+            "  <Sync Start=2000>\n"
+            "    <P Class=enCC>&nbsp;</P>\n"
+            "  </Sync>\n"
+            "</Body>\n"
+            "</SAMI>\n"
+        )
+        captions = SAMIReader().read(content)
+        lang = list(captions.get_languages())[0]
+        nodes = captions.get_captions(lang)[0].nodes
+        text = "".join(n.content for n in nodes)
+        assert "—" in text