nerdd-module 0.3.48__tar.gz → 0.3.49__tar.gz

{nerdd_module-0.3.48 → nerdd_module-0.3.49}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nerdd-module
-Version: 0.3.48
+Version: 0.3.49
 Summary: Base package to create NERDD modules
 Author-email: Steffen Hirte <steffen.hirte@univie.ac.at>
 Maintainer-email: Steffen Hirte <steffen.hirte@univie.ac.at>

{nerdd_module-0.3.48 → nerdd_module-0.3.49}/nerdd_module/input/stream_reader.py

@@ -1,6 +1,6 @@
 from abc import abstractmethod
 from codecs import getreader
-from typing import Any, Iterator
+from typing import Any, Iterator, Optional
 
 import chardet
 
@@ -10,8 +10,9 @@ __all__ = ["StreamReader"]
 
 
 class StreamReader(Reader):
-    def __init__(self) -> None:
+    def __init__(self, encoding: Optional[str] = "utf-8-sig") -> None:
         super().__init__()
+        self.encoding = encoding
 
     def read(self, input_stream: Any, explore: ExploreCallable) -> Iterator[MoleculeEntry]:
         if not hasattr(input_stream, "read") or not hasattr(input_stream, "seek"):
@@ -20,24 +21,28 @@ class StreamReader(Reader):
         input_stream.seek(0)
 
         #
-        # detect file encoding
+        # detect file encoding (if not provided)
         #
 
         # read a portion of the file's content
-        sample = input_stream.read(1_000_000)
-        result = chardet.detect(sample)
-        if result["confidence"] > 0.5 and result["encoding"] is not None:
-            encoding = result["encoding"]
+        if self.encoding is None:
+            sample = input_stream.read(1_000_000)
+            result = chardet.detect(sample)
+            if result["confidence"] > 0.5 and result["encoding"] is not None:
+                encoding = result["encoding"]
+            else:
+                encoding = "utf-8-sig"
+
+            input_stream.seek(0)
         else:
-            encoding = "utf-8"
-
-        input_stream.seek(0)
+            encoding = self.encoding
 
         #
         # read file
         #
         StreamReader = getreader(encoding)
-        reader = StreamReader(input_stream)
+        # errors="replace": replace invalid characters instead of failing
+        reader = StreamReader(input_stream, "replace")
         return self._read_stream(reader, explore)
 
     @abstractmethod

{nerdd_module-0.3.48 → nerdd_module-0.3.49}/nerdd_module/input/tar_reader.py

@@ -1,5 +1,5 @@
 import tarfile
-from typing import Any, Iterator
+from typing import Any, Iterator, Tuple
 
 from .reader import ExploreCallable, MoleculeEntry, Reader
 
@@ -21,7 +21,14 @@ class TarReader(Reader):
                 if not member.isfile():
                     continue
                 for entry in explore(tar.extractfile(member)):
-                    yield entry._replace(source=(member.name, *entry.source))
+                    # the underlying reader only sees the file content as a stream
+                    # -> it might believe that the source is "raw_input"
+                    # -> we need to correct that here
+                    if len(entry.source) == 1 and entry.source[0] == "raw_input":
+                        source: Tuple[str, ...] = tuple()
+                    else:
+                        source = entry.source
+                    yield entry._replace(source=(member.name, *source))
 
     def __repr__(self) -> str:
         return "TarReader()"

{nerdd_module-0.3.48 → nerdd_module-0.3.49}/nerdd_module/input/zip_reader.py

@@ -1,5 +1,5 @@
 import zipfile
-from typing import Any, Iterator
+from typing import Any, Iterator, Tuple
 
 from .reader import ExploreCallable, MoleculeEntry, Reader
 
@@ -23,7 +23,15 @@ class ZipReader(Reader):
                     continue
                 with zipf.open(member, "r") as f:
                     for entry in explore(f):
-                        yield entry._replace(source=(member, *entry.source))
+                        # the underlying reader only sees the file content as a stream
+                        # -> it might believe that the source is "raw_input"
+                        # -> we need to correct that here
+                        if len(entry.source) == 1 and entry.source[0] == "raw_input":
+                            source: Tuple[str, ...] = tuple()
+                        else:
+                            source = entry.source
+
+                        yield entry._replace(source=(member, *source))
 
     def __repr__(self) -> str:
         return "ZipReader()"

{nerdd_module-0.3.48 → nerdd_module-0.3.49}/nerdd_module.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nerdd-module
-Version: 0.3.48
+Version: 0.3.49
 Summary: Base package to create NERDD modules
 Author-email: Steffen Hirte <steffen.hirte@univie.ac.at>
 Maintainer-email: Steffen Hirte <steffen.hirte@univie.ac.at>

{nerdd_module-0.3.48 → nerdd_module-0.3.49}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "nerdd-module"
-version = "0.3.48"
+version = "0.3.49"
 description = "Base package to create NERDD modules"
 readme = "README.md"
 license = "BSD-3-Clause"