chemdataextractor-lite 1.0.0__py3-none-any.whl

chemdataextractor/__init__.py

@@ -0,0 +1,13 @@
+"""Lightweight scientific document loading."""
+
+import logging
+
+from .doc.document import Document
+
+__title__ = "ChemDataExtractor Lite"
+__version__ = "1.0.0"
+__license__ = "MIT"
+
+logging.getLogger(__name__).addHandler(logging.NullHandler())
+
+__all__ = ["Document"]

chemdataextractor/data/README.md

@@ -0,0 +1,3 @@
+# Bundled sentence model
+
+`punkt_chem-1.0.pickle` is the chemistry-trained Punkt sentence tokenizer distributed by the ChemDataExtractor project. It is bundled here so sentence segmentation does not perform network access at runtime.

chemdataextractor/doc/__init__.py

@@ -0,0 +1,17 @@
+"""Structural document model."""
+
+from .document import Document
+from .element import BaseElement, CaptionedElement
+from .figure import Figure
+from .meta import MetaData
+from .table import Table
+from .text import (
+    Caption, Cell, Citation, Footnote, Heading, Paragraph, RichToken, Sentence,
+    Span, Text, Title, Token,
+)
+
+__all__ = [
+    "BaseElement", "Caption", "CaptionedElement", "Cell", "Citation", "Document",
+    "Figure", "Footnote", "Heading", "MetaData", "Paragraph", "RichToken",
+    "Sentence", "Span", "Table", "Text", "Title", "Token",
+]

chemdataextractor/doc/document.py

@@ -0,0 +1,168 @@
+"""Document container and reader dispatch."""
+
+import collections
+import io
+import json
+import logging
+from pathlib import Path
+
+from ..errors import ReaderError
+from ..text import get_encoding
+from .element import CaptionedElement
+from .figure import Figure
+from .meta import MetaData
+from .table import Table
+from .text import Caption, Cell, Citation, Footnote, Heading, Paragraph, Sentence, Title
+
+log = logging.getLogger(__name__)
+
+
+class Document(collections.abc.Sequence):
+    """A format-neutral sequence of structural document elements."""
+
+    def __init__(self, *elements):
+        self._elements = []
+        for element in elements:
+            if isinstance(element, str):
+                element = Paragraph(element)
+            elif isinstance(element, bytes):
+                encoding = get_encoding(element)
+                log.warning("Guessed bytestring encoding as %s", encoding)
+                element = Paragraph(element.decode(encoding))
+            element.document = self
+            self._elements.append(element)
+
+    def __repr__(self):
+        return f"<Document: {len(self)} elements>"
+
+    __str__ = __repr__
+
+    def __getitem__(self, index):
+        return self.elements[index]
+
+    def __len__(self):
+        return len(self.elements)
+
+    @property
+    def elements(self):
+        return self._elements
+
+    @classmethod
+    def from_file(cls, file, fname=None, readers=None):
+        if isinstance(file, (str, bytes, Path)):
+            path = Path(file)
+            with path.open("rb") as stream:
+                return cls.from_string(stream.read(), fname=fname or str(path), readers=readers)
+        if fname is None and hasattr(file, "name"):
+            fname = file.name
+        return cls.from_string(file.read(), fname=fname, readers=readers)
+
+    @classmethod
+    def from_string(cls, content, fname=None, readers=None):
+        if isinstance(content, str):
+            content = content.encode("utf-8")
+        if not isinstance(content, bytes):
+            raise TypeError("content must be bytes or str")
+        if readers is None:
+            from ..reader import DEFAULT_READERS
+            readers = DEFAULT_READERS
+        errors = []
+        for reader in readers:
+            try:
+                if not reader.detect(content, fname=fname):
+                    continue
+                document = reader.readstring(content)
+                log.debug("Parsed document with %s", reader.__class__.__name__)
+                return document
+            except ReaderError as error:
+                errors.append(error)
+        message = "Unable to read document"
+        if errors:
+            message += f": {errors[-1]}"
+        raise ReaderError(message)
+
+    def get_element_with_id(self, id):
+        pending = list(self.elements)
+        while pending:
+            element = pending.pop(0)
+            if element.id == id:
+                return element
+            if element.elements:
+                pending.extend(element.elements)
+        return None
+
+    def _elements_of_type(self, element_type):
+        return [element for element in self.elements if isinstance(element, element_type)]
+
+    @property
+    def figures(self):
+        return self._elements_of_type(Figure)
+
+    @property
+    def tables(self):
+        return self._elements_of_type(Table)
+
+    @property
+    def citations(self):
+        return self._elements_of_type(Citation)
+
+    @property
+    def footnotes(self):
+        return self._elements_of_type(Footnote)
+
+    @property
+    def titles(self):
+        return self._elements_of_type(Title)
+
+    @property
+    def headings(self):
+        return self._elements_of_type(Heading)
+
+    @property
+    def paragraphs(self):
+        return self._elements_of_type(Paragraph)
+
+    @property
+    def captions(self):
+        return self._elements_of_type(Caption)
+
+    @property
+    def captioned_elements(self):
+        return self._elements_of_type(CaptionedElement)
+
+    @property
+    def metadata(self):
+        metadata = self._elements_of_type(MetaData)
+        return metadata[0] if metadata else None
+
+    @property
+    def sentences(self):
+        pending = list(self.elements)
+        sentences = []
+        while pending:
+            element = pending.pop(0)
+            if isinstance(element, Sentence) and not isinstance(element, Cell):
+                sentences.append(element)
+            elif element.elements:
+                pending[0:0] = element.elements
+        return sentences
+
+    def heading_for_sentence(self, sentence):
+        current_heading = None
+        for element in self.elements:
+            if isinstance(element, Heading):
+                current_heading = element
+                continue
+            if sentence in (element.elements or []):
+                return current_heading
+        return None
+
+    def serialize(self):
+        return {"type": "document", "elements": [element.serialize() for element in self.elements]}
+
+    def to_json(self, *args, **kwargs):
+        return json.dumps(self.serialize(), *args, **kwargs)
+
+    def _repr_html_(self):
+        body = "\n".join(element._repr_html_() for element in self.elements if hasattr(element, "_repr_html_"))
+        return f'<div class="cde-document">\n{body}\n</div>'

chemdataextractor/doc/element.py

@@ -0,0 +1,59 @@
+"""Base classes for document elements."""
+
+import json
+from abc import ABCMeta, abstractmethod
+
+
+class BaseElement(metaclass=ABCMeta):
+    def __init__(self, document=None, references=None, id=None, **kwargs):
+        self._document = document
+        self.id = id
+        self.references = list(references or [])
+
+    def __repr__(self):
+        return f"<{self.__class__.__name__}>"
+
+    @property
+    def document(self):
+        return self._document
+
+    @document.setter
+    def document(self, document):
+        self._document = document
+
+    @property
+    def elements(self):
+        return None
+
+    @abstractmethod
+    def serialize(self):
+        raise NotImplementedError
+
+    def to_json(self, *args, **kwargs):
+        return json.dumps(self.serialize(), *args, **kwargs)
+
+
+class CaptionedElement(BaseElement):
+    def __init__(self, caption, label=None, **kwargs):
+        self.caption = caption
+        self.label = label
+        super().__init__(**kwargs)
+        self.caption.document = self.document
+
+    def __str__(self):
+        return self.caption.text
+
+    @BaseElement.document.setter
+    def document(self, document):
+        self._document = document
+        self.caption.document = document
+
+    @property
+    def elements(self):
+        return [self.caption]
+
+    def serialize(self):
+        data = {"type": self.__class__.__name__, "caption": self.caption.serialize()}
+        if self.label is not None:
+            data["label"] = self.label
+        return data

chemdataextractor/doc/figure.py

@@ -0,0 +1,14 @@
+"""Figure document element."""
+
+from .element import CaptionedElement
+
+
+class Figure(CaptionedElement):
+    def __init__(self, caption, label=None, links=None, **kwargs):
+        super().__init__(caption=caption, label=label, **kwargs)
+        self.links = list(links or [])
+
+    def serialize(self):
+        data = super().serialize()
+        data["links"] = self.links
+        return data

chemdataextractor/doc/meta.py

@@ -0,0 +1,30 @@
+"""Bibliographic metadata document element."""
+
+from .element import BaseElement
+
+
+class MetaData(BaseElement):
+    FIELDS = (
+        "title", "authors", "publisher", "journal", "volume", "issue",
+        "firstpage", "lastpage", "doi", "date", "language", "pdf_url", "html_url",
+    )
+
+    def __init__(self, data, **kwargs):
+        super().__init__(**kwargs)
+        normalized = {key.lstrip("_"): value for key, value in data.items()}
+        self._data = {key: normalized.get(key) for key in self.FIELDS}
+
+    def __repr__(self):
+        return repr({key: value for key, value in self.data.items() if value})
+
+    def __getattr__(self, name):
+        if name in self.FIELDS:
+            return self._data[name]
+        raise AttributeError(name)
+
+    @property
+    def data(self):
+        return dict(self._data)
+
+    def serialize(self):
+        return {"MetaData": {key: value for key, value in self._data.items() if value}}

chemdataextractor/doc/table.py

@@ -0,0 +1,36 @@
+"""Structural table document element."""
+
+from .element import CaptionedElement
+from .text import Cell
+
+
+class Table(CaptionedElement):
+    def __init__(self, caption, label=None, table_data=None, footnotes=None, **kwargs):
+        super().__init__(caption=caption, label=label, **kwargs)
+        self.table_data = [
+            [cell if isinstance(cell, Cell) else Cell(str(cell)) for cell in row]
+            for row in (table_data or [])
+        ]
+        self.footnotes = list(footnotes or [])
+        self.document = self.document
+
+    @CaptionedElement.document.setter
+    def document(self, document):
+        self._document = document
+        self.caption.document = document
+        for row in getattr(self, "table_data", []):
+            for cell in row:
+                cell.document = document
+        for footnote in getattr(self, "footnotes", []):
+            footnote.document = document
+
+    @property
+    def elements(self):
+        return [cell for row in self.table_data for cell in row] + [self.caption] + self.footnotes
+
+    def serialize(self):
+        data = super().serialize()
+        data["rows"] = [[cell.text for cell in row] for row in self.table_data]
+        if self.footnotes:
+            data["footnotes"] = [footnote.serialize() for footnote in self.footnotes]
+        return data

chemdataextractor/doc/text.py

@@ -0,0 +1,224 @@
+"""Text document elements with sentence and word segmentation."""
+
+import collections
+import unicodedata
+
+from ..nlp.tokenize import ChemSentenceTokenizer, ChemWordTokenizer
+from .element import BaseElement
+
+
+class Span:
+    def __init__(self, text, start, end):
+        self.text = text
+        self.start = start
+        self.end = end
+
+    def __repr__(self):
+        return f"{self.__class__.__name__}({self.text!r}, {self.start!r}, {self.end!r})"
+
+    def __str__(self):
+        return self.text
+
+    def __eq__(self, other):
+        return (
+            isinstance(other, self.__class__)
+            and self.text == other.text
+            and self.start == other.start
+            and self.end == other.end
+        )
+
+    def __hash__(self):
+        return hash((self.text, self.start, self.end))
+
+    @property
+    def length(self):
+        return self.end - self.start
+
+
+class Token(Span):
+    pass
+
+
+# Kept as an import alias for loader clients that used the old token class name.
+RichToken = Token
+
+
+class BaseText(BaseElement):
+    def __init__(self, text, **kwargs):
+        if not isinstance(text, str):
+            raise TypeError("Text must be a unicode string")
+        super().__init__(**kwargs)
+        self._text = text
+
+    def __repr__(self):
+        return (
+            f"{self.__class__.__name__}(id={self.id!r}, "
+            f"references={self.references!r}, text={self.text!r})"
+        )
+
+    def __str__(self):
+        return self.text
+
+    @property
+    def text(self):
+        return self._text
+
+    def serialize(self):
+        data = {"type": self.__class__.__name__, "content": self.text}
+        if self.id is not None:
+            data["id"] = self.id
+        if self.references:
+            data["references"] = self.references
+        return data
+
+    def _repr_html_(self):
+        return self.text
+
+
+class Text(collections.abc.Sequence, BaseText):
+    sentence_tokenizer = ChemSentenceTokenizer()
+    word_tokenizer = ChemWordTokenizer()
+
+    def __init__(self, text, sentence_tokenizer=None, word_tokenizer=None, **kwargs):
+        super().__init__(text, **kwargs)
+        self.sentence_tokenizer = sentence_tokenizer or self.sentence_tokenizer
+        self.word_tokenizer = word_tokenizer or self.word_tokenizer
+        self._sentences = None
+
+    def __getitem__(self, index):
+        return self.sentences[index]
+
+    def __len__(self):
+        return len(self.sentences)
+
+    @property
+    def sentences(self):
+        if self._sentences is None:
+            spans = self.sentence_tokenizer.span_tokenize(self.text)
+            self._sentences = [
+                Sentence(
+                    self.text[start:end],
+                    start=start,
+                    end=end,
+                    word_tokenizer=self.word_tokenizer,
+                    document=self.document,
+                )
+                for start, end in spans
+            ]
+        return self._sentences
+
+    @BaseElement.document.setter
+    def document(self, document):
+        self._document = document
+        if self._sentences is not None:
+            for sentence in self._sentences:
+                sentence.document = document
+
+    @property
+    def elements(self):
+        return self.sentences
+
+    @property
+    def raw_sentences(self):
+        return [sentence.text for sentence in self.sentences]
+
+    @property
+    def tokens(self):
+        return [sentence.tokens for sentence in self.sentences]
+
+    @property
+    def raw_tokens(self):
+        return [sentence.raw_tokens for sentence in self.sentences]
+
+    def __add__(self, other):
+        if type(self) is type(other):
+            return self.__class__(
+                self.text + other.text,
+                id=self.id or other.id,
+                references=self.references + other.references,
+                sentence_tokenizer=self.sentence_tokenizer,
+                word_tokenizer=self.word_tokenizer,
+            )
+        return NotImplemented
+
+
+class Title(Text):
+    def _repr_html_(self):
+        return f'<h1 class="cde-title">{self.text}</h1>'
+
+
+class Heading(Text):
+    def _repr_html_(self):
+        return f'<h2 class="cde-heading">{self.text}</h2>'
+
+
+class Paragraph(Text):
+    def _repr_html_(self):
+        return f'<p class="cde-paragraph">{self.text}</p>'
+
+
+class Footnote(Text):
+    pass
+
+
+class Citation(Text):
+    pass
+
+
+class Caption(Text):
+    pass
+
+
+class Sentence(collections.abc.Sequence, BaseText):
+    word_tokenizer = ChemWordTokenizer()
+
+    def __init__(self, text, start=0, end=None, word_tokenizer=None, **kwargs):
+        super().__init__(text, **kwargs)
+        self.start = start
+        self.end = len(text) if end is None else end
+        self.word_tokenizer = word_tokenizer or self.word_tokenizer
+        self._tokens = None
+
+    def __getitem__(self, index):
+        return self.tokens[index]
+
+    def __len__(self):
+        return len(self.tokens)
+
+    @property
+    def tokens(self):
+        if self._tokens is None:
+            spans = self.word_tokenizer.span_tokenize(self.text)
+            self._tokens = [
+                Token(
+                    "".join(ch for ch in self.text[left:right] if unicodedata.category(ch)[0] != "C"),
+                    left + self.start,
+                    right + self.start,
+                )
+                for left, right in spans
+            ]
+        return self._tokens
+
+    @property
+    def raw_tokens(self):
+        return [token.text for token in self.tokens]
+
+    @property
+    def elements(self):
+        return None
+
+    def __add__(self, other):
+        if type(self) is type(other):
+            return self.__class__(
+                self.text + other.text,
+                start=self.start,
+                end=self.start + len(self.text + other.text),
+                id=self.id or other.id,
+                references=self.references + other.references,
+                word_tokenizer=self.word_tokenizer,
+            )
+        return NotImplemented
+
+
+class Cell(Sentence):
+    pass

chemdataextractor/errors.py

@@ -0,0 +1,9 @@
+"""Public exceptions raised by document loaders."""
+
+
+class ChemDataExtractorError(Exception):
+    """Base package exception."""
+
+
+class ReaderError(ChemDataExtractorError):
+    """Raised when a reader cannot load a document."""

chemdataextractor/nlp/__init__.py

@@ -0,0 +1,8 @@
+"""Sentence and word segmentation utilities."""
+
+from .tokenize import ChemSentenceTokenizer, ChemWordTokenizer, FineWordTokenizer, SentenceTokenizer, WordTokenizer
+
+__all__ = [
+    "ChemSentenceTokenizer", "ChemWordTokenizer", "FineWordTokenizer",
+    "SentenceTokenizer", "WordTokenizer",
+]