sentence-struct 0.1.0__py3-none-any.whl

sentence_struct/__init__.py

@@ -0,0 +1,15 @@
+"""sentence-struct: multilingual sentence structure analysis."""
+
+from sentence_struct.api import SUPPORTED_LANGUAGES, analyze, analyze_sentences
+from sentence_struct.models import Chunk, Sentence, Token
+
+__all__ = [
+    "SUPPORTED_LANGUAGES",
+    "Chunk",
+    "Sentence",
+    "Token",
+    "analyze",
+    "analyze_sentences",
+]
+
+__version__ = "0.1.0"

sentence_struct/api.py

@@ -0,0 +1,44 @@
+"""Public analyze() entrypoint with language dispatch."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from sentence_struct.models import Sentence
+from sentence_struct.serialize import document_to_dict
+
+SUPPORTED_LANGUAGES = frozenset({"ja"})
+
+
+def analyze_sentences(text: str, *, language: str = "ja") -> list[Sentence]:
+    """Analyze text into Sentence objects (tokens + chunks per sentence)."""
+    lang = language.lower().strip()
+    if lang not in SUPPORTED_LANGUAGES:
+        supported = ", ".join(sorted(SUPPORTED_LANGUAGES))
+        raise ValueError(f"Unsupported language {language!r}. Supported: {supported}")
+
+    if lang == "ja":
+        from sentence_struct.backends.ja.segment import analyze as analyze_ja
+
+        return analyze_ja(text)
+
+    raise ValueError(f"Unsupported language {language!r}")
+
+
+def analyze(text: str, *, language: str = "ja") -> dict[str, Any]:
+    """Analyze text into the public document dict.
+
+    Schema::
+
+        {
+          "language": "ja",
+          "text": "...",
+          "sentences": [
+            {"index": 1, "text": "...", "tokens": [...], "chunks": [...]}
+          ]
+        }
+
+    Chunks are siblings of tokens and reference them via ``tokenIndices``.
+    """
+    sentences = analyze_sentences(text, language=language)
+    return document_to_dict(text, sentences, language=language.lower().strip())

sentence_struct/backends/__init__.py

@@ -0,0 +1 @@
+"""Language backends. Japanese ships first; more languages later."""

sentence_struct/backends/ja/__init__.py

@@ -0,0 +1,5 @@
+"""Japanese backend (GiNza / Sudachi)."""
+
+from sentence_struct.backends.ja.segment import analyze
+
+__all__ = ["analyze"]

sentence_struct/backends/ja/chunks.py

@@ -0,0 +1,163 @@
+"""In-sentence chunking from dependency-based linear bunsetsu groups."""
+
+from __future__ import annotations
+
+from spacy.tokens import Span, Token as SpacyToken
+
+from sentence_struct.models import Chunk
+
+_ATTACH_MAJORS = frozenset({"助詞", "助動詞", "接尾辞", "補助記号", "接続助詞"})
+
+_PHRASE_BY_MAJOR = {
+    "名詞": "NP",
+    "代名詞": "NP",
+    "動詞": "VP",
+    "形容詞": "ADJP",
+    "形状詞": "ADJP",
+    "連体詞": "連体詞",
+    "副詞": "ADVP",
+    "助詞": "助詞",
+    "助動詞": "助動詞",
+    "補助記号": "補助記号",
+}
+_PHRASE_BY_POS = {
+    "NOUN": "NP",
+    "PROPN": "NP",
+    "PRON": "NP",
+    "NUM": "NP",
+    "VERB": "VP",
+    "AUX": "VP",
+    "ADJ": "ADJP",
+    "ADV": "ADVP",
+}
+
+
+def _pos_major(token: SpacyToken) -> str:
+    return token.tag_.split("-", 1)[0]
+
+
+def _attaches_to_previous(token: SpacyToken, current_group: list[SpacyToken]) -> bool:
+    if token.pos_ == "PUNCT":
+        return True
+    if _pos_major(token) in _ATTACH_MAJORS:
+        return True
+    if token.dep_ in {
+        "fixed",
+        "mark",
+        "case",
+        "aux",
+        "cop",
+        "det",
+        "compound",
+        "clf",
+    }:
+        return True
+    if token.dep_ in {"advcl", "xcomp", "ccomp"} and token.head.i < token.i:
+        if any(t.i == token.head.i for t in current_group):
+            return True
+    return False
+
+
+def _merge_nummod_groups(groups: list[list[SpacyToken]]) -> list[list[SpacyToken]]:
+    merged: list[list[SpacyToken]] = []
+    i = 0
+    while i < len(groups):
+        group = groups[i]
+        if i + 1 < len(groups):
+            nxt = groups[i + 1]
+            nxt_ids = {t.i for t in nxt}
+            if any(t.dep_ == "nummod" and t.head.i in nxt_ids for t in group):
+                merged.append(group + nxt)
+                i += 2
+                continue
+        merged.append(group)
+        i += 1
+    return merged
+
+
+def _chunk_head(tokens: list[SpacyToken]) -> SpacyToken:
+    local = {t.i for t in tokens}
+    external_heads = [t for t in tokens if t.head.i not in local]
+    if external_heads:
+        return min(external_heads, key=lambda t: t.i)
+    roots = [t for t in tokens if t.dep_ == "ROOT"]
+    if roots:
+        return roots[0]
+    return tokens[0]
+
+
+def _is_predicative(tokens: list[SpacyToken]) -> bool:
+    ids = {t.i for t in tokens}
+    for t in tokens:
+        if t.dep_ != "ROOT":
+            continue
+        root_major = _pos_major(t)
+        if root_major in ("動詞", "形容詞", "形状詞"):
+            return True
+        if root_major == "名詞":
+            has_suru_verb = any(_pos_major(x) == "動詞" and x.i in ids for x in tokens)
+            if has_suru_verb:
+                return True
+    return False
+
+
+def _is_suru_phrase(tokens: list[SpacyToken]) -> bool:
+    for t in tokens:
+        if _pos_major(t) != "名詞":
+            continue
+        if any(x.head.i == t.i and _pos_major(x) == "動詞" for x in tokens):
+            return True
+    return False
+
+
+def _is_adjectival_predicate(tokens: list[SpacyToken]) -> bool:
+    for t in tokens:
+        if t.dep_ == "ROOT" and _pos_major(t) == "名詞":
+            return any(_pos_major(x) == "助動詞" for x in tokens)
+    return False
+
+
+def _chunk_type(head: SpacyToken, tokens: list[SpacyToken]) -> str:
+    if _is_predicative(tokens) or _is_suru_phrase(tokens):
+        return "VP"
+    if _is_adjectival_predicate(tokens):
+        return "ADJP"
+    if _pos_major(head) == "副詞" or head.dep_ == "advmod":
+        return "ADVP"
+    major = _pos_major(head)
+    if major in _PHRASE_BY_MAJOR:
+        return _PHRASE_BY_MAJOR[major]
+    return _PHRASE_BY_POS.get(head.pos_, "NP")
+
+
+def _group_tokens(tokens: list[SpacyToken]) -> list[list[SpacyToken]]:
+    if not tokens:
+        return []
+    groups: list[list[SpacyToken]] = [[tokens[0]]]
+    for token in tokens[1:]:
+        if _attaches_to_previous(token, groups[-1]):
+            groups[-1].append(token)
+        else:
+            groups.append([token])
+    return groups
+
+
+def build_chunks(span: Span) -> tuple[Chunk, ...]:
+    """Split a spaCy span into linearly adjacent chunks."""
+    tokens = [t for t in span if not t.is_space]
+    chunks: list[Chunk] = []
+    offset = span.start
+
+    for group in _merge_nummod_groups(_group_tokens(tokens)):
+        head = _chunk_head(group)
+        indices = tuple(t.i - offset for t in group)
+        chunks.append(
+            Chunk(
+                type=_chunk_type(head, group),
+                token_indices=indices,
+                text="".join(t.text for t in group),
+                head_index=head.i - offset,
+                role=head.dep_,
+            )
+        )
+    return tuple(chunks)

sentence_struct/backends/ja/segment.py

@@ -0,0 +1,222 @@
+"""Japanese sentence segmentation + tokenization (GiNza / Sudachi)."""
+
+from __future__ import annotations
+
+import re
+from functools import lru_cache
+
+import spacy
+from spacy.tokens import Doc, Span, Token as SpacyToken
+
+from sentence_struct.backends.ja.chunks import build_chunks
+from sentence_struct.backends.ja.token_merge import merge_tokens, remap_chunks
+from sentence_struct.models import Chunk, Sentence, Token
+
+MAX_SENTENCE_CHARS = 26
+_SOFT_BREAK_CHARS = frozenset("、，,")
+_HARD_SPLIT_RE = re.compile(r"\n+|(?<=[。！？])")
+
+
+@lru_cache(maxsize=1)
+def _load_nlp() -> spacy.Language:
+    return spacy.load("ja_ginza")
+
+
+def _reading(token: SpacyToken) -> str | None:
+    for feat in token.morph:
+        if feat.startswith("Reading="):
+            return feat.split("=", 1)[1]
+    return None
+
+
+def _tokenize_span(span: Span, start_index: int = 1) -> tuple[Token, ...]:
+    tokens: list[Token] = []
+    for i, t in enumerate(span, start=start_index):
+        tokens.append(
+            Token(
+                index=i,
+                surface=t.text,
+                lemma=t.lemma_,
+                pos=t.pos_,
+                pos_detail=t.tag_,
+                reading=_reading(t),
+            )
+        )
+    return tuple(tokens)
+
+
+def _finalize_tokens_and_chunks(
+    tokens: tuple[Token, ...],
+    chunks: tuple[Chunk, ...],
+) -> tuple[tuple[Token, ...], tuple[Chunk, ...]]:
+    merged_tokens, old_to_new = merge_tokens(tokens)
+    merged_chunks = remap_chunks(chunks, old_to_new, merged_tokens)
+    return merged_tokens, merged_chunks
+
+
+def _coarse_units(text: str) -> list[str]:
+    return [p.strip() for p in _HARD_SPLIT_RE.split(text.strip()) if p.strip()]
+
+
+def _pack_token_spans(doc: Doc, max_chars: int = MAX_SENTENCE_CHARS) -> list[Span]:
+    if not len(doc):
+        return []
+
+    chunks = build_chunks(doc[:])
+    if not chunks:
+        return _pack_by_tokens(doc, list(doc), max_chars)
+
+    groups = _pack_chunks(list(chunks), max_chars)
+    spans: list[Span] = []
+    for group in groups:
+        if not group:
+            continue
+        if len(group) == 1 and len(group[0].text) > max_chars:
+            c_start = group[0].token_indices[0]
+            c_end = group[0].token_indices[-1] + 1
+            spans.extend(_pack_by_tokens(doc, list(doc[c_start:c_end]), max_chars))
+            continue
+        start = group[0].token_indices[0]
+        end = group[-1].token_indices[-1] + 1
+        spans.append(doc[start:end])
+    return [s for s in spans if s.text.strip()]
+
+
+def _pack_chunks(
+    chunks: list[Chunk],
+    max_chars: int,
+    min_chars: int = 8,
+) -> list[list[Chunk]]:
+    groups: list[list[Chunk]] = []
+    current: list[Chunk] = []
+    length = 0
+
+    for chunk in chunks:
+        clen = len(chunk.text)
+        if clen > max_chars and not current:
+            groups.append([chunk])
+            continue
+        if current and length + clen > max_chars:
+            groups.append(current)
+            current = [chunk]
+            length = clen
+        else:
+            current.append(chunk)
+            length += clen
+            if chunk.text[-1:] in _SOFT_BREAK_CHARS and length >= max_chars // 2:
+                groups.append(current)
+                current = []
+                length = 0
+    if current:
+        groups.append(current)
+
+    return _rebalance_chunk_groups(groups, max_chars, min_chars)
+
+
+def _rebalance_chunk_groups(
+    groups: list[list[Chunk]],
+    max_chars: int,
+    min_chars: int,
+) -> list[list[Chunk]]:
+    if len(groups) < 2:
+        return groups
+
+    result = [list(g) for g in groups]
+    i = 1
+    while i < len(result):
+        right_len = sum(len(c.text) for c in result[i])
+        if right_len >= min_chars or len(result[i - 1]) <= 1:
+            i += 1
+            continue
+
+        while result[i - 1] and right_len < min_chars:
+            if len(result[i - 1]) <= 1:
+                break
+            stolen = result[i - 1][-1]
+            left_len = sum(len(c.text) for c in result[i - 1]) - len(stolen.text)
+            new_right = right_len + len(stolen.text)
+            if left_len < min_chars or new_right > max_chars:
+                break
+            result[i - 1].pop()
+            result[i].insert(0, stolen)
+            right_len = new_right
+
+        if not result[i - 1]:
+            result.pop(i - 1)
+            continue
+        i += 1
+
+    return [g for g in result if g]
+
+
+def _pack_by_tokens(doc: Doc, tokens: list[SpacyToken], max_chars: int) -> list[Span]:
+    if not tokens:
+        return []
+
+    spans: list[Span] = []
+    group_start = tokens[0].i
+    length = 0
+
+    def flush(end_i: int) -> None:
+        nonlocal group_start, length
+        if end_i > group_start:
+            spans.append(doc[group_start:end_i])
+        group_start = end_i
+        length = 0
+
+    for tok in tokens:
+        tlen = len(tok.text)
+        if tlen > max_chars and length == 0:
+            spans.append(doc[tok.i : tok.i + 1])
+            group_start = tok.i + 1
+            length = 0
+            continue
+        if length and length + tlen > max_chars:
+            flush(tok.i)
+        length += tlen
+        if tok.text in _SOFT_BREAK_CHARS and length >= max_chars // 2:
+            flush(tok.i + 1)
+
+    flush(tokens[-1].i + 1)
+    return spans
+
+
+def _sentence_from_span(span: Span, index: int, method: str) -> Sentence | None:
+    stripped = span.text.strip()
+    if not stripped:
+        return None
+    raw_tokens = _tokenize_span(span)
+    raw_chunks = build_chunks(span)
+    tokens, chunks = _finalize_tokens_and_chunks(raw_tokens, raw_chunks)
+    return Sentence(
+        index=index,
+        text=stripped,
+        char_count=len(stripped),
+        method=method,
+        tokens=tokens,
+        chunks=chunks,
+    )
+
+
+def analyze(text: str) -> list[Sentence]:
+    """Segment and tokenize Japanese text.
+
+    Strategy:
+    1. Hard split on newlines and 。！？
+    2. If still over MAX_SENTENCE_CHARS, pack by chunks
+    3. Rebalance short tails by borrowing chunks from the left
+    """
+    nlp = _load_nlp()
+    sentences: list[Sentence] = []
+    for unit in _coarse_units(text):
+        doc = nlp(unit)
+        spans = (
+            [doc[:]]
+            if len(unit) <= MAX_SENTENCE_CHARS
+            else _pack_token_spans(doc, MAX_SENTENCE_CHARS)
+        )
+        for span in spans:
+            sent = _sentence_from_span(span, len(sentences) + 1, method="ginza")
+            if sent is not None:
+                sentences.append(sent)
+    return sentences

sentence_struct/backends/ja/taxonomy.py

@@ -0,0 +1,90 @@
+"""Token / Chunk type normalization for UI highlighting groups."""
+
+from __future__ import annotations
+
+TOKEN_POS_GROUPS: dict[str, str] = {
+    "名詞": "名詞",
+    "代名詞": "名詞",
+    "数詞": "名詞",
+    "動詞": "動詞",
+    "形容詞": "形容詞",
+    "形状詞": "形容詞",
+    "連体詞": "形容詞",
+    "副詞": "副詞",
+    "接続詞": "副詞",
+    "助詞": "助詞",
+    "助動詞": "助動詞",
+    "感動詞": "感動詞",
+    "接頭辞": "_",
+    "接尾辞": "_",
+    "補助記号": "_",
+    "記号": "_",
+    "空白": "_",
+    "フィラー": "_",
+    "未知語": "_",
+}
+
+_TOKEN_UPOS_GROUPS: dict[str, str] = {
+    "NOUN": "名詞",
+    "PROPN": "名詞",
+    "PRON": "名詞",
+    "NUM": "名詞",
+    "VERB": "動詞",
+    "ADJ": "形容詞",
+    "ADV": "副詞",
+    "ADP": "助詞",
+    "AUX": "助動詞",
+    "SCONJ": "副詞",
+    "CCONJ": "副詞",
+    "INTJ": "感動詞",
+    "PUNCT": "_",
+    "X": "_",
+}
+
+CHUNK_TYPE_GROUPS: dict[str, str] = {
+    "NP": "NP",
+    "VP": "VP",
+    "ADJP": "ADJP",
+    "ADVP": "ADVP",
+    "PP": "ADVP",
+    "CP": "VP",
+    "CCONJP": "ADVP",
+    "名詞": "NP",
+    "代名詞": "NP",
+    "動詞": "VP",
+    "形容詞": "ADJP",
+    "形状詞": "ADJP",
+    "連体詞": "ADJP",
+    "副詞": "ADVP",
+    "助詞": "助詞",
+    "助動詞": "VP",
+    "感動詞": "感動詞",
+    "補助記号": "_",
+}
+
+TOKEN_HIGHLIGHT_KEYS = ("名詞", "動詞", "形容詞", "副詞", "助詞", "助動詞", "感動詞")
+CHUNK_HIGHLIGHT_KEYS = ("NP", "VP", "ADJP", "ADVP", "助詞", "感動詞")
+SUDACHI_POS_MAJORS = frozenset(TOKEN_POS_GROUPS)
+CHUNK_SOURCE_TYPES = frozenset(CHUNK_TYPE_GROUPS)
+
+
+def normalize_token_pos(pos: str, *, pos_detail: str | None = None) -> str:
+    """Sudachi major POS → posGroup."""
+    major = pos.split("-", 1)[0] if pos else ""
+    if major in TOKEN_POS_GROUPS:
+        return TOKEN_POS_GROUPS[major]
+    if pos_detail:
+        detail_major = pos_detail.split("-", 1)[0]
+        if detail_major in TOKEN_POS_GROUPS:
+            return TOKEN_POS_GROUPS[detail_major]
+    return "_"
+
+
+def normalize_token_upos(pos: str) -> str:
+    """Universal POS → posGroup."""
+    return _TOKEN_UPOS_GROUPS.get(pos, "_")
+
+
+def normalize_chunk_type(chunk_type: str) -> str:
+    """chunk type → typeGroup."""
+    return CHUNK_TYPE_GROUPS.get(chunk_type, "_")

sentence_struct/backends/ja/token_merge.py

@@ -0,0 +1,125 @@
+"""Merge morpheme tokens into learner-friendly word surfaces."""
+
+from __future__ import annotations
+
+from sentence_struct.models import Chunk, Token
+
+_PREDICATE_HEAD_MAJORS = frozenset({"動詞", "形容詞", "形状詞", "助動詞"})
+_MERGE_ATTACH_MAJORS = frozenset({"助動詞", "接尾辞"})
+
+
+def _pos_major(pos_detail: str) -> str:
+    return pos_detail.split("-", 1)[0]
+
+
+def _merges_into_previous(prev: Token, curr: Token) -> bool:
+    prev_major = _pos_major(prev.pos_detail)
+    curr_major = _pos_major(curr.pos_detail)
+
+    if curr_major in _MERGE_ATTACH_MAJORS:
+        return prev_major in _PREDICATE_HEAD_MAJORS
+
+    if curr_major == "形容詞" and prev_major == "形容詞":
+        return True
+
+    if curr_major == "助詞" and curr.surface in {"て", "で"} and prev_major == "動詞":
+        return True
+
+    if curr_major == "動詞" and prev.surface.endswith(("て", "で")):
+        return True
+
+    return False
+
+
+def _join_reading(prev: str | None, curr: str | None) -> str | None:
+    merged = (prev or "") + (curr or "")
+    return merged or None
+
+
+def merge_tokens(tokens: tuple[Token, ...]) -> tuple[tuple[Token, ...], list[int]]:
+    """Merge morphemes; return (merged_tokens, old_to_new index map)."""
+    if not tokens:
+        return (), []
+
+    merged: list[Token] = []
+    old_to_new: list[int] = []
+
+    for token in tokens:
+        if merged and _merges_into_previous(merged[-1], token):
+            prev = merged[-1]
+            merged[-1] = Token(
+                index=prev.index,
+                surface=prev.surface + token.surface,
+                lemma=prev.lemma,
+                pos=prev.pos,
+                pos_detail=prev.pos_detail,
+                reading=_join_reading(prev.reading, token.reading),
+            )
+            old_to_new.append(len(merged) - 1)
+            continue
+
+        old_to_new.append(len(merged))
+        merged.append(token)
+
+    reindexed = tuple(
+        Token(
+            index=i + 1,
+            surface=t.surface,
+            lemma=t.lemma,
+            pos=t.pos,
+            pos_detail=t.pos_detail,
+            reading=t.reading,
+        )
+        for i, t in enumerate(merged)
+    )
+    return reindexed, old_to_new
+
+
+def remap_chunks(
+    chunks: tuple[Chunk, ...],
+    old_to_new: list[int],
+    merged_tokens: tuple[Token, ...],
+) -> tuple[Chunk, ...]:
+    """Remap chunk indices/text after token merge."""
+    remapped: list[Chunk] = []
+    seen_keys: set[tuple[int, ...]] = set()
+
+    for chunk in chunks:
+        seen: set[int] = set()
+        indices: list[int] = []
+        for old_index in chunk.token_indices:
+            new_index = old_to_new[old_index]
+            if new_index in seen:
+                continue
+            seen.add(new_index)
+            indices.append(new_index)
+
+        if not indices:
+            continue
+
+        key = tuple(indices)
+        if key in seen_keys:
+            continue
+        seen_keys.add(key)
+
+        remapped.append(
+            Chunk(
+                type=chunk.type,
+                token_indices=tuple(indices),
+                text="".join(merged_tokens[i].surface for i in indices),
+                head_index=old_to_new[chunk.head_index],
+                role=chunk.role,
+            )
+        )
+
+    return _drop_subset_chunks(remapped)
+
+
+def _drop_subset_chunks(chunks: list[Chunk]) -> tuple[Chunk, ...]:
+    kept: list[Chunk] = []
+    index_sets = [set(chunk.token_indices) for chunk in chunks]
+    for i, chunk in enumerate(chunks):
+        if any(index_sets[i] < index_sets[j] for j in range(len(chunks)) if i != j):
+            continue
+        kept.append(chunk)
+    return tuple(kept)

sentence_struct/cli.py

@@ -0,0 +1,74 @@
+"""CLI: sentence-struct [options] [text]."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from sentence_struct.api import SUPPORTED_LANGUAGES, analyze
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="sentence-struct",
+        description="Analyze text into sentences with tokens and syntactic chunks.",
+    )
+    parser.add_argument(
+        "text",
+        nargs="?",
+        help="Input text. Omit when using --file.",
+    )
+    parser.add_argument(
+        "-f",
+        "--file",
+        type=Path,
+        help="Read input text from a UTF-8 file.",
+    )
+    parser.add_argument(
+        "-o",
+        "--output",
+        type=Path,
+        help="Write JSON to this path (default: stdout).",
+    )
+    parser.add_argument(
+        "-l",
+        "--language",
+        default="ja",
+        help=f"Language code (default: ja). Supported: {', '.join(sorted(SUPPORTED_LANGUAGES))}",
+    )
+    parser.add_argument(
+        "--indent",
+        type=int,
+        default=2,
+        help="JSON indent (default: 2).",
+    )
+    args = parser.parse_args(argv)
+
+    if args.file is not None:
+        text = args.file.read_text(encoding="utf-8").strip()
+    elif args.text is not None:
+        text = args.text.strip()
+    else:
+        if sys.stdin.isatty():
+            parser.error("Provide text, --file, or pipe stdin.")
+        text = sys.stdin.read().strip()
+
+    if not text:
+        parser.error("Input text is empty.")
+
+    result = analyze(text, language=args.language)
+    payload = json.dumps(result, ensure_ascii=False, indent=args.indent) + "\n"
+
+    if args.output is not None:
+        args.output.parent.mkdir(parents=True, exist_ok=True)
+        args.output.write_text(payload, encoding="utf-8")
+    else:
+        sys.stdout.write(payload)
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

sentence_struct/models.py

@@ -0,0 +1,40 @@
+"""Shared dataclasses for sentence structure analysis."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class Token:
+    """Word / morpheme unit inside a sentence."""
+
+    index: int
+    surface: str
+    lemma: str
+    pos: str
+    pos_detail: str
+    reading: str | None
+
+
+@dataclass(frozen=True)
+class Chunk:
+    """Linear syntactic chunk that references token indices in the same sentence."""
+
+    type: str
+    token_indices: tuple[int, ...]
+    text: str
+    head_index: int
+    role: str
+
+
+@dataclass(frozen=True)
+class Sentence:
+    """One display sentence with sibling token and chunk layers."""
+
+    index: int
+    text: str
+    char_count: int
+    method: str
+    tokens: tuple[Token, ...] = ()
+    chunks: tuple[Chunk, ...] = ()

sentence_struct/serialize.py

@@ -0,0 +1,64 @@
+"""Serialize Sentence objects to the public JSON-friendly dict schema."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from sentence_struct.backends.ja.taxonomy import normalize_chunk_type, normalize_token_pos
+from sentence_struct.models import Sentence
+
+
+def _pos_major_from_detail(pos_detail: str) -> str:
+    return pos_detail.split("-", 1)[0]
+
+
+def sentence_to_dict(sentence: Sentence) -> dict[str, Any]:
+    tokens = []
+    for t in sentence.tokens:
+        pos = _pos_major_from_detail(t.pos_detail)
+        tokens.append(
+            {
+                "text": t.surface,
+                "pos": pos,
+                "posGroup": normalize_token_pos(pos, pos_detail=t.pos_detail),
+                "pos_detail": t.pos_detail,
+                "lemma": t.lemma,
+                "reading": t.reading,
+            }
+        )
+    chunks = [
+        {
+            "type": c.type,
+            "typeGroup": normalize_chunk_type(c.type),
+            "tokenIndices": list(c.token_indices),
+            "text": c.text,
+            "headIndex": c.head_index,
+            "role": c.role,
+        }
+        for c in sentence.chunks
+    ]
+    return {
+        "index": sentence.index,
+        "text": sentence.text,
+        "char_count": sentence.char_count,
+        "method": sentence.method,
+        "tokens": tokens,
+        "chunks": chunks,
+    }
+
+
+def document_to_dict(
+    text: str,
+    sentences: list[Sentence],
+    *,
+    language: str,
+) -> dict[str, Any]:
+    return {
+        "language": language,
+        "text": text,
+        "char_count": len(text),
+        "sentence_count": len(sentences),
+        "token_count": sum(len(s.tokens) for s in sentences),
+        "chunk_count": sum(len(s.chunks) for s in sentences),
+        "sentences": [sentence_to_dict(s) for s in sentences],
+    }

sentence_struct-0.1.0.dist-info/METADATA

@@ -0,0 +1,114 @@
+Metadata-Version: 2.5
+Name: sentence-struct
+Version: 0.1.0
+Summary: Multilingual sentence structure analysis: sentences with tokens and syntactic chunks
+Project-URL: Homepage, https://github.com/memshare-project/sentence-struct
+Project-URL: Repository, https://github.com/memshare-project/sentence-struct
+Project-URL: Issues, https://github.com/memshare-project/sentence-struct/issues
+Project-URL: Changelog, https://github.com/memshare-project/sentence-struct/blob/main/CHANGELOG.md
+Author: sentence-struct contributors
+License-Expression: MIT
+License-File: LICENSE
+Keywords: chunking,japanese,linguistics,nlp,sentence-structure,tokenization
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.11
+Requires-Dist: ginza>=5.2.1
+Requires-Dist: ja-ginza>=5.2.0
+Requires-Dist: spacy>=3.8.0
+Requires-Dist: sudachidict-core>=20240409
+Requires-Dist: sudachipy>=0.6.8
+Provides-Extra: dev
+Requires-Dist: pytest-cov>=6.0; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Provides-Extra: ja-full
+Requires-Dist: sudachidict-full>=20240409; extra == 'ja-full'
+Description-Content-Type: text/markdown
+
+# sentence-struct
+
+Analyze text into a learner-friendly structure:
+
+**document → sentences[] → { tokens[], chunks[] }**
+
+Chunks are siblings of tokens and reference them via `tokenIndices` (not nested).
+
+Japanese (`ja`) is implemented first via GiNza / Sudachi. More languages later.
+
+## Install
+
+```bash
+pip install sentence-struct
+# or
+uv add sentence-struct
+```
+
+Optional larger Sudachi dictionary:
+
+```bash
+pip install "sentence-struct[ja-full]"
+```
+
+Requires Python ≥ 3.11. First install pulls spaCy / GiNza / Sudachi (hundreds of MB).
+
+## Usage
+
+```python
+from sentence_struct import analyze
+
+doc = analyze("秋が近づくにつれ、朝晩の涼しさが心地よい。", language="ja")
+print(doc["sentence_count"], doc["token_count"], doc["chunk_count"])
+print(doc["sentences"][0]["tokens"][0])
+print(doc["sentences"][0]["chunks"][0])
+```
+
+CLI:
+
+```bash
+sentence-struct "秋が近づく。"
+sentence-struct -f essay.txt -o out.json
+```
+
+## Schema (abbreviated)
+
+```json
+{
+  "language": "ja",
+  "text": "...",
+  "sentences": [
+    {
+      "index": 1,
+      "text": "...",
+      "tokens": [
+        {"text": "秋", "pos": "名詞", "posGroup": "名詞", "lemma": "秋", "reading": "アキ"}
+      ],
+      "chunks": [
+        {"type": "NP", "typeGroup": "NP", "tokenIndices": [0, 1], "text": "秋が", "role": "nsubj"}
+      ]
+    }
+  ]
+}
+```
+
+## Develop (uv)
+
+```bash
+uv sync
+uv run pytest -m "not integration"
+uv run pytest -m integration
+uv run sentence-struct "今日は良い天気です。"
+```
+
+## Release
+
+See [RELEASING.md](RELEASING.md). Tag `vX.Y.Z` → GitHub Actions builds with **uv** and publishes to PyPI via Trusted Publishing.
+
+## License
+
+MIT

sentence_struct-0.1.0.dist-info/RECORD

@@ -0,0 +1,16 @@
+sentence_struct/__init__.py,sha256=sXaBB5YZUejoqVBp1ZS4kzQd8CrLK5gNxPyG86yq76k,351
+sentence_struct/api.py,sha256=59pNPMuqKuAS1ANBJV-LOuy5srVpiPX0SAE5ddYPg8k,1372
+sentence_struct/cli.py,sha256=Ot8KdaPaMwsi6nRw_WXqXfQ7HKl-2qH2FbqP5Aj2Mig,1932
+sentence_struct/models.py,sha256=1J8tJh2ZddB2f93DvROGcsXDwAAzvS8qNodQlN5RdvE,791
+sentence_struct/serialize.py,sha256=ZtAW5RvBpzZMEEInQwaa-w5YEjUPfyHPLtwhuFBsOeM,1807
+sentence_struct/backends/__init__.py,sha256=LKpvISA-W-9ghcpzoHFagwIAs_DFZvXj_bbrKo7jtjg,69
+sentence_struct/backends/ja/__init__.py,sha256=ysQT-BfWDlep2Gb4WtuAwil-juQAk3XekDmBM20WJUw,122
+sentence_struct/backends/ja/chunks.py,sha256=6OSO-3lvbOHVIRdl14IHl0b1qG9uSHw0j55bXtASvQE,4696
+sentence_struct/backends/ja/segment.py,sha256=ftp9SuGW9KOeYAYjUMpYBVpWreOTAcsOw1Cm76Ba98Q,6476
+sentence_struct/backends/ja/taxonomy.py,sha256=EDISS7JlXSWkNEmhlJSYgKoOFCNZP7yg59beCWNB5A0,2361
+sentence_struct/backends/ja/token_merge.py,sha256=UKAN5RVg60dt0a1O3ThhmJtixcw5Ms5QLRhSvQhvLDo,3645
+sentence_struct-0.1.0.dist-info/METADATA,sha256=C_BzcpNWKIj9DwqNYdWu4yIWiUpJf0ci36xMPaYrxJU,3131
+sentence_struct-0.1.0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+sentence_struct-0.1.0.dist-info/entry_points.txt,sha256=DC8FZqHbJvFi-N2J4dH8jEHhMzChYZvtLtRfXICKAts,61
+sentence_struct-0.1.0.dist-info/licenses/LICENSE,sha256=RAXd024Sqp6H5sF6exAlGSyS75yYJOYfMuEVRHAGoo0,1085
+sentence_struct-0.1.0.dist-info/RECORD,,

sentence_struct-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.0
+Root-Is-Purelib: true
+Tag: py3-none-any

sentence_struct-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+sentence-struct = sentence_struct.cli:main

sentence_struct-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 sentence-struct contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.