tokmor 1.2.9__py3-none-any.whl

tokmor/morphology/english.py

@@ -0,0 +1,315 @@
+"""
+English Morphological Analyzer - 자체 구현
+==========================================
+
+특징:
+- 어간 추출 (Stemming)
+- 불규칙 동사/명사 처리
+- 접사 분리 (prefix/suffix)
+- 품사 태깅 (POS tagging)
+
+품사 태그 (Penn Treebank):
+- NN: 명사, NNS: 복수명사, NNP: 고유명사
+- VB: 동사원형, VBD: 과거, VBG: 현재분사, VBN: 과거분사, VBZ: 3인칭단수
+- JJ: 형용사, JJR: 비교급, JJS: 최상급
+- RB: 부사, RBR: 비교급, RBS: 최상급
+- DT: 관사, IN: 전치사, CC: 접속사
+- PRP: 인칭대명사, PRP$: 소유대명사
+"""
+
+import re
+from typing import List, Tuple, Dict, Optional
+from dataclasses import dataclass
+
+
+@dataclass
+class Morpheme:
+    surface: str
+    lemma: str
+    pos: str
+    start: int
+    end: int
+
+    def __repr__(self):
+        return f"{self.surface}/{self.pos}"
+
+
+class EnglishAnalyzer:
+    """영어 형태소 분석기"""
+
+    def __init__(self):
+        self._build_dictionary()
+        self._build_rules()
+
+    def _build_dictionary(self):
+        """사전 구축"""
+
+        # 불규칙 동사 (과거형 -> 원형)
+        self.irregular_verbs = {
+            # be
+            'am': 'be', 'is': 'be', 'are': 'be', 'was': 'be', 'were': 'be', 'been': 'be', 'being': 'be',
+            # have
+            'has': 'have', 'had': 'have', 'having': 'have',
+            # do
+            'does': 'do', 'did': 'do', 'done': 'do', 'doing': 'do',
+            # go
+            'goes': 'go', 'went': 'go', 'gone': 'go', 'going': 'go',
+            # come
+            'came': 'come', 'coming': 'come',
+            # get
+            'gets': 'get', 'got': 'get', 'gotten': 'get', 'getting': 'get',
+            # make
+            'makes': 'make', 'made': 'make', 'making': 'make',
+            # say
+            'says': 'say', 'said': 'say', 'saying': 'say',
+            # take
+            'takes': 'take', 'took': 'take', 'taken': 'take', 'taking': 'take',
+            # see
+            'sees': 'see', 'saw': 'see', 'seen': 'see', 'seeing': 'see',
+            # know
+            'knows': 'know', 'knew': 'know', 'known': 'know', 'knowing': 'know',
+            # think
+            'thinks': 'think', 'thought': 'think', 'thinking': 'think',
+            # give
+            'gives': 'give', 'gave': 'give', 'given': 'give', 'giving': 'give',
+            # find
+            'finds': 'find', 'found': 'find', 'finding': 'find',
+            # tell
+            'tells': 'tell', 'told': 'tell', 'telling': 'tell',
+            # become
+            'becomes': 'become', 'became': 'become', 'becoming': 'become',
+            # leave
+            'leaves': 'leave', 'left': 'leave', 'leaving': 'leave',
+            # put
+            'puts': 'put', 'putting': 'put',
+            # keep
+            'keeps': 'keep', 'kept': 'keep', 'keeping': 'keep',
+            # let
+            'lets': 'let', 'letting': 'let',
+            # begin
+            'begins': 'begin', 'began': 'begin', 'begun': 'begin', 'beginning': 'begin',
+            # write
+            'writes': 'write', 'wrote': 'write', 'written': 'write', 'writing': 'write',
+            # run
+            'runs': 'run', 'ran': 'run', 'running': 'run',
+            # read
+            'reads': 'read', 'reading': 'read',
+            # speak
+            'speaks': 'speak', 'spoke': 'speak', 'spoken': 'speak', 'speaking': 'speak',
+            # buy
+            'buys': 'buy', 'bought': 'buy', 'buying': 'buy',
+            # bring
+            'brings': 'bring', 'brought': 'bring', 'bringing': 'bring',
+            # sit
+            'sits': 'sit', 'sat': 'sit', 'sitting': 'sit',
+            # stand
+            'stands': 'stand', 'stood': 'stand', 'standing': 'stand',
+            # lose
+            'loses': 'lose', 'lost': 'lose', 'losing': 'lose',
+            # pay
+            'pays': 'pay', 'paid': 'pay', 'paying': 'pay',
+            # meet
+            'meets': 'meet', 'met': 'meet', 'meeting': 'meet',
+            # send
+            'sends': 'send', 'sent': 'send', 'sending': 'send',
+            # build
+            'builds': 'build', 'built': 'build', 'building': 'build',
+            # fall
+            'falls': 'fall', 'fell': 'fall', 'fallen': 'fall', 'falling': 'fall',
+            # cut
+            'cuts': 'cut', 'cutting': 'cut',
+            # drive
+            'drives': 'drive', 'drove': 'drive', 'driven': 'drive', 'driving': 'drive',
+            # break
+            'breaks': 'break', 'broke': 'break', 'broken': 'break', 'breaking': 'break',
+            # grow
+            'grows': 'grow', 'grew': 'grow', 'grown': 'grow', 'growing': 'grow',
+            # choose
+            'chooses': 'choose', 'chose': 'choose', 'chosen': 'choose', 'choosing': 'choose',
+            # eat
+            'eats': 'eat', 'ate': 'eat', 'eaten': 'eat', 'eating': 'eat',
+            # draw
+            'draws': 'draw', 'drew': 'draw', 'drawn': 'draw', 'drawing': 'draw',
+            # fly
+            'flies': 'fly', 'flew': 'fly', 'flown': 'fly', 'flying': 'fly',
+            # throw
+            'throws': 'throw', 'threw': 'throw', 'thrown': 'throw', 'throwing': 'throw',
+            # catch
+            'catches': 'catch', 'caught': 'catch', 'catching': 'catch',
+            # teach
+            'teaches': 'teach', 'taught': 'teach', 'teaching': 'teach',
+            # wear
+            'wears': 'wear', 'wore': 'wear', 'worn': 'wear', 'wearing': 'wear',
+            # win
+            'wins': 'win', 'won': 'win', 'winning': 'win',
+            # sell
+            'sells': 'sell', 'sold': 'sell', 'selling': 'sell',
+        }
+
+        # 불규칙 복수형
+        self.irregular_plurals = {
+            'men': 'man', 'women': 'woman', 'children': 'child',
+            'feet': 'foot', 'teeth': 'tooth', 'geese': 'goose',
+            'mice': 'mouse', 'people': 'person', 'lives': 'life',
+            'wives': 'wife', 'knives': 'knife', 'leaves': 'leaf',
+            'selves': 'self', 'halves': 'half', 'wolves': 'wolf',
+            'thieves': 'thief', 'shelves': 'shelf', 'loaves': 'loaf',
+            'potatoes': 'potato', 'tomatoes': 'tomato', 'heroes': 'hero',
+            'analyses': 'analysis', 'bases': 'basis', 'crises': 'crisis',
+            'criteria': 'criterion', 'phenomena': 'phenomenon',
+        }
+
+        # 기능어 (closed class)
+        self.determiners = {'the', 'a', 'an', 'this', 'that', 'these', 'those', 'my', 'your', 'his', 'her', 'its', 'our', 'their'}
+        self.pronouns = {'i', 'me', 'my', 'mine', 'you', 'your', 'yours', 'he', 'him', 'his', 'she', 'her', 'hers', 'it', 'its', 'we', 'us', 'our', 'ours', 'they', 'them', 'their', 'theirs', 'who', 'whom', 'whose', 'which', 'what', 'this', 'that', 'these', 'those'}
+        self.prepositions = {'in', 'on', 'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', 'through', 'during', 'before', 'after', 'above', 'below', 'to', 'from', 'up', 'down', 'out', 'off', 'over', 'under', 'again', 'further', 'then', 'once', 'of'}
+        self.conjunctions = {'and', 'but', 'or', 'nor', 'for', 'yet', 'so', 'because', 'although', 'while', 'if', 'when', 'where', 'as', 'than', 'whether', 'that'}
+        self.auxiliaries = {'be', 'am', 'is', 'are', 'was', 'were', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', 'did', 'will', 'would', 'shall', 'should', 'may', 'might', 'must', 'can', 'could'}
+
+        # 접미사 -> 품사
+        self.suffix_pos = {
+            # 명사
+            'tion': 'NN', 'sion': 'NN', 'ment': 'NN', 'ness': 'NN', 'ity': 'NN',
+            'er': 'NN', 'or': 'NN', 'ist': 'NN', 'ism': 'NN', 'ance': 'NN', 'ence': 'NN',
+            # 형용사
+            'able': 'JJ', 'ible': 'JJ', 'ful': 'JJ', 'less': 'JJ', 'ous': 'JJ',
+            'ive': 'JJ', 'al': 'JJ', 'ic': 'JJ', 'ical': 'JJ',
+            # 부사
+            'ly': 'RB',
+        }
+
+    def _build_rules(self):
+        """규칙 구축"""
+        # 동사 활용 패턴
+        self.verb_patterns = [
+            # -ing (현재분사)
+            (r'(.+)ying$', r'\1y', 'VBG'),  # studying -> study
+            (r'(.+)ing$', r'\1e', 'VBG'),   # making -> make
+            (r'(.+)ing$', r'\1', 'VBG'),    # going -> go
+            # -ed (과거/과거분사)
+            (r'(.+)ied$', r'\1y', 'VBD'),   # studied -> study
+            (r'(.+)ed$', r'\1e', 'VBD'),    # liked -> like
+            (r'(.+)ed$', r'\1', 'VBD'),     # walked -> walk
+            # -s/-es (3인칭 단수)
+            (r'(.+)ies$', r'\1y', 'VBZ'),   # studies -> study
+            (r'(.+)es$', r'\1', 'VBZ'),     # goes -> go
+            (r'(.+)s$', r'\1', 'VBZ'),      # walks -> walk
+        ]
+
+        # 명사 복수형 패턴
+        self.plural_patterns = [
+            (r'(.+)ies$', r'\1y'),    # cities -> city
+            (r'(.+)ves$', r'\1f'),    # lives -> life
+            (r'(.+)ves$', r'\1fe'),   # wives -> wife
+            (r'(.+)es$', r'\1'),      # boxes -> box
+            (r'(.+)s$', r'\1'),       # cats -> cat
+        ]
+
+        # 비교급/최상급 패턴
+        self.comparative_patterns = [
+            (r'(.+)ier$', r'\1y', 'JJR'),   # happier -> happy
+            (r'(.+)iest$', r'\1y', 'JJS'),  # happiest -> happy
+            (r'(.+)er$', r'\1', 'JJR'),     # bigger -> big
+            (r'(.+)est$', r'\1', 'JJS'),    # biggest -> big
+        ]
+
+    def analyze(self, text: str) -> List[Morpheme]:
+        """형태소 분석"""
+        if not text:
+            return []
+
+        tokens = self._tokenize(text)
+        result = []
+
+        for token, start, end in tokens:
+            morpheme = self._analyze_token(token, start, end)
+            result.append(morpheme)
+
+        return result
+
+    def _tokenize(self, text: str) -> List[Tuple[str, int, int]]:
+        """토큰화"""
+        tokens = []
+        for match in re.finditer(r"[a-zA-Z]+|[0-9]+|[^\s\w]", text):
+            tokens.append((match.group(), match.start(), match.end()))
+        return tokens
+
+    def _analyze_token(self, token: str, start: int, end: int) -> Morpheme:
+        """단일 토큰 분석"""
+        lower = token.lower()
+
+        # 1. 기능어 체크
+        if lower in self.determiners:
+            return Morpheme(token, lower, 'DT', start, end)
+        if lower in self.pronouns:
+            return Morpheme(token, lower, 'PRP', start, end)
+        if lower in self.prepositions:
+            return Morpheme(token, lower, 'IN', start, end)
+        if lower in self.conjunctions:
+            return Morpheme(token, lower, 'CC', start, end)
+        if lower in self.auxiliaries:
+            lemma = self.irregular_verbs.get(lower, lower)
+            return Morpheme(token, lemma, 'VB', start, end)
+
+        # 2. 불규칙 동사 체크
+        if lower in self.irregular_verbs:
+            lemma = self.irregular_verbs[lower]
+            pos = self._get_verb_form(lower)
+            return Morpheme(token, lemma, pos, start, end)
+
+        # 3. 불규칙 복수형 체크
+        if lower in self.irregular_plurals:
+            lemma = self.irregular_plurals[lower]
+            return Morpheme(token, lemma, 'NNS', start, end)
+
+        # 4. 규칙 기반 분석
+        # 동사 활용
+        for pattern, replacement, pos in self.verb_patterns:
+            if re.match(pattern, lower):
+                lemma = re.sub(pattern, replacement, lower)
+                return Morpheme(token, lemma, pos, start, end)
+
+        # 복수형
+        for pattern, replacement in self.plural_patterns:
+            if re.match(pattern, lower) and len(lower) > 3:
+                lemma = re.sub(pattern, replacement, lower)
+                return Morpheme(token, lemma, 'NNS', start, end)
+
+        # 비교급/최상급
+        for pattern, replacement, pos in self.comparative_patterns:
+            if re.match(pattern, lower):
+                lemma = re.sub(pattern, replacement, lower)
+                return Morpheme(token, lemma, pos, start, end)
+
+        # 5. 접미사 기반 품사 추정
+        for suffix, pos in self.suffix_pos.items():
+            if lower.endswith(suffix) and len(lower) > len(suffix) + 2:
+                return Morpheme(token, lower, pos, start, end)
+
+        # 6. 대문자로 시작하면 고유명사
+        if token[0].isupper() and start > 0:
+            return Morpheme(token, token, 'NNP', start, end)
+
+        # 7. 기본값: 명사
+        return Morpheme(token, lower, 'NN', start, end)
+
+    def _get_verb_form(self, word: str) -> str:
+        """동사 형태 판별"""
+        if word.endswith('ing'):
+            return 'VBG'
+        elif word.endswith('ed') or word in {'was', 'were', 'had', 'did', 'went', 'came', 'saw', 'took', 'made', 'said', 'got', 'gave', 'found', 'thought', 'told', 'became', 'left', 'kept', 'began', 'wrote', 'ran', 'spoke', 'bought', 'brought', 'sat', 'stood', 'lost', 'paid', 'met', 'sent', 'built', 'fell', 'drove', 'broke', 'grew', 'chose', 'ate', 'drew', 'flew', 'threw', 'caught', 'taught', 'wore', 'won', 'sold'}:
+            return 'VBD'
+        elif word.endswith('en') or word in {'been', 'done', 'gone', 'seen', 'known', 'given', 'taken', 'written', 'spoken', 'chosen', 'eaten', 'drawn', 'flown', 'thrown', 'worn', 'driven', 'broken', 'grown', 'fallen', 'forgotten', 'gotten'}:
+            return 'VBN'
+        elif word.endswith('s'):
+            return 'VBZ'
+        else:
+            return 'VB'
+
+    def lemmatize(self, text: str) -> List[str]:
+        """기본형 추출"""
+        return [m.lemma for m in self.analyze(text)]
+
+    def pos_tag(self, text: str) -> List[Tuple[str, str]]:
+        """품사 태깅"""
+        return [(m.surface, m.pos) for m in self.analyze(text)]