sigilyph 0.5.2__py3-none-any.whl

sigilyph/core/text_process.py

@@ -0,0 +1,328 @@
+'''
+FilePath: /python-Sigilyph/sigilyph/core/text_process.py
+Descripttion: 
+Author: Yixiang Chen
+version: 
+Date: 2025-03-31 16:31:26
+LastEditors: Yixiang Chen
+LastEditTime: 2026-01-16 17:36:55
+'''
+
+
+import langid
+import re
+
+import jieba
+import jieba.posseg
+
+from sigilyph.core.g2p_func import g2p_en, g2p_cn
+#from sigilyph.core.norm_func import preprocess_first, text_norm_en, text_norm_cn
+from sigilyph.text_norm.norm_func import preprocess_first, text_norm_en, text_norm_cn
+
+from sigilyph.core.symbols import base_phone_set, cn_phone_set, en_phone_set, punctuation, special_phrase
+
+#all_phone_set = [] + sorted(set(base_phone_set + cn_phone_set + en_phone_set))
+#all_phone_set = [] + list(set(base_phone_set)) + list(set(cn_phone_set + en_phone_set))
+all_phone_set = [] + sorted(set(base_phone_set)) + sorted(set(cn_phone_set)) + sorted(set(en_phone_set))
+all_phone_dict = {xx:idx for idx, xx in enumerate(all_phone_set)}
+
+norm_func_dict = {
+    'en': text_norm_en,
+    'zh': text_norm_cn
+}
+
+g2p_func_dict = {
+    'en': g2p_en,
+    'zh': g2p_cn
+}
+
+sil1symbol='-'
+
+def text_split_lang_old(text, lang):
+    if lang == 'ZH' or lang == 'zh':
+        multi_lang_text_list = [{'lang':'zh', 'text_split': text}]
+    elif lang == 'en':
+        multi_lang_text_list = [{'lang':'en', 'text_split': text}]
+    else:
+        pattern = r'([a-zA-Z ,.\!\?\"]+|[\u4e00-\u9fa5 ，。\！\？\“\”]+)'
+        text_split = re.findall(pattern, text)
+        multi_lang_text_list = []
+        for idx in range(len(text_split)):
+            tmpts = text_split[idx]
+            tmp_lang = langid.classify(tmpts)[0]
+            multi_lang_text_list.append({'lang':tmp_lang, 'text_split': tmpts})
+    return multi_lang_text_list
+
+def text_split_lang_bk0724(text, lang):
+    if lang == 'ZH' or lang == 'zh':
+        multi_lang_text_list = [{'lang':'zh', 'text_split': text}]
+    elif lang == 'en':
+        multi_lang_text_list = [{'lang':'en', 'text_split': text}]
+    else:
+        pretext_split =  re.split("(\[.*?\])", text, re.I|re.M)
+        multi_lang_text_list = []
+        pretext_split = list(filter(None, pretext_split))
+        for utext in pretext_split:
+            if utext[0] != '[':
+                pattern = r'([a-zA-Z ,.\!\?\"]+|[\u4e00-\u9fa5 ，。,.\t \！\？]+)'
+                text_split = re.findall(pattern, utext)
+                for idx in range(len(text_split)):
+                    tmpts = text_split[idx]
+                    tmp_lang = langid.classify(tmpts)[0]
+                    if tmp_lang in ['zh', 'jp', 'ja']:
+                        tmp_lang = 'zh'
+                    else:
+                        tmp_lang = 'en' 
+                    if not tmpts.isspace():
+                        multi_lang_text_list.append({'lang':tmp_lang, 'text_split': tmpts})
+            else:
+                phones = utext[1:-1]
+                multi_lang_text_list.append({'lang':'phone', 'text_split': phones})
+    return multi_lang_text_list
+
+
+def search_ele_mid(flaglist, tf = 'v'):
+    nowidx = -1
+    halflen = (len(flaglist))//2
+    for gap in range(len(flaglist)-halflen):
+        nowidx = halflen - gap
+        if flaglist[nowidx]==tf:
+            return nowidx
+        nowidx = halflen + gap
+        if flaglist[nowidx]==tf:
+            return nowidx
+    return nowidx
+
+def add_pause(text, tf='v'):
+    segment = jieba.posseg.cut(text.strip())
+    wlist = []
+    flist = []
+    for x in segment:
+        wlist.append(x.word)
+        flist.append(x.flag)
+    idx = search_ele_mid(flist, tf)
+    if idx != len(flist)-1:
+        wlist.insert(idx, sil1symbol)
+    outtext = ''.join(wlist)
+    return outtext
+
+def has_punc(text):
+    for char in text:
+        if char in [',', '.', '!', '?', '，','。','？','！', sil1symbol]:
+            return True
+    return False
+
+def text_split_lang(text, lang):
+    if lang == 'ZH' or lang == 'zh':
+        multi_lang_text_list = [{'lang':'zh', 'text_split': text}]
+    elif lang == 'en':
+        multi_lang_text_list = [{'lang':'en', 'text_split': text}]
+    else:
+        pretext_split =  re.split("(\[.*?\])", text, re.I|re.M)
+        multi_lang_text_list = []
+        pretext_split = list(filter(None, pretext_split))
+        for utext in pretext_split:
+            if utext[0] != '[':
+                pattern = r'([a-zA-Z ,.\!\?]+|[\u4e00-\u9fa5 ，。,.\t \"\！\？\“\”\、]+)'
+                text_split = re.findall(pattern, utext)
+                print(text_split)
+                for idx in range(len(text_split)):
+                    tmpts = text_split[idx]
+                    tmp_lang = langid.classify(tmpts)[0]
+                    if len(tmpts)>20:
+                        if not has_punc(tmpts[:-1]):
+                            tmpts = add_pause(tmpts, 'p')
+                        if not has_punc(tmpts[:-1]):
+                            tmpts = add_pause(tmpts, 'v')   
+                    if tmpts in special_phrase:
+                        tmpts = tmpts+sil1symbol
+                    if tmp_lang in ['zh', 'jp', 'ja']:
+                        tmp_lang = 'zh'
+                        tmpts = tmpts.replace(' ', sil1symbol)
+                    else:
+                        tmp_lang = 'en' 
+                    if not tmpts.isspace():
+                        multi_lang_text_list.append({'lang':tmp_lang, 'text_split': tmpts})
+            else:
+                phones = utext[1:-1]
+                multi_lang_text_list.append({'lang':'phone', 'text_split': phones})
+    return multi_lang_text_list
+
+def text_norm(text, lang):
+    outtext = norm_func_dict[lang](text)
+    return outtext
+
+def g2p(text, lang):
+    phoneme_list = g2p_func_dict[lang](text)
+    return phoneme_list
+
+def tokenizer(phoneme_list):
+    #token_list = [all_phone_dict[pho] for pho in phoneme_list]
+    token_list = [all_phone_dict[pho] if pho in all_phone_dict.keys() else all_phone_dict['sil'] for pho in phoneme_list]
+    return token_list 
+
+def postprocess(phonelist):
+    outlist = [xx if xx not in punctuation else 'sil' for xx in phonelist]
+    return outlist
+
+def postprocess_tts(phonelist):
+    #outlist = ['sil', '<sp>']
+    outlist = []
+    print(phonelist)
+    for idx in range(len(phonelist)):
+        pm = phonelist[idx]
+        if pm not in punctuation:
+            outlist.append(pm)
+        elif pm == sil1symbol:
+            outlist.append('sil_1')
+        else:
+            #outlist.append('sil')
+            outlist.append('sil_punc')
+            #outlist.append('<sp>')
+    #if outlist[-1] == 'sil': 
+    #    outlist.append('<sp>')
+    #elif outlist[-2] != 'sil':
+    #    outlist.append('sil')
+    #    outlist.append('<sp>')
+    if phonelist[-2] not in punctuation and outlist[-1].split('_')[0] != 'sil':
+        #outlist.append('sil')
+        outlist.append('sil_end')
+        outlist.append('<sp>')
+    return outlist
+
+def text_process_old(text, lang, spflag=True):
+    multi_lang_text_list = text_split_lang(text, lang) 
+
+    all_phone = []
+    for text_split_dict in multi_lang_text_list:
+        use_lang = text_split_dict['lang']
+        if use_lang not in norm_func_dict.keys():
+            use_lang = 'zh'
+        use_text = text_split_dict['text_split']
+        use_text = text_norm(use_text, use_lang) 
+        phone_list = g2p(use_text, use_lang)
+        #all_phone.append('sil')
+        all_phone.append('sil_lang')
+        all_phone.append('<sp>')
+        all_phone.extend(phone_list)
+    #all_phone = postprocess(all_phone)
+    all_phone = postprocess_tts(all_phone)
+    if not spflag:
+        while '<sp>' in all_phone:
+            all_phone.remove('<sp>')
+    return all_phone
+
+def text_process(text, lang, spflag=True, use_lang='zh'):
+    text = preprocess_first(text, use_lang=use_lang)
+
+    multi_lang_text_list = text_split_lang(text, lang) 
+
+    all_phone = []
+    for text_split_dict in multi_lang_text_list:
+        use_lang = text_split_dict['lang']
+        use_text = text_split_dict['text_split']
+        if use_lang == 'phone':
+            phonelist = use_text.split()
+            all_phone.extend(phonelist) 
+        else:
+            if use_lang not in norm_func_dict.keys():
+                use_lang = 'zh'
+            use_text = text_norm(use_text, use_lang)
+            phone_list = g2p(use_text, use_lang)
+            #all_phone.append('sil')
+            all_phone.append('sil_lang')
+            all_phone.append('<sp>')
+            all_phone.extend(phone_list)
+    #all_phone = postprocess(all_phone)
+    all_phone = postprocess_tts(all_phone)
+    if not spflag:
+        while '<sp>' in all_phone:
+            all_phone.remove('<sp>')
+    return all_phone
+
+def replace_sil2label_old(phones):
+    phones = ['sil_1' if xx == 'sil_lang' else xx for xx in phones]
+    phones = ['sil_2' if xx == 'sil_punc' else xx for xx in phones]
+    phones = ['sil_2' if xx == 'sil_end' else xx for xx in phones]
+    phones = ['sil_1' if xx == 'sil' else xx for xx in phones]
+    outphones = []
+    for ele in phones:
+        if outphones == []:
+            outphones.append(ele)
+        else:
+            if ele.split('_')[0] == 'sil' and outphones[-1].split('_')[0] == 'sil':
+                #outphones[-1] = 'sil_2'
+                outphones[-1] = 'sil_1'
+            else:
+                outphones.append(ele)
+    if outphones[-1].split('_')[0] == 'sil':
+        outphones = outphones[:-1]
+    return outphones
+
+def replace_sil2label_0808(phones):
+    #phones = ['sil_1' if xx == 'sil_lang' else xx for xx in phones]
+    phones = ['' if xx == 'sil_lang' else xx for xx in phones]
+    phones = ['sil_2' if xx == 'sil_punc' else xx for xx in phones]
+    phones = ['sil_2' if xx == 'sil_end' else xx for xx in phones]
+    phones = ['sil_1' if xx == 'sil' else xx for xx in phones]
+    phones = list(filter(None, phones))
+    #outphones = []
+    outphones = ['sil_1']
+    for ele in phones:
+        if outphones == []:
+            outphones.append(ele)
+        else:
+            if ele.split('_')[0] == 'sil' and outphones[-1].split('_')[0] == 'sil':
+                #outphones[-1] = 'sil_2'
+                outphones[-1] = 'sil_1'
+            else:
+                outphones.append(ele)
+    if outphones[-1].split('_')[0] == 'sil':
+        outphones = outphones[:-1]
+    return outphones
+
+def replace_sil2label(phones):
+    #phones = ['sil_1' if xx == 'sil_lang' else xx for xx in phones]
+    phones = ['' if xx == 'sil_lang' else xx for xx in phones]
+    phones = ['sil_2' if xx == 'sil_punc' else xx for xx in phones]
+    phones = ['sil_2' if xx == 'sil_end' else xx for xx in phones]
+    phones = ['sil_1' if xx == 'sil' else xx for xx in phones]
+    phones = list(filter(None, phones))
+    #outphones = []
+    outphones = ['sil_1']
+    for ele in phones:
+        if outphones == []:
+            outphones.append(ele)
+        else:
+            if ele.split('_')[0] == 'sil' and outphones[-1].split('_')[0] == 'sil':
+                outphones[-1] = 'sil_2'
+                #outphones[-1] = 'sil_1'
+            else:
+                outphones.append(ele)
+    #if outphones[-1].split('_')[0] == 'sil':
+    #    outphones = outphones[:-1]
+    return outphones
+
+
+def text_process_asr(text, lang):
+    multi_lang_text_list = text_split_lang(text, lang) 
+
+    all_phone = []
+    for text_split_dict in multi_lang_text_list:
+        use_lang = text_split_dict['lang']
+        use_text = text_split_dict['text_split']
+        use_text = text_norm(use_text, use_lang) 
+        phone_list = g2p(use_text, use_lang)
+        phone_list_new = []
+        for idx in range(len(phone_list)):
+            tmpp = phone_list[idx]
+            if tmpp != '<sp>':
+                phone_list_new.append(tmpp)
+        all_phone.extend(phone_list_new)
+    all_phone = postprocess(all_phone)
+    if all_phone[0] != 'sil':
+        all_phone = ['sil'] + all_phone 
+    if all_phone[-1] != 'sil': 
+        all_phone = all_phone + ['sil']
+    
+    return all_phone

sigilyph/fst_tool/infer_normalizer.py

@@ -0,0 +1,49 @@
+# Copyright (c) 2022 Zhendong Peng (pzd17@tsinghua.org.cn)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from sigilyph.fst_tool.processor import Processor
+from importlib_resources import files
+
+class ZhNormalizer(Processor):
+
+    def __init__(self,
+                 version_id='v1',
+                 cache_dir=None,
+                 overwrite_cache=False,
+                 remove_interjections=True,
+                 remove_erhua=True,
+                 traditional_to_simple=True,
+                 remove_puncts=False,
+                 full_to_half=True,
+                 tag_oov=False):
+        super().__init__(name='zh_normalizer')
+        self.remove_interjections = remove_interjections
+        self.remove_erhua = remove_erhua
+        self.traditional_to_simple = traditional_to_simple
+        self.remove_puncts = remove_puncts
+        self.full_to_half = full_to_half
+        self.tag_oov = tag_oov
+        if cache_dir is None:
+            cache_dir = files("no_fst")
+        #self.build_fst('zh_tn', cache_dir, overwrite_cache)
+        self.build_fst('textnorm_zh_' + version_id, cache_dir, overwrite_cache)
+
+
+class EnNormalizer(Processor):
+    def __init__(self, version_id='v1', cache_dir=None, overwrite_cache=False):
+        super().__init__(name='en_normalizer', ordertype="en_tn")
+        if cache_dir is None:
+            cache_dir = files("no_fst")
+        #self.build_fst('en_tn', cache_dir, overwrite_cache)
+        self.build_fst('textnorm_en_' + version_id, cache_dir, overwrite_cache)

sigilyph/fst_tool/processor.py

@@ -0,0 +1,122 @@
+# Copyright (c) 2022 Zhendong Peng (pzd17@tsinghua.org.cn)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import string
+import logging
+
+from sigilyph.fst_tool.token_parser import TokenParser
+
+from pynini import (cdrewrite, cross, difference, escape, Fst, shortestpath,
+                    union, closure, invert)
+from pynini.lib import byte, utf8
+from pynini.lib.pynutil import delete, insert
+
+class Processor:
+
+    def __init__(self, name, ordertype="tn"):
+        self.ALPHA = byte.ALPHA
+        self.DIGIT = byte.DIGIT
+        self.PUNCT = byte.PUNCT
+        self.SPACE = byte.SPACE | u'\u00A0'
+        self.VCHAR = utf8.VALID_UTF8_CHAR
+        self.VSIGMA = self.VCHAR.star
+        self.LOWER = byte.LOWER
+        self.UPPER = byte.UPPER
+
+        CHAR = difference(self.VCHAR, union('\\', '"'))
+        self.CHAR = (CHAR | cross('\\', '\\\\\\') | cross('"', '\\"'))
+        self.SIGMA = (CHAR | cross('\\\\\\', '\\') | cross('\\"', '"')).star
+        self.NOT_QUOTE = difference(self.VCHAR, r'"').optimize()
+        self.NOT_SPACE = difference(self.VCHAR, self.SPACE).optimize()
+        self.INSERT_SPACE = insert(" ")
+        self.DELETE_SPACE = delete(self.SPACE).star
+        self.DELETE_EXTRA_SPACE = cross(closure(self.SPACE, 1), " ")
+        self.DELETE_ZERO_OR_ONE_SPACE = delete(closure(self.SPACE, 0, 1))
+        self.MIN_NEG_WEIGHT = -0.0001
+        self.TO_LOWER = union(*[
+            cross(x, y)
+            for x, y in zip(string.ascii_uppercase, string.ascii_lowercase)
+        ])
+        self.TO_UPPER = invert(self.TO_LOWER)
+
+        self.name = name
+        self.ordertype = ordertype
+        self.tagger = None
+        self.verbalizer = None
+    
+    def build_rule(self, fst, l='', r=''):
+        rule = cdrewrite(fst, l, r, self.VSIGMA)
+        return rule
+
+    def add_tokens(self, tagger):
+        tagger = insert(f"{self.name} {{ ") + tagger + insert(' } ')
+        return tagger.optimize()
+
+    def delete_tokens(self, verbalizer):
+        verbalizer = (delete(f"{self.name}") + delete(' { ') + verbalizer +
+                      delete(' }') + delete(' ').ques)
+        return verbalizer.optimize()
+
+    def build_verbalizer(self):
+        verbalizer = delete('value: "') + self.SIGMA + delete('"')
+        self.verbalizer = self.delete_tokens(verbalizer)
+    
+    def build_fst(self, prefix, cache_dir, overwrite_cache):
+        logger = logging.getLogger('textnorm-{}'.format(self.name))
+        logger.setLevel(logging.INFO)
+        handler = logging.StreamHandler()
+        #fmt = logging.Formatter('%(asctime)s WETEXT %(levelname)s %(message)s')
+        fmt = logging.Formatter('TextNorm %(levelname)s %(message)s')
+        handler.setFormatter(fmt)
+        logger.addHandler(handler)
+
+        os.makedirs(cache_dir, exist_ok=True)
+        tagger_name = '{}_tagger.fst'.format(prefix)
+        verbalizer_name = '{}_verbalizer.fst'.format(prefix)
+
+        tagger_path = os.path.join(cache_dir, tagger_name)
+        verbalizer_path = os.path.join(cache_dir, verbalizer_name)
+
+        exists = os.path.exists(tagger_path) and os.path.exists(
+            verbalizer_path)
+        if exists and not overwrite_cache:
+            logger.info("found existing fst: {}".format(tagger_path))
+            logger.info("                    {}".format(verbalizer_path))
+            #logger.info("skip building fst for {} ...".format(self.name))
+            self.tagger = Fst.read(tagger_path).optimize()
+            self.verbalizer = Fst.read(verbalizer_path).optimize()
+        else:
+            logger.info("NO fst")
+
+    def tag(self, input):
+        if len(input) == 0:
+            return ''
+        input = escape(input)
+        lattice = input @ self.tagger
+        return shortestpath(lattice, nshortest=1, unique=True).string()
+
+    def verbalize(self, input):
+        # Only words from the blacklist are contained.
+        if len(input) == 0:
+            return ''
+        output = TokenParser(self.ordertype).reorder(input)
+        # We need escape for pynini to build the fst from string.
+        lattice = escape(output) @ self.verbalizer
+        return shortestpath(lattice, nshortest=1, unique=True).string()
+
+    def normalize(self, input):
+        return self.verbalize(self.tag(input))
+
+

sigilyph/fst_tool/token_parser.py

@@ -0,0 +1,159 @@
+# Copyright (c) 2022 Zhendong Peng (pzd17@tsinghua.org.cn)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import string
+
+EOS = '<EOS>'
+TN_ORDERS = {
+    'date': ['year', 'month', 'day'],
+    'fraction': ['denominator', 'numerator'],
+    'measure': ['denominator', 'numerator', 'value'],
+    'money': ['value', 'currency'],
+    'time': ['noon', 'hour', 'minute', 'second']
+}
+EN_TN_ORDERS = {
+    'date': ['preserve_order', 'text', 'day', 'month', 'year'],
+    'money': ['integer_part', 'fractional_part', 'quantity', 'currency_maj'],
+}
+ITN_ORDERS = {
+    'date': ['year', 'month', 'day'],
+    'fraction': ['sign', 'numerator', 'denominator'],
+    'measure': ['numerator', 'denominator', 'value'],
+    'money': ['currency', 'value', 'decimal'],
+    'time': ['hour', 'minute', 'second', 'noon']
+}
+
+class Token:
+
+    def __init__(self, name):
+        self.name = name
+        self.order = []
+        self.members = {}
+
+    def append(self, key, value):
+        self.order.append(key)
+        self.members[key] = value
+
+    def string(self, orders):
+        output = self.name + ' {'
+        if self.name in orders.keys():
+            if "preserve_order" not in self.members.keys() or \
+                    self.members["preserve_order"] != "true":
+                self.order = orders[self.name]
+
+        for key in self.order:
+            if key not in self.members.keys():
+                continue
+            output += ' {}: "{}"'.format(key, self.members[key])
+        return output + ' }'
+    
+
+class TokenParser:
+
+    def __init__(self, ordertype="tn"):
+        if ordertype == "tn":
+            self.orders = TN_ORDERS
+        elif ordertype == "itn":
+            self.orders = ITN_ORDERS
+        elif ordertype == "en_tn":
+            self.orders = EN_TN_ORDERS
+        else:
+            raise NotImplementedError()
+
+    def load(self, input):
+        assert len(input) > 0
+        self.index = 0
+        self.text = input
+        self.char = input[0]
+        self.tokens = []
+
+    def read(self):
+        if self.index < len(self.text) - 1:
+            self.index += 1
+            self.char = self.text[self.index]
+            return True
+        self.char = EOS
+        return False
+
+    def parse_ws(self):
+        not_eos = self.char != EOS
+        while not_eos and self.char == ' ':
+            not_eos = self.read()
+        return not_eos
+
+    def parse_char(self, exp):
+        if self.char == exp:
+            self.read()
+            return True
+        return False
+
+    def parse_chars(self, exp):
+        ok = False
+        for x in exp:
+            ok |= self.parse_char(x)
+        return ok
+
+    def parse_key(self):
+        assert self.char != EOS
+        assert self.char not in string.whitespace
+
+        key = ''
+        while self.char in string.ascii_letters + '_':
+            key += self.char
+            self.read()
+        return key
+
+    def parse_value(self):
+        assert self.char != EOS
+        escape = False
+
+        value = ''
+        while self.char != '"':
+            value += self.char
+            escape = self.char == '\\'
+            self.read()
+            if escape:
+                escape = False
+                value += self.char
+                self.read()
+        return value
+
+    def parse(self, input):
+        self.load(input)
+        while self.parse_ws():
+            name = self.parse_key()
+            self.parse_chars(' { ')
+
+            token = Token(name)
+            while self.parse_ws():
+                if self.char == '}':
+                    self.parse_char('}')
+                    break
+                key = self.parse_key()
+                self.parse_chars(': "')
+                value = self.parse_value()
+                self.parse_char('"')
+                token.append(key, value)
+            self.tokens.append(token)
+
+    def reorder(self, input):
+        self.parse(input)
+        output = ''
+        for token in self.tokens:
+            output += token.string(self.orders) + ' '
+        return output.strip()
+
+
+
+