unitok 4.4.4__tar.gz

unitok-4.4.4/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Jyonn
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

unitok-4.4.4/PKG-INFO

@@ -0,0 +1,179 @@
+Metadata-Version: 2.1
+Name: unitok
+Version: 4.4.4
+Summary: Unified Tokenizer
+Home-page: https://github.com/Jyonn/UnifiedTokenizer
+Author: Jyonn Liu
+Author-email: liu@qijiong.work
+License: MIT Licence
+Keywords: token,tokenizer,NLP,transformers,glove,bert,llama
+Platform: any
+Description-Content-Type: text/markdown
+License-File: LICENSE
+
+# UniTok v4
+
+Unified preprocessing for heterogeneous ML tables: text, categorical, and numerical columns in one pipeline.
+
+- Python package: `unitok`
+- Current package version: 4.4.2 (from `setup.py`)
+- Legacy v3 docs: `README_v3.md`
+
+## Why UniTok
+
+UniTok turns raw tabular data into model-ready numeric tables while preserving:
+
+- Consistent vocabularies across multiple datasets
+- Clear feature definitions (column -> tokenizer -> output feature)
+- Reproducible metadata and saved artifacts
+- Simple unions across datasets via shared keys
+
+## Core Ideas
+
+- **UniTok**: Orchestrates preprocessing lifecycle and holds processed data.
+- **Feature**: Binds a column to a tokenizer and output name.
+- **Tokenizer**: Encodes objects to ids (entity, split, digit, transformers).
+- **Vocab**: Global index for tokens; shared across datasets.
+- **Meta**: Stores schema, tokenizers, vocabularies, and feature definitions.
+- **State**: `initialized` -> `tokenized` -> `organized`.
+
+## Install
+
+```bash
+pip install unitok
+```
+
+Requirements: Python 3.7+, pandas, transformers, tqdm, rich.
+
+## Quickstart
+
+```python
+import pandas as pd
+from unitok import UniTok, Vocab
+from unitok.tokenizer import BertTokenizer, TransformersTokenizer, EntityTokenizer, SplitTokenizer, DigitTokenizer
+
+item = pd.read_csv(
+    'news-sample.tsv', sep='\t',
+    names=['nid', 'category', 'subcategory', 'title', 'abstract'],
+    usecols=['nid', 'category', 'subcategory', 'title', 'abstract'],
+)
+item['abstract'] = item['abstract'].fillna('')
+
+user = pd.read_csv(
+    'user-sample.tsv', sep='\t',
+    names=['uid', 'history'],
+)
+
+interaction = pd.read_csv(
+    'interaction-sample.tsv', sep='\t',
+    names=['uid', 'nid', 'click'],
+)
+
+item_vocab = Vocab(name='nid')
+user_vocab = Vocab(name='uid')
+
+with UniTok() as item_ut:
+    bert = BertTokenizer(vocab='bert')
+    llama = TransformersTokenizer(vocab='llama', key='huggyllama/llama-7b')
+
+    item_ut.add_feature(tokenizer=EntityTokenizer(vocab=item_vocab), column='nid', key=True)
+    item_ut.add_feature(tokenizer=bert, column='title', name='title@bert', truncate=20)
+    item_ut.add_feature(tokenizer=llama, column='title', name='title@llama', truncate=20)
+    item_ut.add_feature(tokenizer=bert, column='abstract', name='abstract@bert', truncate=50)
+    item_ut.add_feature(tokenizer=llama, column='abstract', name='abstract@llama', truncate=50)
+    item_ut.add_feature(tokenizer=EntityTokenizer(vocab='category'), column='category')
+    item_ut.add_feature(tokenizer=EntityTokenizer(vocab='subcategory'), column='subcategory')
+
+with UniTok() as user_ut:
+    user_ut.add_feature(tokenizer=EntityTokenizer(vocab=user_vocab), column='uid', key=True)
+    user_ut.add_feature(tokenizer=SplitTokenizer(vocab=item_vocab, sep=','), column='history', truncate=30)
+
+with UniTok() as inter_ut:
+    inter_ut.add_index_feature(name='index')
+    inter_ut.add_feature(tokenizer=EntityTokenizer(vocab=user_vocab), column='uid')
+    inter_ut.add_feature(tokenizer=EntityTokenizer(vocab=item_vocab), column='nid')
+    inter_ut.add_feature(tokenizer=DigitTokenizer(vocab='click', vocab_size=2), column='click')
+
+item_ut.tokenize(item).save('sample-ut/item')
+item_vocab.deny_edit()
+user_ut.tokenize(user).save('sample-ut/user')
+inter_ut.tokenize(interaction).save('sample-ut/interaction')
+```
+
+## Loading Saved Data
+
+```python
+from unitok import UniTok
+
+ut = UniTok.load('sample-ut/item')
+print(len(ut))
+print(ut[0])
+```
+
+## Combining Datasets (Union)
+
+```python
+with inter_ut:
+    inter_ut.union(user_ut)
+    print(inter_ut[0])
+```
+
+- **Soft union** (default): links tables and resolves on access
+- **Hard union**: materializes merged columns
+
+## CLI
+
+Summarize a saved table:
+
+```bash
+unitok path/to/data
+```
+
+Add a feature into an existing table (integrate):
+
+```bash
+unitok integrate path/to/data --file data.tsv --column title --name title@bert \
+  --vocab bert --tokenizer transformers --t.key bert-base-uncased
+```
+
+Remove a feature from a saved table:
+
+```bash
+unitok remove path/to/data --name title@bert
+```
+
+## Data Artifacts
+
+Saved directories include:
+
+- `meta.json` with schema, tokenizers, vocabularies
+- `data.pkl` with tokenized columns
+- `*.vocab` pickled vocabularies
+
+## Migration From v3
+
+If you have v3 artifacts:
+
+```bash
+unidep-upgrade-v4 <path>
+```
+
+## Notes and Constraints
+
+- Key feature must be **atomic** (tokenizer returns a single id, not a list).
+- Shared vocabularies must match for unions.
+- `truncate=None` means an atomic feature; list features must use a truncate.
+- `Feature` supersedes the deprecated `Job` class.
+
+## Repository Layout (High-Level)
+
+- `unitok/` core library
+- `UniTokv3/` legacy v3 code
+- `dist/` built distributions
+- `setup.py`, `requirements.txt`
+
+## License
+
+MIT License. See `LICENSE`.
+
+

unitok-4.4.4/README.md

@@ -0,0 +1,164 @@
+# UniTok v4
+
+Unified preprocessing for heterogeneous ML tables: text, categorical, and numerical columns in one pipeline.
+
+- Python package: `unitok`
+- Current package version: 4.4.2 (from `setup.py`)
+- Legacy v3 docs: `README_v3.md`
+
+## Why UniTok
+
+UniTok turns raw tabular data into model-ready numeric tables while preserving:
+
+- Consistent vocabularies across multiple datasets
+- Clear feature definitions (column -> tokenizer -> output feature)
+- Reproducible metadata and saved artifacts
+- Simple unions across datasets via shared keys
+
+## Core Ideas
+
+- **UniTok**: Orchestrates preprocessing lifecycle and holds processed data.
+- **Feature**: Binds a column to a tokenizer and output name.
+- **Tokenizer**: Encodes objects to ids (entity, split, digit, transformers).
+- **Vocab**: Global index for tokens; shared across datasets.
+- **Meta**: Stores schema, tokenizers, vocabularies, and feature definitions.
+- **State**: `initialized` -> `tokenized` -> `organized`.
+
+## Install
+
+```bash
+pip install unitok
+```
+
+Requirements: Python 3.7+, pandas, transformers, tqdm, rich.
+
+## Quickstart
+
+```python
+import pandas as pd
+from unitok import UniTok, Vocab
+from unitok.tokenizer import BertTokenizer, TransformersTokenizer, EntityTokenizer, SplitTokenizer, DigitTokenizer
+
+item = pd.read_csv(
+    'news-sample.tsv', sep='\t',
+    names=['nid', 'category', 'subcategory', 'title', 'abstract'],
+    usecols=['nid', 'category', 'subcategory', 'title', 'abstract'],
+)
+item['abstract'] = item['abstract'].fillna('')
+
+user = pd.read_csv(
+    'user-sample.tsv', sep='\t',
+    names=['uid', 'history'],
+)
+
+interaction = pd.read_csv(
+    'interaction-sample.tsv', sep='\t',
+    names=['uid', 'nid', 'click'],
+)
+
+item_vocab = Vocab(name='nid')
+user_vocab = Vocab(name='uid')
+
+with UniTok() as item_ut:
+    bert = BertTokenizer(vocab='bert')
+    llama = TransformersTokenizer(vocab='llama', key='huggyllama/llama-7b')
+
+    item_ut.add_feature(tokenizer=EntityTokenizer(vocab=item_vocab), column='nid', key=True)
+    item_ut.add_feature(tokenizer=bert, column='title', name='title@bert', truncate=20)
+    item_ut.add_feature(tokenizer=llama, column='title', name='title@llama', truncate=20)
+    item_ut.add_feature(tokenizer=bert, column='abstract', name='abstract@bert', truncate=50)
+    item_ut.add_feature(tokenizer=llama, column='abstract', name='abstract@llama', truncate=50)
+    item_ut.add_feature(tokenizer=EntityTokenizer(vocab='category'), column='category')
+    item_ut.add_feature(tokenizer=EntityTokenizer(vocab='subcategory'), column='subcategory')
+
+with UniTok() as user_ut:
+    user_ut.add_feature(tokenizer=EntityTokenizer(vocab=user_vocab), column='uid', key=True)
+    user_ut.add_feature(tokenizer=SplitTokenizer(vocab=item_vocab, sep=','), column='history', truncate=30)
+
+with UniTok() as inter_ut:
+    inter_ut.add_index_feature(name='index')
+    inter_ut.add_feature(tokenizer=EntityTokenizer(vocab=user_vocab), column='uid')
+    inter_ut.add_feature(tokenizer=EntityTokenizer(vocab=item_vocab), column='nid')
+    inter_ut.add_feature(tokenizer=DigitTokenizer(vocab='click', vocab_size=2), column='click')
+
+item_ut.tokenize(item).save('sample-ut/item')
+item_vocab.deny_edit()
+user_ut.tokenize(user).save('sample-ut/user')
+inter_ut.tokenize(interaction).save('sample-ut/interaction')
+```
+
+## Loading Saved Data
+
+```python
+from unitok import UniTok
+
+ut = UniTok.load('sample-ut/item')
+print(len(ut))
+print(ut[0])
+```
+
+## Combining Datasets (Union)
+
+```python
+with inter_ut:
+    inter_ut.union(user_ut)
+    print(inter_ut[0])
+```
+
+- **Soft union** (default): links tables and resolves on access
+- **Hard union**: materializes merged columns
+
+## CLI
+
+Summarize a saved table:
+
+```bash
+unitok path/to/data
+```
+
+Add a feature into an existing table (integrate):
+
+```bash
+unitok integrate path/to/data --file data.tsv --column title --name title@bert \
+  --vocab bert --tokenizer transformers --t.key bert-base-uncased
+```
+
+Remove a feature from a saved table:
+
+```bash
+unitok remove path/to/data --name title@bert
+```
+
+## Data Artifacts
+
+Saved directories include:
+
+- `meta.json` with schema, tokenizers, vocabularies
+- `data.pkl` with tokenized columns
+- `*.vocab` pickled vocabularies
+
+## Migration From v3
+
+If you have v3 artifacts:
+
+```bash
+unidep-upgrade-v4 <path>
+```
+
+## Notes and Constraints
+
+- Key feature must be **atomic** (tokenizer returns a single id, not a list).
+- Shared vocabularies must match for unions.
+- `truncate=None` means an atomic feature; list features must use a truncate.
+- `Feature` supersedes the deprecated `Job` class.
+
+## Repository Layout (High-Level)
+
+- `unitok/` core library
+- `UniTokv3/` legacy v3 code
+- `dist/` built distributions
+- `setup.py`, `requirements.txt`
+
+## License
+
+MIT License. See `LICENSE`.

unitok-4.4.4/UniTok.egg-info/PKG-INFO

@@ -0,0 +1,179 @@
+Metadata-Version: 2.1
+Name: unitok
+Version: 4.4.4
+Summary: Unified Tokenizer
+Home-page: https://github.com/Jyonn/UnifiedTokenizer
+Author: Jyonn Liu
+Author-email: liu@qijiong.work
+License: MIT Licence
+Keywords: token,tokenizer,NLP,transformers,glove,bert,llama
+Platform: any
+Description-Content-Type: text/markdown
+License-File: LICENSE
+
+# UniTok v4
+
+Unified preprocessing for heterogeneous ML tables: text, categorical, and numerical columns in one pipeline.
+
+- Python package: `unitok`
+- Current package version: 4.4.2 (from `setup.py`)
+- Legacy v3 docs: `README_v3.md`
+
+## Why UniTok
+
+UniTok turns raw tabular data into model-ready numeric tables while preserving:
+
+- Consistent vocabularies across multiple datasets
+- Clear feature definitions (column -> tokenizer -> output feature)
+- Reproducible metadata and saved artifacts
+- Simple unions across datasets via shared keys
+
+## Core Ideas
+
+- **UniTok**: Orchestrates preprocessing lifecycle and holds processed data.
+- **Feature**: Binds a column to a tokenizer and output name.
+- **Tokenizer**: Encodes objects to ids (entity, split, digit, transformers).
+- **Vocab**: Global index for tokens; shared across datasets.
+- **Meta**: Stores schema, tokenizers, vocabularies, and feature definitions.
+- **State**: `initialized` -> `tokenized` -> `organized`.
+
+## Install
+
+```bash
+pip install unitok
+```
+
+Requirements: Python 3.7+, pandas, transformers, tqdm, rich.
+
+## Quickstart
+
+```python
+import pandas as pd
+from unitok import UniTok, Vocab
+from unitok.tokenizer import BertTokenizer, TransformersTokenizer, EntityTokenizer, SplitTokenizer, DigitTokenizer
+
+item = pd.read_csv(
+    'news-sample.tsv', sep='\t',
+    names=['nid', 'category', 'subcategory', 'title', 'abstract'],
+    usecols=['nid', 'category', 'subcategory', 'title', 'abstract'],
+)
+item['abstract'] = item['abstract'].fillna('')
+
+user = pd.read_csv(
+    'user-sample.tsv', sep='\t',
+    names=['uid', 'history'],
+)
+
+interaction = pd.read_csv(
+    'interaction-sample.tsv', sep='\t',
+    names=['uid', 'nid', 'click'],
+)
+
+item_vocab = Vocab(name='nid')
+user_vocab = Vocab(name='uid')
+
+with UniTok() as item_ut:
+    bert = BertTokenizer(vocab='bert')
+    llama = TransformersTokenizer(vocab='llama', key='huggyllama/llama-7b')
+
+    item_ut.add_feature(tokenizer=EntityTokenizer(vocab=item_vocab), column='nid', key=True)
+    item_ut.add_feature(tokenizer=bert, column='title', name='title@bert', truncate=20)
+    item_ut.add_feature(tokenizer=llama, column='title', name='title@llama', truncate=20)
+    item_ut.add_feature(tokenizer=bert, column='abstract', name='abstract@bert', truncate=50)
+    item_ut.add_feature(tokenizer=llama, column='abstract', name='abstract@llama', truncate=50)
+    item_ut.add_feature(tokenizer=EntityTokenizer(vocab='category'), column='category')
+    item_ut.add_feature(tokenizer=EntityTokenizer(vocab='subcategory'), column='subcategory')
+
+with UniTok() as user_ut:
+    user_ut.add_feature(tokenizer=EntityTokenizer(vocab=user_vocab), column='uid', key=True)
+    user_ut.add_feature(tokenizer=SplitTokenizer(vocab=item_vocab, sep=','), column='history', truncate=30)
+
+with UniTok() as inter_ut:
+    inter_ut.add_index_feature(name='index')
+    inter_ut.add_feature(tokenizer=EntityTokenizer(vocab=user_vocab), column='uid')
+    inter_ut.add_feature(tokenizer=EntityTokenizer(vocab=item_vocab), column='nid')
+    inter_ut.add_feature(tokenizer=DigitTokenizer(vocab='click', vocab_size=2), column='click')
+
+item_ut.tokenize(item).save('sample-ut/item')
+item_vocab.deny_edit()
+user_ut.tokenize(user).save('sample-ut/user')
+inter_ut.tokenize(interaction).save('sample-ut/interaction')
+```
+
+## Loading Saved Data
+
+```python
+from unitok import UniTok
+
+ut = UniTok.load('sample-ut/item')
+print(len(ut))
+print(ut[0])
+```
+
+## Combining Datasets (Union)
+
+```python
+with inter_ut:
+    inter_ut.union(user_ut)
+    print(inter_ut[0])
+```
+
+- **Soft union** (default): links tables and resolves on access
+- **Hard union**: materializes merged columns
+
+## CLI
+
+Summarize a saved table:
+
+```bash
+unitok path/to/data
+```
+
+Add a feature into an existing table (integrate):
+
+```bash
+unitok integrate path/to/data --file data.tsv --column title --name title@bert \
+  --vocab bert --tokenizer transformers --t.key bert-base-uncased
+```
+
+Remove a feature from a saved table:
+
+```bash
+unitok remove path/to/data --name title@bert
+```
+
+## Data Artifacts
+
+Saved directories include:
+
+- `meta.json` with schema, tokenizers, vocabularies
+- `data.pkl` with tokenized columns
+- `*.vocab` pickled vocabularies
+
+## Migration From v3
+
+If you have v3 artifacts:
+
+```bash
+unidep-upgrade-v4 <path>
+```
+
+## Notes and Constraints
+
+- Key feature must be **atomic** (tokenizer returns a single id, not a list).
+- Shared vocabularies must match for unions.
+- `truncate=None` means an atomic feature; list features must use a truncate.
+- `Feature` supersedes the deprecated `Job` class.
+
+## Repository Layout (High-Level)
+
+- `unitok/` core library
+- `UniTokv3/` legacy v3 code
+- `dist/` built distributions
+- `setup.py`, `requirements.txt`
+
+## License
+
+MIT License. See `LICENSE`.
+
+

unitok-4.4.4/UniTok.egg-info/SOURCES.txt

@@ -0,0 +1,78 @@
+LICENSE
+README.md
+setup.py
+UniTok.egg-info/PKG-INFO
+UniTok.egg-info/SOURCES.txt
+UniTok.egg-info/dependency_links.txt
+UniTok.egg-info/entry_points.txt
+UniTok.egg-info/requires.txt
+UniTok.egg-info/top_level.txt
+UniTokv3/__init__.py
+UniTokv3/__main__.py
+UniTokv3/cols.py
+UniTokv3/column.py
+UniTokv3/fut.py
+UniTokv3/global_setting.py
+UniTokv3/meta.py
+UniTokv3/unidep.py
+UniTokv3/unitok.py
+UniTokv3/vocab.py
+UniTokv3/vocabs.py
+UniTokv3/analysis/__init__.py
+UniTokv3/analysis/lengths.py
+UniTokv3/analysis/plot.py
+UniTokv3/tok/__init__.py
+UniTokv3/tok/bert_tok.py
+UniTokv3/tok/ent_tok.py
+UniTokv3/tok/id_tok.py
+UniTokv3/tok/number_tok.py
+UniTokv3/tok/seq_tok.py
+UniTokv3/tok/split_tok.py
+UniTokv3/tok/tok.py
+unitok/__init__.py
+unitok/__main__.py
+unitok/feature.py
+unitok/job.py
+unitok/meta.py
+unitok/selector.py
+unitok/status.py
+unitok/unitok.py
+unitok.egg-info/PKG-INFO
+unitok.egg-info/SOURCES.txt
+unitok.egg-info/dependency_links.txt
+unitok.egg-info/entry_points.txt
+unitok.egg-info/requires.txt
+unitok.egg-info/top_level.txt
+unitok/tokenizer/__init__.py
+unitok/tokenizer/base_tokenizer.py
+unitok/tokenizer/digit_tokenizer.py
+unitok/tokenizer/entity_tokenizer.py
+unitok/tokenizer/glove_tokenizer.py
+unitok/tokenizer/split_tokenizer.py
+unitok/tokenizer/transformers_tokenizer.py
+unitok/tokenizer/union_tokenizer.py
+unitok/tokenizer/unknown_tokenizer.py
+unitok/utils/__init__.py
+unitok/utils/class_pool.py
+unitok/utils/data.py
+unitok/utils/function.py
+unitok/utils/instance.py
+unitok/utils/map.py
+unitok/utils/space.py
+unitok/utils/symbol.py
+unitok/utils/verbose.py
+unitok/utils/handler/__init__.py
+unitok/utils/handler/json_handler.py
+unitok/utils/handler/pkl_handler.py
+unitok/utils/hub/__init__.py
+unitok/utils/hub/hub.py
+unitok/utils/hub/param_hub.py
+unitok/utils/index_set/__init__.py
+unitok/utils/index_set/feature_set.py
+unitok/utils/index_set/index_set.py
+unitok/utils/index_set/job_set.py
+unitok/utils/index_set/tokenizer_set.py
+unitok/utils/index_set/vocabulary_set.py
+unitok/vocabulary/__init__.py
+unitok/vocabulary/counter.py
+unitok/vocabulary/vocabulary.py

unitok-4.4.4/UniTok.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

unitok-4.4.4/UniTok.egg-info/entry_points.txt

@@ -0,0 +1,5 @@
+[console_scripts]
+unidep-upgrade-v4 = UniTokv3.__main__:upgrade
+unitok = unitok.__main__:main
+unitokv3 = UniTokv3.__main__:main
+

unitok-4.4.4/UniTok.egg-info/requires.txt

@@ -0,0 +1,9 @@
+termplot==0.0.2
+tqdm
+numpy
+pandas
+transformers
+oba
+prettytable
+rich
+fastparquet

unitok-4.4.4/UniTok.egg-info/top_level.txt

@@ -0,0 +1,2 @@
+UniTokv3
+unitok

unitok-4.4.4/UniTokv3/__init__.py

@@ -0,0 +1,26 @@
+from .unitok import UniTok
+from .vocab import Vocab
+from .vocabs import Vocabs
+from .cols import Cols
+from .column import Column
+from .analysis import Lengths, Plot
+from .fut import Fut
+
+from .unidep import UniDep
+from .meta import Meta, Col, Voc
+
+from .global_setting import Global
+
+__all__ = [
+    UniTok,
+    UniDep,
+    Lengths,
+    Plot,
+    Column,
+    column,
+    analysis,
+    Vocab,
+    Vocabs,
+    Global,
+    Fut,
+]