wowool-lingware-build 1.1.3.dev6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wowool/lingware/build/build.py +88 -0
- wowool/lingware/build/domain/__init__.py +11 -0
- wowool/lingware/build/domain/build.py +455 -0
- wowool/lingware/build/domain/templates.py +70 -0
- wowool/lingware/build/git.py +34 -0
- wowool/lingware/build/language/__init__.py +1 -0
- wowool/lingware/build/language/build.py +196 -0
- wowool/lingware/build/language/templates.py +19 -0
- wowool/lingware/build/lid/__init__.py +1 -0
- wowool/lingware/build/lid/build.py +110 -0
- wowool/lingware/build/lid/templates.py +19 -0
- wowool/lingware/build/pytask.py +235 -0
- wowool/lingware/build/templates.py +67 -0
- wowool/lingware/build/tools/afst/__main__.py +20 -0
- wowool/lingware/build/tools/convert_domain_info/__init__.py +1 -0
- wowool/lingware/build/tools/convert_domain_info/__main__.py +51 -0
- wowool/lingware/build/tools/convert_domain_info/convert.py +31 -0
- wowool/lingware/build/tools/convert_language_info/__main__.py +44 -0
- wowool/lingware/build/utilities.py +120 -0
- wowool/lingware/unittest/__init__.py +1 -0
- wowool/lingware/unittest/unittest.py +437 -0
- wowool_lingware_build-1.1.3.dev6.dist-info/METADATA +12 -0
- wowool_lingware_build-1.1.3.dev6.dist-info/RECORD +26 -0
- wowool_lingware_build-1.1.3.dev6.dist-info/WHEEL +5 -0
- wowool_lingware_build-1.1.3.dev6.dist-info/entry_points.txt +4 -0
- wowool_lingware_build-1.1.3.dev6.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
from subprocess import run
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
import os
|
|
4
|
+
import shutil
|
|
5
|
+
import logging
|
|
6
|
+
from wowool.lingware.build.git import get_version
|
|
7
|
+
import json
|
|
8
|
+
|
|
9
|
+
from wowool.lingware.build.utilities import (
|
|
10
|
+
resolve_template,
|
|
11
|
+
PREFIX_DO_NOT_EDIT,
|
|
12
|
+
eot_namespace,
|
|
13
|
+
wowool_module,
|
|
14
|
+
get_filename_version,
|
|
15
|
+
get_stage_lxware_folder,
|
|
16
|
+
update_file,
|
|
17
|
+
get_lxcommon,
|
|
18
|
+
get_language,
|
|
19
|
+
set_build_tools_environment,
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
import re
|
|
23
|
+
|
|
24
|
+
logger = logging.getLogger(__name__)
|
|
25
|
+
WOWOOL_NROF_THREADS: int = int(os.environ.get("WOWOOL_NROF_THREADS", "8"))
|
|
26
|
+
thread_option = f" -j {WOWOOL_NROF_THREADS} " if WOWOOL_NROF_THREADS > 0 else " "
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
lxcommon_pattern = re.compile("""\\${LXCOMMON}/([a-zA-Z_0-9\\-/\\.]+)""", re.MULTILINE)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def collect_makefile_lxcommon_files(mf_fn):
|
|
33
|
+
files = set()
|
|
34
|
+
with open(mf_fn) as fh:
|
|
35
|
+
mf_data = fh.read()
|
|
36
|
+
|
|
37
|
+
for fn in lxcommon_pattern.findall(mf_data):
|
|
38
|
+
files.add(fn)
|
|
39
|
+
|
|
40
|
+
return files
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def force_debug_message_in_github_actions(msg: str):
|
|
44
|
+
Path("dummy.log").write_text(msg)
|
|
45
|
+
run("cat dummy.log", shell=True)
|
|
46
|
+
Path("dummy.log").unlink(missing_ok=True)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def make_language(folder: Path, target: str = "", requires_lxcommon=True):
|
|
50
|
+
set_build_tools_environment()
|
|
51
|
+
repo_version = get_version(folder)
|
|
52
|
+
filename_version = get_filename_version(repo_version)
|
|
53
|
+
lxcommon = get_lxcommon(folder) if requires_lxcommon else ""
|
|
54
|
+
print(f"{repo_version=} {filename_version=} {lxcommon=}")
|
|
55
|
+
|
|
56
|
+
lingware_bin = get_stage_lxware_folder(folder) / "wowool"
|
|
57
|
+
|
|
58
|
+
cmd = f"make {thread_option} {target} VERSION=@{filename_version} LXCOMMON={lxcommon} LINGWARE_BIN={lingware_bin}"
|
|
59
|
+
force_debug_message_in_github_actions(f"{cmd=}")
|
|
60
|
+
result = run(
|
|
61
|
+
cmd,
|
|
62
|
+
cwd=folder,
|
|
63
|
+
shell=True,
|
|
64
|
+
)
|
|
65
|
+
if result.returncode != 0:
|
|
66
|
+
raise RuntimeError(f"!!! Could not build {folder} {cmd=}")
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def languages(folder: Path):
|
|
70
|
+
for language in folder.glob("languages/*"):
|
|
71
|
+
if language.is_dir():
|
|
72
|
+
yield language
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class LanguageBuildInfo:
|
|
76
|
+
def __init__(self, folder: Path, dialect_path: Path, package_lxware_wowool: Path | None = None):
|
|
77
|
+
self.folder = dialect_path
|
|
78
|
+
self.stage_lxware = get_stage_lxware_folder(folder)
|
|
79
|
+
self.stage_lxware_eot = self.stage_lxware / "wowool"
|
|
80
|
+
if package_lxware_wowool is None:
|
|
81
|
+
self.package_lxware_wowool = dialect_path / "wowool" / "lxware" / "wowool"
|
|
82
|
+
else:
|
|
83
|
+
self.package_lxware_wowool = package_lxware_wowool
|
|
84
|
+
|
|
85
|
+
def clean(self):
|
|
86
|
+
shutil.rmtree(f"{self.folder}/dist", ignore_errors=True)
|
|
87
|
+
shutil.rmtree(f"{self.folder}/build", ignore_errors=True)
|
|
88
|
+
shutil.rmtree(f"{self.folder}/var", ignore_errors=True)
|
|
89
|
+
shutil.rmtree(f"{self.folder}/eot", ignore_errors=True)
|
|
90
|
+
for egg_info in self.folder.glob("*.egg-info"):
|
|
91
|
+
shutil.rmtree(egg_info, ignore_errors=True)
|
|
92
|
+
Path(f"{self.folder}/setup.py").unlink(missing_ok=True)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def add_version_info(fn: Path, filename_version: str) -> Path:
|
|
96
|
+
return Path(f"wowool/{fn.stem}@{filename_version}{fn.suffix}")
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def package_language(folder: Path, dialect_path: Path, package_lxware_wowool: Path | None = None, run_setup: bool = True):
|
|
100
|
+
logger.info(f"build: {dialect_path.name=}")
|
|
101
|
+
repo_version = get_version(folder)
|
|
102
|
+
filename_version = get_filename_version(repo_version)
|
|
103
|
+
language_root = get_language(folder)
|
|
104
|
+
lbi = LanguageBuildInfo(folder, dialect_path, package_lxware_wowool=package_lxware_wowool)
|
|
105
|
+
lbi.clean()
|
|
106
|
+
|
|
107
|
+
files = [lc for lc in dialect_path.glob("*.language")]
|
|
108
|
+
|
|
109
|
+
assert len(files) == 1, "to many language file in folder"
|
|
110
|
+
language_fn = files[0]
|
|
111
|
+
|
|
112
|
+
project_fn = dialect_path / "project.json"
|
|
113
|
+
package_config = {}
|
|
114
|
+
if project_fn.exists():
|
|
115
|
+
with open(project_fn) as fh:
|
|
116
|
+
package_config = json.load(fh)
|
|
117
|
+
|
|
118
|
+
from wowool.native.core.language_info import Parser
|
|
119
|
+
|
|
120
|
+
exculde_pattern = package_config["exclude"] if "exclude" in package_config else []
|
|
121
|
+
|
|
122
|
+
def add_namespace(name: str):
|
|
123
|
+
if name in exculde_pattern:
|
|
124
|
+
return name
|
|
125
|
+
|
|
126
|
+
fn = Path(name)
|
|
127
|
+
fn_version = add_version_info(fn, filename_version)
|
|
128
|
+
stage_full_fn = lbi.stage_lxware / fn_version
|
|
129
|
+
if fn.suffix == ".tkz":
|
|
130
|
+
# need to do something spesial to the tkz_cache file
|
|
131
|
+
fn_tkz_cache = lbi.stage_lxware / Path(fn_version).with_suffix(".tkz_cache")
|
|
132
|
+
|
|
133
|
+
assert fn_tkz_cache.exists(), f"Could not find {fn_tkz_cache}"
|
|
134
|
+
fn_version_tkz_cache = fn_version.with_suffix(".tkz_cache")
|
|
135
|
+
shutil.copy(fn_tkz_cache, lbi.package_lxware_wowool / fn_version_tkz_cache.name)
|
|
136
|
+
|
|
137
|
+
assert stage_full_fn.exists(), f"Could not find {stage_full_fn}"
|
|
138
|
+
shutil.copy(stage_full_fn, lbi.package_lxware_wowool / fn_version.name)
|
|
139
|
+
return str(fn_version)
|
|
140
|
+
|
|
141
|
+
language_parser = Parser(add_namespace)
|
|
142
|
+
|
|
143
|
+
with open(language_fn) as fh:
|
|
144
|
+
jo_language = json.load(fh)
|
|
145
|
+
|
|
146
|
+
# print(f"{jo_language=}\n{lbi.stage_lxware=}")
|
|
147
|
+
|
|
148
|
+
lbi.package_lxware_wowool.mkdir(parents=True, exist_ok=True)
|
|
149
|
+
language_parser(jo_language)
|
|
150
|
+
fn_version = add_version_info(language_fn, filename_version)
|
|
151
|
+
shutil.copy(lbi.stage_lxware_eot / fn_version.name, lbi.package_lxware_wowool / fn_version.name)
|
|
152
|
+
|
|
153
|
+
if run_setup:
|
|
154
|
+
setup_fn = Path(dialect_path, "setup.py").resolve()
|
|
155
|
+
print(f"setup_fn: {setup_fn}")
|
|
156
|
+
setup_fn.unlink(missing_ok=True)
|
|
157
|
+
|
|
158
|
+
# !!! do not remove this variable it's used in the resolve function
|
|
159
|
+
name = None
|
|
160
|
+
if package_config and "name" in package_config:
|
|
161
|
+
package_name = package_config["name"]
|
|
162
|
+
elif language_root != dialect_path.name:
|
|
163
|
+
package_name = f"language-{language_root}-{dialect_path.name}"
|
|
164
|
+
else:
|
|
165
|
+
package_name = language_root
|
|
166
|
+
|
|
167
|
+
setup_dependencies = []
|
|
168
|
+
if package_config and "dependencies" in package_config:
|
|
169
|
+
setup_dependencies = [f"wowool-lxware-language-{name}=={repo_version}" for name in package_config["dependencies"]]
|
|
170
|
+
|
|
171
|
+
# !!! do not remove this variable it's used in the resolve function
|
|
172
|
+
setup_py_requirement = f"install_requires={json.dumps(setup_dependencies)}"
|
|
173
|
+
|
|
174
|
+
with open(setup_fn, "w") as fh:
|
|
175
|
+
from wowool.lingware.build.language.templates import LANGUAGE_SETUP_TEMPLATE
|
|
176
|
+
|
|
177
|
+
fh.write(resolve_template(LANGUAGE_SETUP_TEMPLATE, globals(), locals()))
|
|
178
|
+
|
|
179
|
+
logger.debug(f"Run setup.py in {dialect_path}")
|
|
180
|
+
run("python setup.py bdist_wheel", cwd=dialect_path, shell=True)
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def package_languages(folder: Path, package_lxware_wowool: Path | None = None, run_setup=True):
|
|
184
|
+
for dialect_path in languages(folder):
|
|
185
|
+
package_language(folder, dialect_path, package_lxware_wowool=package_lxware_wowool, run_setup=run_setup)
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def upload_language(language_folder: Path):
|
|
189
|
+
from wowool.build.codeartifact import upload_pypi
|
|
190
|
+
|
|
191
|
+
upload_pypi(language_folder)
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def upload_languages(folder: Path):
|
|
195
|
+
for language_folder in languages(folder):
|
|
196
|
+
upload_language(language_folder)
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
LANGUAGE_SETUP_TEMPLATE = """{PREFIX_DO_NOT_EDIT}
|
|
2
|
+
# This file is generated.
|
|
3
|
+
#
|
|
4
|
+
from setuptools import setup
|
|
5
|
+
|
|
6
|
+
wowool_packages = ["{wowool_module}"]
|
|
7
|
+
results = setup(
|
|
8
|
+
name="{eot_namespace}-{package_name}",
|
|
9
|
+
version="{repo_version}",
|
|
10
|
+
packages=wowool_packages,
|
|
11
|
+
author="EyeOnText",
|
|
12
|
+
author_email = "info@eyeontext.com",
|
|
13
|
+
description="Wowool NLP {' '.join(package_name.split('-'))} Package",
|
|
14
|
+
zip_safe=False,
|
|
15
|
+
package_data={{"{wowool_module}": ["*"]}},
|
|
16
|
+
include_package_data=True,
|
|
17
|
+
{setup_py_requirement}
|
|
18
|
+
)
|
|
19
|
+
"""
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from .build import make_lid, package_lid, upload_lid
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
from subprocess import run
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
import os
|
|
4
|
+
import shutil
|
|
5
|
+
import logging
|
|
6
|
+
from wowool.lingware.build.git import get_version
|
|
7
|
+
import json
|
|
8
|
+
from wowool.lingware.build.utilities import (
|
|
9
|
+
resolve_template,
|
|
10
|
+
get_filename_version,
|
|
11
|
+
get_stage_lxware_folder,
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
eot_namespace = "wowool"
|
|
15
|
+
wowool_module = "wowool.lxware"
|
|
16
|
+
|
|
17
|
+
import re
|
|
18
|
+
|
|
19
|
+
logger = logging.getLogger(__name__)
|
|
20
|
+
|
|
21
|
+
WOWOOL_NROF_THREADS: int = int(os.environ.get("WOWOOL_NROF_THREADS", "8"))
|
|
22
|
+
thread_option = f" -j {WOWOOL_NROF_THREADS} " if WOWOOL_NROF_THREADS > 0 else " "
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
lxcommon_pattern = re.compile("""\\${LXCOMMON}/([a-zA-Z_0-9\\-/\\.]+)""", re.MULTILINE)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def collect_makefile_lxcommon_files(mf_fn):
|
|
29
|
+
files = set()
|
|
30
|
+
with open(mf_fn) as fh:
|
|
31
|
+
mf_data = fh.read()
|
|
32
|
+
|
|
33
|
+
for fn in lxcommon_pattern.findall(mf_data):
|
|
34
|
+
files.add(fn)
|
|
35
|
+
|
|
36
|
+
return files
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def make_lid(folder: Path, target: str = ""):
|
|
40
|
+
repo_version = get_version(folder)
|
|
41
|
+
filename_version = get_filename_version(repo_version)
|
|
42
|
+
print(f"{repo_version=} {filename_version=}")
|
|
43
|
+
|
|
44
|
+
lingware_bin = get_stage_lxware_folder(folder)
|
|
45
|
+
cmd = f"make {thread_option} {target} VERSION=@{filename_version} LINGWARE_BIN={lingware_bin}"
|
|
46
|
+
print(f"{cmd=}")
|
|
47
|
+
result = run(cmd, cwd=folder, shell=True)
|
|
48
|
+
if result.returncode != 0:
|
|
49
|
+
raise RuntimeError(f"!!! Could not build {folder} {cmd=}")
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class LidBuildInfo:
|
|
53
|
+
def __init__(self, folder: Path, dialect_path: Path):
|
|
54
|
+
self.folder = dialect_path
|
|
55
|
+
self.stage_lxware = get_stage_lxware_folder(folder)
|
|
56
|
+
self.package_lxware = dialect_path / "wowool" / "lxware"
|
|
57
|
+
|
|
58
|
+
def clean(self):
|
|
59
|
+
shutil.rmtree(f"{self.folder}/dist", ignore_errors=True)
|
|
60
|
+
shutil.rmtree(f"{self.folder}/build", ignore_errors=True)
|
|
61
|
+
shutil.rmtree(f"{self.folder}/var", ignore_errors=True)
|
|
62
|
+
shutil.rmtree(f"{self.folder}/wowool", ignore_errors=True)
|
|
63
|
+
for egg_info in self.folder.glob("*.egg-info"):
|
|
64
|
+
shutil.rmtree(egg_info, ignore_errors=True)
|
|
65
|
+
Path(f"{self.folder}/setup.py").unlink(missing_ok=True)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def package_lid(folder: Path):
|
|
69
|
+
repo_version = get_version(folder)
|
|
70
|
+
filename_version = get_filename_version(repo_version)
|
|
71
|
+
lbi = LidBuildInfo(folder, folder)
|
|
72
|
+
lbi.clean()
|
|
73
|
+
|
|
74
|
+
lbi.package_lxware.mkdir(parents=True)
|
|
75
|
+
shutil.copy(lbi.stage_lxware / "lid.fst", lbi.package_lxware / "lid.fst")
|
|
76
|
+
|
|
77
|
+
setup_fn = Path(folder, "setup.py").resolve()
|
|
78
|
+
print(f"setup_fn: {setup_fn}")
|
|
79
|
+
setup_fn.unlink(missing_ok=True)
|
|
80
|
+
|
|
81
|
+
project_fn = folder / "project.json"
|
|
82
|
+
package_config = None
|
|
83
|
+
if project_fn.exists():
|
|
84
|
+
with open(project_fn) as fh:
|
|
85
|
+
package_config = json.load(fh)
|
|
86
|
+
|
|
87
|
+
if package_config and "name" in package_config:
|
|
88
|
+
package_name = package_config["name"]
|
|
89
|
+
else:
|
|
90
|
+
package_name = "lid"
|
|
91
|
+
|
|
92
|
+
setup_dependencies = []
|
|
93
|
+
# !!! do not remove this variable it's used in the resolve function
|
|
94
|
+
setup_py_requirement = f"install_requires={json.dumps(setup_dependencies)}"
|
|
95
|
+
|
|
96
|
+
with open(setup_fn, "w") as fh:
|
|
97
|
+
from wowool.lingware.build.language.templates import LANGUAGE_SETUP_TEMPLATE
|
|
98
|
+
|
|
99
|
+
fh.write(resolve_template(LANGUAGE_SETUP_TEMPLATE, globals(), locals()))
|
|
100
|
+
|
|
101
|
+
from wowool.build.pydist import build_wheel_package
|
|
102
|
+
|
|
103
|
+
logger.debug(f"Run setup.py in {folder}")
|
|
104
|
+
build_wheel_package(folder)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def upload_lid(language_folder: Path):
|
|
108
|
+
from wowool.build.codeartifact import upload_pypi
|
|
109
|
+
|
|
110
|
+
upload_pypi(language_folder)
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
LANGUAGE_SETUP_TEMPLATE = """{PREFIX_DO_NOT_EDIT}
|
|
2
|
+
# This file is generated.
|
|
3
|
+
#
|
|
4
|
+
from setuptools import setup
|
|
5
|
+
|
|
6
|
+
wowool_packages = ["{wowool_module}"]
|
|
7
|
+
results = setup(
|
|
8
|
+
name="{eot_namespace}-{package_name}",
|
|
9
|
+
version="{repo_version}",
|
|
10
|
+
packages=wowool_packages,
|
|
11
|
+
author="EyeOnText",
|
|
12
|
+
author_email = "info@eyeontext.com",
|
|
13
|
+
description="Wowool NLP {' '.join(package_name.split('-'))} Package",
|
|
14
|
+
zip_safe=False,
|
|
15
|
+
package_data={{"{wowool_module}": ["*"]}},
|
|
16
|
+
include_package_data=True,
|
|
17
|
+
{setup_py_requirement}
|
|
18
|
+
)
|
|
19
|
+
"""
|
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
from os import getcwd
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
from typing import Union
|
|
4
|
+
|
|
5
|
+
from invoke.collection import Collection
|
|
6
|
+
from invoke.tasks import task
|
|
7
|
+
from logging import getLogger
|
|
8
|
+
from shutil import rmtree
|
|
9
|
+
|
|
10
|
+
logger = getLogger("wowool.lingware.build")
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def create_py_tasks(fp_root: Union[str, Path, None] = None) -> Collection:
|
|
14
|
+
fp_root = Path(fp_root) if fp_root is not None else Path(getcwd())
|
|
15
|
+
if fp_root.is_file():
|
|
16
|
+
fp_root = fp_root.parent
|
|
17
|
+
|
|
18
|
+
_ns = Collection()
|
|
19
|
+
|
|
20
|
+
@task
|
|
21
|
+
def version(c):
|
|
22
|
+
from wowool.lingware.build.git import get_version
|
|
23
|
+
|
|
24
|
+
print(get_version())
|
|
25
|
+
|
|
26
|
+
_ns.add_task(version)
|
|
27
|
+
|
|
28
|
+
@task
|
|
29
|
+
def build_languages(_):
|
|
30
|
+
"""
|
|
31
|
+
build the basic morphology files.
|
|
32
|
+
"""
|
|
33
|
+
from wowool.lingware.build.language import make_language
|
|
34
|
+
|
|
35
|
+
make_language(fp_root)
|
|
36
|
+
|
|
37
|
+
_ns.add_task(build_languages)
|
|
38
|
+
|
|
39
|
+
@task
|
|
40
|
+
def build_domains(_):
|
|
41
|
+
"""Build all the domain files."""
|
|
42
|
+
|
|
43
|
+
from wowool.lingware.build.domain import build_domains
|
|
44
|
+
|
|
45
|
+
build_domains(fp_root)
|
|
46
|
+
|
|
47
|
+
_ns.add_task(build_domains)
|
|
48
|
+
|
|
49
|
+
@task(pre=[build_languages, build_domains])
|
|
50
|
+
def build(_):
|
|
51
|
+
"""
|
|
52
|
+
build the complete language module.
|
|
53
|
+
"""
|
|
54
|
+
pass
|
|
55
|
+
|
|
56
|
+
_ns.add_task(build)
|
|
57
|
+
|
|
58
|
+
# --------------------------------------------------------------------------------------
|
|
59
|
+
|
|
60
|
+
@task(pre=[build_languages])
|
|
61
|
+
def package_languages(_):
|
|
62
|
+
"""
|
|
63
|
+
build the basic morphology files.
|
|
64
|
+
"""
|
|
65
|
+
from wowool.lingware.build.language import package_languages
|
|
66
|
+
|
|
67
|
+
package_languages(fp_root)
|
|
68
|
+
|
|
69
|
+
_ns.add_task(package_languages)
|
|
70
|
+
|
|
71
|
+
@task(pre=[build_domains])
|
|
72
|
+
def package_domains(_):
|
|
73
|
+
"""upload domains all the domain file."""
|
|
74
|
+
from wowool.lingware.build.domain import package_domains
|
|
75
|
+
|
|
76
|
+
package_domains(fp_root)
|
|
77
|
+
|
|
78
|
+
_ns.add_task(package_domains)
|
|
79
|
+
|
|
80
|
+
@task(pre=[package_languages, package_domains])
|
|
81
|
+
def package(_):
|
|
82
|
+
pass
|
|
83
|
+
|
|
84
|
+
@task
|
|
85
|
+
def package_all(_, clean: bool = True):
|
|
86
|
+
from wowool.lingware.build.language import package_languages
|
|
87
|
+
from wowool.lingware.build.domain import package_domains
|
|
88
|
+
from wowool.lingware.build.build import generate_pyproject_file
|
|
89
|
+
from subprocess import run
|
|
90
|
+
|
|
91
|
+
output_folder = fp_root / "_build_package"
|
|
92
|
+
if clean:
|
|
93
|
+
rmtree(output_folder, ignore_errors=True)
|
|
94
|
+
lxware_src = output_folder / "src/wowool/lxware/wowool"
|
|
95
|
+
|
|
96
|
+
print(f"Package languages and domains in {fp_root} to {lxware_src}")
|
|
97
|
+
package_languages(fp_root, package_lxware_wowool=lxware_src, run_setup=False)
|
|
98
|
+
print(f"Package domains and domains in {fp_root} to {lxware_src}")
|
|
99
|
+
package_domains(fp_root, package_lxware_wowool=lxware_src, run_setup=False)
|
|
100
|
+
generate_pyproject_file(fp_root, fp_root.name, output_folder=output_folder)
|
|
101
|
+
logger.debug(f"Run build package in {output_folder}")
|
|
102
|
+
run(f"python -m build --wheel {output_folder}", cwd=output_folder, shell=True)
|
|
103
|
+
|
|
104
|
+
_ns.add_task(package_all)
|
|
105
|
+
|
|
106
|
+
# --------------------------------------------------------------------------------------
|
|
107
|
+
|
|
108
|
+
@task
|
|
109
|
+
def upload_all(_):
|
|
110
|
+
"""
|
|
111
|
+
deploy all the language and domains packages.
|
|
112
|
+
"""
|
|
113
|
+
from wowool.build.pypi import upload_pypi
|
|
114
|
+
|
|
115
|
+
output_folder = fp_root / "_build_package"
|
|
116
|
+
if output_folder.exists():
|
|
117
|
+
upload_pypi(output_folder)
|
|
118
|
+
else:
|
|
119
|
+
raise FileNotFoundError(f"Folder {output_folder} not found.")
|
|
120
|
+
|
|
121
|
+
_ns.add_task(upload_all)
|
|
122
|
+
|
|
123
|
+
@task
|
|
124
|
+
def upload_languages(_):
|
|
125
|
+
"""
|
|
126
|
+
deploy all the language packages.
|
|
127
|
+
"""
|
|
128
|
+
from wowool.lingware.build.language import upload_languages
|
|
129
|
+
|
|
130
|
+
upload_languages(fp_root)
|
|
131
|
+
|
|
132
|
+
_ns.add_task(upload_languages)
|
|
133
|
+
|
|
134
|
+
@task
|
|
135
|
+
def upload_domains(_):
|
|
136
|
+
"""
|
|
137
|
+
deploy all the domains packages.
|
|
138
|
+
"""
|
|
139
|
+
from wowool.lingware.build.domain import upload_domains
|
|
140
|
+
|
|
141
|
+
upload_domains(fp_root)
|
|
142
|
+
|
|
143
|
+
_ns.add_task(upload_domains)
|
|
144
|
+
|
|
145
|
+
@task(post=[upload_languages, upload_domains])
|
|
146
|
+
def upload(_):
|
|
147
|
+
"""
|
|
148
|
+
deploy all the package.
|
|
149
|
+
"""
|
|
150
|
+
from wowool.lingware.build.language import upload_languages
|
|
151
|
+
from wowool.lingware.build.domain import upload_domains
|
|
152
|
+
|
|
153
|
+
upload_languages(fp_root)
|
|
154
|
+
upload_domains(fp_root)
|
|
155
|
+
|
|
156
|
+
_ns.add_task(upload)
|
|
157
|
+
|
|
158
|
+
# --------------------------------------------------------------------------------------
|
|
159
|
+
@task
|
|
160
|
+
def clean_domains(_):
|
|
161
|
+
"""Clean all the domain files."""
|
|
162
|
+
from wowool.lingware.build.domain import clean_domains
|
|
163
|
+
|
|
164
|
+
clean_domains(fp_root)
|
|
165
|
+
|
|
166
|
+
_ns.add_task(clean_domains)
|
|
167
|
+
|
|
168
|
+
@task
|
|
169
|
+
def clean_languages(_):
|
|
170
|
+
"""
|
|
171
|
+
cleans the language folder.
|
|
172
|
+
"""
|
|
173
|
+
from wowool.lingware.build.language import make_language
|
|
174
|
+
|
|
175
|
+
make_language(fp_root, "clean")
|
|
176
|
+
|
|
177
|
+
_ns.add_task(clean_languages)
|
|
178
|
+
|
|
179
|
+
@task(pre=[clean_languages, clean_domains])
|
|
180
|
+
def clean(_):
|
|
181
|
+
"""
|
|
182
|
+
cleans the language and domains.
|
|
183
|
+
"""
|
|
184
|
+
pass
|
|
185
|
+
|
|
186
|
+
_ns.add_task(clean)
|
|
187
|
+
|
|
188
|
+
@task
|
|
189
|
+
def test_domains(context, domains="*", target="", verbose=False, j=1):
|
|
190
|
+
"""test all the domain files."""
|
|
191
|
+
|
|
192
|
+
from wowool.lingware.unittest.unittest import test_domains
|
|
193
|
+
|
|
194
|
+
test_domains(fp_root)
|
|
195
|
+
|
|
196
|
+
_ns.add_task(test_domains)
|
|
197
|
+
|
|
198
|
+
@task
|
|
199
|
+
def test_wheels(context, domains="*", target="", verbose=False, j=1):
|
|
200
|
+
"""test all the domain files."""
|
|
201
|
+
|
|
202
|
+
from wowool.lingware.unittest.unittest import test_wheels
|
|
203
|
+
|
|
204
|
+
test_wheels(fp_root)
|
|
205
|
+
|
|
206
|
+
_ns.add_task(test_wheels)
|
|
207
|
+
|
|
208
|
+
@task(
|
|
209
|
+
help={
|
|
210
|
+
"verbose": "print the results on screen",
|
|
211
|
+
"diff": "app used for the diff, default(meld)",
|
|
212
|
+
"overwrite": "generate the expected result file",
|
|
213
|
+
"pattern": "glob pattern to run, default **/*.txt",
|
|
214
|
+
}
|
|
215
|
+
)
|
|
216
|
+
def test(context, verbose=False, diff="meld", overwrite=False, pattern="**/*.txt", integration=False):
|
|
217
|
+
"""
|
|
218
|
+
start the unit-test.
|
|
219
|
+
"""
|
|
220
|
+
from wowool.lingware.unittest.unittest import run_test_folder
|
|
221
|
+
|
|
222
|
+
run_test_folder(context, fp_root, verbose, diff, overwrite, pattern, integration)
|
|
223
|
+
|
|
224
|
+
_ns.add_task(test)
|
|
225
|
+
|
|
226
|
+
from wowool.lingware.build.domain import load_domain_tasks
|
|
227
|
+
|
|
228
|
+
exec(load_domain_tasks(fp_root, "_ns"))
|
|
229
|
+
|
|
230
|
+
return _ns
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
# Allow both default Invoke namespaces to be available for import
|
|
234
|
+
ns = create_py_tasks()
|
|
235
|
+
namespace = ns
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
PACKAGE_SETUP_TEMPLATE = """#
|
|
2
|
+
{PREFIX_DO_NOT_EDIT}
|
|
3
|
+
# This file is generated.
|
|
4
|
+
#
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from setuptools import setup
|
|
7
|
+
|
|
8
|
+
THIS_DIR = Path(__file__).parent.resolve()
|
|
9
|
+
language = THIS_DIR.name
|
|
10
|
+
|
|
11
|
+
wowool_packages = ["{wowool_module}"]
|
|
12
|
+
|
|
13
|
+
results = setup(
|
|
14
|
+
name="wowool-lxware-{package_name}",
|
|
15
|
+
version="{repo_version}",
|
|
16
|
+
packages=wowool_packages,
|
|
17
|
+
author="Wowool",
|
|
18
|
+
author_email = "info@wowool.com",
|
|
19
|
+
description="Wowool NLP {package_name} Package",
|
|
20
|
+
zip_safe=False,
|
|
21
|
+
package_data={{"{wowool_module}": ["*"]}},
|
|
22
|
+
include_package_data=True,
|
|
23
|
+
{setup_py_install_requirement}
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
PACKAGE_PYPROJECT_TEMPLATE = """
|
|
30
|
+
[project]
|
|
31
|
+
name = "wowool-{package_name}"
|
|
32
|
+
version = "{repo_version}"
|
|
33
|
+
description = "Wowool NLP {package_name} Package"
|
|
34
|
+
readme = {{ file = "README.md", content-type = "text/markdown" }}
|
|
35
|
+
|
|
36
|
+
[build-system]
|
|
37
|
+
requires = ["setuptools"]
|
|
38
|
+
build-backend = "setuptools.build_meta"
|
|
39
|
+
|
|
40
|
+
[tool.setuptools.packages.find]
|
|
41
|
+
where = ["src"]
|
|
42
|
+
|
|
43
|
+
[tool.setuptools.package-data]
|
|
44
|
+
"wowool.lxware.wowool" = ['*.*']
|
|
45
|
+
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
README_TEMPLATE = """# Wowool NLP {package_name} Package
|
|
49
|
+
This package contains the language file and domains for {language}.
|
|
50
|
+
To use this package, you need to install the wowool-sdk package and request a license file at info@wowool.com
|
|
51
|
+
|
|
52
|
+
## Usage
|
|
53
|
+
|
|
54
|
+
```python
|
|
55
|
+
from wowool.sdk import Pipeline
|
|
56
|
+
from pathlib import Path
|
|
57
|
+
input_text = Path("input.txt").read_text()
|
|
58
|
+
pipeline = Pipeline("{language},entity")
|
|
59
|
+
doc = pipeline(input_text)
|
|
60
|
+
print(doc)
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
## Domains
|
|
64
|
+
|
|
65
|
+
{domains}
|
|
66
|
+
|
|
67
|
+
"""
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
def main():
|
|
2
|
+
|
|
3
|
+
import subprocess
|
|
4
|
+
import sys
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
import platform
|
|
7
|
+
import os
|
|
8
|
+
|
|
9
|
+
ext = ".exe" if "Windows" == platform.system() else ""
|
|
10
|
+
cpp_app = Path(__file__).parent.parent.parent.parent.parent / "package" / "lib" / f"afst++{ext}"
|
|
11
|
+
if not cpp_app.exists():
|
|
12
|
+
if "WOWOOL_ROOT" in os.environ:
|
|
13
|
+
cpp_app = Path(os.environ["WOWOOL_ROOT"]) / "bin" / f"afst{ext}"
|
|
14
|
+
assert cpp_app.exists(), "Could not find afst application."
|
|
15
|
+
sys.argv[0] = str(cpp_app)
|
|
16
|
+
subprocess.run(sys.argv)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
if __name__ == "__main__":
|
|
20
|
+
main()
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from wowool.lingware.build.tools.convert_domain_info.convert import convert
|