wisal-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wisal/__init__.py +0 -0
- wisal/arabic_utils.py +58 -0
- wisal/cli.py +148 -0
- wisal/errors.py +6 -0
- wisal/formats.py +156 -0
- wisal/messages.py +111 -0
- wisal/validate.py +59 -0
- wisal_cli-0.1.0.dist-info/METADATA +121 -0
- wisal_cli-0.1.0.dist-info/RECORD +13 -0
- wisal_cli-0.1.0.dist-info/WHEEL +5 -0
- wisal_cli-0.1.0.dist-info/entry_points.txt +2 -0
- wisal_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
- wisal_cli-0.1.0.dist-info/top_level.txt +1 -0
wisal/__init__.py
ADDED
|
File without changes
|
wisal/arabic_utils.py
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Helper utilities for handling Arabic-specific text quirks in data.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
|
|
7
|
+
# Arabic-Indic digit to Latin digit mapping
|
|
8
|
+
ARABIC_INDIC_DIGITS = "٠١٢٣٤٥٦٧٨٩"
|
|
9
|
+
LATIN_DIGITS = "0123456789"
|
|
10
|
+
_DIGIT_TRANSLATION = str.maketrans(ARABIC_INDIC_DIGITS, LATIN_DIGITS)
|
|
11
|
+
|
|
12
|
+
_ARABIC_DIGIT_RE = re.compile(f"[{ARABIC_INDIC_DIGITS}]")
|
|
13
|
+
_LATIN_DIGIT_RE = re.compile(r"[0-9]")
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def contains_arabic_indic_digits(text: str) -> bool:
|
|
17
|
+
"""Return True if the text contains Arabic-Indic digits."""
|
|
18
|
+
return bool(_ARABIC_DIGIT_RE.search(text))
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def has_mixed_digits(text: str) -> bool:
|
|
22
|
+
"""Return True if the text mixes Arabic-Indic and Latin digits."""
|
|
23
|
+
return bool(_ARABIC_DIGIT_RE.search(text)) and bool(_LATIN_DIGIT_RE.search(text))
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def normalize_digits(text: str) -> str:
|
|
27
|
+
"""Convert all Arabic-Indic digits in the text to Latin digits."""
|
|
28
|
+
if not isinstance(text, str):
|
|
29
|
+
return text
|
|
30
|
+
return text.translate(_DIGIT_TRANSLATION)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def likely_encoding_issue(raw_bytes: bytes) -> str | None:
|
|
34
|
+
"""
|
|
35
|
+
Try to detect whether the file uses a common non-UTF-8 encoding for
|
|
36
|
+
Arabic text (such as Windows-1256), returning the likely encoding name,
|
|
37
|
+
or None if the bytes are valid UTF-8.
|
|
38
|
+
"""
|
|
39
|
+
try:
|
|
40
|
+
raw_bytes.decode("utf-8")
|
|
41
|
+
return None
|
|
42
|
+
except UnicodeDecodeError:
|
|
43
|
+
pass
|
|
44
|
+
|
|
45
|
+
for encoding in ("windows-1256", "iso-8859-6", "cp720"):
|
|
46
|
+
try:
|
|
47
|
+
raw_bytes.decode(encoding)
|
|
48
|
+
return encoding
|
|
49
|
+
except UnicodeDecodeError:
|
|
50
|
+
continue
|
|
51
|
+
return None
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def detect_column_digit_issues(values: list[str]) -> bool:
|
|
55
|
+
"""Check a single column's values across rows for mixed digit systems."""
|
|
56
|
+
joined_has_arabic = any(contains_arabic_indic_digits(v) for v in values if isinstance(v, str))
|
|
57
|
+
joined_has_latin = any(_LATIN_DIGIT_RE.search(v) for v in values if isinstance(v, str))
|
|
58
|
+
return joined_has_arabic and joined_has_latin
|
wisal/cli.py
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Command-line interface for the Wisal tool.
|
|
3
|
+
|
|
4
|
+
Usage:
|
|
5
|
+
wisal convert input.json output.csv
|
|
6
|
+
wisal convert data.csv data.yaml --flatten
|
|
7
|
+
wisal convert data.json out.csv --dry-run --lang ar
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import argparse
|
|
11
|
+
import os
|
|
12
|
+
import sys
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
from wisal.arabic_utils import normalize_digits
|
|
16
|
+
from wisal.errors import WisalError
|
|
17
|
+
from wisal.formats import format_from_extension, load_data, write_data, write_stdout
|
|
18
|
+
from wisal.messages import Lang, detect_language, t
|
|
19
|
+
from wisal.validate import validate_rows
|
|
20
|
+
|
|
21
|
+
__version__ = "0.1.0"
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _apply_digit_normalization(data):
|
|
25
|
+
"""Recursively apply digit normalization to every string in the structure."""
|
|
26
|
+
if isinstance(data, dict):
|
|
27
|
+
return {k: _apply_digit_normalization(v) for k, v in data.items()}
|
|
28
|
+
if isinstance(data, list):
|
|
29
|
+
return [_apply_digit_normalization(v) for v in data]
|
|
30
|
+
if isinstance(data, str):
|
|
31
|
+
return normalize_digits(data)
|
|
32
|
+
return data
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
36
|
+
lang_parent = argparse.ArgumentParser(add_help=False)
|
|
37
|
+
lang_parent.add_argument(
|
|
38
|
+
"--lang",
|
|
39
|
+
choices=["ar", "en"],
|
|
40
|
+
default=None,
|
|
41
|
+
help="Message language (ar/en). Default: auto-detect from system locale.",
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
parser = argparse.ArgumentParser(
|
|
45
|
+
prog="wisal",
|
|
46
|
+
description=(
|
|
47
|
+
"Wisal — a bilingual (Arabic/English) data conversion tool for JSON, CSV, and YAML."
|
|
48
|
+
),
|
|
49
|
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
50
|
+
parents=[lang_parent],
|
|
51
|
+
)
|
|
52
|
+
parser.add_argument(
|
|
53
|
+
"--version", action="version", version=f"wisal {__version__}"
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
57
|
+
|
|
58
|
+
convert = subparsers.add_parser(
|
|
59
|
+
"convert",
|
|
60
|
+
help="Convert a file from one format to another",
|
|
61
|
+
parents=[lang_parent],
|
|
62
|
+
)
|
|
63
|
+
convert.add_argument("input", help="Input file")
|
|
64
|
+
convert.add_argument("output", help="Output file")
|
|
65
|
+
convert.add_argument(
|
|
66
|
+
"--flatten",
|
|
67
|
+
action="store_true",
|
|
68
|
+
help="Flatten nested fields into columns (for CSV output)",
|
|
69
|
+
)
|
|
70
|
+
convert.add_argument(
|
|
71
|
+
"--normalize-digits",
|
|
72
|
+
action="store_true",
|
|
73
|
+
help="Normalize Arabic-Indic digits (٠-٩) to Latin digits",
|
|
74
|
+
)
|
|
75
|
+
convert.add_argument(
|
|
76
|
+
"--dry-run",
|
|
77
|
+
action="store_true",
|
|
78
|
+
help="Preview output without writing a file",
|
|
79
|
+
)
|
|
80
|
+
convert.add_argument(
|
|
81
|
+
"--no-validate",
|
|
82
|
+
action="store_true",
|
|
83
|
+
help="Skip data quality checks",
|
|
84
|
+
)
|
|
85
|
+
convert.add_argument(
|
|
86
|
+
"--quiet",
|
|
87
|
+
action="store_true",
|
|
88
|
+
help="Suppress warnings, show only errors",
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
return parser
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def run(argv: list[str] | None = None) -> int:
|
|
95
|
+
parser = build_parser()
|
|
96
|
+
args = parser.parse_args(argv)
|
|
97
|
+
|
|
98
|
+
if args.lang:
|
|
99
|
+
os.environ["WISAL_LANG"] = args.lang
|
|
100
|
+
lang = Lang(args.lang) if args.lang else detect_language()
|
|
101
|
+
|
|
102
|
+
def warn(message: str) -> None:
|
|
103
|
+
if not getattr(args, "quiet", False):
|
|
104
|
+
print(f"⚠ {message}", file=sys.stderr)
|
|
105
|
+
|
|
106
|
+
if args.command == "convert":
|
|
107
|
+
input_path = Path(args.input)
|
|
108
|
+
output_path = Path(args.output)
|
|
109
|
+
|
|
110
|
+
try:
|
|
111
|
+
data = load_data(input_path, warn=warn)
|
|
112
|
+
|
|
113
|
+
if args.normalize_digits:
|
|
114
|
+
data = _apply_digit_normalization(data)
|
|
115
|
+
|
|
116
|
+
if not args.no_validate:
|
|
117
|
+
rows = data if isinstance(data, list) else [data]
|
|
118
|
+
validate_rows(rows, warn=warn)
|
|
119
|
+
|
|
120
|
+
if args.dry_run:
|
|
121
|
+
out_fmt = format_from_extension(output_path)
|
|
122
|
+
write_stdout(data, out_fmt, flatten=args.flatten, warn=warn)
|
|
123
|
+
print(t("dry_run_notice", lang=lang), file=sys.stderr)
|
|
124
|
+
else:
|
|
125
|
+
write_data(data, output_path, flatten=args.flatten, warn=warn)
|
|
126
|
+
row_count = len(data) if isinstance(data, list) else 1
|
|
127
|
+
print(
|
|
128
|
+
t(
|
|
129
|
+
"conversion_success",
|
|
130
|
+
lang=lang,
|
|
131
|
+
source=str(input_path),
|
|
132
|
+
target=str(output_path),
|
|
133
|
+
rows=row_count,
|
|
134
|
+
)
|
|
135
|
+
)
|
|
136
|
+
except WisalError as e:
|
|
137
|
+
print(f"✗ {e}", file=sys.stderr)
|
|
138
|
+
return 1
|
|
139
|
+
|
|
140
|
+
return 0
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def main() -> None:
|
|
144
|
+
sys.exit(run())
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
if __name__ == "__main__":
|
|
148
|
+
main()
|
wisal/errors.py
ADDED
wisal/formats.py
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Reading and writing supported data formats: JSON, CSV, YAML.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import csv
|
|
6
|
+
import json
|
|
7
|
+
import sys
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
import yaml
|
|
12
|
+
|
|
13
|
+
from wisal.arabic_utils import likely_encoding_issue
|
|
14
|
+
from wisal.errors import WisalError
|
|
15
|
+
from wisal.messages import t
|
|
16
|
+
|
|
17
|
+
SUPPORTED_FORMATS = ("json", "csv", "yaml", "yml")
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def format_from_extension(path: Path) -> str:
|
|
21
|
+
ext = path.suffix.lstrip(".").lower()
|
|
22
|
+
if ext == "yml":
|
|
23
|
+
ext = "yaml"
|
|
24
|
+
if ext not in SUPPORTED_FORMATS:
|
|
25
|
+
raise WisalError(t("unsupported_format", ext=ext or "(none)"))
|
|
26
|
+
return ext
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def read_file_bytes(path: Path) -> bytes:
|
|
30
|
+
if not path.exists():
|
|
31
|
+
raise WisalError(t("file_not_found", path=str(path)))
|
|
32
|
+
raw = path.read_bytes()
|
|
33
|
+
if len(raw) == 0:
|
|
34
|
+
raise WisalError(t("empty_input", path=str(path)))
|
|
35
|
+
return raw
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def load_data(path: Path, warn) -> Any:
|
|
39
|
+
"""
|
|
40
|
+
Load data from a file based on its extension, checking encoding along the way.
|
|
41
|
+
"""
|
|
42
|
+
fmt = format_from_extension(path)
|
|
43
|
+
raw = read_file_bytes(path)
|
|
44
|
+
|
|
45
|
+
detected_encoding = likely_encoding_issue(raw)
|
|
46
|
+
if detected_encoding:
|
|
47
|
+
warn(t("encoding_mismatch_warning", detected=detected_encoding))
|
|
48
|
+
text = raw.decode(detected_encoding, errors="replace")
|
|
49
|
+
else:
|
|
50
|
+
text = raw.decode("utf-8")
|
|
51
|
+
|
|
52
|
+
if fmt == "json":
|
|
53
|
+
try:
|
|
54
|
+
return json.loads(text)
|
|
55
|
+
except json.JSONDecodeError as e:
|
|
56
|
+
raise WisalError(
|
|
57
|
+
t("invalid_json", line=e.lineno, column=e.colno, detail=e.msg)
|
|
58
|
+
) from e
|
|
59
|
+
|
|
60
|
+
if fmt == "yaml":
|
|
61
|
+
try:
|
|
62
|
+
return yaml.safe_load(text)
|
|
63
|
+
except yaml.YAMLError as e:
|
|
64
|
+
raise WisalError(str(e)) from e
|
|
65
|
+
|
|
66
|
+
if fmt == "csv":
|
|
67
|
+
reader = csv.DictReader(text.splitlines())
|
|
68
|
+
return list(reader)
|
|
69
|
+
|
|
70
|
+
raise WisalError(t("unsupported_format", ext=fmt))
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _flatten_dict(d: dict, parent_key: str = "", sep: str = ".") -> dict:
|
|
74
|
+
items = {}
|
|
75
|
+
for k, v in d.items():
|
|
76
|
+
new_key = f"{parent_key}{sep}{k}" if parent_key else str(k)
|
|
77
|
+
if isinstance(v, dict):
|
|
78
|
+
items.update(_flatten_dict(v, new_key, sep=sep))
|
|
79
|
+
else:
|
|
80
|
+
items[new_key] = v
|
|
81
|
+
return items
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _rows_from_data(data: Any, flatten: bool, warn) -> list[dict]:
|
|
85
|
+
"""Normalize any data structure (list or object) into a list of CSV-ready rows."""
|
|
86
|
+
if isinstance(data, dict):
|
|
87
|
+
data = [data]
|
|
88
|
+
if not isinstance(data, list):
|
|
89
|
+
raise WisalError(t("unsupported_format", ext="(structure)"))
|
|
90
|
+
|
|
91
|
+
rows = []
|
|
92
|
+
for row in data:
|
|
93
|
+
if not isinstance(row, dict):
|
|
94
|
+
rows.append({"value": row})
|
|
95
|
+
continue
|
|
96
|
+
new_row = {}
|
|
97
|
+
for key, value in row.items():
|
|
98
|
+
if isinstance(value, (dict, list)):
|
|
99
|
+
if flatten and isinstance(value, dict):
|
|
100
|
+
new_row.update(_flatten_dict({key: value}))
|
|
101
|
+
else:
|
|
102
|
+
warn(t("nested_structure_warning", field=key))
|
|
103
|
+
new_row[key] = json.dumps(value, ensure_ascii=False)
|
|
104
|
+
else:
|
|
105
|
+
new_row[key] = value
|
|
106
|
+
rows.append(new_row)
|
|
107
|
+
return rows
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def write_data(data: Any, path: Path, flatten: bool, warn) -> None:
|
|
111
|
+
"""
|
|
112
|
+
Write data to a file based on its extension.
|
|
113
|
+
"""
|
|
114
|
+
fmt = format_from_extension(path)
|
|
115
|
+
|
|
116
|
+
if fmt == "json":
|
|
117
|
+
path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
118
|
+
return
|
|
119
|
+
|
|
120
|
+
if fmt == "yaml":
|
|
121
|
+
path.write_text(
|
|
122
|
+
yaml.dump(data, allow_unicode=True, sort_keys=False), encoding="utf-8"
|
|
123
|
+
)
|
|
124
|
+
return
|
|
125
|
+
|
|
126
|
+
if fmt == "csv":
|
|
127
|
+
rows = _rows_from_data(data, flatten=flatten, warn=warn)
|
|
128
|
+
if not rows:
|
|
129
|
+
path.write_text("", encoding="utf-8")
|
|
130
|
+
return
|
|
131
|
+
fieldnames = list(dict.fromkeys(k for row in rows for k in row.keys()))
|
|
132
|
+
with path.open("w", newline="", encoding="utf-8") as f:
|
|
133
|
+
writer = csv.DictWriter(f, fieldnames=fieldnames)
|
|
134
|
+
writer.writeheader()
|
|
135
|
+
for row in rows:
|
|
136
|
+
writer.writerow(row)
|
|
137
|
+
return
|
|
138
|
+
|
|
139
|
+
raise WisalError(t("unsupported_format", ext=fmt))
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def write_stdout(data: Any, fmt: str, flatten: bool, warn) -> None:
|
|
143
|
+
"""Write the output to stdout (for dry-run preview mode)."""
|
|
144
|
+
if fmt == "json":
|
|
145
|
+
sys.stdout.write(json.dumps(data, ensure_ascii=False, indent=2) + "\n")
|
|
146
|
+
elif fmt == "yaml":
|
|
147
|
+
sys.stdout.write(yaml.dump(data, allow_unicode=True, sort_keys=False))
|
|
148
|
+
elif fmt == "csv":
|
|
149
|
+
rows = _rows_from_data(data, flatten=flatten, warn=warn)
|
|
150
|
+
if not rows:
|
|
151
|
+
return
|
|
152
|
+
fieldnames = list(dict.fromkeys(k for row in rows for k in row.keys()))
|
|
153
|
+
writer = csv.DictWriter(sys.stdout, fieldnames=fieldnames)
|
|
154
|
+
writer.writeheader()
|
|
155
|
+
for row in rows:
|
|
156
|
+
writer.writerow(row)
|
wisal/messages.py
ADDED
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Bilingual (Arabic/English) message system for the Wisal CLI tool.
|
|
3
|
+
|
|
4
|
+
Language is auto-detected from the LANG environment variable, or forced via --lang.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import os
|
|
8
|
+
from enum import Enum
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class Lang(str, Enum):
|
|
12
|
+
AR = "ar"
|
|
13
|
+
EN = "en"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def detect_language() -> Lang:
|
|
17
|
+
"""Detect the appropriate language from the system environment. Defaults to English."""
|
|
18
|
+
env_lang = os.environ.get("WISAL_LANG", "").lower()
|
|
19
|
+
if env_lang in ("ar", "arabic"):
|
|
20
|
+
return Lang.AR
|
|
21
|
+
if env_lang in ("en", "english"):
|
|
22
|
+
return Lang.EN
|
|
23
|
+
|
|
24
|
+
system_lang = os.environ.get("LANG", "") + os.environ.get("LC_ALL", "")
|
|
25
|
+
if "ar" in system_lang.lower():
|
|
26
|
+
return Lang.AR
|
|
27
|
+
return Lang.EN
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
# كل رسالة لها مفتاح، وقيمة لكل لغة.
|
|
31
|
+
# Each message has a key, and a value per language.
|
|
32
|
+
MESSAGES = {
|
|
33
|
+
"file_not_found": {
|
|
34
|
+
Lang.AR: "خطأ: الملف '{path}' غير موجود. تأكد من المسار وحاول مرة أخرى.",
|
|
35
|
+
Lang.EN: "Error: File '{path}' was not found. Check the path and try again.",
|
|
36
|
+
},
|
|
37
|
+
"unsupported_format": {
|
|
38
|
+
Lang.AR: "خطأ: الصيغة '{ext}' غير مدعومة. الصيغ المدعومة: json, csv, yaml.",
|
|
39
|
+
Lang.EN: "Error: Format '{ext}' is not supported. Supported formats: json, csv, yaml.",
|
|
40
|
+
},
|
|
41
|
+
"key_missing_in_row": {
|
|
42
|
+
Lang.AR: "تحذير: الحقل '{field}' غير موجود في السطر {line}. تم استخدام قيمة فارغة بدلاً منه.",
|
|
43
|
+
Lang.EN: "Warning: Field '{field}' is missing in row {line}. An empty value was used instead.",
|
|
44
|
+
},
|
|
45
|
+
"encoding_detected": {
|
|
46
|
+
Lang.AR: "تم اكتشاف ترميز النص التالي: {encoding}. سيتم استخدامه لقراءة الملف.",
|
|
47
|
+
Lang.EN: "Detected text encoding: {encoding}. It will be used to read the file.",
|
|
48
|
+
},
|
|
49
|
+
"encoding_mismatch_warning": {
|
|
50
|
+
Lang.AR: "تحذير: الملف يبدو أنه بترميز '{detected}' وليس UTF-8. "
|
|
51
|
+
"قد يظهر النص العربي بشكل غير صحيح إذا لم تتم معالجته.",
|
|
52
|
+
Lang.EN: "Warning: The file appears to be encoded as '{detected}', not UTF-8. "
|
|
53
|
+
"Arabic text may display incorrectly if this is not handled.",
|
|
54
|
+
},
|
|
55
|
+
"mixed_digits_warning": {
|
|
56
|
+
Lang.AR: "تحذير: تم العثور على أرقام عربية-هندية (٠-٩) مختلطة مع أرقام لاتينية في العمود '{column}'. "
|
|
57
|
+
"استخدم --normalize-digits لتوحيدها.",
|
|
58
|
+
Lang.EN: "Warning: Arabic-Indic digits (٠-٩) found mixed with Latin digits in column '{column}'. "
|
|
59
|
+
"Use --normalize-digits to unify them.",
|
|
60
|
+
},
|
|
61
|
+
"conversion_success": {
|
|
62
|
+
Lang.AR: "تم تحويل '{source}' إلى '{target}' بنجاح. ({rows} سطر/عنصر)",
|
|
63
|
+
Lang.EN: "Successfully converted '{source}' to '{target}'. ({rows} rows/items)",
|
|
64
|
+
},
|
|
65
|
+
"invalid_json": {
|
|
66
|
+
Lang.AR: "خطأ في تحليل JSON بالقرب من السطر {line}، العمود {column}: {detail}",
|
|
67
|
+
Lang.EN: "JSON parsing error near line {line}, column {column}: {detail}",
|
|
68
|
+
},
|
|
69
|
+
"nested_structure_warning": {
|
|
70
|
+
Lang.AR: "تحذير: الحقل '{field}' يحتوي على بنية متداخلة ولا يمكن تمثيله مباشرة في CSV. "
|
|
71
|
+
"تم تحويله إلى نص JSON مضمّن. استخدم --flatten لتفكيك البنية إلى أعمدة.",
|
|
72
|
+
Lang.EN: "Warning: Field '{field}' contains a nested structure that cannot be represented "
|
|
73
|
+
"directly in CSV. It was embedded as a JSON string. Use --flatten to expand it into columns.",
|
|
74
|
+
},
|
|
75
|
+
"empty_input": {
|
|
76
|
+
Lang.AR: "خطأ: الملف المدخل '{path}' فارغ. لا يوجد شيء لتحويله.",
|
|
77
|
+
Lang.EN: "Error: Input file '{path}' is empty. Nothing to convert.",
|
|
78
|
+
},
|
|
79
|
+
"output_written": {
|
|
80
|
+
Lang.AR: "تم حفظ الناتج في: {path}",
|
|
81
|
+
Lang.EN: "Output saved to: {path}",
|
|
82
|
+
},
|
|
83
|
+
"dry_run_notice": {
|
|
84
|
+
Lang.AR: "وضع المعاينة فقط (dry-run): لم يتم حفظ أي ملف.",
|
|
85
|
+
Lang.EN: "Dry-run mode: no file was written.",
|
|
86
|
+
},
|
|
87
|
+
"validation_type_mismatch": {
|
|
88
|
+
Lang.AR: "تحذير: العمود '{column}' يحتوي على أنواع بيانات مختلطة "
|
|
89
|
+
"(مثال: أرقام ونصوص معاً) في الأسطر: {lines}.",
|
|
90
|
+
Lang.EN: "Warning: Column '{column}' has mixed data types "
|
|
91
|
+
"(e.g. numbers and text together) in rows: {lines}.",
|
|
92
|
+
},
|
|
93
|
+
"diff_summary": {
|
|
94
|
+
Lang.AR: "ملخص المقارنة: {added} حقل مُضاف، {removed} حقل محذوف، {changed} حقل مُغيّر.",
|
|
95
|
+
Lang.EN: "Diff summary: {added} field(s) added, {removed} field(s) removed, {changed} field(s) changed.",
|
|
96
|
+
},
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def t(key: str, lang: Lang | None = None, **kwargs) -> str:
|
|
101
|
+
"""
|
|
102
|
+
ترجمة مفتاح رسالة إلى نص باللغة المطلوبة، مع تعبئة المتغيرات.
|
|
103
|
+
Translate a message key into text in the requested language, filling in variables.
|
|
104
|
+
"""
|
|
105
|
+
if lang is None:
|
|
106
|
+
lang = detect_language()
|
|
107
|
+
template = MESSAGES.get(key, {}).get(lang)
|
|
108
|
+
if template is None:
|
|
109
|
+
# احتياط: استخدم الإنجليزية إذا لم توجد الرسالة باللغة المطلوبة
|
|
110
|
+
template = MESSAGES.get(key, {}).get(Lang.EN, key)
|
|
111
|
+
return template.format(**kwargs)
|
wisal/validate.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Data quality checks prior to conversion: mixed types, mixed digits, missing fields.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from wisal.arabic_utils import detect_column_digit_issues
|
|
8
|
+
from wisal.messages import t
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _python_type_name(value: Any) -> str:
|
|
12
|
+
if value is None:
|
|
13
|
+
return "null"
|
|
14
|
+
if isinstance(value, bool):
|
|
15
|
+
return "bool"
|
|
16
|
+
if isinstance(value, (int, float)):
|
|
17
|
+
return "number"
|
|
18
|
+
return "text"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def validate_rows(rows: list[dict], warn) -> None:
|
|
22
|
+
"""
|
|
23
|
+
Run all quality checks on a list of rows (already normalized to dicts)
|
|
24
|
+
and emit warnings through the provided warn function.
|
|
25
|
+
"""
|
|
26
|
+
if not rows:
|
|
27
|
+
return
|
|
28
|
+
|
|
29
|
+
columns: dict[str, list] = {}
|
|
30
|
+
for row in rows:
|
|
31
|
+
if not isinstance(row, dict):
|
|
32
|
+
continue
|
|
33
|
+
for key, value in row.items():
|
|
34
|
+
columns.setdefault(key, []).append(value)
|
|
35
|
+
|
|
36
|
+
for column, values in columns.items():
|
|
37
|
+
# Check for mixed types
|
|
38
|
+
types_seen = {}
|
|
39
|
+
for idx, value in enumerate(values):
|
|
40
|
+
type_name = _python_type_name(value)
|
|
41
|
+
if type_name in ("null",):
|
|
42
|
+
continue
|
|
43
|
+
types_seen.setdefault(type_name, []).append(idx + 1)
|
|
44
|
+
|
|
45
|
+
non_null_types = [k for k in types_seen if k != "null"]
|
|
46
|
+
if len(non_null_types) > 1:
|
|
47
|
+
all_lines = sorted({ln for lns in types_seen.values() for ln in lns})
|
|
48
|
+
warn(
|
|
49
|
+
t(
|
|
50
|
+
"validation_type_mismatch",
|
|
51
|
+
column=column,
|
|
52
|
+
lines=", ".join(str(x) for x in all_lines[:10]),
|
|
53
|
+
)
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
# Check for mixed Arabic-Indic/Latin digits
|
|
57
|
+
str_values = [v for v in values if isinstance(v, str)]
|
|
58
|
+
if str_values and detect_column_digit_issues(str_values):
|
|
59
|
+
warn(t("mixed_digits_warning", column=column))
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: wisal-cli
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Bilingual (Arabic/English) CLI tool to convert between JSON, CSV, and YAML
|
|
5
|
+
Author: Wisal Contributors
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/amgdsaleh66/wisal
|
|
8
|
+
Project-URL: Issues, https://github.com/amgdsaleh66/wisal/issues
|
|
9
|
+
Keywords: cli,json,csv,yaml,convert,arabic,bilingual,i18n
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Environment :: Console
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Natural Language :: Arabic
|
|
15
|
+
Classifier: Natural Language :: English
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Topic :: Software Development :: Localization
|
|
21
|
+
Classifier: Topic :: Utilities
|
|
22
|
+
Requires-Python: >=3.10
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
License-File: LICENSE
|
|
25
|
+
Requires-Dist: pyyaml>=6.0
|
|
26
|
+
Provides-Extra: dev
|
|
27
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
28
|
+
Dynamic: license-file
|
|
29
|
+
|
|
30
|
+
# Wisal (وصال)
|
|
31
|
+
|
|
32
|
+
**A bilingual (Arabic/English) CLI tool to convert data between JSON, CSV, and YAML.**
|
|
33
|
+
|
|
34
|
+
## Why Wisal?
|
|
35
|
+
|
|
36
|
+
Most data conversion tools (jc, Miller, dasel...) are excellent but entirely English: error messages, warnings, even documentation. This creates a real barrier for thousands of Arabic-speaking developers, especially when debugging complex data.
|
|
37
|
+
|
|
38
|
+
Wisal is a small, focused tool that solves exactly this problem:
|
|
39
|
+
|
|
40
|
+
- 🌐 Every message (error, warning, help) is available in Arabic and English, auto-detected from the system locale (defaults to English)
|
|
41
|
+
- 🔢 Detects and handles Arabic-Indic digits (٠-٩) mixed with Latin digits in the same data
|
|
42
|
+
- 🔤 Detects common encoding issues (Windows-1256 vs UTF-8) that corrupt Arabic text
|
|
43
|
+
- ✅ Validates data quality before converting (mixed types, nested fields) instead of silently failing
|
|
44
|
+
- 📦 No bloat: one tool, one command, predictable behavior
|
|
45
|
+
|
|
46
|
+
## Installation
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
pip install wisal-cli
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
Or from source:
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
git clone https://github.com/amgdsaleh66/wisal.git
|
|
56
|
+
cd wisal
|
|
57
|
+
pip install -e .
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
## Usage
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
# Simple conversion
|
|
64
|
+
wisal convert data.json data.csv
|
|
65
|
+
|
|
66
|
+
# Force Arabic messages
|
|
67
|
+
wisal convert data.json data.csv --lang ar
|
|
68
|
+
|
|
69
|
+
# Flatten nested fields when converting to CSV
|
|
70
|
+
wisal convert data.json data.csv --flatten
|
|
71
|
+
|
|
72
|
+
# Normalize Arabic-Indic digits to Latin
|
|
73
|
+
wisal convert data.json data.csv --normalize-digits
|
|
74
|
+
|
|
75
|
+
# Preview output without saving
|
|
76
|
+
wisal convert data.json data.csv --dry-run
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
Run `wisal --help` or `wisal convert --help` for the full list of options.
|
|
80
|
+
|
|
81
|
+
### Example
|
|
82
|
+
|
|
83
|
+
**input.json:**
|
|
84
|
+
```json
|
|
85
|
+
[
|
|
86
|
+
{"name": "Ahmad", "age": "٣٠", "city": "Riyadh"},
|
|
87
|
+
{"name": "Sara", "age": 25, "city": "Dubai"}
|
|
88
|
+
]
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
wisal convert input.json output.csv --normalize-digits
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
```
|
|
96
|
+
⚠ Warning: Column 'age' has mixed data types (e.g. numbers and text together) in rows: 1, 2.
|
|
97
|
+
Successfully converted 'input.json' to 'output.csv'. (2 rows/items)
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
## Supported Formats
|
|
101
|
+
|
|
102
|
+
| From \ To | JSON | CSV | YAML |
|
|
103
|
+
|-----------|------|-----|------|
|
|
104
|
+
| JSON | — | ✅ | ✅ |
|
|
105
|
+
| CSV | ✅ | — | ✅ |
|
|
106
|
+
| YAML | ✅ | ✅ | — |
|
|
107
|
+
|
|
108
|
+
## Contributing
|
|
109
|
+
|
|
110
|
+
Contributions welcome! Open an issue or pull request.
|
|
111
|
+
|
|
112
|
+
```bash
|
|
113
|
+
pip install -e ".[dev]"
|
|
114
|
+
pytest tests/
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md) for details.
|
|
118
|
+
|
|
119
|
+
## License
|
|
120
|
+
|
|
121
|
+
MIT
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
wisal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
wisal/arabic_utils.py,sha256=_PL6umZ-i_AHnHn7_pHrKdjeKS7oYJl1O02UQ-9v45U,1919
|
|
3
|
+
wisal/cli.py,sha256=1U-c4zME1Nt2sZPF0OYMR9uTVhSDBX0SGxesLIkMngc,4407
|
|
4
|
+
wisal/errors.py,sha256=gCAzesp9CHWuRy_J7ppLGvrCi2jADxCDr0vvqpuoPso,162
|
|
5
|
+
wisal/formats.py,sha256=qnD3JexvDg28UI7fFoTNuQeYybZ92OChae8HIrTMEgI,4865
|
|
6
|
+
wisal/messages.py,sha256=MdbTBoK7iO1voks7fHsew7Vg6DOdVhyc8Qd1xeGFizA,5480
|
|
7
|
+
wisal/validate.py,sha256=SFJ18mmgJwvY4d-QUPiufa9tgPNsQ8PATGAauoA6gf4,1838
|
|
8
|
+
wisal_cli-0.1.0.dist-info/licenses/LICENSE,sha256=Sz3Wb_sXW4NFFX8-xUvuCAvhLM5SDBU33rxL_LjtoNo,1075
|
|
9
|
+
wisal_cli-0.1.0.dist-info/METADATA,sha256=Gp4xoMbcQXIjMnulyjF7Yj1dBwOtFdBnpx9zneTHLYE,3449
|
|
10
|
+
wisal_cli-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
11
|
+
wisal_cli-0.1.0.dist-info/entry_points.txt,sha256=l_MMVZQs0p_91XQO5N4Z1hIBWXSOr6y7gyCvXu_tAko,41
|
|
12
|
+
wisal_cli-0.1.0.dist-info/top_level.txt,sha256=GBW7UDmzQT_SdSnxPPzsvt6qpmAV8BSSFBN_q8QGbH4,6
|
|
13
|
+
wisal_cli-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Wisal Contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
wisal
|