excel2api 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- excel2api/__init__.py +1 -0
- excel2api/__main__.py +4 -0
- excel2api/api.py +98 -0
- excel2api/cli.py +255 -0
- excel2api/config.py +30 -0
- excel2api/converter.py +18 -0
- excel2api/dependencies.py +120 -0
- excel2api/doctor.py +79 -0
- excel2api/errors.py +17 -0
- excel2api/reader.py +36 -0
- excel2api/report.py +45 -0
- excel2api/schema.py +69 -0
- excel2api/sync.py +154 -0
- excel2api/validator.py +160 -0
- excel2api-1.0.0.dist-info/METADATA +398 -0
- excel2api-1.0.0.dist-info/RECORD +19 -0
- excel2api-1.0.0.dist-info/WHEEL +4 -0
- excel2api-1.0.0.dist-info/entry_points.txt +2 -0
- excel2api-1.0.0.dist-info/licenses/LICENSE +16 -0
excel2api/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "1.0.0"
|
excel2api/__main__.py
ADDED
excel2api/api.py
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
from typing import Any
|
|
3
|
+
|
|
4
|
+
import requests
|
|
5
|
+
from requests.adapters import HTTPAdapter
|
|
6
|
+
from urllib3.util.retry import Retry
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@dataclass
|
|
10
|
+
class APIResult:
|
|
11
|
+
row: int
|
|
12
|
+
operation: str
|
|
13
|
+
success: bool
|
|
14
|
+
status_code: int | None = None
|
|
15
|
+
response: Any = None
|
|
16
|
+
error: str | None = None
|
|
17
|
+
sheet: str | None = None
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class APIClient:
|
|
21
|
+
RETRY_STATUS_CODES = (408, 429, 500, 502, 503, 504)
|
|
22
|
+
|
|
23
|
+
def __init__(
|
|
24
|
+
self,
|
|
25
|
+
base_url: str,
|
|
26
|
+
token: str | None = None,
|
|
27
|
+
timeout: int = 30,
|
|
28
|
+
retries: int = 0,
|
|
29
|
+
retry_create: bool = False,
|
|
30
|
+
headers: dict[str, str] | None = None,
|
|
31
|
+
):
|
|
32
|
+
self.base_url = base_url.rstrip("/")
|
|
33
|
+
self.timeout = timeout
|
|
34
|
+
self.session = requests.Session()
|
|
35
|
+
|
|
36
|
+
if token:
|
|
37
|
+
self.session.headers.update({"Authorization": f"Bearer {token}"})
|
|
38
|
+
|
|
39
|
+
self.session.headers.update({
|
|
40
|
+
"Content-Type": "application/json",
|
|
41
|
+
"Accept": "application/json",
|
|
42
|
+
})
|
|
43
|
+
if headers:
|
|
44
|
+
self.session.headers.update(headers)
|
|
45
|
+
|
|
46
|
+
if retries > 0:
|
|
47
|
+
allowed_methods = {"PUT", "PATCH", "DELETE", "GET", "HEAD", "OPTIONS"}
|
|
48
|
+
if retry_create:
|
|
49
|
+
allowed_methods.add("POST")
|
|
50
|
+
|
|
51
|
+
retry = Retry(
|
|
52
|
+
total=retries,
|
|
53
|
+
connect=retries,
|
|
54
|
+
read=retries,
|
|
55
|
+
status=retries,
|
|
56
|
+
backoff_factor=0.5,
|
|
57
|
+
status_forcelist=self.RETRY_STATUS_CODES,
|
|
58
|
+
allowed_methods=allowed_methods,
|
|
59
|
+
respect_retry_after_header=True,
|
|
60
|
+
raise_on_status=False,
|
|
61
|
+
)
|
|
62
|
+
adapter = HTTPAdapter(max_retries=retry)
|
|
63
|
+
self.session.mount("http://", adapter)
|
|
64
|
+
self.session.mount("https://", adapter)
|
|
65
|
+
|
|
66
|
+
def request(self, method: str, path: str = "", **kwargs):
|
|
67
|
+
url = f"{self.base_url}/{path.lstrip('/')}" if path else self.base_url
|
|
68
|
+
return self.session.request(method, url, timeout=self.timeout, **kwargs)
|
|
69
|
+
|
|
70
|
+
def execute(self, operation: str, record: dict, identifier=None, endpoints: dict[str, object] | None = None):
|
|
71
|
+
operation = operation.upper()
|
|
72
|
+
endpoints = endpoints or {}
|
|
73
|
+
default = {
|
|
74
|
+
"CREATE": {"method": "POST", "path": ""},
|
|
75
|
+
"UPDATE": {"method": "PUT", "path": "/{id}"},
|
|
76
|
+
"PATCH": {"method": "PATCH", "path": "/{id}"},
|
|
77
|
+
"DELETE": {"method": "DELETE", "path": "/{id}"},
|
|
78
|
+
}
|
|
79
|
+
if operation not in default and operation not in endpoints:
|
|
80
|
+
raise ValueError(f"Unsupported operation: {operation}")
|
|
81
|
+
spec = endpoints.get(operation, default.get(operation))
|
|
82
|
+
if isinstance(spec, str):
|
|
83
|
+
spec = {"method": default.get(operation, {"method": "POST"})["method"], "path": spec}
|
|
84
|
+
spec = spec or {}
|
|
85
|
+
method = str(spec.get("method", default.get(operation, {}).get("method", "POST"))).upper()
|
|
86
|
+
path_template = str(spec.get("path", default.get(operation, {}).get("path", "")))
|
|
87
|
+
if "{id}" in path_template or "{identifier}" in path_template:
|
|
88
|
+
if identifier is None:
|
|
89
|
+
raise ValueError(f"{operation} requires an identifier")
|
|
90
|
+
values = dict(record)
|
|
91
|
+
values.update({"id": identifier, "identifier": identifier})
|
|
92
|
+
try:
|
|
93
|
+
path = path_template.format(**values)
|
|
94
|
+
except KeyError as exc:
|
|
95
|
+
raise ValueError(f"Missing endpoint template field: {exc.args[0]}") from exc
|
|
96
|
+
if method == "DELETE":
|
|
97
|
+
return self.request(method, path)
|
|
98
|
+
return self.request(method, path, json=record)
|
excel2api/cli.py
ADDED
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import os
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
import typer
|
|
5
|
+
|
|
6
|
+
from .converter import convert_file
|
|
7
|
+
from .config import load_sync_config
|
|
8
|
+
from .sync import sync_file
|
|
9
|
+
from .schema import load_config
|
|
10
|
+
from .report import write_error_report, write_sync_report
|
|
11
|
+
from .dependencies import dependency_order, resolve_overrides
|
|
12
|
+
from .doctor import validate_sync_config
|
|
13
|
+
from . import __version__
|
|
14
|
+
|
|
15
|
+
app = typer.Typer(help="Convert Excel/CSV into validated API-ready data.")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@app.command()
|
|
19
|
+
def doctor(
|
|
20
|
+
config: Path = typer.Argument(..., exists=True),
|
|
21
|
+
):
|
|
22
|
+
"""Validate a sync configuration before running a migration."""
|
|
23
|
+
try:
|
|
24
|
+
problems = validate_sync_config(config)
|
|
25
|
+
except Exception as exc:
|
|
26
|
+
typer.echo(f"ERROR: {exc}")
|
|
27
|
+
raise typer.Exit(code=2)
|
|
28
|
+
if problems:
|
|
29
|
+
typer.echo("Configuration check failed:")
|
|
30
|
+
for problem in problems:
|
|
31
|
+
typer.echo(f"- {problem}")
|
|
32
|
+
raise typer.Exit(code=2)
|
|
33
|
+
typer.echo(f"Excel2API {__version__}: configuration OK")
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@app.callback(invoke_without_command=True)
|
|
37
|
+
def main(version: bool = typer.Option(False, "--version", help="Show version and exit.")):
|
|
38
|
+
if version:
|
|
39
|
+
typer.echo(__version__)
|
|
40
|
+
raise typer.Exit()
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@app.command()
|
|
44
|
+
def convert(
|
|
45
|
+
input_file: Path = typer.Argument(..., exists=True),
|
|
46
|
+
schema: Path = typer.Option(..., "--schema", "-s", exists=True),
|
|
47
|
+
output: Path | None = typer.Option(None, "--output", "-o"),
|
|
48
|
+
):
|
|
49
|
+
"""Validate and convert Excel/CSV data to JSON."""
|
|
50
|
+
records, errors = convert_file(str(input_file), str(schema))
|
|
51
|
+
if output:
|
|
52
|
+
output.write_text(json.dumps(records, indent=2, default=str), encoding="utf-8")
|
|
53
|
+
typer.echo(f"Output written to {output}")
|
|
54
|
+
else:
|
|
55
|
+
typer.echo(json.dumps(records, indent=2, default=str))
|
|
56
|
+
typer.echo(f"Valid records: {len(records)}")
|
|
57
|
+
typer.echo(f"Errors: {len(errors)}")
|
|
58
|
+
if errors:
|
|
59
|
+
for error in errors:
|
|
60
|
+
typer.echo(f"Row {error.row} | {error.field} | {error.message}")
|
|
61
|
+
raise typer.Exit(code=1)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@app.command()
|
|
65
|
+
def validate(
|
|
66
|
+
input_file: Path = typer.Argument(..., exists=True),
|
|
67
|
+
schema: Path = typer.Option(..., "--schema", "-s", exists=True),
|
|
68
|
+
):
|
|
69
|
+
"""Validate Excel/CSV data without writing JSON output."""
|
|
70
|
+
records, errors = convert_file(str(input_file), str(schema))
|
|
71
|
+
typer.echo(f"Valid records: {len(records)}")
|
|
72
|
+
typer.echo(f"Errors: {len(errors)}")
|
|
73
|
+
for error in errors:
|
|
74
|
+
typer.echo(f"Row {error.row} | {error.field} | {error.message}")
|
|
75
|
+
if errors:
|
|
76
|
+
raise typer.Exit(code=1)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _parse_headers(items: list[str], configured: dict[str, str]) -> dict[str, str]:
|
|
80
|
+
headers = dict(configured or {})
|
|
81
|
+
for item in items:
|
|
82
|
+
if "=" not in item:
|
|
83
|
+
raise typer.BadParameter("Header must use NAME=VALUE format", param_hint="--header")
|
|
84
|
+
name, value = item.split("=", 1)
|
|
85
|
+
headers[name.strip()] = value
|
|
86
|
+
return headers
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _auth_token(cfg_api: dict, token: str | None) -> str | None:
|
|
90
|
+
if token is not None:
|
|
91
|
+
return token
|
|
92
|
+
auth = cfg_api.get("auth", {}) or {}
|
|
93
|
+
auth_type = str(auth.get("type", "none")).lower()
|
|
94
|
+
if auth_type == "bearer":
|
|
95
|
+
token_env = auth.get("token_env", "EXCEL2API_TOKEN")
|
|
96
|
+
return os.getenv(token_env) or auth.get("token")
|
|
97
|
+
return os.getenv("EXCEL2API_TOKEN")
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
@app.command()
|
|
101
|
+
def sync(
|
|
102
|
+
input_file: Path | None = typer.Argument(None, exists=True),
|
|
103
|
+
schema: Path | None = typer.Option(None, "--schema", "-s", exists=True),
|
|
104
|
+
api: str | None = typer.Option(None, "--api"),
|
|
105
|
+
config: Path | None = typer.Option(None, "--config", "-c", exists=True),
|
|
106
|
+
operation_field: str | None = typer.Option(None, "--operation-field"),
|
|
107
|
+
identifier_field: str | None = typer.Option(None, "--id-field"),
|
|
108
|
+
token: str | None = typer.Option(None, "--token"),
|
|
109
|
+
dry_run: bool | None = typer.Option(None, "--dry-run/--no-dry-run"),
|
|
110
|
+
timeout: int | None = typer.Option(None, "--timeout", min=1),
|
|
111
|
+
retries: int | None = typer.Option(None, "--retries", min=0, max=10),
|
|
112
|
+
retry_create: bool | None = typer.Option(None, "--retry-create/--no-retry-create"),
|
|
113
|
+
report: Path | None = typer.Option(None, "--report"),
|
|
114
|
+
header: list[str] = typer.Option([], "--header"),
|
|
115
|
+
rate_limit: float | None = typer.Option(None, "--rate-limit", min=0.0),
|
|
116
|
+
batch_size: int | None = typer.Option(None, "--batch-size", min=0),
|
|
117
|
+
checkpoint: Path | None = typer.Option(None, "--checkpoint"),
|
|
118
|
+
resume: bool | None = typer.Option(None, "--resume/--no-resume"),
|
|
119
|
+
stop_on_error: bool | None = typer.Option(None, "--stop-on-error/--continue-on-error"),
|
|
120
|
+
excel_report: Path | None = typer.Option(None, "--excel-report"),
|
|
121
|
+
error_report: Path | None = typer.Option(None, "--error-report"),
|
|
122
|
+
):
|
|
123
|
+
"""Execute one or multiple Excel sheets against API endpoints."""
|
|
124
|
+
cfg = load_sync_config(str(config)) if config else {}
|
|
125
|
+
cfg_api = cfg.get("api", {}) or {}
|
|
126
|
+
cfg_sync = cfg.get("sync", {}) or {}
|
|
127
|
+
|
|
128
|
+
input_file = input_file or (Path(cfg["input"]) if cfg.get("input") else None)
|
|
129
|
+
schema = schema or (Path(cfg["schema"]) if cfg.get("schema") else None)
|
|
130
|
+
api = api or cfg_api.get("base_url")
|
|
131
|
+
if not input_file or not input_file.exists():
|
|
132
|
+
raise typer.BadParameter("Input file is required (argument or config.input)")
|
|
133
|
+
if not api and not cfg.get("sheets"):
|
|
134
|
+
raise typer.BadParameter("API URL is required (--api or config.api.base_url)")
|
|
135
|
+
if not schema and not cfg.get("sheets"):
|
|
136
|
+
raise typer.BadParameter("Schema file is required (--schema or config.schema)")
|
|
137
|
+
|
|
138
|
+
operation_field = operation_field or cfg_sync.get("operation_field", "operation")
|
|
139
|
+
identifier_field = identifier_field or cfg_sync.get("identifier_field", "id")
|
|
140
|
+
timeout = timeout if timeout is not None else int(cfg_api.get("timeout", 30))
|
|
141
|
+
retries = retries if retries is not None else int(cfg_api.get("retries", 0))
|
|
142
|
+
retry_create = retry_create if retry_create is not None else bool(cfg_api.get("retry_create", False))
|
|
143
|
+
rate_limit = rate_limit if rate_limit is not None else float(cfg_api.get("rate_limit", 0.0))
|
|
144
|
+
batch_size = batch_size if batch_size is not None else int(cfg_sync.get("batch_size", 0))
|
|
145
|
+
resume = resume if resume is not None else bool(cfg_sync.get("resume", False))
|
|
146
|
+
stop_on_error = stop_on_error if stop_on_error is not None else bool(cfg_sync.get("stop_on_error", False))
|
|
147
|
+
dry_run = dry_run if dry_run is not None else bool(cfg_sync.get("dry_run", False))
|
|
148
|
+
checkpoint = checkpoint or (Path(cfg_sync["checkpoint"]) if cfg_sync.get("checkpoint") else None)
|
|
149
|
+
report = report or (Path(cfg_sync["report"]) if cfg_sync.get("report") else None)
|
|
150
|
+
excel_report = excel_report or (Path(cfg_sync["excel_report"]) if cfg_sync.get("excel_report") else None)
|
|
151
|
+
error_report = error_report or (Path(cfg_sync["error_report"]) if cfg_sync.get("error_report") else None)
|
|
152
|
+
headers = _parse_headers(header, cfg_api.get("headers", {}))
|
|
153
|
+
token = _auth_token(cfg_api, token)
|
|
154
|
+
|
|
155
|
+
sheets = cfg.get("sheets") or {}
|
|
156
|
+
jobs = []
|
|
157
|
+
if sheets:
|
|
158
|
+
for sheet_name, sheet_cfg in sheets.items():
|
|
159
|
+
sheet_cfg = sheet_cfg or {}
|
|
160
|
+
sheet_schema = Path(sheet_cfg["schema"])
|
|
161
|
+
sheet_api = str(api or cfg_api.get("base_url", "")).rstrip("/")
|
|
162
|
+
endpoint = sheet_cfg.get("endpoint")
|
|
163
|
+
if endpoint:
|
|
164
|
+
sheet_api += "/" + str(endpoint).lstrip("/")
|
|
165
|
+
jobs.append((sheet_name, sheet_schema, sheet_api, sheet_cfg))
|
|
166
|
+
else:
|
|
167
|
+
jobs.append((None, schema, api, {}))
|
|
168
|
+
|
|
169
|
+
# Resolve multi-sheet dependencies before execution. Sheets without references
|
|
170
|
+
# retain their configured order; dependencies are topologically sorted.
|
|
171
|
+
if sheets:
|
|
172
|
+
ordered_names = dependency_order(sheets)
|
|
173
|
+
jobs_by_name = {job[0]: job for job in jobs}
|
|
174
|
+
jobs = [jobs_by_name[name] for name in ordered_names]
|
|
175
|
+
for name, _, _, cfg_item in jobs:
|
|
176
|
+
cfg_item["_all_sheets"] = sheets
|
|
177
|
+
|
|
178
|
+
all_results = []
|
|
179
|
+
all_errors = []
|
|
180
|
+
results_by_sheet = {}
|
|
181
|
+
for sheet_name, job_schema, job_api, sheet_cfg in jobs:
|
|
182
|
+
if not job_schema.exists():
|
|
183
|
+
raise typer.BadParameter(f"Schema file not found for sheet '{sheet_name}': {job_schema}")
|
|
184
|
+
job_sync = sheet_cfg.get("sync", {}) or {}
|
|
185
|
+
job_endpoints = sheet_cfg.get("endpoints", cfg_api.get("endpoints", {})) or {}
|
|
186
|
+
job_operation = sheet_cfg.get("operation_field", job_sync.get("operation_field", operation_field))
|
|
187
|
+
job_identifier = sheet_cfg.get("identifier_field", job_sync.get("identifier_field", identifier_field))
|
|
188
|
+
job_response = sheet_cfg.get("response", cfg.get("response", {})) or {}
|
|
189
|
+
job_mapping = job_response.get("mapping", {}) or {}
|
|
190
|
+
job_checkpoint = sheet_cfg.get("checkpoint")
|
|
191
|
+
if job_checkpoint is None:
|
|
192
|
+
job_checkpoint = str(checkpoint) if checkpoint and not sheets else None
|
|
193
|
+
job_resume = bool(sheet_cfg.get("resume", resume))
|
|
194
|
+
job_batch = int(sheet_cfg.get("batch_size", batch_size))
|
|
195
|
+
|
|
196
|
+
overrides = {}
|
|
197
|
+
if sheet_name:
|
|
198
|
+
overrides = resolve_overrides(str(input_file), sheet_name, sheet_cfg, results_by_sheet)
|
|
199
|
+
|
|
200
|
+
results, errors = sync_file(
|
|
201
|
+
str(input_file), str(job_schema), job_api,
|
|
202
|
+
operation_field=job_operation, identifier_field=job_identifier,
|
|
203
|
+
token=token, dry_run=bool(sheet_cfg.get("dry_run", dry_run)),
|
|
204
|
+
timeout=int(sheet_cfg.get("timeout", timeout)), retries=int(sheet_cfg.get("retries", retries)),
|
|
205
|
+
retry_create=bool(sheet_cfg.get("retry_create", retry_create)), headers=headers,
|
|
206
|
+
rate_limit=float(sheet_cfg.get("rate_limit", rate_limit)), batch_size=job_batch,
|
|
207
|
+
checkpoint=job_checkpoint, resume=job_resume,
|
|
208
|
+
stop_on_error=bool(sheet_cfg.get("stop_on_error", stop_on_error)),
|
|
209
|
+
response_mapping=job_mapping, endpoints=job_endpoints, sheet_name=sheet_name,
|
|
210
|
+
record_overrides=overrides,
|
|
211
|
+
)
|
|
212
|
+
for r in results:
|
|
213
|
+
r.sheet = sheet_name
|
|
214
|
+
for e in errors:
|
|
215
|
+
e.sheet = sheet_name
|
|
216
|
+
all_results.extend(results)
|
|
217
|
+
all_errors.extend(errors)
|
|
218
|
+
if sheet_name:
|
|
219
|
+
results_by_sheet[sheet_name] = results
|
|
220
|
+
typer.echo(f"Sheet: {sheet_name or 'default'} | processed: {len(results)} | validation errors: {len(errors)}")
|
|
221
|
+
|
|
222
|
+
if all_errors:
|
|
223
|
+
if error_report:
|
|
224
|
+
write_error_report(error_report, all_errors)
|
|
225
|
+
typer.echo(f"Error report written to {error_report}")
|
|
226
|
+
for error in all_errors:
|
|
227
|
+
typer.echo(f"Sheet {getattr(error, 'sheet', None)} | Row {error.row} | {error.field} | {error.message}")
|
|
228
|
+
raise typer.Exit(code=1)
|
|
229
|
+
|
|
230
|
+
counts, failures = {}, 0
|
|
231
|
+
for result in all_results:
|
|
232
|
+
counts[result.operation] = counts.get(result.operation, 0) + 1
|
|
233
|
+
failures += int(not result.success)
|
|
234
|
+
status = "OK" if result.success else "FAILED"
|
|
235
|
+
detail = f"HTTP {result.status_code}" if result.status_code is not None else result.error or "DRY RUN"
|
|
236
|
+
typer.echo(f"Sheet {getattr(result, 'sheet', None) or 'default'} | Row {result.row} | {result.operation} | {status} | {detail}")
|
|
237
|
+
|
|
238
|
+
typer.echo("\nSummary")
|
|
239
|
+
typer.echo(f"Total: {len(all_results)}")
|
|
240
|
+
for operation, count in sorted(counts.items()):
|
|
241
|
+
typer.echo(f"{operation}: {count}")
|
|
242
|
+
typer.echo(f"Failed: {failures}")
|
|
243
|
+
|
|
244
|
+
if report:
|
|
245
|
+
data = [{"sheet": getattr(r, "sheet", None), "row": r.row, "operation": r.operation,
|
|
246
|
+
"success": r.success, "status_code": r.status_code, "response": r.response,
|
|
247
|
+
"error": r.error} for r in all_results]
|
|
248
|
+
report.write_text(json.dumps({"results": data, "summary": {"total": len(data), "failed": failures}}, indent=2, default=str), encoding="utf-8")
|
|
249
|
+
typer.echo(f"Report written to {report}")
|
|
250
|
+
if excel_report:
|
|
251
|
+
# Response mappings can differ per sheet; use a generic combined report.
|
|
252
|
+
write_sync_report(excel_report, all_results)
|
|
253
|
+
typer.echo(f"Excel report written to {excel_report}")
|
|
254
|
+
if failures:
|
|
255
|
+
raise typer.Exit(code=1)
|
excel2api/config.py
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
import yaml
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def load_sync_config(path: str | Path) -> dict:
|
|
6
|
+
"""Load and validate a sync configuration file."""
|
|
7
|
+
with open(path, "r", encoding="utf-8") as file:
|
|
8
|
+
data = yaml.safe_load(file) or {}
|
|
9
|
+
|
|
10
|
+
if not isinstance(data, dict):
|
|
11
|
+
raise ValueError("Sync config must be a YAML mapping")
|
|
12
|
+
|
|
13
|
+
for section in ("api", "sync", "response"):
|
|
14
|
+
value = data.get(section, {}) or {}
|
|
15
|
+
if not isinstance(value, dict):
|
|
16
|
+
raise ValueError(f"{section} section must be a mapping")
|
|
17
|
+
|
|
18
|
+
sheets = data.get("sheets", {}) or {}
|
|
19
|
+
if sheets and not isinstance(sheets, dict):
|
|
20
|
+
raise ValueError("sheets section must be a mapping")
|
|
21
|
+
for name, spec in sheets.items():
|
|
22
|
+
if not isinstance(spec, dict):
|
|
23
|
+
raise ValueError(f"Sheet '{name}' configuration must be a mapping")
|
|
24
|
+
if not spec.get("schema"):
|
|
25
|
+
raise ValueError(f"Sheet '{name}' must define a schema")
|
|
26
|
+
refs = spec.get("references", []) or []
|
|
27
|
+
if not isinstance(refs, list):
|
|
28
|
+
raise ValueError(f"Sheet '{name}' references must be a list")
|
|
29
|
+
|
|
30
|
+
return data
|
excel2api/converter.py
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
from .reader import read_file
|
|
2
|
+
from .schema import load_schema
|
|
3
|
+
from .validator import validate_and_convert
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def convert_file(input_path: str, schema_path: str, sheet_name=None):
|
|
7
|
+
rows = read_file(input_path, sheet_name=sheet_name)
|
|
8
|
+
fields = load_schema(schema_path)
|
|
9
|
+
|
|
10
|
+
records = []
|
|
11
|
+
errors = []
|
|
12
|
+
for row_number, row in enumerate(rows, start=2):
|
|
13
|
+
record, row_errors = validate_and_convert(row, fields, row_number)
|
|
14
|
+
if row_errors:
|
|
15
|
+
errors.extend(row_errors)
|
|
16
|
+
else:
|
|
17
|
+
records.append(record)
|
|
18
|
+
return records, errors
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from .report import extract_path
|
|
7
|
+
from .schema import load_schema
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass(frozen=True)
|
|
11
|
+
class ReferenceSpec:
|
|
12
|
+
target_field: str
|
|
13
|
+
source_sheet: str
|
|
14
|
+
source_key: str
|
|
15
|
+
target_key: str
|
|
16
|
+
source_value: str
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def parse_references(sheet_cfg: dict[str, Any]) -> list[ReferenceSpec]:
|
|
20
|
+
refs = sheet_cfg.get("references", []) or []
|
|
21
|
+
if not isinstance(refs, list):
|
|
22
|
+
raise ValueError("references must be a list")
|
|
23
|
+
result = []
|
|
24
|
+
for ref in refs:
|
|
25
|
+
if not isinstance(ref, dict):
|
|
26
|
+
raise ValueError("each reference must be a mapping")
|
|
27
|
+
required = ("target_field", "source_sheet", "source_key", "target_key", "source_value")
|
|
28
|
+
missing = [key for key in required if not ref.get(key)]
|
|
29
|
+
if missing:
|
|
30
|
+
raise ValueError(f"reference missing required fields: {', '.join(missing)}")
|
|
31
|
+
result.append(ReferenceSpec(**{key: ref[key] for key in required}))
|
|
32
|
+
return result
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def dependency_order(sheets: dict[str, dict[str, Any]]) -> list[str]:
|
|
36
|
+
graph = {name: set() for name in sheets}
|
|
37
|
+
for name, cfg in sheets.items():
|
|
38
|
+
for ref in parse_references(cfg or {}):
|
|
39
|
+
if ref.source_sheet not in sheets:
|
|
40
|
+
raise ValueError(f"Sheet '{name}' references unknown sheet '{ref.source_sheet}'")
|
|
41
|
+
if ref.source_sheet == name:
|
|
42
|
+
raise ValueError(f"Sheet '{name}' cannot depend on itself")
|
|
43
|
+
graph[name].add(ref.source_sheet)
|
|
44
|
+
|
|
45
|
+
ordered = []
|
|
46
|
+
remaining = {name: set(deps) for name, deps in graph.items()}
|
|
47
|
+
while remaining:
|
|
48
|
+
ready = sorted(name for name, deps in remaining.items() if not deps)
|
|
49
|
+
if not ready:
|
|
50
|
+
cycle = ", ".join(sorted(remaining))
|
|
51
|
+
raise ValueError(f"Circular sheet dependency detected involving: {cycle}")
|
|
52
|
+
ordered.extend(ready)
|
|
53
|
+
for name in ready:
|
|
54
|
+
remaining.pop(name)
|
|
55
|
+
for deps in remaining.values():
|
|
56
|
+
deps.difference_update(ready)
|
|
57
|
+
return ordered
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def build_reference_indexes(
|
|
61
|
+
sheet_name: str,
|
|
62
|
+
results: list[Any],
|
|
63
|
+
response_mapping: dict[str, str],
|
|
64
|
+
) -> dict[str, dict[Any, Any]]:
|
|
65
|
+
indexes: dict[str, dict[Any, Any]] = {}
|
|
66
|
+
for result in results:
|
|
67
|
+
if not result.success or not result.record:
|
|
68
|
+
continue
|
|
69
|
+
for key_name in {
|
|
70
|
+
key for key in result.record.keys()
|
|
71
|
+
}:
|
|
72
|
+
indexes.setdefault(key_name, {})[result.record.get(key_name)] = result.response
|
|
73
|
+
return indexes
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def resolve_overrides(
|
|
77
|
+
input_path: str,
|
|
78
|
+
sheet_name: str,
|
|
79
|
+
sheet_cfg: dict[str, Any],
|
|
80
|
+
results_by_sheet: dict[str, list[Any]],
|
|
81
|
+
) -> dict[int, dict[str, Any]]:
|
|
82
|
+
"""Build row-numbered target-field overrides from completed dependency sheets."""
|
|
83
|
+
refs = parse_references(sheet_cfg)
|
|
84
|
+
if not refs:
|
|
85
|
+
return {}
|
|
86
|
+
|
|
87
|
+
target_fields = {f.name: f.column for f in load_schema(sheet_cfg["schema"])}
|
|
88
|
+
all_rows = __import__("excel2api.reader", fromlist=["read_file"]).read_file(input_path, sheet_name=sheet_name)
|
|
89
|
+
overrides: dict[int, dict[str, Any]] = {}
|
|
90
|
+
|
|
91
|
+
for ref in refs:
|
|
92
|
+
if ref.target_field not in target_fields:
|
|
93
|
+
raise ValueError(f"Reference target field '{ref.target_field}' is not in sheet '{sheet_name}' schema")
|
|
94
|
+
source_results = results_by_sheet.get(ref.source_sheet)
|
|
95
|
+
if source_results is None:
|
|
96
|
+
raise ValueError(f"Dependency '{ref.source_sheet}' has not been processed")
|
|
97
|
+
|
|
98
|
+
source_map: dict[Any, Any] = {}
|
|
99
|
+
source_mapping = {}
|
|
100
|
+
# Read the dependency's configured response mapping if present.
|
|
101
|
+
source_cfg = sheet_cfg.get("_all_sheets", {}).get(ref.source_sheet, {}) if isinstance(sheet_cfg.get("_all_sheets"), dict) else {}
|
|
102
|
+
source_mapping = (source_cfg.get("response", {}) or {}).get("mapping", {}) or {}
|
|
103
|
+
if ref.source_value in source_mapping:
|
|
104
|
+
response_path = source_mapping[ref.source_value]
|
|
105
|
+
else:
|
|
106
|
+
response_path = ref.source_value
|
|
107
|
+
|
|
108
|
+
for result in source_results:
|
|
109
|
+
if not result.success or not result.record:
|
|
110
|
+
continue
|
|
111
|
+
key = result.record.get(ref.source_key)
|
|
112
|
+
value = extract_path(result.response, response_path, None)
|
|
113
|
+
if key is not None and value is not None:
|
|
114
|
+
source_map[key] = value
|
|
115
|
+
|
|
116
|
+
for row_number, raw_row in enumerate(all_rows, start=2):
|
|
117
|
+
target_key = raw_row.get(target_fields.get(ref.target_key, ref.target_key))
|
|
118
|
+
if target_key in source_map:
|
|
119
|
+
overrides.setdefault(row_number, {})[ref.target_field] = source_map[target_key]
|
|
120
|
+
return overrides
|
excel2api/doctor.py
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
from typing import Any
|
|
3
|
+
|
|
4
|
+
from .config import load_sync_config
|
|
5
|
+
from .reader import list_sheets
|
|
6
|
+
from .schema import load_schema
|
|
7
|
+
from .dependencies import dependency_order
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def validate_sync_config(path: str | Path) -> list[str]:
|
|
11
|
+
"""Return configuration problems; an empty list means the config is healthy."""
|
|
12
|
+
config_path = Path(path).resolve()
|
|
13
|
+
cfg = load_sync_config(config_path)
|
|
14
|
+
problems: list[str] = []
|
|
15
|
+
base = config_path.parent
|
|
16
|
+
|
|
17
|
+
def resolve(value: Any) -> Path:
|
|
18
|
+
p = Path(str(value))
|
|
19
|
+
return p if p.is_absolute() else base / p
|
|
20
|
+
|
|
21
|
+
input_value = cfg.get("input")
|
|
22
|
+
if input_value:
|
|
23
|
+
input_path = resolve(input_value)
|
|
24
|
+
if not input_path.exists():
|
|
25
|
+
problems.append(f"Input file not found: {input_path}")
|
|
26
|
+
elif input_path.suffix.lower() not in {".xlsx", ".xls", ".csv"}:
|
|
27
|
+
problems.append(f"Unsupported input format: {input_path.suffix}")
|
|
28
|
+
elif not cfg.get("sheets"):
|
|
29
|
+
problems.append("Missing config.input")
|
|
30
|
+
|
|
31
|
+
api = cfg.get("api", {}) or {}
|
|
32
|
+
if not api.get("base_url") and not cfg.get("sheets"):
|
|
33
|
+
problems.append("Missing api.base_url")
|
|
34
|
+
|
|
35
|
+
sheets = cfg.get("sheets", {}) or {}
|
|
36
|
+
try:
|
|
37
|
+
dependency_order(sheets)
|
|
38
|
+
except ValueError as exc:
|
|
39
|
+
problems.append(str(exc))
|
|
40
|
+
|
|
41
|
+
for sheet_name, spec in sheets.items():
|
|
42
|
+
schema_path = resolve(spec.get("schema", ""))
|
|
43
|
+
if not schema_path.exists():
|
|
44
|
+
problems.append(f"Schema for sheet '{sheet_name}' not found: {schema_path}")
|
|
45
|
+
else:
|
|
46
|
+
try:
|
|
47
|
+
load_schema(schema_path)
|
|
48
|
+
except Exception as exc:
|
|
49
|
+
problems.append(f"Invalid schema for sheet '{sheet_name}': {exc}")
|
|
50
|
+
if input_value and Path(str(input_value)).suffix.lower() in {".xlsx", ".xls"}:
|
|
51
|
+
try:
|
|
52
|
+
names = list_sheets(resolve(input_value))
|
|
53
|
+
if sheet_name not in names:
|
|
54
|
+
problems.append(f"Sheet '{sheet_name}' not found in workbook")
|
|
55
|
+
except Exception as exc:
|
|
56
|
+
problems.append(f"Cannot inspect workbook: {exc}")
|
|
57
|
+
|
|
58
|
+
if not sheets and cfg.get("schema"):
|
|
59
|
+
schema_path = resolve(cfg["schema"])
|
|
60
|
+
if not schema_path.exists():
|
|
61
|
+
problems.append(f"Schema file not found: {schema_path}")
|
|
62
|
+
else:
|
|
63
|
+
try:
|
|
64
|
+
load_schema(schema_path)
|
|
65
|
+
except Exception as exc:
|
|
66
|
+
problems.append(f"Invalid schema: {exc}")
|
|
67
|
+
elif not sheets and not cfg.get("schema"):
|
|
68
|
+
problems.append("Missing config.schema")
|
|
69
|
+
|
|
70
|
+
for operation, spec in (api.get("endpoints", {}) or {}).items():
|
|
71
|
+
if not isinstance(spec, (str, dict)):
|
|
72
|
+
problems.append(f"Endpoint '{operation}' must be a string or mapping")
|
|
73
|
+
continue
|
|
74
|
+
if isinstance(spec, dict) and not spec.get("method"):
|
|
75
|
+
problems.append(f"Endpoint '{operation}' is missing method")
|
|
76
|
+
if isinstance(spec, dict) and "path" not in spec:
|
|
77
|
+
problems.append(f"Endpoint '{operation}' is missing path")
|
|
78
|
+
|
|
79
|
+
return problems
|
excel2api/errors.py
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
@dataclass
|
|
5
|
+
class ValidationError:
|
|
6
|
+
row: int
|
|
7
|
+
field: str
|
|
8
|
+
message: str
|
|
9
|
+
sheet: str | None = None
|
|
10
|
+
|
|
11
|
+
def as_dict(self) -> dict:
|
|
12
|
+
return {
|
|
13
|
+
"row": self.row,
|
|
14
|
+
"field": self.field,
|
|
15
|
+
"message": self.message,
|
|
16
|
+
"sheet": self.sheet,
|
|
17
|
+
}
|
excel2api/reader.py
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
import pandas as pd
|
|
3
|
+
|
|
4
|
+
SUPPORTED_EXTENSIONS = {".xlsx", ".xls", ".csv"}
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def _validate_path(path: str | Path) -> Path:
|
|
8
|
+
path = Path(path)
|
|
9
|
+
if not path.exists():
|
|
10
|
+
raise FileNotFoundError(f"Input file not found: {path}")
|
|
11
|
+
if path.suffix.lower() not in SUPPORTED_EXTENSIONS:
|
|
12
|
+
raise ValueError(
|
|
13
|
+
f"Unsupported file type: {path.suffix}. "
|
|
14
|
+
f"Supported: {', '.join(sorted(SUPPORTED_EXTENSIONS))}"
|
|
15
|
+
)
|
|
16
|
+
return path
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def list_sheets(path: str | Path) -> list[str]:
|
|
20
|
+
path = _validate_path(path)
|
|
21
|
+
if path.suffix.lower() == ".csv":
|
|
22
|
+
return ["CSV"]
|
|
23
|
+
return list(pd.ExcelFile(path).sheet_names)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def read_file(path: str | Path, sheet_name: str | int | None = None) -> list[dict]:
|
|
27
|
+
path = _validate_path(path)
|
|
28
|
+
if path.suffix.lower() == ".csv":
|
|
29
|
+
if sheet_name not in (None, "CSV"):
|
|
30
|
+
raise ValueError("CSV files do not contain multiple sheets")
|
|
31
|
+
frame = pd.read_csv(path)
|
|
32
|
+
else:
|
|
33
|
+
frame = pd.read_excel(path, sheet_name=sheet_name if sheet_name is not None else 0)
|
|
34
|
+
|
|
35
|
+
frame = frame.where(pd.notna(frame), None)
|
|
36
|
+
return frame.to_dict(orient="records")
|