csvai 0.1.0__py3-none-any.whl → 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- csvai/cli.py +17 -0
- csvai/processor.py +87 -5
- csvai/settings.py +1 -1
- csvai/ui.py +15 -0
- csvai-0.2.0.dist-info/METADATA +453 -0
- csvai-0.2.0.dist-info/RECORD +14 -0
- csvai-0.1.0.dist-info/METADATA +0 -751
- csvai-0.1.0.dist-info/RECORD +0 -14
- {csvai-0.1.0.dist-info → csvai-0.2.0.dist-info}/WHEEL +0 -0
- {csvai-0.1.0.dist-info → csvai-0.2.0.dist-info}/entry_points.txt +0 -0
- {csvai-0.1.0.dist-info → csvai-0.2.0.dist-info}/licenses/LICENSE +0 -0
- {csvai-0.1.0.dist-info → csvai-0.2.0.dist-info}/top_level.txt +0 -0
csvai/cli.py
CHANGED
@@ -25,6 +25,20 @@ def main() -> None:
|
|
25
25
|
)
|
26
26
|
parser.add_argument("--limit", type=int, help="Limit number of new rows to process")
|
27
27
|
parser.add_argument("--model", default=settings.default_model, help="Model to use")
|
28
|
+
# Image processing options (Option A)
|
29
|
+
parser.add_argument(
|
30
|
+
"--process-image",
|
31
|
+
action="store_true",
|
32
|
+
help="Enable image processing; attaches image from column (default 'image') or URL",
|
33
|
+
)
|
34
|
+
parser.add_argument(
|
35
|
+
"--image-col",
|
36
|
+
help="Name of the image column (default: 'image')",
|
37
|
+
)
|
38
|
+
parser.add_argument(
|
39
|
+
"--image-root",
|
40
|
+
help="Directory to resolve local image filenames (default: ./images)",
|
41
|
+
)
|
28
42
|
args = parser.parse_args()
|
29
43
|
|
30
44
|
config = ProcessorConfig(
|
@@ -34,6 +48,9 @@ def main() -> None:
|
|
34
48
|
schema=args.schema,
|
35
49
|
limit=args.limit,
|
36
50
|
model=args.model,
|
51
|
+
process_image=args.process_image,
|
52
|
+
image_col=args.image_col,
|
53
|
+
image_root=args.image_root,
|
37
54
|
)
|
38
55
|
processor = CSVAIProcessor(config, settings=settings)
|
39
56
|
|
csvai/processor.py
CHANGED
@@ -7,6 +7,8 @@ import json
|
|
7
7
|
import logging
|
8
8
|
import re
|
9
9
|
from dataclasses import dataclass
|
10
|
+
import base64
|
11
|
+
import mimetypes
|
10
12
|
from pathlib import Path
|
11
13
|
from typing import Any, Dict, Iterable, List, Optional, Set, Tuple
|
12
14
|
|
@@ -101,7 +103,7 @@ def _pick_text_from_response(resp: Any) -> str:
|
|
101
103
|
|
102
104
|
|
103
105
|
async def call_openai_responses(
|
104
|
-
|
106
|
+
input_payload: Any,
|
105
107
|
client: AsyncOpenAI,
|
106
108
|
model: str,
|
107
109
|
schema: Optional[Dict[str, Any]],
|
@@ -123,7 +125,7 @@ async def call_openai_responses(
|
|
123
125
|
text_cfg = {"format": {"type": "json_object"}}
|
124
126
|
resp = await client.responses.create(
|
125
127
|
model=model,
|
126
|
-
input=
|
128
|
+
input=input_payload,
|
127
129
|
temperature=settings.temperature,
|
128
130
|
max_output_tokens=settings.max_output_tokens,
|
129
131
|
text=text_cfg,
|
@@ -217,6 +219,9 @@ async def process_row(
|
|
217
219
|
model: str,
|
218
220
|
schema: Optional[Dict[str, Any]],
|
219
221
|
settings: Settings,
|
222
|
+
process_image: bool,
|
223
|
+
image_col: Optional[str],
|
224
|
+
image_root: Optional[str],
|
220
225
|
) -> RowResult:
|
221
226
|
row_id = (raw_row.get("id") or str(row_idx)).strip()
|
222
227
|
sanitized = sanitize_keys(raw_row)
|
@@ -225,7 +230,70 @@ async def process_row(
|
|
225
230
|
prompt = render_prompt(prompt_template, sanitized, raw_row)
|
226
231
|
except Exception as e:
|
227
232
|
return RowResult(id=row_id, error=f"prompt_error: {e}")
|
228
|
-
|
233
|
+
# Build input for Responses API (text-only by default; multimodal if enabled and available)
|
234
|
+
input_payload: Any
|
235
|
+
|
236
|
+
if process_image:
|
237
|
+
col = (image_col or "image").strip()
|
238
|
+
val = (raw_row.get(col) or "").strip() if col else ""
|
239
|
+
|
240
|
+
def _resolve_image_part(v: str) -> Optional[Dict[str, Any]]:
|
241
|
+
if not v:
|
242
|
+
return None
|
243
|
+
lv = v.lower()
|
244
|
+
if lv.startswith("http://") or lv.startswith("https://"):
|
245
|
+
return {
|
246
|
+
"type": "input_image",
|
247
|
+
"image_url": v,
|
248
|
+
}
|
249
|
+
# Local file resolution: absolute/relative or under image_root
|
250
|
+
p = Path(v)
|
251
|
+
if not p.is_absolute():
|
252
|
+
# try relative to CWD first
|
253
|
+
p = Path.cwd() / v
|
254
|
+
if not p.exists():
|
255
|
+
root = Path(image_root) if image_root else (Path.cwd() / "images")
|
256
|
+
cand = root / v
|
257
|
+
if cand.exists():
|
258
|
+
p = cand
|
259
|
+
if not p.exists():
|
260
|
+
logging.warning("Row %s: image not found '%s' → proceeding text-only", row_id, v)
|
261
|
+
return None
|
262
|
+
try:
|
263
|
+
data = p.read_bytes()
|
264
|
+
b64 = base64.b64encode(data).decode("ascii")
|
265
|
+
mime = mimetypes.guess_type(str(p))[0] or "image/jpeg"
|
266
|
+
data_url = f"data:{mime};base64,{b64}"
|
267
|
+
return {
|
268
|
+
"type": "input_image",
|
269
|
+
"image_url": data_url,
|
270
|
+
}
|
271
|
+
except Exception as e:
|
272
|
+
logging.warning(
|
273
|
+
"Row %s: failed to read image '%s' (%s) → proceeding text-only",
|
274
|
+
row_id,
|
275
|
+
v,
|
276
|
+
e,
|
277
|
+
)
|
278
|
+
return None
|
279
|
+
|
280
|
+
img_part = _resolve_image_part(val)
|
281
|
+
if img_part is not None:
|
282
|
+
input_payload = [
|
283
|
+
{
|
284
|
+
"role": "user",
|
285
|
+
"content": [
|
286
|
+
{"type": "input_text", "text": prompt},
|
287
|
+
img_part,
|
288
|
+
],
|
289
|
+
}
|
290
|
+
]
|
291
|
+
else:
|
292
|
+
input_payload = prompt
|
293
|
+
else:
|
294
|
+
input_payload = prompt
|
295
|
+
|
296
|
+
raw = await call_openai_responses(input_payload, client, model, schema, settings)
|
229
297
|
if not raw:
|
230
298
|
return RowResult(id=row_id, error="api_empty")
|
231
299
|
try:
|
@@ -254,6 +322,10 @@ class ProcessorConfig:
|
|
254
322
|
schema: Optional[str] = None
|
255
323
|
limit: Optional[int] = None
|
256
324
|
model: Optional[str] = None
|
325
|
+
# Image options (Option A)
|
326
|
+
process_image: bool = False
|
327
|
+
image_col: Optional[str] = None # default to 'image' when None
|
328
|
+
image_root: Optional[str] = None # default to CWD/images when None
|
257
329
|
|
258
330
|
|
259
331
|
class CSVAIProcessor:
|
@@ -348,7 +420,18 @@ class CSVAIProcessor:
|
|
348
420
|
|
349
421
|
async def run_one(idx: int, raw: Dict[str, Any]) -> RowResult:
|
350
422
|
async with sem:
|
351
|
-
return await process_row(
|
423
|
+
return await process_row(
|
424
|
+
idx,
|
425
|
+
raw,
|
426
|
+
client,
|
427
|
+
prompt_template,
|
428
|
+
args.model,
|
429
|
+
schema,
|
430
|
+
self.settings,
|
431
|
+
process_image=self.config.process_image,
|
432
|
+
image_col=(self.config.image_col or "image"),
|
433
|
+
image_root=self.config.image_root,
|
434
|
+
)
|
352
435
|
|
353
436
|
results = await asyncio.gather(
|
354
437
|
*(run_one(idx, raw) for idx, raw in batch), return_exceptions=True
|
@@ -417,4 +500,3 @@ class CSVAIProcessor:
|
|
417
500
|
if writer is not None:
|
418
501
|
writer.close()
|
419
502
|
await client.close()
|
420
|
-
|
csvai/settings.py
CHANGED
@@ -22,7 +22,7 @@ class Settings:
|
|
22
22
|
backoff_factor: float = 1.7
|
23
23
|
|
24
24
|
def __post_init__(self) -> None:
|
25
|
-
load_dotenv(find_dotenv())
|
25
|
+
load_dotenv(find_dotenv(usecwd=True), override=True)
|
26
26
|
self.openai_api_key = os.getenv("OPENAI_API_KEY", self.openai_api_key)
|
27
27
|
self.default_model = os.getenv("DEFAULT_MODEL", self.default_model)
|
28
28
|
self.max_output_tokens = int(
|
csvai/ui.py
CHANGED
@@ -80,6 +80,18 @@ schema_file = st.file_uploader("Schema (optional, .json)", type=["json"], key="s
|
|
80
80
|
model = st.text_input("Model", value=settings.default_model, key="model")
|
81
81
|
limit = st.number_input("Row limit (0 = all new)", min_value=0, value=0, step=1, key="limit")
|
82
82
|
|
83
|
+
# Image processing options
|
84
|
+
process_image = st.toggle("Process images", value=False, key="process_image")
|
85
|
+
image_col = None
|
86
|
+
image_root = None
|
87
|
+
if process_image:
|
88
|
+
image_col = st.text_input("Image column", value="image", key="image_col")
|
89
|
+
image_root = st.text_input(
|
90
|
+
"Image root (for local filenames)",
|
91
|
+
value=str(Path.cwd() / "images"),
|
92
|
+
key="image_root",
|
93
|
+
)
|
94
|
+
|
83
95
|
c1, c2 = st.columns(2)
|
84
96
|
with c1:
|
85
97
|
run_clicked = st.button("▶ Run", use_container_width=True, key="run_btn")
|
@@ -148,6 +160,9 @@ if run_clicked:
|
|
148
160
|
schema=str(schema_path) if schema_path else None,
|
149
161
|
limit=int(limit) if limit > 0 else None,
|
150
162
|
model=model,
|
163
|
+
process_image=bool(process_image),
|
164
|
+
image_col=str(image_col) if image_col else None,
|
165
|
+
image_root=str(image_root) if image_root else None,
|
151
166
|
)
|
152
167
|
processor = CSVAIProcessor(cfg, settings=settings)
|
153
168
|
|
@@ -0,0 +1,453 @@
|
|
1
|
+
Metadata-Version: 2.4
|
2
|
+
Name: csvai
|
3
|
+
Version: 0.2.0
|
4
|
+
Summary: Enrich CSV or Excel rows using OpenAI models (text or image analysis).
|
5
|
+
Author: Zyxware Technologies, Vimal Joseph
|
6
|
+
License-Expression: GPL-2.0-only
|
7
|
+
Project-URL: Homepage, https://www.zyxware.com/article/6935/csvai-automate-data-enrichment-any-csv-or-excel-file-generative-ai
|
8
|
+
Project-URL: Bug Tracker, https://github.com/zyxware/csvai/issues
|
9
|
+
Project-URL: Source, https://github.com/zyxware/csvai
|
10
|
+
Project-URL: Contact, https://www.zyxware.com/contact-us
|
11
|
+
Keywords: csv,excel,ai,openai,data enrichment,llm,automation,vision,image analysis
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
13
|
+
Classifier: Operating System :: OS Independent
|
14
|
+
Requires-Python: >=3.9
|
15
|
+
Description-Content-Type: text/markdown
|
16
|
+
License-File: LICENSE
|
17
|
+
Requires-Dist: openai>=1.99.0
|
18
|
+
Requires-Dist: jinja2>=3.1.4
|
19
|
+
Requires-Dist: python-dotenv>=1.0.1
|
20
|
+
Requires-Dist: pandas>=2.0.0
|
21
|
+
Requires-Dist: openpyxl>=3.1.0
|
22
|
+
Provides-Extra: ui
|
23
|
+
Requires-Dist: streamlit; extra == "ui"
|
24
|
+
Dynamic: license-file
|
25
|
+
|
26
|
+
# CSVAI — Apply an AI prompt to each row in a CSV or Excel file and write enriched results
|
27
|
+
|
28
|
+
The `csvai` library reads an input CSV or Excel file, renders a prompt for each row (you can use raw column names like `{{ Address }}`), calls an **OpenAI model via the Responses API**, and writes the original columns plus AI-generated fields to an output CSV or Excel file. It also support **image analysis** (vision) when enabled.
|
29
|
+
|
30
|
+
The tool is **async + concurrent**, **resumable**, and **crash-safe**. It supports **Structured Outputs** with a **JSON Schema** for reliable JSON, or **JSON mode** (without a schema) if you prefer a lighter setup.
|
31
|
+
|
32
|
+
We also have a **CSV AI Prompt Builder** (a Custom GPT) to help you generate prompts and JSON Schemas tailored to your CSVs.
|
33
|
+
|
34
|
+
---
|
35
|
+
|
36
|
+
## Features
|
37
|
+
|
38
|
+
* **Structured Outputs**: enforce exact JSON with a schema for consistent, validated results.
|
39
|
+
* **JSON mode**: force a single JSON object without defining a schema.
|
40
|
+
* **Async & concurrent**: process many rows in parallel for faster throughput.
|
41
|
+
* **Resumable**: rows already written (by `id`) are skipped on re-run.
|
42
|
+
* **CSV or Excel**: handle `.csv` and `.xlsx` inputs and outputs.
|
43
|
+
* **image analysis**: add `--process-image` to attach an image per row (via URL or local file) to multimodal models like `gpt-4o-mini`.
|
44
|
+
|
45
|
+
---
|
46
|
+
|
47
|
+
## Installation
|
48
|
+
|
49
|
+
Requires Python **3.9+**.
|
50
|
+
OpenAI API Key: Create a key - https://platform.openai.com/api-keys and use it in the .env file with OPENAI_API_KEY=
|
51
|
+
See example.env in the [project repo](https://github.com/zyxware/csvai).
|
52
|
+
|
53
|
+
### From PyPI
|
54
|
+
|
55
|
+
```bash
|
56
|
+
pip install csvai
|
57
|
+
# Include Streamlit UI dependencies
|
58
|
+
pip install "csvai[ui]"
|
59
|
+
```
|
60
|
+
|
61
|
+
### From GitHub
|
62
|
+
|
63
|
+
```bash
|
64
|
+
# Install directly from the repository
|
65
|
+
pip install git+https://github.com/zyxware/csvai.git
|
66
|
+
# With Streamlit UI dependencies
|
67
|
+
pip install "csvai[ui] @ git+https://github.com/zyxware/csvai.git"
|
68
|
+
```
|
69
|
+
|
70
|
+
### For local development
|
71
|
+
|
72
|
+
```bash
|
73
|
+
git clone https://github.com/zyxware/csvai
|
74
|
+
cd csvai
|
75
|
+
python -m venv .venv
|
76
|
+
source .venv/bin/activate
|
77
|
+
pip install -r requirements.txt
|
78
|
+
pip install -e .
|
79
|
+
cp example.env .env
|
80
|
+
# Edit .env and set OPENAI_API_KEY=sk-...
|
81
|
+
```
|
82
|
+
|
83
|
+
Installing the package exposes the `csvai` CLI and the `csvai-ui` command.
|
84
|
+
|
85
|
+
---
|
86
|
+
|
87
|
+
## Usage
|
88
|
+
|
89
|
+
### CLI
|
90
|
+
|
91
|
+
#### Auto-discovery
|
92
|
+
|
93
|
+
If you name your files like this:
|
94
|
+
|
95
|
+
```
|
96
|
+
input.csv # or input.xlsx
|
97
|
+
input.prompt.txt
|
98
|
+
input.schema.json # optional
|
99
|
+
```
|
100
|
+
|
101
|
+
Run:
|
102
|
+
|
103
|
+
```bash
|
104
|
+
csvai input.csv # or input.xlsx
|
105
|
+
```
|
106
|
+
|
107
|
+
#### Or specify prompt & schema explicitly
|
108
|
+
|
109
|
+
```bash
|
110
|
+
# With a prompt and a strict schema (best reliability)
|
111
|
+
csvai address.xlsx --prompt address.prompt.txt --schema address.schema.json
|
112
|
+
|
113
|
+
# Or JSON mode (no schema; still a single JSON object)
|
114
|
+
csvai address.xlsx --prompt address.prompt.txt
|
115
|
+
```
|
116
|
+
|
117
|
+
Sample datasets (`address.csv` and `address.xlsx`) with the matching prompt and schema live in the `example/` directory.
|
118
|
+
|
119
|
+
### Streamlit UI
|
120
|
+
|
121
|
+
After installing with the `ui` extra, launch the web interface:
|
122
|
+
|
123
|
+
```bash
|
124
|
+
csvai-ui
|
125
|
+
```
|
126
|
+
|
127
|
+
The UI lets you upload a CSV/Excel file, provide a prompt and optional schema.
|
128
|
+
A "Process images" toggle is available to attach an image per row; you can set the image column (default `image`) and the image root directory (default `./images`).
|
129
|
+
|
130
|
+
---
|
131
|
+
|
132
|
+
## Example Prompt & Schema
|
133
|
+
|
134
|
+
### Prompt (`address.prompt.txt`)
|
135
|
+
|
136
|
+
```text
|
137
|
+
Extract city, state, and country from the given address.
|
138
|
+
|
139
|
+
Rules:
|
140
|
+
- city: city/town/locality (preserve accents, proper case)
|
141
|
+
- state: ISO-standard name of the state/region/province or "" if none
|
142
|
+
- country: ISO 3166 English short name of the country; infer if obvious, else ""
|
143
|
+
- Ignore descriptors like "(EU)"
|
144
|
+
- Do not guess street-level info
|
145
|
+
|
146
|
+
Inputs:
|
147
|
+
Address: {{Address}}
|
148
|
+
```
|
149
|
+
|
150
|
+
### Schema (`address.schema.json`)
|
151
|
+
|
152
|
+
```json
|
153
|
+
{
|
154
|
+
"type": "object",
|
155
|
+
"properties": {
|
156
|
+
"city": {
|
157
|
+
"type": "string",
|
158
|
+
"description": "City, town, or locality name with correct casing and accents preserved"
|
159
|
+
},
|
160
|
+
"state": {
|
161
|
+
"type": "string",
|
162
|
+
"description": "ISO-standard name of the state, region, or province, or empty string if none"
|
163
|
+
},
|
164
|
+
"country": {
|
165
|
+
"type": "string",
|
166
|
+
"description": "ISO 3166 English short name of the country, inferred if obvious, else empty string"
|
167
|
+
}
|
168
|
+
},
|
169
|
+
"required": ["city", "state", "country"],
|
170
|
+
"additionalProperties": false
|
171
|
+
}
|
172
|
+
```
|
173
|
+
---
|
174
|
+
|
175
|
+
## Creating Prompts & Schemas with the CSV AI Prompt Builder
|
176
|
+
|
177
|
+
You can use the **[CSV AI Prompt Builder](https://chat.openai.com/g/g-689d8067bd888191a896d2cfdab27a39-csv-ai-prompt-builder)** custom GPT to:
|
178
|
+
|
179
|
+
* Quickly design a **prompt** tailored to your CSV data.
|
180
|
+
* Generate a **JSON Schema** that matches your desired structured output.
|
181
|
+
|
182
|
+
**Example input to the builder:**
|
183
|
+
|
184
|
+
```
|
185
|
+
File: reviews.csv. Inputs: title,body,stars. Output: sentiment,summary.
|
186
|
+
```
|
187
|
+
|
188
|
+
**Example result:**
|
189
|
+
|
190
|
+
**Prompt**
|
191
|
+
|
192
|
+
```
|
193
|
+
Analyze each review and produce sentiment and a concise summary.
|
194
|
+
|
195
|
+
Rules:
|
196
|
+
- sentiment: one of positive, neutral, negative.
|
197
|
+
- Star mapping: stars ≤ 2 ⇒ negative; 3 ⇒ neutral; ≥ 4 ⇒ positive. If stars is blank or invalid, infer from tone.
|
198
|
+
- summary: 1–2 sentences, factual, include key pros/cons, no emojis, no first person, no marketing fluff.
|
199
|
+
- Use the same language as the Body.
|
200
|
+
- Return only the fields required by the tool schema.
|
201
|
+
|
202
|
+
Inputs:
|
203
|
+
Title: {{title}}
|
204
|
+
Body: {{body}}
|
205
|
+
Stars: {{stars}}
|
206
|
+
```
|
207
|
+
|
208
|
+
**Schema**
|
209
|
+
|
210
|
+
```json
|
211
|
+
{
|
212
|
+
"type": "object",
|
213
|
+
"properties": {
|
214
|
+
"sentiment": {
|
215
|
+
"type": "string",
|
216
|
+
"description": "Overall sentiment derived from stars and/or tone; one of positive, neutral, negative",
|
217
|
+
"enum": ["positive", "neutral", "negative"]
|
218
|
+
},
|
219
|
+
"summary": {
|
220
|
+
"type": "string",
|
221
|
+
"description": "Concise 1–2 sentence summary capturing key pros/cons without opinionated fluff"
|
222
|
+
}
|
223
|
+
},
|
224
|
+
"required": ["sentiment", "summary"],
|
225
|
+
"additionalProperties": false
|
226
|
+
}
|
227
|
+
```
|
228
|
+
|
229
|
+
**Command to execute**
|
230
|
+
|
231
|
+
```bash
|
232
|
+
python -m csvai.cli reviews.csv --prompt reviews.prompt.txt --schema reviews.schema.json
|
233
|
+
```
|
234
|
+
**Tip — have the builder generate a schema for you**
|
235
|
+
* **I have `products.csv` with Product Title, Product Description, Category, and Sub Category. Help me enrich with SEO meta fields.**
|
236
|
+
* **I have `reviews.csv` with Title, Body, and Stars. Help me extract sentiment and generate a short summary.**
|
237
|
+
* **I have `address.csv` with an Address field. Help me extract City, State, and Country using ISO-standard names.**
|
238
|
+
* **I have `tickets.csv` with Subject and Description. Help me classify each ticket into predefined support categories.**
|
239
|
+
* **I have `posts.csv` with Title, Body, URL, Image URL, Brand, and Platform. Help me generate social media captions, hashtags, emojis, CTAs, and alt text.**
|
240
|
+
* **I have `jobs.csv` with Job Title and Description. Help me categorize jobs into sectors and identify the level of seniority.**
|
241
|
+
|
242
|
+
|
243
|
+
---
|
244
|
+
|
245
|
+
## CLI
|
246
|
+
|
247
|
+
```bash
|
248
|
+
csvai INPUT.csv [--prompt PROMPT_FILE] [--output OUTPUT_FILE]
|
249
|
+
[--limit N] [--model MODEL] [--schema SCHEMA_FILE]
|
250
|
+
[--process-image] [--image-col COL] [--image-root DIR]
|
251
|
+
```
|
252
|
+
|
253
|
+
**Flags**
|
254
|
+
|
255
|
+
* `--prompt, -p` — path to a plaintext prompt file (Jinja template).
|
256
|
+
* `--output, -o` — output CSV path (default: `<input>_enriched.csv`).
|
257
|
+
* `--limit` — process only the first `N` new/pending rows.
|
258
|
+
* `--model` — model name (default from `.env`, falls back to `gpt-4o-mini`).
|
259
|
+
* `--schema` — path to a JSON Schema for structured outputs (optional).
|
260
|
+
* `--process-image` — enable image analysis; when set, attaches an image per row if available.
|
261
|
+
* `--image-col` — name of the image column (default: `image`).
|
262
|
+
* `--image-root` — directory to resolve local image filenames (default: `./images`).
|
263
|
+
|
264
|
+
Notes on images:
|
265
|
+
- If the image cell is blank, the row is processed as text-only.
|
266
|
+
- If the cell is a full URL (`http(s)://...`), the model fetches it.
|
267
|
+
- Otherwise the value is treated as a filename: resolved as an absolute/relative path first, then `./images/<filename>`.
|
268
|
+
- If a referenced file is missing/unreadable, the tool logs a warning and proceeds text-only.
|
269
|
+
|
270
|
+
---
|
271
|
+
|
272
|
+
## Environment Variables (`.env`)
|
273
|
+
|
274
|
+
See [`example.env`](example.env) for all configurable variables.
|
275
|
+
|
276
|
+
```ini
|
277
|
+
OPENAI_API_KEY=sk-...
|
278
|
+
DEFAULT_MODEL=gpt-4o-mini
|
279
|
+
MAX_OUTPUT_TOKENS=600
|
280
|
+
TEMPERATURE=0.2
|
281
|
+
MAX_CONCURRENT_REQUESTS=12
|
282
|
+
PROCESSING_BATCH_SIZE=100
|
283
|
+
REQUEST_TIMEOUT=45
|
284
|
+
ALT_PROMPT_SUFFIX=.prompt.txt
|
285
|
+
OUTPUT_FILE_SUFFIX=_enriched.csv
|
286
|
+
```
|
287
|
+
|
288
|
+
---
|
289
|
+
|
290
|
+
## Input/Output Behavior
|
291
|
+
|
292
|
+
* **Input CSV**: the script reads all rows. If an `id` column exists, it’s used to resume. If not, rows are indexed `0..N-1` internally for this run.
|
293
|
+
* **Prompt rendering**: every row is sanitized so `{{ Raw Header }}` becomes `{{ Raw_Header }}`. You can also reference the raw values as `{{ raw["Raw Header"] }}` if needed.
|
294
|
+
* **Output CSV**: contains the original columns plus AI-generated fields. The **header is fixed** after the first successful batch; later rows are written with the same header order.
|
295
|
+
* **Resume**: rerunning skips rows whose `id` is already present in the output file.
|
296
|
+
|
297
|
+
---
|
298
|
+
|
299
|
+
## Image Analysis Example
|
300
|
+
|
301
|
+
Files in `examples/`:
|
302
|
+
|
303
|
+
- `image.csv` — demo rows with an image URL, a local filename, and a blank image.
|
304
|
+
- `image.prompt.txt` — prompt to produce a one-sentence `description`.
|
305
|
+
- `image.schema.json` — schema requiring the `description` field.
|
306
|
+
|
307
|
+
Local image (for row 2): place a file at `./images/sample.jpg` (relative to your current working directory). For convenience you can download a sample image, for example:
|
308
|
+
|
309
|
+
```bash
|
310
|
+
mkdir -p images
|
311
|
+
curl -L -o images/sample.jpg https://upload.wikimedia.org/wikipedia/commons/3/3f/JPEG_example_flower.jpg
|
312
|
+
```
|
313
|
+
|
314
|
+
Run the example (multimodal enabled):
|
315
|
+
|
316
|
+
```bash
|
317
|
+
csvai examples/image.csv \
|
318
|
+
--prompt examples/image.prompt.txt \
|
319
|
+
--schema examples/image.schema.json \
|
320
|
+
--process-image
|
321
|
+
```
|
322
|
+
|
323
|
+
Notes:
|
324
|
+
- The image column defaults to `image`; override with `--image-col` if needed.
|
325
|
+
- Local filenames are resolved as-is first; if not found, `./images/<filename>` is tried.
|
326
|
+
- If an image is missing or unreadable, the row is processed as text-only and a warning is logged.
|
327
|
+
|
328
|
+
|
329
|
+
## Structured Outputs vs JSON Mode
|
330
|
+
|
331
|
+
### Structured Outputs (recommended)
|
332
|
+
|
333
|
+
When you pass `--schema`, the request includes:
|
334
|
+
|
335
|
+
```python
|
336
|
+
text={
|
337
|
+
"format": {
|
338
|
+
"type": "json_schema",
|
339
|
+
"name": "row_schema",
|
340
|
+
"schema": schema,
|
341
|
+
"strict": true
|
342
|
+
}
|
343
|
+
}
|
344
|
+
```
|
345
|
+
|
346
|
+
This guarantees the model returns **exactly** the keys/types you expect.
|
347
|
+
|
348
|
+
### JSON Mode (no schema)
|
349
|
+
|
350
|
+
When no schema is provided, the request includes:
|
351
|
+
|
352
|
+
```python
|
353
|
+
text={"format": {"type": "json_object"}}
|
354
|
+
```
|
355
|
+
|
356
|
+
The model must still return a single JSON object, but no exact schema is enforced.
|
357
|
+
|
358
|
+
> **Prompting tip:** mention the word **JSON** in your prompt and explicitly list the expected fields to improve compliance in JSON mode.
|
359
|
+
|
360
|
+
---
|
361
|
+
|
362
|
+
## Performance & Concurrency
|
363
|
+
|
364
|
+
* Concurrency is controlled by `MAX_CONCURRENT_REQUESTS`.
|
365
|
+
* Increase gradually; too high can trigger API rate limits.
|
366
|
+
* `PROCESSING_BATCH_SIZE` controls how many results are written per batch.
|
367
|
+
* `REQUEST_TIMEOUT` guards slow requests; the script retries with backoff.
|
368
|
+
|
369
|
+
---
|
370
|
+
|
371
|
+
## Troubleshooting
|
372
|
+
|
373
|
+
**“Missing required parameter: `text.format.name`”**
|
374
|
+
You used structured outputs but didn’t include `name` **alongside** `type` and `schema`. The script already sends this correctly; ensure you’re on the latest version and that `--schema` points to the right file.
|
375
|
+
|
376
|
+
**“Invalid schema … `required` must include every key”**
|
377
|
+
The Responses structured-outputs path expects `required` to include **all** keys in `properties`. Either (a) add them all to `required`, (b) remove non-required keys from `properties`, or (c) use JSON mode.
|
378
|
+
|
379
|
+
**Rows not resuming**
|
380
|
+
Ensure there’s an `id` column in both input and output. If not present, the script uses positional IDs for the current run only.
|
381
|
+
|
382
|
+
---
|
383
|
+
|
384
|
+
## FAQ
|
385
|
+
|
386
|
+
**Q: Does it run concurrently?**
|
387
|
+
Yes. Concurrency is controlled via `MAX_CONCURRENT_REQUESTS` (default 10).
|
388
|
+
|
389
|
+
**Q: Can I rely on an `id` column?**
|
390
|
+
Yes. If present in the input CSV, it’s used for resumability. Otherwise rows are indexed for the session.
|
391
|
+
|
392
|
+
**Q: Can I output nested JSON?**
|
393
|
+
The schema can be nested, but CSV is flat. If you want nested data, extend the script with a flattener (e.g., convert `address.street` → `address_street`).
|
394
|
+
|
395
|
+
**Q: Which models work?**
|
396
|
+
Recent `gpt-4o*` models support Responses + Structured Outputs. If a model doesn’t support it, use JSON mode.
|
397
|
+
|
398
|
+
**Q: Do I need a JSON schema?**
|
399
|
+
No, but it’s strongly recommended for stable columns and fewer parse failures.
|
400
|
+
|
401
|
+
---
|
402
|
+
|
403
|
+
## Support
|
404
|
+
|
405
|
+
This application was developed as an internal tool and we will continue to improve and optimize it as long as we use it. If you would like us to customize this or build a similar or related system to automate your tasks with AI, we are available for **commercial support**.
|
406
|
+
|
407
|
+
---
|
408
|
+
|
409
|
+
### About Zyxware Technologies
|
410
|
+
|
411
|
+
At **Zyxware Technologies**, our mission is to help organizations harness the power of technology to solve real-world problems. Guided by our founding values of honesty and fairness, we are committed to delivering genuine value to our clients and the free and open-source community.
|
412
|
+
|
413
|
+
**CSVAI** is a direct result of this philosophy. We originally developed it to automate and streamline our own internal data-enrichment tasks. Realizing its potential to help others, we are sharing it as a free tool in the spirit of our commitment to Free Software.
|
414
|
+
|
415
|
+
Our expertise is centered around our **AI & Automation Services**. We specialize in building intelligent solutions that reduce manual effort, streamline business operations, and unlock data-driven insights. While we provide powerful free tools like this one, we also offer **custom development and commercial support** for businesses that require tailored AI solutions.
|
416
|
+
|
417
|
+
If you're looking to automate a unique business process or build a similar system, we invite you to [**reach out to us**](https://www.zyxware.com/contact-us) to schedule a free discovery call.
|
418
|
+
|
419
|
+
---
|
420
|
+
|
421
|
+
## Updates
|
422
|
+
|
423
|
+
For updates and new versions, visit: [Project Page @ Zyxware](https://www.zyxware.com/article/6935/csvai-automate-data-enrichment-any-csv-or-excel-file-generative-ai)
|
424
|
+
|
425
|
+
---
|
426
|
+
|
427
|
+
## Contact
|
428
|
+
|
429
|
+
[https://www.zyxware.com/contact-us](https://www.zyxware.com/contact-us)
|
430
|
+
|
431
|
+
---
|
432
|
+
|
433
|
+
## Source Repository
|
434
|
+
|
435
|
+
[https://github.com/zyxware/csvai](https://github.com/zyxware/csvai)
|
436
|
+
|
437
|
+
---
|
438
|
+
|
439
|
+
## Reporting Issues
|
440
|
+
|
441
|
+
[https://github.com/zyxware/csvai/issues](https://github.com/zyxware/csvai/issues)
|
442
|
+
|
443
|
+
---
|
444
|
+
|
445
|
+
## License and Disclaimer
|
446
|
+
|
447
|
+
**GPL v2** – Free to use & modify. Use it at your own risk. We are not collecting any user data.
|
448
|
+
|
449
|
+
---
|
450
|
+
|
451
|
+
## Need Help or Commercial Support?
|
452
|
+
|
453
|
+
If you have any questions, feel free to [contact us](https://www.zyxware.com/contact-us).
|
@@ -0,0 +1,14 @@
|
|
1
|
+
csvai/__init__.py,sha256=e-mhAI44n21eES-x_va8ek_V-kGjq66Av6qezPnEu7s,238
|
2
|
+
csvai/__main__.py,sha256=n2wYQ4W8MNAIZ8jldmLYMqNGc-fPtRTasUfK2l_TBHM,92
|
3
|
+
csvai/cli.py,sha256=g8rGMGShjs-ND24rrSds1zg6DgTO5t5jsldEmB0cuAw,2258
|
4
|
+
csvai/io_utils.py,sha256=zVQtJlDp3C-qCrMmgV721TrFAPaMMJfBvMMXYkdMAsE,4499
|
5
|
+
csvai/launch.py,sha256=vScv6J1wyp-VSU1z5ghauINODgQGPt-9sd0uPTwkHKU,356
|
6
|
+
csvai/processor.py,sha256=l4QvqkxSQo6s6nnQlLleQzd00DbxVYY8M6v2PyYaTd0,17516
|
7
|
+
csvai/settings.py,sha256=-Jf7z90xugPCkK39NzHbVrMKQFmCpqQJ2XXSccm8gWY,1819
|
8
|
+
csvai/ui.py,sha256=7NpJ4blUnBqM-gwP7ecBq566DWnbaqRuimmTox-435I,9003
|
9
|
+
csvai-0.2.0.dist-info/licenses/LICENSE,sha256=gXf5dRMhNSbfLPYYTY_5hsZ1r7UU1OaKQEAQUhuIBkM,18092
|
10
|
+
csvai-0.2.0.dist-info/METADATA,sha256=ywVQi2q7cLxpAKZYK9CUMKSC-pEiAwM5pGGBkLUR-PI,15579
|
11
|
+
csvai-0.2.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
12
|
+
csvai-0.2.0.dist-info/entry_points.txt,sha256=FPtxQaCmYAivxE5mCR7xZ4qRHql0WkR5zFGsa6uzNtU,70
|
13
|
+
csvai-0.2.0.dist-info/top_level.txt,sha256=5zoOsPtoSVx3vJiHHobEuEe2oFoEBpX3w6FIvLLU0xg,6
|
14
|
+
csvai-0.2.0.dist-info/RECORD,,
|