seed-data 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- doc_gen_agent/__init__.py +24 -0
- doc_gen_agent/__main__.py +104 -0
- doc_gen_agent/augment.py +258 -0
- doc_gen_agent/batch.py +213 -0
- doc_gen_agent/cli.py +27 -0
- doc_gen_agent/critique.py +147 -0
- doc_gen_agent/nodes.py +53 -0
- doc_gen_agent/orchestrate.py +447 -0
- doc_gen_agent/packet.py +785 -0
- doc_gen_agent/packets/insurance-claim-packet/packet.json +34 -0
- doc_gen_agent/packets/lending-package/packet.json +78 -0
- doc_gen_agent/prompts/__init__.py +17 -0
- doc_gen_agent/prompts/aug_critic.j2 +37 -0
- doc_gen_agent/prompts/augmentor.j2 +51 -0
- doc_gen_agent/prompts/critic.j2 +14 -0
- doc_gen_agent/prompts/critic_vision.j2 +94 -0
- doc_gen_agent/prompts/data_critic.j2 +21 -0
- doc_gen_agent/prompts/data_generator.j2 +55 -0
- doc_gen_agent/prompts/doc_generator.j2 +102 -0
- doc_gen_agent/prompts/doc_generator_html.j2 +123 -0
- doc_gen_agent/schemas/bank-statement/generation_guidance.md +53 -0
- doc_gen_agent/schemas/bank-statement/schema.json +116 -0
- doc_gen_agent/schemas/cable-bill/generation_guidance.md +90 -0
- doc_gen_agent/schemas/cable-bill/samples/README.md +37 -0
- doc_gen_agent/schemas/cable-bill/schema.json +201 -0
- doc_gen_agent/schemas/commercial-lease/generation_guidance.md +45 -0
- doc_gen_agent/schemas/commercial-lease/schema.json +89 -0
- doc_gen_agent/schemas/credit-report/generation_guidance.md +60 -0
- doc_gen_agent/schemas/credit-report/schema.json +218 -0
- doc_gen_agent/schemas/employment-verification-letter/generation_guidance.md +44 -0
- doc_gen_agent/schemas/employment-verification-letter/schema.json +129 -0
- doc_gen_agent/schemas/fcc-invoice/generation_guidance.md +61 -0
- doc_gen_agent/schemas/fcc-invoice/samples/README.md +37 -0
- doc_gen_agent/schemas/fcc-invoice/schema.json +182 -0
- doc_gen_agent/schemas/homeowners-insurance-declaration/generation_guidance.md +63 -0
- doc_gen_agent/schemas/homeowners-insurance-declaration/schema.json +160 -0
- doc_gen_agent/schemas/insurance-claim/generation_guidance.md +38 -0
- doc_gen_agent/schemas/insurance-claim/schema.json +115 -0
- doc_gen_agent/schemas/invoice/generation_guidance.md +55 -0
- doc_gen_agent/schemas/invoice/schema.json +161 -0
- doc_gen_agent/schemas/loan-application/generation_guidance.md +59 -0
- doc_gen_agent/schemas/loan-application/schema.json +195 -0
- doc_gen_agent/schemas/medical-discharge/generation_guidance.md +32 -0
- doc_gen_agent/schemas/medical-discharge/schema.json +100 -0
- doc_gen_agent/schemas/pay-stub/generation_guidance.md +57 -0
- doc_gen_agent/schemas/pay-stub/schema.json +166 -0
- doc_gen_agent/schemas/police-report/generation_guidance.md +36 -0
- doc_gen_agent/schemas/police-report/schema.json +148 -0
- doc_gen_agent/schemas/property-appraisal/generation_guidance.md +59 -0
- doc_gen_agent/schemas/property-appraisal/schema.json +185 -0
- doc_gen_agent/schemas/statement-of-work/generation_guidance.md +126 -0
- doc_gen_agent/schemas/statement-of-work/schema.json +154 -0
- doc_gen_agent/schemas/title-report/generation_guidance.md +59 -0
- doc_gen_agent/schemas/title-report/schema.json +189 -0
- doc_gen_agent/schemas/w2/generation_guidance.md +51 -0
- doc_gen_agent/schemas/w2/schema.json +156 -0
- doc_gen_agent/session.py +12 -0
- doc_gen_agent/tools.py +127 -0
- doc_gen_agent/utils.py +91 -0
- seed_data-0.0.1.dist-info/METADATA +492 -0
- seed_data-0.0.1.dist-info/RECORD +66 -0
- seed_data-0.0.1.dist-info/WHEEL +5 -0
- seed_data-0.0.1.dist-info/entry_points.txt +2 -0
- seed_data-0.0.1.dist-info/licenses/LICENSE +175 -0
- seed_data-0.0.1.dist-info/licenses/NOTICE +1 -0
- seed_data-0.0.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""doc-gen-agent — AI-powered document generation pipeline."""
|
|
2
|
+
|
|
3
|
+
MODELS = {
|
|
4
|
+
"haiku": {"model_id": "us.anthropic.claude-haiku-4-5-20251001-v1:0", "max_tokens": 63999},
|
|
5
|
+
"sonnet": {"model_id": "us.anthropic.claude-sonnet-4-6", "max_tokens": 63999},
|
|
6
|
+
"opus": {"model_id": "us.anthropic.claude-opus-4-6-v1", "max_tokens": 63999},
|
|
7
|
+
"nova-pro": {"model_id": "us.amazon.nova-pro-v1:0", "max_tokens": 16383},
|
|
8
|
+
"nova-lite": {"model_id": "us.amazon.nova-lite-v1:0", "max_tokens": 16383},
|
|
9
|
+
"nova2-lite": {"model_id": "us.amazon.nova-2-lite-v1:0", "max_tokens": 63999},
|
|
10
|
+
"nova2-pro": {"model_id": "us.amazon.nova-pro-v2:0", "max_tokens": 63999},
|
|
11
|
+
"nova-premier": {"model_id": "us.amazon.nova-premier-v1:0", "max_tokens": 16383},
|
|
12
|
+
"deepseek-v3": {"model_id": "deepseek.v3.2", "max_tokens": 16384, "streaming": False},
|
|
13
|
+
"qwen3-vl": {"model_id": "qwen.qwen3-vl-235b-a22b", "max_tokens": 32767, "streaming": False},
|
|
14
|
+
"qwen3-coder": {"model_id": "qwen.qwen3-coder-next", "max_tokens": 32767, "streaming": False},
|
|
15
|
+
"qwen3-coder-480b": {"model_id": "qwen.qwen3-coder-480b-a35b-v1:0", "max_tokens": 32767, "streaming": False},
|
|
16
|
+
"qwen3-80b": {"model_id": "qwen.qwen3-next-80b-a3b", "max_tokens": 32767, "streaming": False},
|
|
17
|
+
"gpt-oss": {"model_id": "openai.gpt-oss-120b-1:0", "max_tokens": 16383, "streaming": False},
|
|
18
|
+
"gpt-oss-20b": {"model_id": "openai.gpt-oss-20b-1:0", "max_tokens": 16383, "streaming": False},
|
|
19
|
+
"gpt-oss-sg": {"model_id": "openai.gpt-oss-safeguard-120b", "max_tokens": 16383, "streaming": False},
|
|
20
|
+
"gpt-oss-sg-20b": {"model_id": "openai.gpt-oss-safeguard-20b", "max_tokens": 16383, "streaming": False},
|
|
21
|
+
"nemotron": {"model_id": "nvidia.nemotron-nano-12b-v2", "max_tokens": 16383, "streaming": False},
|
|
22
|
+
"nemotron-super": {"model_id": "nvidia.nemotron-super-3-120b", "max_tokens": 16383, "streaming": False},
|
|
23
|
+
"nemotron-nano-30b": {"model_id": "nvidia.nemotron-nano-3-30b", "max_tokens": 16383, "streaming": False},
|
|
24
|
+
}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
"""CLI entrypoint: python -m doc_gen_agent"""
|
|
2
|
+
import argparse
|
|
3
|
+
import json
|
|
4
|
+
import sys
|
|
5
|
+
import time
|
|
6
|
+
|
|
7
|
+
from dotenv import load_dotenv
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def main():
|
|
11
|
+
load_dotenv()
|
|
12
|
+
|
|
13
|
+
from doc_gen_agent import MODELS
|
|
14
|
+
from doc_gen_agent.orchestrate import orchestrate
|
|
15
|
+
|
|
16
|
+
model_choices = list(MODELS.keys())
|
|
17
|
+
parser = argparse.ArgumentParser(
|
|
18
|
+
prog="doc-gen-agent",
|
|
19
|
+
description="AI-powered document generation pipeline",
|
|
20
|
+
)
|
|
21
|
+
parser.add_argument("--schema-dir", required=True, help="Schema directory")
|
|
22
|
+
parser.add_argument("--output", default="./output", help="Output directory")
|
|
23
|
+
parser.add_argument("--extra", default=None, help="Extra instructions / brief")
|
|
24
|
+
parser.add_argument("--count", type=int, default=1, help="Number of docs (>1 uses batch agent)")
|
|
25
|
+
parser.add_argument("--batch-name", default=None, help="Batch directory name (default: timestamp)")
|
|
26
|
+
parser.add_argument("--data-model", default="gpt-oss", choices=model_choices)
|
|
27
|
+
parser.add_argument("--doc-model", default="gpt-oss", choices=model_choices)
|
|
28
|
+
parser.add_argument("--critic-model", default="sonnet", choices=model_choices)
|
|
29
|
+
parser.add_argument("--batch-model", default="nova2-lite", choices=model_choices,
|
|
30
|
+
help="Model for batch orchestrator")
|
|
31
|
+
parser.add_argument("--augment", action="store_true")
|
|
32
|
+
parser.add_argument("--no-critic-samples", action="store_true",
|
|
33
|
+
help="Disable reference sample PDFs in doc critic")
|
|
34
|
+
parser.add_argument("--aug-model", default="deepseek-v3", choices=model_choices)
|
|
35
|
+
parser.add_argument("--threshold", type=int, default=5)
|
|
36
|
+
parser.add_argument("--max-attempts", type=int, default=5)
|
|
37
|
+
parser.add_argument("--timeout", type=int, default=3600)
|
|
38
|
+
parser.add_argument("--quiet", action="store_true")
|
|
39
|
+
args = parser.parse_args()
|
|
40
|
+
|
|
41
|
+
if args.count > 1:
|
|
42
|
+
# Batch mode: LLM orchestrator generates diverse documents
|
|
43
|
+
from doc_gen_agent.batch import run_batch
|
|
44
|
+
brief = args.extra or "Generate diverse, realistic documents"
|
|
45
|
+
result = run_batch(
|
|
46
|
+
schema_dir=args.schema_dir,
|
|
47
|
+
count=args.count,
|
|
48
|
+
brief=brief,
|
|
49
|
+
batch_name=args.batch_name,
|
|
50
|
+
output_dir=args.output,
|
|
51
|
+
batch_model=args.batch_model,
|
|
52
|
+
data_model=args.data_model,
|
|
53
|
+
doc_model=args.doc_model,
|
|
54
|
+
critic_model=args.critic_model,
|
|
55
|
+
aug_model=args.aug_model,
|
|
56
|
+
threshold=args.threshold,
|
|
57
|
+
max_attempts=args.max_attempts,
|
|
58
|
+
timeout=args.timeout,
|
|
59
|
+
augment=args.augment,
|
|
60
|
+
critic_samples=not args.no_critic_samples,
|
|
61
|
+
)
|
|
62
|
+
print(f"\n{'=' * 60}")
|
|
63
|
+
print(f"Batch time: {result['elapsed_s']}s")
|
|
64
|
+
else:
|
|
65
|
+
# Single doc mode
|
|
66
|
+
start = time.time()
|
|
67
|
+
result = orchestrate(
|
|
68
|
+
schema_dir=args.schema_dir,
|
|
69
|
+
output_dir=args.output,
|
|
70
|
+
extra_instructions=args.extra,
|
|
71
|
+
data_model=args.data_model,
|
|
72
|
+
doc_model=args.doc_model,
|
|
73
|
+
critic_model=args.critic_model,
|
|
74
|
+
aug_model=args.aug_model,
|
|
75
|
+
threshold=args.threshold,
|
|
76
|
+
max_attempts=args.max_attempts,
|
|
77
|
+
timeout=args.timeout,
|
|
78
|
+
augment=args.augment,
|
|
79
|
+
verbose=not args.quiet,
|
|
80
|
+
critic_samples=not args.no_critic_samples,
|
|
81
|
+
)
|
|
82
|
+
elapsed = time.time() - start
|
|
83
|
+
|
|
84
|
+
print(f"\n{'=' * 60}")
|
|
85
|
+
print(json.dumps(result, indent=2))
|
|
86
|
+
|
|
87
|
+
if result["success"]:
|
|
88
|
+
print(f"\nPDF: {result['path']}")
|
|
89
|
+
if result.get("augmented_path"):
|
|
90
|
+
print(f"Augmented: {result['augmented_path']}")
|
|
91
|
+
print(f"Verdict: {result['verdict']}")
|
|
92
|
+
|
|
93
|
+
usage = result.get("token_usage", {})
|
|
94
|
+
in_tok = usage.get("inputTokens", 0)
|
|
95
|
+
out_tok = usage.get("outputTokens", 0)
|
|
96
|
+
cost = (in_tok * 3 + out_tok * 15) / 1_000_000
|
|
97
|
+
print(f"Tokens: {in_tok + out_tok:,} Cost: ${cost:.4f} Time: {elapsed:.1f}s")
|
|
98
|
+
|
|
99
|
+
if not result["success"]:
|
|
100
|
+
sys.exit(1)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
if __name__ == "__main__":
|
|
104
|
+
main()
|
doc_gen_agent/augment.py
ADDED
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
"""Augraphy augmentation tool wrapper.
|
|
2
|
+
|
|
3
|
+
Handles PDF → PNG → augraphy pipeline → PNG → PDF conversion.
|
|
4
|
+
The LLM agent decides the augmentation config; this module executes it.
|
|
5
|
+
"""
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
|
|
9
|
+
import augraphy
|
|
10
|
+
import numpy as np
|
|
11
|
+
import pypdfium2 as pdfium
|
|
12
|
+
from PIL import Image
|
|
13
|
+
from strands import tool
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
# Registry of available augraphy augmentations the agent can choose from
|
|
18
|
+
AUGMENTATION_REGISTRY = {
|
|
19
|
+
# Ink phase (affects text/ink layer)
|
|
20
|
+
"InkBleed": augraphy.InkBleed,
|
|
21
|
+
"InkColorSwap": augraphy.InkColorSwap,
|
|
22
|
+
"InkMottling": augraphy.InkMottling,
|
|
23
|
+
"InkShifter": augraphy.InkShifter,
|
|
24
|
+
"BleedThrough": augraphy.BleedThrough,
|
|
25
|
+
"Dithering": augraphy.Dithering,
|
|
26
|
+
"Hollow": augraphy.Hollow,
|
|
27
|
+
"Letterpress": augraphy.Letterpress,
|
|
28
|
+
"LinesDegradation": augraphy.LinesDegradation,
|
|
29
|
+
"LowInkPeriodicLines": augraphy.LowInkPeriodicLines,
|
|
30
|
+
"LowInkRandomLines": augraphy.LowInkRandomLines,
|
|
31
|
+
# Paper phase (affects paper/background)
|
|
32
|
+
"BadPhotoCopy": augraphy.BadPhotoCopy,
|
|
33
|
+
"ColorPaper": augraphy.ColorPaper,
|
|
34
|
+
"NoiseTexturize": augraphy.NoiseTexturize,
|
|
35
|
+
"BrightnessTexturize": augraphy.BrightnessTexturize,
|
|
36
|
+
"DelaunayTessellation": augraphy.DelaunayTessellation,
|
|
37
|
+
"VoronoiTessellation": augraphy.VoronoiTessellation,
|
|
38
|
+
# Post phase (affects final output)
|
|
39
|
+
"BadPhotoCopy_post": augraphy.BadPhotoCopy,
|
|
40
|
+
"Brightness": augraphy.Brightness,
|
|
41
|
+
"ColorShift": augraphy.ColorShift,
|
|
42
|
+
"DirtyDrum": augraphy.DirtyDrum,
|
|
43
|
+
"DirtyRollers": augraphy.DirtyRollers,
|
|
44
|
+
"Faxify": augraphy.Faxify,
|
|
45
|
+
"Folding": augraphy.Folding,
|
|
46
|
+
"Gamma": augraphy.Gamma,
|
|
47
|
+
"Geometric": augraphy.Geometric,
|
|
48
|
+
"GlitchEffect": augraphy.GlitchEffect,
|
|
49
|
+
"Jpeg": augraphy.Jpeg,
|
|
50
|
+
"Markup": augraphy.Markup,
|
|
51
|
+
"PageBorder": augraphy.PageBorder,
|
|
52
|
+
"Scribbles": augraphy.Scribbles,
|
|
53
|
+
"ShadowCast": augraphy.ShadowCast,
|
|
54
|
+
"SubtleNoise": augraphy.SubtleNoise,
|
|
55
|
+
"WaterMark": augraphy.WaterMark,
|
|
56
|
+
"BookBinding": augraphy.BookBinding,
|
|
57
|
+
"Stains": augraphy.Stains,
|
|
58
|
+
"DepthSimulatedBlur": augraphy.DepthSimulatedBlur,
|
|
59
|
+
"LightingGradient": augraphy.LightingGradient,
|
|
60
|
+
"Moire": augraphy.Moire,
|
|
61
|
+
"DotMatrix": augraphy.DotMatrix,
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _build_augmentation(name: str, params: dict):
|
|
66
|
+
"""Instantiate an augraphy augmentation by name with given params."""
|
|
67
|
+
cls = AUGMENTATION_REGISTRY.get(name)
|
|
68
|
+
if cls is None:
|
|
69
|
+
raise ValueError(f"Unknown augmentation: {name}. Available: {list(AUGMENTATION_REGISTRY.keys())}")
|
|
70
|
+
# Convert list params to tuples, and ensure integer params stay as ints
|
|
71
|
+
# (JSON parse turns everything to float)
|
|
72
|
+
converted = {}
|
|
73
|
+
for k, v in params.items():
|
|
74
|
+
if isinstance(v, list) and len(v) == 2 and all(isinstance(x, (int, float)) for x in v):
|
|
75
|
+
# If both values are whole numbers, convert to int tuple
|
|
76
|
+
if all(x == int(x) for x in v):
|
|
77
|
+
converted[k] = (int(v[0]), int(v[1]))
|
|
78
|
+
else:
|
|
79
|
+
converted[k] = (v[0], v[1])
|
|
80
|
+
elif isinstance(v, float) and v == int(v):
|
|
81
|
+
converted[k] = int(v)
|
|
82
|
+
else:
|
|
83
|
+
converted[k] = v
|
|
84
|
+
return cls(**converted)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _pdf_to_images(pdf_path: str, dpi: int = 200) -> list[np.ndarray]:
|
|
88
|
+
"""Convert PDF pages to numpy arrays."""
|
|
89
|
+
doc = pdfium.PdfDocument(pdf_path)
|
|
90
|
+
images = []
|
|
91
|
+
for page in doc:
|
|
92
|
+
bitmap = page.render(scale=dpi / 72)
|
|
93
|
+
img = np.asarray(bitmap.to_pil().convert("RGB"))
|
|
94
|
+
images.append(img)
|
|
95
|
+
page.close()
|
|
96
|
+
doc.close()
|
|
97
|
+
return images
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _images_to_pdf(images: list[np.ndarray], output_path: str, dpi: int = 200):
|
|
101
|
+
"""Convert numpy arrays back to a PDF."""
|
|
102
|
+
pages = [Image.fromarray(img) for img in images]
|
|
103
|
+
# resolution embeds DPI so the physical page size matches the source render
|
|
104
|
+
pages[0].save(
|
|
105
|
+
output_path,
|
|
106
|
+
format="PDF",
|
|
107
|
+
save_all=True,
|
|
108
|
+
append_images=pages[1:],
|
|
109
|
+
resolution=float(dpi),
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def run_augmentation(pdf_path: str, config: dict, dpi: int = 150, timeout: int = 90) -> str:
|
|
114
|
+
"""Apply augraphy augmentations to a PDF.
|
|
115
|
+
|
|
116
|
+
Args:
|
|
117
|
+
pdf_path: Path to the input PDF.
|
|
118
|
+
config: Dict with keys "ink_phase", "paper_phase", "post_phase".
|
|
119
|
+
Each is a list of {"name": str, "params": dict}.
|
|
120
|
+
dpi: Resolution for PDF rendering (lower = faster).
|
|
121
|
+
timeout: Max seconds per page augmentation.
|
|
122
|
+
|
|
123
|
+
Returns:
|
|
124
|
+
Path to the augmented PDF.
|
|
125
|
+
"""
|
|
126
|
+
|
|
127
|
+
ink_phase = [_build_augmentation(a["name"], a.get("params", {}))
|
|
128
|
+
for a in config.get("ink_phase", [])]
|
|
129
|
+
paper_phase = [_build_augmentation(a["name"], a.get("params", {}))
|
|
130
|
+
for a in config.get("paper_phase", [])]
|
|
131
|
+
post_phase = [_build_augmentation(a["name"], a.get("params", {}))
|
|
132
|
+
for a in config.get("post_phase", [])]
|
|
133
|
+
|
|
134
|
+
pipeline = augraphy.AugraphyPipeline(
|
|
135
|
+
ink_phase=ink_phase,
|
|
136
|
+
paper_phase=paper_phase,
|
|
137
|
+
post_phase=post_phase,
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
images = _pdf_to_images(pdf_path, dpi=dpi)
|
|
141
|
+
|
|
142
|
+
augmented_images = []
|
|
143
|
+
for i, img in enumerate(images):
|
|
144
|
+
augmented = pipeline(img)
|
|
145
|
+
augmented_images.append(augmented)
|
|
146
|
+
print(f" Page {i+1}: augmented successfully")
|
|
147
|
+
# Explicitly free original image to reduce peak memory
|
|
148
|
+
del img
|
|
149
|
+
|
|
150
|
+
images.clear()
|
|
151
|
+
|
|
152
|
+
base, ext = os.path.splitext(pdf_path)
|
|
153
|
+
output_path = f"{base}_augmented.pdf"
|
|
154
|
+
_images_to_pdf(augmented_images, output_path, dpi=dpi)
|
|
155
|
+
|
|
156
|
+
# Explicitly free augmented images after writing
|
|
157
|
+
augmented_images.clear()
|
|
158
|
+
import gc
|
|
159
|
+
gc.collect()
|
|
160
|
+
|
|
161
|
+
# Clean up augraphy cache
|
|
162
|
+
import shutil as _shutil
|
|
163
|
+
cache_dir = os.path.join(os.getcwd(), "augraphy_cache")
|
|
164
|
+
if os.path.isdir(cache_dir):
|
|
165
|
+
_shutil.rmtree(cache_dir, ignore_errors=True)
|
|
166
|
+
|
|
167
|
+
return output_path
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
@tool
|
|
171
|
+
def apply_augmentation(pdf_path: str, augmentation_config: str) -> dict:
|
|
172
|
+
"""Apply augraphy augmentations to a PDF document.
|
|
173
|
+
|
|
174
|
+
Takes a clean PDF and applies document aging/degradation effects.
|
|
175
|
+
|
|
176
|
+
Args:
|
|
177
|
+
pdf_path: Path to the clean PDF to augment.
|
|
178
|
+
augmentation_config: JSON string with augmentation configuration.
|
|
179
|
+
Must have keys "ink_phase", "paper_phase", "post_phase".
|
|
180
|
+
Each is a list of objects with "name" and "params".
|
|
181
|
+
ONLY use these safe augmentations:
|
|
182
|
+
- ink_phase: InkBleed, LowInkRandomLines, LowInkPeriodicLines, LinesDegradation, Dithering
|
|
183
|
+
- paper_phase: NoiseTexturize, BrightnessTexturize, ColorPaper
|
|
184
|
+
- post_phase: SubtleNoise, Jpeg, Brightness, Gamma, DirtyDrum, DirtyRollers, Faxify, BadPhotoCopy
|
|
185
|
+
|
|
186
|
+
Returns:
|
|
187
|
+
Path to the augmented PDF, or error message.
|
|
188
|
+
"""
|
|
189
|
+
try:
|
|
190
|
+
config = json.loads(augmentation_config)
|
|
191
|
+
|
|
192
|
+
# Log the augmentation chain
|
|
193
|
+
all_augs = []
|
|
194
|
+
for phase in ("ink_phase", "paper_phase", "post_phase"):
|
|
195
|
+
for aug in config.get(phase, []):
|
|
196
|
+
all_augs.append(f" [{phase}] {aug['name']}: {aug.get('params', {})}")
|
|
197
|
+
chain_log = "\n".join(all_augs) if all_augs else " (none)"
|
|
198
|
+
print(f"\nAugmentation chain:\n{chain_log}")
|
|
199
|
+
|
|
200
|
+
output_path = run_augmentation(pdf_path, config)
|
|
201
|
+
if os.path.exists(output_path):
|
|
202
|
+
size = os.path.getsize(output_path)
|
|
203
|
+
# Save the aug config for reproducibility
|
|
204
|
+
config_path = output_path.replace("_augmented.pdf", "_aug_config.json")
|
|
205
|
+
with open(config_path, "w") as f:
|
|
206
|
+
json.dump(config, f, indent=2)
|
|
207
|
+
return {
|
|
208
|
+
"status": "success",
|
|
209
|
+
"content": [{"text": f"Augmented PDF saved to: {output_path} ({size:,} bytes)\nConfig saved to: {config_path}"}],
|
|
210
|
+
}
|
|
211
|
+
else:
|
|
212
|
+
return {
|
|
213
|
+
"status": "error",
|
|
214
|
+
"content": [{"text": "Augmentation completed but output file was not created."}],
|
|
215
|
+
}
|
|
216
|
+
except Exception as e:
|
|
217
|
+
return {
|
|
218
|
+
"status": "error",
|
|
219
|
+
"content": [{"text": f"Augmentation failed: {e}. Try simpler augmentations (SubtleNoise, Jpeg, Brightness)."}],
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def critique_augmented_document(pdf_path: str, model: str = "haiku", threshold: int = 7) -> dict:
|
|
224
|
+
"""Critique an augmented PDF using structured_output."""
|
|
225
|
+
from doc_gen_agent import prompts as p
|
|
226
|
+
from doc_gen_agent.critique import CritiqueResult
|
|
227
|
+
from doc_gen_agent.utils import make_model as _make_model
|
|
228
|
+
from strands import Agent
|
|
229
|
+
|
|
230
|
+
with open(pdf_path, "rb") as f:
|
|
231
|
+
pdf_bytes = f.read()
|
|
232
|
+
|
|
233
|
+
system_prompt = p.render("aug_critic", threshold=threshold)
|
|
234
|
+
agent = Agent(model=_make_model(model), system_prompt=system_prompt)
|
|
235
|
+
|
|
236
|
+
user_message = [
|
|
237
|
+
{"document": {"format": "pdf", "name": "augmented_doc",
|
|
238
|
+
"source": {"bytes": pdf_bytes}}},
|
|
239
|
+
{"text": f"Evaluate this augmented document. Threshold: {threshold}/10."},
|
|
240
|
+
]
|
|
241
|
+
|
|
242
|
+
result = agent(user_message, structured_output_model=CritiqueResult)
|
|
243
|
+
critique = result.structured_output
|
|
244
|
+
verdict = "accepted" if critique.score >= threshold else "rejected"
|
|
245
|
+
|
|
246
|
+
print(f"\n--- Aug Critique ({model}) ---")
|
|
247
|
+
print(f" Score: {critique.score}/10 → {verdict}")
|
|
248
|
+
print(f" Summary: {critique.summary}")
|
|
249
|
+
for issue in critique.issues:
|
|
250
|
+
print(f" [{issue.severity}] ({issue.category}) {issue.description}")
|
|
251
|
+
print("--- End Aug Critique ---\n")
|
|
252
|
+
|
|
253
|
+
return {
|
|
254
|
+
"score": critique.score,
|
|
255
|
+
"verdict": verdict,
|
|
256
|
+
"issues": [i.model_dump() for i in critique.issues],
|
|
257
|
+
"summary": critique.summary,
|
|
258
|
+
}
|
doc_gen_agent/batch.py
ADDED
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
"""Batch orchestrator — LLM agent that generates diverse documents using orchestrate as a tool."""
|
|
2
|
+
import glob
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
import shutil
|
|
6
|
+
import sys
|
|
7
|
+
import time
|
|
8
|
+
from datetime import datetime
|
|
9
|
+
|
|
10
|
+
from strands import Agent, tool
|
|
11
|
+
|
|
12
|
+
from doc_gen_agent.orchestrate import orchestrate
|
|
13
|
+
from doc_gen_agent.utils import make_model
|
|
14
|
+
|
|
15
|
+
_pipeline_config = {}
|
|
16
|
+
_batch_results = []
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@tool
|
|
20
|
+
def generate_document(extra_instructions: str) -> dict:
|
|
21
|
+
"""Generate a single document using the full pipeline.
|
|
22
|
+
|
|
23
|
+
Args:
|
|
24
|
+
extra_instructions: Specific scenario for this document. Be detailed:
|
|
25
|
+
include advertiser name, industry, station/location, time period,
|
|
26
|
+
and any other specifics that make this document unique.
|
|
27
|
+
|
|
28
|
+
Returns:
|
|
29
|
+
Summary of the generation result.
|
|
30
|
+
"""
|
|
31
|
+
cfg = _pipeline_config
|
|
32
|
+
start = time.time()
|
|
33
|
+
|
|
34
|
+
result = orchestrate(
|
|
35
|
+
schema_dir=cfg["schema_dir"],
|
|
36
|
+
output_dir=cfg["batch_dir"],
|
|
37
|
+
extra_instructions=extra_instructions,
|
|
38
|
+
data_model=cfg["data_model"],
|
|
39
|
+
doc_model=cfg["doc_model"],
|
|
40
|
+
critic_model=cfg["critic_model"],
|
|
41
|
+
aug_model=cfg["aug_model"],
|
|
42
|
+
threshold=cfg["threshold"],
|
|
43
|
+
max_attempts=cfg["max_attempts"],
|
|
44
|
+
timeout=cfg["timeout"],
|
|
45
|
+
augment=cfg["augment"],
|
|
46
|
+
critic_samples=cfg.get("critic_samples", True),
|
|
47
|
+
verbose=False,
|
|
48
|
+
)
|
|
49
|
+
elapsed = time.time() - start
|
|
50
|
+
|
|
51
|
+
doc_id = result.get("doc_id", "unknown")
|
|
52
|
+
|
|
53
|
+
# Track result
|
|
54
|
+
usage = result.get("token_usage", {})
|
|
55
|
+
entry = {
|
|
56
|
+
"doc_id": doc_id,
|
|
57
|
+
"success": result["success"],
|
|
58
|
+
"verdict": result.get("verdict", "error"),
|
|
59
|
+
"extra": extra_instructions,
|
|
60
|
+
"pdf": result.get("path"),
|
|
61
|
+
"data_json": result.get("data_json"),
|
|
62
|
+
"augmented": result.get("augmented_path"),
|
|
63
|
+
"time_s": round(elapsed, 1),
|
|
64
|
+
"tokens": usage.get("inputTokens", 0) + usage.get("outputTokens", 0),
|
|
65
|
+
}
|
|
66
|
+
if not result["success"]:
|
|
67
|
+
entry["error"] = result.get("error", "unknown")
|
|
68
|
+
_batch_results.append(entry)
|
|
69
|
+
|
|
70
|
+
status = "✓" if result["success"] else "✗"
|
|
71
|
+
print(f" {status} [{len(_batch_results)}] {extra_instructions[:60]}... ({elapsed:.0f}s)")
|
|
72
|
+
if not result["success"]:
|
|
73
|
+
print(f" ERROR: {result.get('error', 'unknown')[:120]}")
|
|
74
|
+
|
|
75
|
+
# Bail early if first 2 docs both fail (likely a config/auth issue)
|
|
76
|
+
if len(_batch_results) >= 2 and all(not r["success"] for r in _batch_results):
|
|
77
|
+
msg = "First 2 documents failed — likely a configuration issue. Aborting batch."
|
|
78
|
+
print(f"\n ⚠ {msg}")
|
|
79
|
+
return {"status": "error", "content": [{"text": msg}]}
|
|
80
|
+
|
|
81
|
+
# Return concise summary to keep batch agent context small
|
|
82
|
+
summary = {k: entry[k] for k in ("success", "verdict", "doc_id", "time_s")}
|
|
83
|
+
return {"status": "success", "content": [{"text": json.dumps(summary)}]}
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def run_batch(
|
|
87
|
+
schema_dir: str,
|
|
88
|
+
count: int,
|
|
89
|
+
brief: str,
|
|
90
|
+
batch_name: str = None,
|
|
91
|
+
output_dir: str = "./output",
|
|
92
|
+
batch_model: str = "sonnet",
|
|
93
|
+
data_model: str = "sonnet",
|
|
94
|
+
doc_model: str = "sonnet",
|
|
95
|
+
critic_model: str = "haiku",
|
|
96
|
+
aug_model: str = "sonnet",
|
|
97
|
+
threshold: int = 7,
|
|
98
|
+
max_attempts: int = 5,
|
|
99
|
+
timeout: int = 3600,
|
|
100
|
+
augment: bool = False,
|
|
101
|
+
critic_samples: bool = True,
|
|
102
|
+
) -> dict:
|
|
103
|
+
"""Run the batch orchestrator agent.
|
|
104
|
+
|
|
105
|
+
Creates a batch directory under output_dir with subdirectories for
|
|
106
|
+
pdfs/, data/, and augmented/. Writes a batch_manifest.json at the end.
|
|
107
|
+
"""
|
|
108
|
+
global _pipeline_config, _batch_results
|
|
109
|
+
_batch_results = []
|
|
110
|
+
|
|
111
|
+
# Create batch directory
|
|
112
|
+
if not batch_name:
|
|
113
|
+
batch_name = datetime.now().strftime("batch_%Y%m%d_%H%M%S")
|
|
114
|
+
batch_dir = os.path.join(output_dir, batch_name)
|
|
115
|
+
os.makedirs(batch_dir, exist_ok=True)
|
|
116
|
+
|
|
117
|
+
# Copy schema and steering docs into config/ folder for reproducibility
|
|
118
|
+
config_dir = os.path.join(batch_dir, "config")
|
|
119
|
+
os.makedirs(config_dir, exist_ok=True)
|
|
120
|
+
for f in glob.glob(os.path.join(schema_dir, "*.json")) + glob.glob(os.path.join(schema_dir, "*.md")):
|
|
121
|
+
shutil.copy2(f, config_dir)
|
|
122
|
+
|
|
123
|
+
_pipeline_config = {
|
|
124
|
+
"schema_dir": schema_dir,
|
|
125
|
+
"batch_dir": batch_dir,
|
|
126
|
+
"data_model": data_model,
|
|
127
|
+
"doc_model": doc_model,
|
|
128
|
+
"critic_model": critic_model,
|
|
129
|
+
"aug_model": aug_model,
|
|
130
|
+
"threshold": threshold,
|
|
131
|
+
"max_attempts": max_attempts,
|
|
132
|
+
"timeout": timeout,
|
|
133
|
+
"augment": augment,
|
|
134
|
+
"critic_samples": critic_samples,
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
system_prompt = f"""You are a batch document generation orchestrator. Your job is to
|
|
138
|
+
generate {count} diverse, unique documents by calling generate_document repeatedly.
|
|
139
|
+
|
|
140
|
+
RULES:
|
|
141
|
+
- Call generate_document exactly {count} times
|
|
142
|
+
- Each call MUST have completely different extra_instructions
|
|
143
|
+
- Vary: advertiser names, industries, station locations, time periods, spot types
|
|
144
|
+
- Track what you've generated to avoid repetition
|
|
145
|
+
- If a generation fails, note it and continue with the next one
|
|
146
|
+
- After all {count} calls, output a brief summary line: "BATCH COMPLETE: X/Y succeeded"
|
|
147
|
+
- Do NOT explain your plan — just start generating immediately
|
|
148
|
+
"""
|
|
149
|
+
|
|
150
|
+
agent = Agent(
|
|
151
|
+
model=make_model(batch_model),
|
|
152
|
+
system_prompt=system_prompt,
|
|
153
|
+
tools=[generate_document],
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
print(f"Batch: {batch_name}")
|
|
157
|
+
print(f"Count: {count}")
|
|
158
|
+
print(f"Brief: {brief}")
|
|
159
|
+
print(f"Output: {batch_dir}")
|
|
160
|
+
print("=" * 60)
|
|
161
|
+
|
|
162
|
+
batch_start = time.time()
|
|
163
|
+
agent(f"Generate {count} diverse documents. Brief: {brief}")
|
|
164
|
+
batch_elapsed = time.time() - batch_start
|
|
165
|
+
|
|
166
|
+
# Write manifest
|
|
167
|
+
succeeded = sum(1 for r in _batch_results if r["success"])
|
|
168
|
+
total_tokens = sum(r.get("tokens", 0) for r in _batch_results)
|
|
169
|
+
total_cost = total_tokens * 3 / 1_000_000 # rough estimate
|
|
170
|
+
|
|
171
|
+
# Get git hash for reproducibility
|
|
172
|
+
import subprocess as _sp
|
|
173
|
+
try:
|
|
174
|
+
git_hash = _sp.check_output(
|
|
175
|
+
["git", "rev-parse", "HEAD"], stderr=_sp.DEVNULL, text=True
|
|
176
|
+
).strip()
|
|
177
|
+
except Exception:
|
|
178
|
+
git_hash = "unknown"
|
|
179
|
+
|
|
180
|
+
manifest = {
|
|
181
|
+
"batch_name": batch_name,
|
|
182
|
+
"brief": brief,
|
|
183
|
+
"command": " ".join(sys.argv),
|
|
184
|
+
"git_hash": git_hash,
|
|
185
|
+
"count_requested": count,
|
|
186
|
+
"count_succeeded": succeeded,
|
|
187
|
+
"count_failed": len(_batch_results) - succeeded,
|
|
188
|
+
"total_tokens": total_tokens,
|
|
189
|
+
"est_cost_usd": round(total_cost, 4),
|
|
190
|
+
"elapsed_s": round(batch_elapsed, 1),
|
|
191
|
+
"config": {
|
|
192
|
+
"schema_dir": schema_dir,
|
|
193
|
+
"data_model": data_model,
|
|
194
|
+
"doc_model": doc_model,
|
|
195
|
+
"critic_model": critic_model,
|
|
196
|
+
"aug_model": aug_model,
|
|
197
|
+
"threshold": threshold,
|
|
198
|
+
"augment": augment,
|
|
199
|
+
},
|
|
200
|
+
"documents": _batch_results,
|
|
201
|
+
}
|
|
202
|
+
manifest_path = os.path.join(batch_dir, "config", "batch_manifest.json")
|
|
203
|
+
os.makedirs(os.path.dirname(manifest_path), exist_ok=True)
|
|
204
|
+
with open(manifest_path, "w") as f:
|
|
205
|
+
json.dump(manifest, f, indent=2)
|
|
206
|
+
|
|
207
|
+
print(f"\n{'=' * 60}")
|
|
208
|
+
print(f"Batch complete: {succeeded}/{count} succeeded")
|
|
209
|
+
print(f"Manifest: {manifest_path}")
|
|
210
|
+
print(f"Tokens: {total_tokens:,} Est cost: ${total_cost:.4f}")
|
|
211
|
+
print(f"Time: {batch_elapsed:.1f}s ({batch_elapsed/max(count,1):.1f}s avg)")
|
|
212
|
+
|
|
213
|
+
return manifest
|
doc_gen_agent/cli.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"""Shared CLI argument parser for doc-gen-agent scripts."""
|
|
2
|
+
import argparse
|
|
3
|
+
from doc_gen_agent import MODELS
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def base_parser(description="doc-gen-agent"):
|
|
7
|
+
"""Create the base argument parser with common options."""
|
|
8
|
+
model_choices = list(MODELS.keys())
|
|
9
|
+
parser = argparse.ArgumentParser(description=description)
|
|
10
|
+
parser.add_argument("--schema-dir", required=True, help="Schema directory")
|
|
11
|
+
parser.add_argument("--output", default="./output", help="Output directory")
|
|
12
|
+
parser.add_argument("--extra", default=None, help="Extra instructions / brief")
|
|
13
|
+
parser.add_argument("--data-model", default="gpt-oss", choices=model_choices)
|
|
14
|
+
parser.add_argument("--doc-model", default="nova2-lite", choices=model_choices)
|
|
15
|
+
parser.add_argument("--critic-model", default="sonnet", choices=model_choices)
|
|
16
|
+
parser.add_argument("--augment", action="store_true")
|
|
17
|
+
parser.add_argument("--aug-model", default="gpt-oss", choices=model_choices)
|
|
18
|
+
parser.add_argument("--no-critic-samples", action="store_true",
|
|
19
|
+
help="Disable reference sample PDFs in doc critic")
|
|
20
|
+
parser.add_argument("--renderer", default="weasyprint", choices=["reportlab", "weasyprint"],
|
|
21
|
+
help="PDF rendering engine (default: weasyprint)")
|
|
22
|
+
parser.add_argument("--preview", action="store_true",
|
|
23
|
+
help="Enable preview_pdf tool for visual self-inspection (requires VL model)")
|
|
24
|
+
parser.add_argument("--threshold", type=int, default=5)
|
|
25
|
+
parser.add_argument("--max-attempts", type=int, default=5)
|
|
26
|
+
parser.add_argument("--timeout", type=int, default=3600)
|
|
27
|
+
return parser
|