webpage-parser 0.1.0__py3-none-any.whl

webpage_parser/extract.py

@@ -0,0 +1,63 @@
+"""CLI 入口：python3 system/extract.py --input <rows.jsonl> --output <preds.jsonl>
+
+契约：
+  - 输出行数与顺序严格对齐输入
+  - 每行 JSON 对象至少含 title/content/pub_time/author 四个字符串键
+  - 单行异常吞掉并降级为空值；进程退出码恒为 0
+  - 无网络访问；流式读写，绝不整体加载
+"""
+import argparse
+import json
+import sys
+
+try:
+    from .pipeline import process_row, empty_result
+except ImportError:
+    sys.path.insert(0, __file__.rsplit("/", 1)[0])
+    from pipeline import process_row, empty_result
+
+
+def _safe_row(line):
+    try:
+        obj = json.loads(line)
+        return obj if isinstance(obj, dict) else None
+    except Exception:
+        return None
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--input", required=True)
+    ap.add_argument("--output", required=True)
+    args = ap.parse_args()
+
+    with open(args.input, "r", encoding="utf-8", errors="replace") as fin, \
+            open(args.output, "w", encoding="utf-8") as fout:
+        for line in fin:
+            if not line.strip():
+                continue
+            row = _safe_row(line)
+            try:
+                if row is None:
+                    res = empty_result()
+                else:
+                    res = process_row(row)
+            except Exception:
+                res = empty_result()
+            for k in ("title", "content", "pub_time", "author"):
+                v = res.get(k)
+                if not isinstance(v, str):
+                    res[k] = "" if v is None else str(v)
+            try:
+                fout.write(json.dumps(res, ensure_ascii=False) + "\n")
+            except Exception:
+                fout.write(json.dumps(empty_result(), ensure_ascii=False) + "\n")
+    return 0
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except Exception:
+        pass
+    sys.exit(0)