endnote-utils 0.1.0__py3-none-any.whl

endnote-utils/__init__.py

@@ -0,0 +1,14 @@
+from .core import (
+    CSV_QUOTING_MAP,
+    DEFAULT_FIELDNAMES,
+    export,
+    export_files_to_csv_with_report,
+    export_folder,
+)
+
+__all__ = [
+    "export", "export_folder", "export_files_to_csv_with_report",
+    "DEFAULT_FIELDNAMES", "CSV_QUOTING_MAP",
+]
+
+__version__ = "0.1.0"

endnote-utils/cli.py

@@ -0,0 +1,54 @@
+from __future__ import annotations
+
+import argparse
+import logging
+import sys
+from pathlib import Path
+
+from .core import DEFAULT_FIELDNAMES, export, export_folder
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(description="Export EndNote XML (file or folder) to CSV + TXT report.")
+    g = p.add_mutually_exclusive_group(required=True)
+    g.add_argument("--xml", help="Path to a single EndNote XML file.")
+    g.add_argument("--folder", help="Path to a folder containing *.xml files.")
+    p.add_argument("--csv", required=True, help="Path to CSV output file.")
+    p.add_argument("--report", required=False, help="Path to TXT report (default: <csv>_report.txt).")
+    p.add_argument("--delimiter", default=",")
+    p.add_argument("--quoting", default="minimal", choices=["minimal","all","nonnumeric","none"])
+    p.add_argument("--no-header", action="store_true")
+    p.add_argument("--encoding", default="utf-8")
+    p.add_argument("--ref-type", default=None)
+    p.add_argument("--year", default=None)
+    p.add_argument("--max-records", type=int, default=None)
+    p.add_argument("--verbose", action="store_true")
+    return p
+
+def main() -> None:
+    args = build_parser().parse_args()
+    logging.basicConfig(
+        level=logging.DEBUG if args.verbose else logging.INFO,
+        format="%(levelname)s: %(message)s", stream=sys.stderr
+    )
+    csv_path = Path(args.csv)
+    report_path = Path(args.report) if args.report else csv_path.with_name(csv_path.stem + "_report.txt")
+    kwargs = dict(
+        report_path=report_path,
+        fieldnames=DEFAULT_FIELDNAMES,
+        delimiter=args.delimiter,
+        quoting=args.quoting,
+        include_header=not args.no_header,
+        encoding=args.encoding,
+        ref_type=args.ref_type,
+        year=args.year,
+        max_records_per_file=args.max_records,
+    )
+
+    if args.xml:
+        total, csv_out, rep_out = export(Path(args.xml), csv_path, **kwargs)
+    else:
+        total, csv_out, rep_out = export_folder(Path(args.folder), csv_path, **kwargs)
+
+    logging.info("Exported %d record(s) → %s", total, csv_out)
+    logging.info("Report → %s", rep_out)

endnote-utils/core.py

@@ -0,0 +1,209 @@
+# src/endnote_exporter/core.py
+from __future__ import annotations
+
+import csv
+import logging
+import time
+import xml.etree.ElementTree as ET
+from datetime import datetime
+from pathlib import Path
+from typing import Dict, Iterable, List, Optional, Tuple
+
+DEFAULT_FIELDNAMES: List[str] = [
+    "database", "ref_type", "title", "journal", "authors", "year",
+    "volume", "number", "abstract", "doi", "urls", "extracted_date",
+]
+
+CSV_QUOTING_MAP = {
+    "minimal": csv.QUOTE_MINIMAL,
+    "all": csv.QUOTE_ALL,
+    "nonnumeric": csv.QUOTE_NONNUMERIC,
+    "none": csv.QUOTE_NONE,
+}
+
+def ensure_parent_dir(p: Path) -> None:
+    p.parent.mkdir(parents=True, exist_ok=True)
+
+# ----------------------------
+# Utilities
+# ----------------------------
+
+def clean_text(text: Optional[str]) -> str:
+    """
+    Trim, collapse internal whitespace, remove stray CRs, keep punctuation intact.
+    Safer for CSV fields than aggressive normalization.
+    """
+    if not text:
+        return ""
+    text = text.replace("\r", " ")
+    return " ".join(text.split()).strip()
+
+
+def safe_find_text(node: ET.Element, path: str) -> str:
+    """Find text with XPath and return cleaned string."""
+    elem = node.find(path)
+    return clean_text(elem.text) if elem is not None and elem.text is not None else ""
+
+
+def join_nonempty(items: Iterable[str], sep: str) -> str:
+    return sep.join(x for x in (i.strip() for i in items) if x)
+
+def ensure_parent_dir(p: Path) -> None:
+    """Create parent directory if it doesn't exist."""
+    if not p.parent.exists():
+        p.parent.mkdir(parents=True, exist_ok=True)
+
+
+# ----------------------------
+# Record processing
+# ----------------------------
+
+def process_doi(record: ET.Element) -> str:
+    """Extract and format DOI information to a canonical URL if possible."""
+    doi_raw = safe_find_text(record, ".//electronic-resource-num/style")
+    if not doi_raw:
+        return ""
+    if doi_raw.startswith("10."):
+        return f"https://doi.org/{doi_raw}"
+    if doi_raw.startswith(("http://", "https://")):
+        return doi_raw
+    return ""
+
+
+def extract_authors(record: ET.Element) -> str:
+    """Collect authors from //author/style, joined by '; '."""
+    authors: List[str] = []
+    for author in record.findall(".//author"):
+        style = author.find("style")
+        if style is not None and style.text:
+            authors.append(clean_text(style.text))
+    return join_nonempty(authors, "; ")
+
+
+def extract_urls(record: ET.Element) -> str:
+    """Collect related URLs from //urls/related-urls/url/style, joined by ' | '."""
+    urls: List[str] = []
+    for url in record.findall(".//urls/related-urls/url"):
+        style = url.find("style")
+        if style is not None and style.text:
+            urls.append(clean_text(style.text))
+    # Deduplicate while preserving order
+    seen = set()
+    deduped = []
+    for u in urls:
+        if u not in seen:
+            seen.add(u)
+            deduped.append(u)
+    return join_nonempty(deduped, " | ")
+
+
+def process_record(record: ET.Element, database: str) -> Dict[str, str]:
+    """Transform a <record> element into a dictionary for CSV."""
+    ref_type_name = ""
+    ref_type = record.find("ref-type")
+    if ref_type is not None:
+        ref_type_name = ref_type.get("name") or ""
+
+    return {
+        "database": database,
+        "ref_type": clean_text(ref_type_name),
+        "title": safe_find_text(record, ".//title/style"),
+        "journal": safe_find_text(record, ".//secondary-title/style"),
+        "authors": extract_authors(record),
+        "year": safe_find_text(record, ".//year/style"),
+        "volume": safe_find_text(record, ".//volume/style"),
+        "number": safe_find_text(record, ".//number/style"),
+        "abstract": safe_find_text(record, ".//abstract/style"),
+        "doi": process_doi(record),
+        "urls": extract_urls(record),
+        "extracted_date": datetime.now().strftime("%Y-%m-%d"),
+    }
+
+def iter_records(xml_path: Path) -> Iterable[ET.Element]:
+    context = ET.iterparse(str(xml_path), events=("start", "end"))
+    _, root = next(context)
+    for event, elem in context:
+        if event == "end" and elem.tag == "record":
+            yield elem
+            elem.clear()
+            root.clear()
+
+def record_matches_filters(row: Dict[str, str], ref_type: Optional[str], year: Optional[str]) -> bool:
+    if ref_type and row.get("ref_type") != ref_type: return False
+    if year and row.get("year") != str(year): return False
+    return True
+
+def export_files_to_csv_with_report(
+    inputs: List[Path],
+    csv_path: Path,
+    report_path: Optional[Path] = None,
+    *,
+    fieldnames: List[str] = None,
+    delimiter: str = ",",
+    quoting: str = "minimal",
+    include_header: bool = True,
+    encoding: str = "utf-8",
+    ref_type: Optional[str] = None,
+    year: Optional[str] = None,
+    max_records_per_file: Optional[int] = None,
+) -> Tuple[int, Path, Path]:
+    """Primary library API: export one or many XML files to a single CSV + TXT report."""
+    fieldnames = fieldnames or DEFAULT_FIELDNAMES
+    qmode = CSV_QUOTING_MAP[quoting]
+    report_path = report_path or csv_path.with_name(csv_path.stem + "_report.txt")
+
+    ensure_parent_dir(csv_path)
+    ensure_parent_dir(report_path)
+
+    total_written, report_lines = 0, []
+    start_ts = time.time()
+    run_start = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+    with open(csv_path, "w", newline="", encoding=encoding) as f:
+        writer = csv.DictWriter(f, fieldnames=fieldnames, delimiter=delimiter, quoting=qmode)
+        if include_header:
+            writer.writeheader()
+
+        for xml_path in inputs:
+            database = xml_path.stem
+            logging.info("Processing %s (database=%s)", xml_path.name, database)
+            file_written = file_skipped = 0
+
+            for rec in iter_records(xml_path):
+                try:
+                    row = process_record(rec, database=database)  # your existing function
+                    if record_matches_filters(row, ref_type, year):
+                        writer.writerow({k: row.get(k, "") for k in fieldnames})
+                        file_written += 1
+                        total_written += 1
+                        if max_records_per_file and file_written >= max_records_per_file:
+                            break
+                except Exception:
+                    file_skipped += 1
+                    logging.debug("Record error in %s", xml_path, exc_info=True)
+
+            report_lines.append(f"{xml_path.name}: {file_written} exported, {file_skipped} skipped")
+
+    dur = time.time() - start_ts
+    report_lines = [
+        f"Run started: {run_start}",
+        *report_lines,
+        f"TOTAL exported: {total_written}",
+        f"Files processed: {len(inputs)}",
+        f"Duration: {dur:.2f} seconds",
+    ]
+    with open(report_path, "w", encoding="utf-8") as rf:
+        rf.write("\n".join(report_lines))
+
+    return total_written, csv_path, report_path
+
+def export(xml_file: Path, csv_path: Path, **kwargs):
+    """Convenience: single XML file to CSV (+report)."""
+    return export_files_to_csv_with_report([xml_file], csv_path, **kwargs)
+
+def export_folder(folder: Path, csv_path: Path, **kwargs):
+    """Convenience: all *.xml in folder to CSV (+report)."""
+    inputs = sorted(p for p in Path(folder).glob("*.xml") if p.is_file())
+    if not inputs:
+        raise FileNotFoundError(f"No *.xml found in {folder}")
+    return export_files_to_csv_with_report(inputs, csv_path, **kwargs)

endnote_utils-0.1.0.dist-info/METADATA

@@ -0,0 +1,154 @@
+Metadata-Version: 2.4
+Name: endnote-utils
+Version: 0.1.0
+Summary: Convert EndNote XML to CSV with streaming parse and TXT report.
+Author-email: Minh Quach <minhquach8@gmail.com>
+License: MIT
+Keywords: endnote,xml,csv,bibliography,research
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+
+# EndNote Exporter
+
+Convert **EndNote XML files** into clean CSVs with automatic TXT reports.  
+Supports both **Python API** and **command-line interface (CLI)**.
+
+---
+
+## Features
+
+- ✅ Parse one XML file (`--xml`) or an entire folder of `*.xml` (`--folder`)
+- ✅ Streams `<record>` elements using `iterparse` (low memory usage)
+- ✅ Extracts fields:  
+  `database, ref_type, title, journal, authors, year, volume, number, abstract, doi, urls, extracted_date`
+- ✅ Adds a `database` column from the XML filename stem (`IEEE.xml → IEEE`)
+- ✅ Normalizes DOI (`10.xxxx` → `https://doi.org/...`)
+- ✅ Always generates a **TXT report** (default: `<csv>_report.txt`) with:
+  - per-file counts (exported/skipped records)  
+  - totals, files processed  
+  - run timestamp & duration
+- ✅ Auto-creates output folders if missing
+- ✅ CLI options for CSV formatting, filters, verbosity
+- ✅ Importable Python API for scripting & integration
+
+---
+
+## Installation
+
+### From PyPI (recommended)
+
+```bash
+pip install endnote-exporter
+````
+
+### From local source
+
+If you have the source code in a folder or `.zip`:
+
+```bash
+cd /path/to/endnote-exporter
+pip install .
+```
+
+Requires **Python 3.8+**.
+
+---
+
+## Usage
+
+### Command Line
+
+#### Single file
+
+```bash
+endnote-xml-export --xml data/IEEE.xml --csv output/ieee.csv
+```
+
+#### Folder with multiple files
+
+```bash
+endnote-xml-export --folder data/xmls --csv output/all_records.csv
+```
+
+#### Custom report path
+
+```bash
+endnote-xml-export \
+  --xml data/Scopus.xml \
+  --csv output/scopus.csv \
+  --report reports/scopus_run.txt
+```
+
+If `--report` is not provided, it defaults to `<csv>_report.txt`.
+
+---
+
+### CLI Options
+
+| Option          | Description                                         | Default            |
+| --------------- | --------------------------------------------------- | ------------------ |
+| `--xml`         | Path to a single EndNote XML file                   | –                  |
+| `--folder`      | Path to a folder containing multiple `*.xml` files  | –                  |
+| `--csv`         | Output CSV path                                     | –                  |
+| `--report`      | Output TXT report path                              | `<csv>_report.txt` |
+| `--delimiter`   | CSV delimiter                                       | `,`                |
+| `--quoting`     | CSV quoting: `minimal`, `all`, `nonnumeric`, `none` | `minimal`          |
+| `--no-header`   | Suppress CSV header row                             | –                  |
+| `--encoding`    | Output CSV encoding                                 | `utf-8`            |
+| `--ref-type`    | Only include records with this `ref_type` name      | –                  |
+| `--year`        | Only include records with this year                 | –                  |
+| `--max-records` | Stop after N records per file (useful for testing)  | –                  |
+| `--verbose`     | Verbose logging with debug details                  | –                  |
+
+---
+
+### Example Report
+
+```
+Run started: 2025-09-11 14:30:22
+IEEE.xml: 120 exported, 0 skipped
+Scopus.xml: 95 exported, 2 skipped
+TOTAL exported: 215
+Files processed: 2
+Duration: 3.14 seconds
+```
+
+---
+
+## Python API
+
+You can also use it directly in Python scripts:
+
+```python
+from pathlib import Path
+from endnote_exporter import export, export_folder
+
+# Single file
+total, csv_out, report_out = export(
+    Path("data/IEEE.xml"), Path("output/ieee.csv")
+)
+
+# Folder
+total, csv_out, report_out = export_folder(
+    Path("data/xmls"), Path("output/all.csv"),
+    ref_type="Conference Proceedings", year="2024"
+)
+```
+
+---
+
+## Development Notes
+
+* Pure Python, uses only standard library (`argparse`, `csv`, `xml.etree.ElementTree`, `logging`, `pathlib`).
+* Streaming XML parsing avoids high memory usage.
+* Robust error handling: skips malformed records but logs them in verbose mode.
+* Follows [PEP 621](https://peps.python.org/pep-0621/) packaging (`pyproject.toml`).
+
+---
+
+## License
+
+MIT License © 2025 Minh Quach

endnote_utils-0.1.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+endnote-utils/__init__.py,sha256=yuPzjVRcBiQPUCFG5DbSvwqz1O42_hYJ3_7dzWotE20,284
+endnote-utils/cli.py,sha256=TQxdO7IlaRXwNTm0MpBVk9CeUTUGgtlcI0O3O9xhgdM,2160
+endnote-utils/core.py,sha256=cddpuRMF5RC5mp3Lll0eTA9MXLzcVDnDl1Z7IMHOr0k,7480
+endnote_utils-0.1.0.dist-info/METADATA,sha256=oBeBNLi7xPptTfTk6FvjtLv576YK0FTGj4_zrWZhebw,4486
+endnote_utils-0.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+endnote_utils-0.1.0.dist-info/entry_points.txt,sha256=1mpe6VMBMZwSGdNPdcDPMesi8QEovtEUtf-_slQF8g8,65
+endnote_utils-0.1.0.dist-info/top_level.txt,sha256=Bs12r-xSbXfACyOO_DFAg4TmG8xY_3iM4cx5skHB3-g,14
+endnote_utils-0.1.0.dist-info/RECORD,,

endnote_utils-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

endnote_utils-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+endnote-xml-export = endnote_exporter.cli:main

endnote_utils-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+endnote-utils