open-source-repository-sampler 0.1.0__py3-none-any.whl

open_source_repository_sampler/__init__.py

@@ -0,0 +1,20 @@
+"""Reproducible, seeded, stratified samples of open-source repositories, each pinned to the
+commit that met the sampling criteria.
+
+For Python experiments:
+
+    from open_source_repository_sampler import read_sample
+    for repo in read_sample("sample/open-source-repository-sample.csv"):
+        repo["url"], repo["commit"], repo["language"]
+
+or let the sampler run the loop: each usable repository is cloned at its pinned commit (shallow,
+or deep with every commit up to it and none after), handed to the experiment, then deleted:
+
+    from open_source_repository_sampler import for_each
+    def experiment(repo):            # repo.path, repo.commit, repo.full_name, repo.row, ...
+        ...
+    for_each("sample/open-source-repository-sample.csv", experiment, deep=True,
+             run_log="results/run-log.jsonl")
+"""
+from .checkout import Repository, clone_at, for_each  # noqa: F401
+from .files import read_sample  # noqa: F401

open_source_repository_sampler/_version.py

@@ -0,0 +1,4 @@
+# Written by setuptools-scm at build time. Do not edit or commit.
+__version__ = version = '0.1.0'
+__commit__ = 'g7502a15812cd704a5ab562e58b030f7279fd72a3'
+__dirty__ = False

open_source_repository_sampler/checkout.py

@@ -0,0 +1,265 @@
+"""Use a sample: get each usable repository exactly as it was sampled, and run an experiment on it.
+
+The clone holds nothing after the pinned commit, so an experiment run years later sees exactly
+the code, and the history, that was sampled:
+
+  * shallow  only the pinned commit's tree (`git fetch --depth 1 <url> <commit>`);
+  * deep     every commit up to and including the pinned commit, and nothing after it
+             (`git fetch <url> <commit>`: only that commit and its ancestors are transferred).
+
+Both fetch by commit from the URL directly, with no tags and no remote configured, so no later
+commit is in the clone and none can be fetched into it by accident.
+
+A commit id is a hash over the commit's content and its parents, so the history up to a pinned
+commit can never change: if a project later rewrites its history, the pinned commit is either
+still served, with exactly the sampled history, or no longer served, and the clone fails loudly.
+A deep clone is also checked against the `n_commits` measured at sampling (when the sample was
+drawn with a history criterion), which catches an incomplete clone (a host that served grafted
+or partial history).
+
+`for_each` is the loop an experiment runs inside: for each usable repository it clones (shallow
+or deep), hands the clone to the experiment, then deletes it, so disk use stays at one
+repository at a time. The clone can always be made again from the sample.
+"""
+from __future__ import annotations
+
+import dataclasses
+import datetime
+import json
+import os
+import shutil
+import subprocess
+import tempfile
+import time
+
+from . import files
+from .measure import network_git
+
+
+def _git(args: list[str], cwd: str, timeout: float | None = None) -> subprocess.CompletedProcess:
+    return subprocess.run(["git", *args], cwd=cwd, capture_output=True, text=True,
+                          timeout=timeout)
+
+
+def dir_name(full_name: str) -> str:
+    return full_name.replace("/", "__")
+
+
+def clone_at(url: str, commit: str, dest: str, deep: bool = False,
+             expected_commits: int | None = None, timeout: float | None = 3600) -> str | None:
+    """`url` into `dest`, detached at `commit`, holding nothing after it. Shallow: that commit
+    only. Deep: all of its history. With `expected_commits` (the sample's `n_commits`), a deep
+    clone whose history differs from the one sampled is refused. None on success, else an
+    error."""
+    shutil.rmtree(dest, ignore_errors=True)
+    os.makedirs(dest)
+    fetch = ["fetch", "--quiet", "--no-tags", url, commit]
+    if not deep:
+        fetch[1:1] = ["--depth", "1"]
+    for args in (["init", "--quiet"], fetch, ["checkout", "--quiet", "--detach", "FETCH_HEAD"]):
+        if args is fetch:
+            err = network_git(fetch, dest, timeout)     # stall-aborted and retried
+        else:
+            r = _git(args, dest, timeout)
+            err = None if r.returncode == 0 else r.stderr.strip()[:160]
+        if err:
+            return f"git {args[0]} failed: {err}"
+    head = _git(["rev-parse", "HEAD"], dest).stdout.strip()
+    if head != commit:
+        return f"landed on {head[:12]}, not {commit[:12]}"
+    n_head = int(_git(["rev-list", "--count", "HEAD"], dest).stdout.strip() or 0)
+    n_all = int(_git(["rev-list", "--count", "--all"], dest).stdout.strip() or 0)
+    if n_all != n_head:
+        return f"clone holds {n_all - n_head} commits beyond {commit[:12]}"
+    if deep and expected_commits is not None and n_head != expected_commits:
+        return (f"history up to {commit[:12]} has {n_head} commits, the sample recorded "
+                f"{expected_commits}: the clone is incomplete")
+    return None
+
+
+def _at(dest: str, commit: str) -> bool:
+    if not os.path.isdir(os.path.join(dest, ".git")):
+        return False
+    return _git(["rev-parse", "HEAD"], dest).stdout.strip() == commit
+
+
+def _rows(sample: str, langs: list[str] | None) -> list[dict]:
+    return [r for r in files.read_sample(sample) if not langs or r["language"] in langs]
+
+
+def checkout(sample: str, dest: str, langs: list[str] | None = None, deep: bool = False,
+             log=print) -> list[dict]:
+    """Clone the usable repositories of `sample` (optionally only `langs`) into `dest`, and
+    keep them. Returns one {full_name, path, commit, error} per repository."""
+    os.makedirs(dest, exist_ok=True)
+    out = []
+    rows = _rows(sample, langs)
+    for i, r in enumerate(rows, 1):
+        path = os.path.join(dest, dir_name(r["full_name"]))
+        if _at(path, r["commit"]):
+            err = None
+            log(f"[{i}/{len(rows)}] {r['full_name']}: already at {r['commit'][:12]}")
+        else:
+            log(f"[{i}/{len(rows)}] {r['full_name']} @ {r['commit'][:12]} "
+                f"({'deep' if deep else 'shallow'}) ...")
+            err = clone_at(r["url"], r["commit"], path, deep, r.get("n_commits") or None)
+            if err:
+                log(f"        ERROR {err}")
+        out.append({"full_name": r["full_name"], "path": path, "commit": r["commit"],
+                    "error": err})
+    bad = [o for o in out if o["error"]]
+    log(f"{len(out) - len(bad)}/{len(out)} repositories at their pinned commit in {dest}")
+    return out
+
+
+@dataclasses.dataclass
+class Repository:
+    """One usable repository of a sample, cloned at its pinned commit, as an experiment gets it."""
+    full_name: str
+    url: str
+    commit: str
+    language: str
+    path: str            # the clone; deleted after the experiment returns (unless kept)
+    deep: bool           # whether `path` holds the history up to `commit`
+    row: dict            # the whole CSV row: stars, tier, criteria measures, ...
+
+
+def _done(log_path: str | None) -> set:
+    """(full_name, commit) pairs the run log records as done."""
+    done = set()
+    if log_path and os.path.exists(log_path):
+        with open(log_path) as fh:
+            for line in fh:
+                if line.strip():
+                    rec = json.loads(line)
+                    if rec.get("status") == "done":
+                        done.add((rec["full_name"], rec["commit"]))
+    return done
+
+
+def for_each(sample: str, experiment, deep: bool = False, langs: list[str] | None = None,
+             work_dir: str | None = None, keep: bool = False, run_log: str | None = None,
+             fail_fast: bool = False, log=print) -> list[dict]:
+    """Run `experiment(repository)` on every usable repository of `sample`, one at a time: clone
+    it at its pinned commit (deep or shallow), call the experiment, delete the clone.
+
+    A failure (clone, or the experiment raising) is recorded and the loop moves on, unless
+    `fail_fast`. With `run_log`, each outcome is appended to that JSON Lines file and a re-run
+    skips the repositories it records as done, so an interrupted run resumes.
+    Returns one {full_name, commit, status, error, result, seconds} per repository."""
+    rows = _rows(sample, langs)
+    done = _done(run_log)
+    own_work = work_dir is None
+    work = tempfile.mkdtemp(prefix="osrs-run-") if own_work else work_dir
+    os.makedirs(work, exist_ok=True)
+    out = []
+    try:
+        for i, r in enumerate(rows, 1):
+            tag = f"[{i}/{len(rows)}] {r['full_name']} @ {r['commit'][:12]}"
+            if (r["full_name"], r["commit"]) in done:
+                log(f"{tag}: done in an earlier run, skipped")
+                out.append({"full_name": r["full_name"], "commit": r["commit"],
+                            "status": "skipped", "error": None, "result": None, "seconds": 0})
+                continue
+            path = os.path.join(work, dir_name(r["full_name"]))
+            t0 = time.time()
+            log(f"{tag}: cloning ({'deep' if deep else 'shallow'}) ...")
+            err = clone_at(r["url"], r["commit"], path, deep, r.get("n_commits") or None)
+            result = None
+            if not err:
+                log(f"{tag}: running experiment ...")
+                try:
+                    result = experiment(Repository(r["full_name"], r["url"], r["commit"],
+                                                   r["language"], path, deep, r))
+                except Exception as e:
+                    err = f"experiment raised {type(e).__name__}: {e}"
+            if not keep:
+                shutil.rmtree(path, ignore_errors=True)
+            rec = {"full_name": r["full_name"], "commit": r["commit"],
+                   "status": "failed" if err else "done", "error": err,
+                   "seconds": round(time.time() - t0, 1)}
+            log(f"{tag}: {rec['status']}" + (f" ({err})" if err else "")
+                + f" in {rec['seconds']}s")
+            if run_log:
+                with open(run_log, "a") as fh:
+                    fh.write(json.dumps({**rec, "finished_utc": datetime.datetime.now(
+                        datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")}) + "\n")
+            out.append({**rec, "result": result})
+            if err and fail_fast:
+                break
+    finally:
+        if own_work and not keep:
+            shutil.rmtree(work, ignore_errors=True)
+    n = {s: sum(1 for o in out if o["status"] == s) for s in ("done", "skipped", "failed")}
+    log(f"{n['done']} done, {n['skipped']} skipped (already done), {n['failed']} failed "
+        f"of {len(rows)} repositories")
+    return out
+
+
+# Placeholders a `run` command may use in its arguments, and the environment variables every
+# run command gets, for experiments not written in Python.
+def _fields(repo: Repository) -> dict:
+    return {"path": repo.path, "url": repo.url, "commit": repo.commit,
+            "full_name": repo.full_name, "language": repo.language,
+            "deep": "true" if repo.deep else "false"}
+
+
+def command_experiment(command: list[str]):
+    """An experiment that runs `command` once per repository: `{path}`, `{url}`, `{commit}`,
+    `{full_name}`, `{language}` and `{deep}` in its arguments are replaced by the repository's
+    values, which are also in the environment as SAMPLED_REPOSITORY_PATH, ..._URL, ..._COMMIT,
+    ..._FULL_NAME, ..._LANGUAGE and ..._DEEP. A non-zero exit is a failure."""
+    def run(repo: Repository) -> int:
+        fields = _fields(repo)
+        args = []
+        for a in command:
+            for k, v in fields.items():
+                a = a.replace("{" + k + "}", v)
+            args.append(a)
+        env = {**os.environ, **{f"SAMPLED_REPOSITORY_{k.upper()}": v for k, v in fields.items()}}
+        code = subprocess.run(args, env=env).returncode
+        if code != 0:
+            raise RuntimeError(f"command exited {code}")
+        return code
+    return run
+
+
+def verify(sample: str, fetch: bool = False, log=print) -> list[str]:
+    """Problems with a sample; empty when it is intact. Checks the CSV against the hash in its
+    derivation, and with `fetch`, that every pinned commit can still be fetched."""
+    if os.path.isdir(sample):
+        sample = os.path.join(sample, files.names()[0])
+    deriv = sample[:-len(".csv")] + ".derivation.json"
+    problems = []
+    if not sample.endswith(".csv") or not os.path.exists(deriv):
+        return [f"no derivation file beside {sample} (expected {deriv})"]
+    with open(deriv) as fh:
+        d = json.load(fh)
+    got = files.sha256(sample)
+    if got != d.get("csv_sha256"):
+        problems.append(f"{os.path.basename(sample)} sha256 {got} does not match the "
+                        f"derivation's {d.get('csv_sha256')}: the CSV was changed after sampling")
+    else:
+        log(f"OK  {os.path.basename(sample)} matches its derivation (sha256 {got[:16]}...)")
+    s = d.get("sampler", {})
+    log(f"    drawn by {s.get('name')} {s.get('version')} @ {s.get('commit')}"
+        + ("  (DIRTY)" if s.get("dirty") else ""))
+    rows = files.read_sample(sample)
+    commits = {r["sampler_commit"] for r in rows}
+    if commits - {s.get("commit") or ""}:
+        problems.append(f"rows name sampler commits {sorted(commits)}, the derivation "
+                        f"{s.get('commit')}")
+    if fetch:
+        with tempfile.TemporaryDirectory(prefix="osrs-verify-") as tmp:
+            _git(["init", "--quiet", "--bare"], tmp)
+            for r in rows:
+                # Stall-aborted and retried, so a transient outage is not reported as a lost
+                # commit.
+                err = network_git(["fetch", "--depth", "1", "--quiet", r["url"], r["commit"]],
+                                  tmp, 600)
+                if err:
+                    problems.append(f"{r['full_name']}: commit {r['commit'][:12]} can no "
+                                    f"longer be fetched ({err})")
+                else:
+                    log(f"OK  {r['full_name']} @ {r['commit'][:12]}")
+    return problems

open_source_repository_sampler/cli.py

@@ -0,0 +1,136 @@
+"""open-source-repository-sampler: draw, use and verify reproducible samples of repositories.
+
+    open-source-repository-sampler sample   --langs java --target 30 --seed 1 --out-dir sample
+    open-source-repository-sampler checkout sample --dest repos [--deep]
+    open-source-repository-sampler run      sample [--deep] -- my-experiment {path} -o out/{full_name}
+    open-source-repository-sampler verify   sample --fetch
+"""
+from __future__ import annotations
+
+import argparse
+import sys
+
+from . import checkout, frame, provenance, sample
+
+
+def _list(s: str) -> list[str]:
+    return [x.strip() for x in s.split(",") if x.strip()]
+
+
+def _sample_args(sp: argparse.ArgumentParser) -> None:
+    d = sample.DEFAULTS
+    g = sp.add_argument_group("what to sample")
+    g.add_argument("--langs", type=_list, default=d["langs"],
+                   help=f"comma list of languages (default all: {','.join(frame.LANGS)})")
+    g.add_argument("--tiers", type=_list, default=d["tiers"],
+                   help="comma list of popularity tiers: "
+                        + ", ".join(f"{k} ({v})" for k, v in frame.TIERS.items())
+                        + " (default all)")
+    g.add_argument("--target", type=int, default=d["target"],
+                   help=f"usable repositories wanted per language (default {d['target']})")
+    g.add_argument("--seed", type=int, default=d["seed"],
+                   help=f"the one value that fixes the draw; fix it before the first draw and "
+                        f"keep the first sample (default {d['seed']})")
+    g.add_argument("--exclude-owners", type=_list, default=d["exclude_owners"],
+                   help="comma list of GitHub owners never sampled, e.g. your own (a "
+                        "conflict-of-interest guard; default none)")
+    c = sp.add_argument_group("criteria a repository must meet to be usable")
+    for p, col, desc in sample.CRITERIA:
+        typ = float if isinstance(d[p], float) else int
+        c.add_argument("--" + p.replace("_", "-"), type=typ, default=d[p],
+                       help=f"minimum {desc} (default {d[p]})")
+    r = sp.add_argument_group("output and run control")
+    r.add_argument("--out-dir", default=".",
+                   help="directory for the sample files (default: the current directory)")
+    r.add_argument("--label", default=None,
+                   help="optional label in the file names, to keep several samples side by side")
+    r.add_argument("--max-clones", type=int, default=d["max_clones"],
+                   help=f"safety stop on total candidates drawn (default {d['max_clones']})")
+    r.add_argument("--pause", type=float, default=d["pause"],
+                   help=f"seconds between GitHub searches (default {d['pause']})")
+    r.add_argument("--allow-dirty", action="store_true",
+                   help="run even though the sampler has uncommitted changes (recorded)")
+    r.add_argument("--overwrite", action="store_true",
+                   help="replace existing sample files in --out-dir")
+
+
+def main(argv: list[str] | None = None) -> int:
+    argv = sys.argv[1:] if argv is None else argv
+    command = []
+    if "--" in argv:                      # `run ... -- <experiment command>`
+        i = argv.index("--")
+        argv, command = argv[:i], argv[i + 1:]
+    ap = argparse.ArgumentParser(prog=provenance.NAME, description=__doc__.splitlines()[0])
+    ap.add_argument("--version", action="store_true", help="print the sampler version and commit")
+    sub = ap.add_subparsers(dest="cmd")
+
+    sp = sub.add_parser("sample", help="draw a sample (writes the CSV and its derivation)")
+    _sample_args(sp)
+
+    cp = sub.add_parser("checkout", help="clone a sample's usable repositories at their commits")
+    cp.add_argument("sample", help="the sample CSV, or the directory holding it")
+    cp.add_argument("--dest", default="repos", help="directory to clone into (default repos)")
+    cp.add_argument("--langs", type=_list, default=None, help="only these detected languages")
+    cp.add_argument("--deep", action="store_true",
+                    help="every commit up to each pinned commit (default: that commit only)")
+
+    rp = sub.add_parser(
+        "run", help="run an experiment command on each usable repository, one clone at a time",
+        usage=f"{provenance.NAME} run SAMPLE [options] -- COMMAND [ARGS with {{path}} ...]",
+        description="Clone each usable repository at its pinned commit, run the command given "
+                    "after `--` on it, then delete the clone. In the command, {path}, {url}, "
+                    "{commit}, {full_name}, {language} and {deep} are replaced by the "
+                    "repository's values, which are also in the environment as "
+                    "SAMPLED_REPOSITORY_PATH, _URL, _COMMIT, _FULL_NAME, _LANGUAGE and _DEEP.")
+    rp.add_argument("sample", help="the sample CSV, or the directory holding it")
+    rp.add_argument("--deep", action="store_true",
+                    help="every commit up to each pinned commit (default: that commit only)")
+    rp.add_argument("--langs", type=_list, default=None, help="only these detected languages")
+    rp.add_argument("--work-dir", default=None,
+                    help="where clones are made (default: a temporary directory)")
+    rp.add_argument("--keep", action="store_true", help="keep each clone after its run")
+    rp.add_argument("--run-log", default=None,
+                    help="JSON Lines file of outcomes; a re-run skips repositories it records "
+                         "as done, so an interrupted run resumes")
+    rp.add_argument("--fail-fast", action="store_true", help="stop at the first failure")
+
+    vp = sub.add_parser("verify", help="check a sample's CSV against its derivation")
+    vp.add_argument("sample", help="the sample CSV, or the directory holding it")
+    vp.add_argument("--fetch", action="store_true",
+                    help="also check every pinned commit can still be fetched")
+
+    args = ap.parse_args(argv)
+    if args.version:
+        s = provenance.sampler()
+        print(f"{s['name']} {s['version']} @ {s['commit']}" + ("  (dirty)" if s["dirty"] else ""))
+        return 0
+    if args.cmd == "sample":
+        params = {k: getattr(args, k) for k in sample.DEFAULTS}
+        try:
+            sample.run(params, args.out_dir, args.label, argv, args.allow_dirty, args.overwrite)
+        except (RuntimeError, ValueError, FileExistsError) as e:
+            print(f"ERROR: {e}", file=sys.stderr)
+            return 2
+        return 0
+    if args.cmd == "checkout":
+        res = checkout.checkout(args.sample, args.dest, args.langs, args.deep)
+        return 1 if any(r["error"] for r in res) else 0
+    if args.cmd == "run":
+        if not command:
+            print("ERROR: give the experiment command after `--`", file=sys.stderr)
+            return 2
+        res = checkout.for_each(args.sample, checkout.command_experiment(command), args.deep,
+                                args.langs, args.work_dir, args.keep, args.run_log,
+                                args.fail_fast)
+        return 1 if any(r["status"] == "failed" for r in res) else 0
+    if args.cmd == "verify":
+        problems = checkout.verify(args.sample, args.fetch)
+        for p in problems:
+            print(f"PROBLEM {p}", file=sys.stderr)
+        return 1 if problems else 0
+    ap.print_help()
+    return 2
+
+
+if __name__ == "__main__":
+    sys.exit(main())

open_source_repository_sampler/files.py

@@ -0,0 +1,106 @@
+"""The two sample files, their fixed names, and reading a sample back.
+
+The names are fixed so a sample can be dropped into a paper's ancillary data unchanged and be
+recognised on sight:
+
+  open-source-repository-sample.csv              one row per candidate drawn
+  open-source-repository-sample.derivation.json  how the sample was derived, and how to redo it
+
+An optional label is inserted before the suffix (`open-source-repository-sample.java-v2.csv`)
+so several samples can sit side by side without breaking the convention.
+"""
+from __future__ import annotations
+
+import csv
+import hashlib
+import os
+import re
+
+STEM = "open-source-repository-sample"
+
+# Column order of the CSV. Every row of every sample has all of these.
+COLUMNS = [
+    "draw",                  # 1-based order in which the candidate was drawn
+    "status",                # usable | excluded (by the criteria) | error (could not inspect)
+    "reason",                # why excluded, or the error (empty when usable)
+    "url",                   # clone URL
+    "full_name",             # owner/name
+    "commit",                # the pinned commit: the exact code checked against the criteria
+    "commit_date",           # committer date of that commit (ISO 8601)
+    "language",              # language detected from the project's own source
+    "language_share",        # that language's fraction of the non-header source files
+    "github_language",       # GitHub's label at sampling time
+    "mirror",                # True when GitHub lists it as a mirror of a repository elsewhere
+    "stratum_language",      # the stratum the candidate was drawn from
+    "tier",
+    "star_band",             # the frame band (star range, maybe a creation-date range) it was in
+    "stars",                 # at sampling time
+    "rank",                  # rank by stars within its band's search results
+    "n_source_files",
+    "n_functions",
+    "n_files_with_functions",
+    "nloc",
+    "n_unparsed_files",      # source files lizard could not parse (no functions counted)
+    # The history columns are measured only when a history criterion (--min-commits,
+    # --min-contributors) is set, which is when the full history is cloned; else empty.
+    "n_commits",             # reachable from the pinned commit
+    "n_contributors",        # distinct author emails reachable from the pinned commit
+    "first_commit_date",
+    "sampler_version",
+    "sampler_commit",        # the commit of the sampler that produced this row
+]
+INT_COLUMNS = {"draw", "stars", "rank", "n_source_files", "n_functions",
+               "n_files_with_functions", "nloc", "n_unparsed_files", "n_commits",
+               "n_contributors"}
+FLOAT_COLUMNS = {"language_share"}
+
+_LABEL = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$")
+
+
+def check_label(label: str | None) -> None:
+    if label is not None and (not _LABEL.match(label) or label.endswith(".derivation")):
+        raise ValueError(f"label {label!r}: use letters, digits, '.', '_' or '-' only")
+
+
+def names(label: str | None = None) -> tuple[str, str, str]:
+    """(csv, derivation, progress) file names for a sample."""
+    check_label(label)
+    stem = STEM if not label else f"{STEM}.{label}"
+    return f"{stem}.csv", f"{stem}.derivation.json", f".{stem}.progress.jsonl"
+
+
+def sha256(path: str) -> str:
+    h = hashlib.sha256()
+    with open(path, "rb") as fh:
+        for chunk in iter(lambda: fh.read(1 << 16), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def write_csv(path: str, rows: list[dict]) -> None:
+    with open(path, "w", newline="") as fh:
+        w = csv.DictWriter(fh, fieldnames=COLUMNS, extrasaction="ignore", lineterminator="\n")
+        w.writeheader()
+        for r in rows:
+            w.writerow({c: "" if r.get(c) is None else r.get(c) for c in COLUMNS})
+
+
+def read_sample(path: str, usable_only: bool = True) -> list[dict]:
+    """The rows of a sample CSV (only the usable ones by default), numbers typed. `path` is the
+    CSV itself or the directory holding an unlabelled sample."""
+    if os.path.isdir(path):
+        path = os.path.join(path, names()[0])
+    with open(path, newline="") as fh:
+        rows = list(csv.DictReader(fh))
+    out = []
+    for r in rows:
+        if usable_only and r.get("status") != "usable":
+            continue
+        for c in INT_COLUMNS:
+            if r.get(c) not in (None, ""):
+                r[c] = int(r[c])
+        for c in FLOAT_COLUMNS:
+            if r.get(c) not in (None, ""):
+                r[c] = float(r[c])
+        out.append(r)
+    return out