redcap-toolbox 2025.7.9__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- redcap_toolbox/__init__.py +4 -0
- redcap_toolbox/download_redcap.py +68 -0
- redcap_toolbox/download_redcap_report.py +103 -0
- redcap_toolbox/minchange.py +116 -0
- redcap_toolbox/split_redcap_data.py +158 -0
- redcap_toolbox/update_redcap_diff.py +94 -0
- redcap_toolbox-2025.7.9.dist-info/METADATA +144 -0
- redcap_toolbox-2025.7.9.dist-info/RECORD +11 -0
- redcap_toolbox-2025.7.9.dist-info/WHEEL +5 -0
- redcap_toolbox-2025.7.9.dist-info/entry_points.txt +5 -0
- redcap_toolbox-2025.7.9.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
# Copyright 2023 Board of Regents of the University of Wisconsin System
|
|
2
|
+
# Written by Nate Vack <njvack@wisc.edu>, Nicholas Vanhaute <nvanhaute@wisc.edu>, and
|
|
3
|
+
# Stuti Shrivastava <sshrivastav6@wisc.edu>
|
|
4
|
+
# Source is available at https://github.com/uwmadison-chm/redcap-tools
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
|
|
3
|
+
"""
|
|
4
|
+
Download the data for a REDCap study to a CSV files. Optionally takes a
|
|
5
|
+
filename with a list of forms to skip from the download, one per line.
|
|
6
|
+
|
|
7
|
+
Relies on the environment variables REDCAP_API_URL and REDCAP_API_TOKEN.
|
|
8
|
+
|
|
9
|
+
Written by Nate Vack <njvack@wisc.edu> and Dan Fitch <dfitch@wisc.edu>
|
|
10
|
+
|
|
11
|
+
Usage:
|
|
12
|
+
download_redcap.py [options] <output_file>
|
|
13
|
+
|
|
14
|
+
Options:
|
|
15
|
+
-h --help Show this screen.
|
|
16
|
+
--forms=<file> A file containing a list of forms to download, one per line;
|
|
17
|
+
if not specified, download all forms.
|
|
18
|
+
--survey-fields Include survey timestamps in output
|
|
19
|
+
-d --debug Print debugging output
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
import logging
|
|
23
|
+
import os
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
|
|
26
|
+
import redcap
|
|
27
|
+
from docopt import docopt
|
|
28
|
+
|
|
29
|
+
logging.basicConfig(format="%(message)s")
|
|
30
|
+
logger = logging.getLogger(__name__)
|
|
31
|
+
logger.setLevel(logging.INFO)
|
|
32
|
+
|
|
33
|
+
API_URL = os.environ["REDCAP_API_URL"]
|
|
34
|
+
API_TOK = os.environ["REDCAP_API_TOKEN"]
|
|
35
|
+
PROJ = redcap.Project(API_URL, API_TOK)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def file_to_list(filename):
|
|
39
|
+
out = []
|
|
40
|
+
with open(filename) as f:
|
|
41
|
+
out = [line.strip() for line in f.readlines()]
|
|
42
|
+
return out
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def download_redcap(out_file, form_list_file, export_survey_fields):
|
|
46
|
+
forms = None
|
|
47
|
+
if form_list_file:
|
|
48
|
+
forms = file_to_list(form_list_file)
|
|
49
|
+
logger.debug(f"Instruments: {forms}")
|
|
50
|
+
data = PROJ.export_records(
|
|
51
|
+
format_type="csv", forms=forms, export_survey_fields=export_survey_fields
|
|
52
|
+
)
|
|
53
|
+
with open(out_file, "w", encoding="utf8") as f:
|
|
54
|
+
f.write(data)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def main():
|
|
58
|
+
args = docopt(__doc__)
|
|
59
|
+
if args["--debug"]:
|
|
60
|
+
logger.setLevel(logging.DEBUG)
|
|
61
|
+
logger.debug(args)
|
|
62
|
+
download_redcap(
|
|
63
|
+
Path(args["<output_file>"]), args["--forms"], args["--survey-fields"]
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
if __name__ == "__main__":
|
|
68
|
+
main()
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
|
|
3
|
+
"""
|
|
4
|
+
Download the reports in a REDCap study to a CSV files. Takes in either a CSV
|
|
5
|
+
file with a list of report IDs, one per line, or a list of report IDs seperated
|
|
6
|
+
by commas.
|
|
7
|
+
|
|
8
|
+
If your report IDs look like: 32001, 32004 with no prefix provided, the output report filenames
|
|
9
|
+
will look like this in the output directory:
|
|
10
|
+
redcap__report_32001.csv
|
|
11
|
+
redcap__report_32004.csv
|
|
12
|
+
|
|
13
|
+
Relies on the environment variables REDCAP_API_URL and REDCAP_API_TOKEN.
|
|
14
|
+
|
|
15
|
+
Written by Stuti Shrivastava <sshrivastav6@wisc.edu>
|
|
16
|
+
|
|
17
|
+
Usage:
|
|
18
|
+
download_redcap_report.py [options] ([--id=<id>]... | [--file=<file>]) <output_dir>
|
|
19
|
+
download_redcap_report.py -h | --help
|
|
20
|
+
|
|
21
|
+
Options:
|
|
22
|
+
-h --help Show this screen.
|
|
23
|
+
-d --debug Print debugging output
|
|
24
|
+
--file=<file> A file containing a list of report IDs, one per line
|
|
25
|
+
--id=<id> A list of report IDs separated by commas
|
|
26
|
+
--prefix=<prefix> A filename prefix for the output [default: redcap]
|
|
27
|
+
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
import logging
|
|
31
|
+
import os
|
|
32
|
+
import traceback
|
|
33
|
+
from pathlib import Path
|
|
34
|
+
|
|
35
|
+
import redcap
|
|
36
|
+
from docopt import docopt
|
|
37
|
+
from requests import RequestException
|
|
38
|
+
|
|
39
|
+
logging.basicConfig(format="%(message)s")
|
|
40
|
+
logger = logging.getLogger(__name__)
|
|
41
|
+
logger.setLevel(logging.INFO)
|
|
42
|
+
|
|
43
|
+
API_URL = os.environ["REDCAP_API_URL"]
|
|
44
|
+
API_TOK = os.environ["REDCAP_API_TOKEN"]
|
|
45
|
+
PROJ = redcap.Project(API_URL, API_TOK)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def file_to_list(filename):
|
|
49
|
+
out = []
|
|
50
|
+
with open(filename) as f:
|
|
51
|
+
out = [line.strip() for line in f.readlines()]
|
|
52
|
+
out = [str(rep_id) for rep_id in out if rep_id != ""]
|
|
53
|
+
return out
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def download_redcap_report(
|
|
57
|
+
report_ids: list[str], out_dir: Path, prefix: str, verbose: bool
|
|
58
|
+
) -> None:
|
|
59
|
+
for rep_id in report_ids:
|
|
60
|
+
logger.debug(f"Downloading report for record ID: {rep_id}")
|
|
61
|
+
try:
|
|
62
|
+
data = PROJ.export_report(report_id=rep_id, format_type="csv")
|
|
63
|
+
out_file = Path(out_dir).joinpath(f"{prefix}__report_{rep_id}.csv")
|
|
64
|
+
with open(out_file, "w", encoding="utf8") as f:
|
|
65
|
+
f.write(data)
|
|
66
|
+
logger.info(f"Report {rep_id} downloaded to {out_file}")
|
|
67
|
+
except RequestException as e:
|
|
68
|
+
logger.warning(e)
|
|
69
|
+
logger.warning(f"Report {rep_id} not found! Skipping ...")
|
|
70
|
+
if verbose:
|
|
71
|
+
logger.warning(traceback.format_exc())
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def main():
|
|
75
|
+
args = docopt(__doc__)
|
|
76
|
+
if args["--debug"]:
|
|
77
|
+
logger.setLevel(logging.DEBUG)
|
|
78
|
+
logger.debug(args)
|
|
79
|
+
|
|
80
|
+
out_dir = Path(args["<output_dir>"])
|
|
81
|
+
if not out_dir.is_dir():
|
|
82
|
+
logger.warning(
|
|
83
|
+
f"Output directory {out_dir} does not exist! Making directory ..."
|
|
84
|
+
)
|
|
85
|
+
out_dir.mkdir(parents=True, exist_ok=True)
|
|
86
|
+
|
|
87
|
+
if args["--file"]:
|
|
88
|
+
report_ids = file_to_list(args["--file"])
|
|
89
|
+
elif args["--id"]:
|
|
90
|
+
report_ids = [str(rep_id) for rep_id in args["--id"]]
|
|
91
|
+
else:
|
|
92
|
+
raise ValueError("No report IDs provided!")
|
|
93
|
+
logger.debug(f"Report IDs: {report_ids}")
|
|
94
|
+
if len(report_ids) == 0:
|
|
95
|
+
raise ValueError(
|
|
96
|
+
f"No report IDs provided! {report_ids}. Provide either --id or --file."
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
download_redcap_report(report_ids, out_dir, args["--prefix"], args["--debug"])
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
if __name__ == "__main__":
|
|
103
|
+
main()
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
|
|
3
|
+
"""
|
|
4
|
+
A function to return a dict with the minimal changes to transform
|
|
5
|
+
one dataframe to another (identically-shaped) dataframe.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from collections.abc import Iterable
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def transformation_dicts(source_df, target_df, allow_new=False):
|
|
12
|
+
"""
|
|
13
|
+
Calculates the minimal set of changes to transform source_df into target_df.
|
|
14
|
+
source_df and target_df must be identically-shaped -- in particular,
|
|
15
|
+
source_df.index must equal target_df.index, and source_df.columns must equal
|
|
16
|
+
target_df.columns.
|
|
17
|
+
|
|
18
|
+
Each dict looks like:
|
|
19
|
+
{
|
|
20
|
+
"index_name_1": index_value_1,
|
|
21
|
+
...
|
|
22
|
+
"index_name_n": index_value_n,
|
|
23
|
+
"column_name_1": new_value_1,
|
|
24
|
+
...
|
|
25
|
+
"column_name_n": new_value_n
|
|
26
|
+
}
|
|
27
|
+
Rows that have no changes are not included.
|
|
28
|
+
"""
|
|
29
|
+
# Validate index formats match
|
|
30
|
+
if not _index_formats_match(source_df, target_df):
|
|
31
|
+
raise ValueError("Index formats do not match")
|
|
32
|
+
|
|
33
|
+
# Check columns match
|
|
34
|
+
if not source_df.columns.equals(target_df.columns):
|
|
35
|
+
raise ValueError("Source and target dfs have different columns")
|
|
36
|
+
|
|
37
|
+
# Handle different cases based on allow_new
|
|
38
|
+
if allow_new:
|
|
39
|
+
return _handle_with_new_rows(source_df, target_df)
|
|
40
|
+
else:
|
|
41
|
+
# Original behavior - indexes must be identical
|
|
42
|
+
if not source_df.index.equals(target_df.index):
|
|
43
|
+
raise ValueError("Source and target dfs have different indexes")
|
|
44
|
+
diff = source_df.ne(target_df)
|
|
45
|
+
return [
|
|
46
|
+
d
|
|
47
|
+
for (idx, vals) in diff.iterrows()
|
|
48
|
+
if (d := diff_dict(idx, vals, target_df))
|
|
49
|
+
]
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _index_formats_match(source_df, target_df):
|
|
53
|
+
"""Check if index formats match (same names and number of levels)."""
|
|
54
|
+
return source_df.index.names == target_df.index.names and len(
|
|
55
|
+
source_df.index.names
|
|
56
|
+
) == len(target_df.index.names)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _handle_with_new_rows(source_df, target_df):
|
|
60
|
+
"""Handle transformation when new rows are allowed."""
|
|
61
|
+
results = []
|
|
62
|
+
|
|
63
|
+
# Handle existing rows (rows in both source and target)
|
|
64
|
+
common_idx = source_df.index.intersection(target_df.index)
|
|
65
|
+
if len(common_idx) > 0:
|
|
66
|
+
source_common = source_df.loc[common_idx]
|
|
67
|
+
target_common = target_df.loc[common_idx]
|
|
68
|
+
diff = source_common.ne(target_common)
|
|
69
|
+
results.extend(
|
|
70
|
+
[
|
|
71
|
+
d
|
|
72
|
+
for (idx, vals) in diff.iterrows()
|
|
73
|
+
if (d := diff_dict(idx, vals, target_df))
|
|
74
|
+
]
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
# Handle new rows (rows in target but not in source)
|
|
78
|
+
new_idx = target_df.index.difference(source_df.index)
|
|
79
|
+
if len(new_idx) > 0:
|
|
80
|
+
new_rows = target_df.loc[new_idx]
|
|
81
|
+
for idx, row in new_rows.iterrows():
|
|
82
|
+
new_row_dict = _create_new_row_dict(idx, row, target_df)
|
|
83
|
+
if new_row_dict:
|
|
84
|
+
results.append(new_row_dict)
|
|
85
|
+
|
|
86
|
+
return results
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _create_new_row_dict(idx, row, target_df):
|
|
90
|
+
"""Create dictionary for new row, filtering out blank columns."""
|
|
91
|
+
if not isinstance(idx, Iterable) or isinstance(idx, str):
|
|
92
|
+
idx = (idx,)
|
|
93
|
+
|
|
94
|
+
# Start with index
|
|
95
|
+
result = dict(zip(target_df.index.names, idx))
|
|
96
|
+
|
|
97
|
+
# Add non-blank columns
|
|
98
|
+
for col_name, value in row.items():
|
|
99
|
+
if value != "" and str(value) != "nan":
|
|
100
|
+
result[col_name] = value
|
|
101
|
+
|
|
102
|
+
# Always return the dict (even if only index) - this handles the case
|
|
103
|
+
# where all columns are blank but we still want to include the index
|
|
104
|
+
return result
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def diff_dict(idx, vals, target_df):
|
|
108
|
+
if not isinstance(idx, Iterable) or isinstance(idx, str):
|
|
109
|
+
idx = (idx,)
|
|
110
|
+
true_indexes = vals[vals].index
|
|
111
|
+
if len(true_indexes) == 0:
|
|
112
|
+
return None
|
|
113
|
+
return (
|
|
114
|
+
dict(zip(target_df.index.names, idx))
|
|
115
|
+
| target_df.loc[idx, true_indexes].to_dict()
|
|
116
|
+
)
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
|
|
3
|
+
"""
|
|
4
|
+
Takes a REDCap CSV file and splits it into:
|
|
5
|
+
|
|
6
|
+
* A file for each event
|
|
7
|
+
* A file for repeated instruments in events where they happen
|
|
8
|
+
|
|
9
|
+
So, if your data has events 'scr,' 'pre,' and 'post', and 'pre' and 'post'
|
|
10
|
+
each have a repeated instrument called 'meds', you can expect
|
|
11
|
+
|
|
12
|
+
redcap__scr.csv
|
|
13
|
+
redcap__pre.csv
|
|
14
|
+
redcap__pre__meds.csv
|
|
15
|
+
redcap__post.csv
|
|
16
|
+
redcap__post__meds.csv
|
|
17
|
+
|
|
18
|
+
In addition, if you don't like the whole _arm_1 appended to your event names
|
|
19
|
+
(who does like that?) or you're using events to denote arms and want all your
|
|
20
|
+
event's data together, you can use the event_map file for this. That file
|
|
21
|
+
should be a CSV file and contain the columns 'redcap_event' and 'filename_event'
|
|
22
|
+
|
|
23
|
+
Example event maps might look like:
|
|
24
|
+
scr__all_arm_1,scr
|
|
25
|
+
pre__control_arm_1,pre
|
|
26
|
+
pre__intervention_arm_1,pre
|
|
27
|
+
|
|
28
|
+
Usage:
|
|
29
|
+
split_redcap_data.py [options] <input_file> <output_directory>
|
|
30
|
+
|
|
31
|
+
Options:
|
|
32
|
+
--event-map=<event_file> A file mapping redcap events to file events
|
|
33
|
+
--prefix=<prefix> A filename prefix for the output [default: redcap]
|
|
34
|
+
--no-condense Don't filter empty rows, columns and files
|
|
35
|
+
-h --help Show this screen
|
|
36
|
+
-d --debug Print debugging output
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
import logging
|
|
40
|
+
from collections import defaultdict
|
|
41
|
+
from pathlib import Path
|
|
42
|
+
|
|
43
|
+
import docopt
|
|
44
|
+
import pandas as pd
|
|
45
|
+
|
|
46
|
+
logging.basicConfig(format="%(message)s")
|
|
47
|
+
logger = logging.getLogger(__name__)
|
|
48
|
+
logger.setLevel(logging.INFO)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def make_event_map(mapping_file):
|
|
52
|
+
if mapping_file is None:
|
|
53
|
+
return {}
|
|
54
|
+
map_df = pd.read_csv(mapping_file, index_col="redcap_event", dtype=str)
|
|
55
|
+
event_map = map_df["filename_event"].to_dict()
|
|
56
|
+
return event_map
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def combine_names(event_name, rep_name):
|
|
60
|
+
parts = [name for name in [event_name, rep_name] if name != ""]
|
|
61
|
+
joined = "__".join(parts)
|
|
62
|
+
logger.debug(f"Made name {joined} from {event_name} and {rep_name}")
|
|
63
|
+
return joined
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def split_data(data, event_map):
|
|
67
|
+
"""
|
|
68
|
+
Returns a dict of nonprefixed_filename: data pairs
|
|
69
|
+
"""
|
|
70
|
+
data_lists = defaultdict(list)
|
|
71
|
+
index_col = data.columns[0]
|
|
72
|
+
event_groups = data.groupby(by="redcap_event_name")
|
|
73
|
+
for rc_event, event_data in event_groups:
|
|
74
|
+
out_event_name = event_map.get(rc_event, rc_event)
|
|
75
|
+
rep_groups = event_data.groupby(by="redcap_repeat_instrument")
|
|
76
|
+
for rep_group, rep_data in rep_groups:
|
|
77
|
+
out_name = combine_names(out_event_name, rep_group)
|
|
78
|
+
data_lists[out_name].append(rep_data)
|
|
79
|
+
# Now, we need to concat the dataframes and sort them by the index column
|
|
80
|
+
dataframes = {
|
|
81
|
+
name: pd.concat(df_list).sort_values(by=index_col)
|
|
82
|
+
for name, df_list in data_lists.items()
|
|
83
|
+
}
|
|
84
|
+
for name, df in dataframes.items():
|
|
85
|
+
logger.debug(f"Name: {name}, Shape: {df.shape})")
|
|
86
|
+
return dataframes
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def condense_df(df, condense_rows=True, condense_cols=True):
|
|
90
|
+
"""
|
|
91
|
+
Drop any rows and columns with only blank values
|
|
92
|
+
This is not a super great way to do this, as it means the shape of the data we write
|
|
93
|
+
(columns in particular) will depend on the content of the data.
|
|
94
|
+
A better way to do this would be to make a list of what columns should appear in
|
|
95
|
+
which events using the API and filter using that list. That's not so simple, though.
|
|
96
|
+
"""
|
|
97
|
+
index_col = df.columns[0]
|
|
98
|
+
reserved_cols = {
|
|
99
|
+
index_col,
|
|
100
|
+
"redcap_event_name",
|
|
101
|
+
"redcap_repeat_instrument",
|
|
102
|
+
"redcap_repeat_instance",
|
|
103
|
+
}
|
|
104
|
+
rowdrop_cols = list(set(df.columns) - reserved_cols)
|
|
105
|
+
|
|
106
|
+
df_rows_cleaned = df
|
|
107
|
+
if condense_rows:
|
|
108
|
+
df_rows_cleaned = df[~(df[rowdrop_cols] == "").all(axis=1)]
|
|
109
|
+
cols_to_keep = df.columns
|
|
110
|
+
if condense_cols:
|
|
111
|
+
cols_to_keep = cols_to_keep = ~(df_rows_cleaned == "").all(axis=0)
|
|
112
|
+
|
|
113
|
+
return df_rows_cleaned.loc[:, cols_to_keep].copy()
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def split_redcap_data(
|
|
117
|
+
input_file, output_directory, prefix="redcap", mapping_file=None, condense=True
|
|
118
|
+
):
|
|
119
|
+
event_map = make_event_map(mapping_file)
|
|
120
|
+
logger.debug(f"Event map: {event_map}")
|
|
121
|
+
data = pd.read_csv(input_file, index_col=None, dtype=str, na_filter=False)
|
|
122
|
+
# Make sure the event and repeating columns are present, so we can process
|
|
123
|
+
# the data the same in all cases.
|
|
124
|
+
if "redcap_event_name" not in data.columns:
|
|
125
|
+
logger.debug("Single event file, adding event column")
|
|
126
|
+
data["redcap_event_name"] = ""
|
|
127
|
+
if "redcap_repeat_instrument" not in data.columns:
|
|
128
|
+
data["redcap_repeat_instrument"] = ""
|
|
129
|
+
data["redcap_repeat_instance"] = ""
|
|
130
|
+
logger.debug("Non-repeating file, added repeat columns")
|
|
131
|
+
named_dataframes = split_data(data, event_map)
|
|
132
|
+
logger.debug(named_dataframes)
|
|
133
|
+
for name, df in named_dataframes.items():
|
|
134
|
+
if condense:
|
|
135
|
+
df = condense_df(df)
|
|
136
|
+
file_base = combine_names(prefix, name)
|
|
137
|
+
filename = f"{file_base}.csv"
|
|
138
|
+
out_path = Path(output_directory) / filename
|
|
139
|
+
logger.info(f"Saving dataframe with shape {df.shape} to {out_path}")
|
|
140
|
+
df.to_csv(out_path, index=False)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def main():
|
|
144
|
+
args = docopt.docopt(__doc__)
|
|
145
|
+
if args["--debug"]:
|
|
146
|
+
logger.setLevel(logging.DEBUG)
|
|
147
|
+
logger.debug(args)
|
|
148
|
+
split_redcap_data(
|
|
149
|
+
Path(args["<input_file>"]),
|
|
150
|
+
Path(args["<output_directory>"]),
|
|
151
|
+
args["--prefix"],
|
|
152
|
+
args["--event-map"],
|
|
153
|
+
not args["--no-condense"],
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
if __name__ == "__main__":
|
|
158
|
+
main()
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
|
|
3
|
+
"""
|
|
4
|
+
Update a REDCap database with the minimum changes needed to make the system in
|
|
5
|
+
sync.
|
|
6
|
+
|
|
7
|
+
Requires the following environment variables to be set:
|
|
8
|
+
REDCAP_API_URL
|
|
9
|
+
REDCAP_API_TOKEN
|
|
10
|
+
|
|
11
|
+
Usage: update_redcap_diff.py [options] <base_csv> <updated_csv>
|
|
12
|
+
|
|
13
|
+
Options:
|
|
14
|
+
--allow-new Allow adding new rows to REDCap.
|
|
15
|
+
--dry-run Don't actually make changes.
|
|
16
|
+
-h --help Show this screen.
|
|
17
|
+
-v --verbose Show debug logging.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
import logging
|
|
21
|
+
import os
|
|
22
|
+
import sys
|
|
23
|
+
|
|
24
|
+
import docopt
|
|
25
|
+
import pandas as pd
|
|
26
|
+
import redcap
|
|
27
|
+
import redcap_toolbox.minchange
|
|
28
|
+
|
|
29
|
+
logging.basicConfig(format="%(message)s")
|
|
30
|
+
logger = logging.getLogger(__name__)
|
|
31
|
+
logger.setLevel(logging.INFO)
|
|
32
|
+
|
|
33
|
+
# Initialize these as None - they'll be set in main()
|
|
34
|
+
API_URL = None
|
|
35
|
+
API_TOK = None
|
|
36
|
+
PROJ = None
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def update_redcap_diff(base_csv, updated_csv, dry_run, allow_new=False):
|
|
40
|
+
base_df = pd.read_csv(
|
|
41
|
+
base_csv, dtype=str, na_values=["nan", "NaN"], keep_default_na=False
|
|
42
|
+
)
|
|
43
|
+
index_cols = [base_df.columns[0]]
|
|
44
|
+
if "redcap_event_name" in base_df.columns:
|
|
45
|
+
index_cols.append("redcap_event_name")
|
|
46
|
+
base_df.set_index(index_cols, inplace=True)
|
|
47
|
+
logger.debug(f"Using index: {index_cols}")
|
|
48
|
+
updated_df = pd.read_csv(
|
|
49
|
+
updated_csv, dtype=str, na_values=["nan", "NaN"], keep_default_na=False
|
|
50
|
+
)
|
|
51
|
+
updated_df.set_index(index_cols, inplace=True)
|
|
52
|
+
|
|
53
|
+
diffs = redcap_toolbox.minchange.transformation_dicts(
|
|
54
|
+
base_df, updated_df, allow_new=allow_new
|
|
55
|
+
)
|
|
56
|
+
if len(diffs) == 0:
|
|
57
|
+
logger.info("No changes to make")
|
|
58
|
+
return
|
|
59
|
+
logger.debug(f"Diffs: {diffs}")
|
|
60
|
+
|
|
61
|
+
if dry_run:
|
|
62
|
+
logger.warning("DRY RUN, NOT UPDATING ANYTHING")
|
|
63
|
+
logger.warning(f"First change would have been {diffs[0]}")
|
|
64
|
+
else:
|
|
65
|
+
result = PROJ.import_records(diffs)
|
|
66
|
+
logger.info(f"Import record result: {result}")
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def main():
|
|
70
|
+
global API_URL, API_TOK, PROJ
|
|
71
|
+
|
|
72
|
+
# Initialize API connection
|
|
73
|
+
try:
|
|
74
|
+
API_URL = os.environ["REDCAP_API_URL"]
|
|
75
|
+
API_TOK = os.environ["REDCAP_API_TOKEN"]
|
|
76
|
+
except KeyError:
|
|
77
|
+
logger.error("REDCAP_API_URL and REDCAP_API_TOKEN must both be set!")
|
|
78
|
+
sys.exit(1)
|
|
79
|
+
PROJ = redcap.Project(API_URL, API_TOK)
|
|
80
|
+
|
|
81
|
+
args = docopt.docopt(__doc__)
|
|
82
|
+
if args["--verbose"]:
|
|
83
|
+
logger.setLevel(logging.DEBUG)
|
|
84
|
+
logger.debug(args)
|
|
85
|
+
update_redcap_diff(
|
|
86
|
+
args["<base_csv>"],
|
|
87
|
+
args["<updated_csv>"],
|
|
88
|
+
args["--dry-run"],
|
|
89
|
+
args["--allow-new"],
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
if __name__ == "__main__":
|
|
94
|
+
main()
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: redcap_toolbox
|
|
3
|
+
Version: 2025.7.9
|
|
4
|
+
Summary: A collection of tools to interact with the REDCap API
|
|
5
|
+
Project-URL: Homepage, https://github.com/uwmadison-chm/redcap-toolbox
|
|
6
|
+
Project-URL: Issues, https://github.com/uwmadison-chm/redcap-toolbox/issues
|
|
7
|
+
Author-email: Nate Vack <njvack@wisc.edu>
|
|
8
|
+
Maintainer-email: Nate Vack <njvack@wisc.edu>, Nicholas Vanhaute <nvanhaute@wisc.edu>, Stuti Shrivastava <sshrivastav6@wisc.edu>
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Natural Language :: English
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Requires-Dist: docopt-ng>=0.9.0
|
|
17
|
+
Requires-Dist: pandas>=2.2.3
|
|
18
|
+
Requires-Dist: pycap>=2.6.0
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
|
|
21
|
+
# redcap-toolbox
|
|
22
|
+
|
|
23
|
+
Python package of REDCap tools
|
|
24
|
+
|
|
25
|
+
## Background
|
|
26
|
+
|
|
27
|
+
The goal of `redcap-toolbox` is to provide researchers with the tools to download and upload data to existing REDCap
|
|
28
|
+
builds. One key aspect of this toolbox is that the minimum number of changes needed to the records are calculated prior
|
|
29
|
+
to importing data in to REDCap, thereby reducing the API load.
|
|
30
|
+
|
|
31
|
+
## Installing REDCap Toolbox and Supported Versions
|
|
32
|
+
|
|
33
|
+
redcap-toolbox is available on PyPI:
|
|
34
|
+
|
|
35
|
+
```console
|
|
36
|
+
$ pip install redcap-toolbox
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
redcap-toolbox officially supports Python 3.8+.
|
|
40
|
+
|
|
41
|
+
## Getting started
|
|
42
|
+
|
|
43
|
+
`redcap-toolbox` relies on the environmental variables `REDCAP_API_URL` and `REDCAP_API_TOKEN`.
|
|
44
|
+
|
|
45
|
+
### Downloading data
|
|
46
|
+
|
|
47
|
+
* Download REDCap dataset using `download_redcap` and provide an output file name. By default, all forms are exported.
|
|
48
|
+
* If you want to include survey timestamps, add the optional flag: `--survey-fields`
|
|
49
|
+
* To download specific instrument forms, enter form names to export in a text file, one per line. If in the web UI, the
|
|
50
|
+
form name has a space in it, replace the space with an underscore. Then, provide that file using the optional flag:
|
|
51
|
+
`--forms get_forms.csv`
|
|
52
|
+
|
|
53
|
+
An example call might look like this:
|
|
54
|
+
|
|
55
|
+
`download_redcap --survey-fields --forms get_forms.csv source_data/full_data.csv`
|
|
56
|
+
|
|
57
|
+
which will download the data set with only the forms defined in the `get_forms.csv` with timestamps
|
|
58
|
+
|
|
59
|
+
### Downloading reports
|
|
60
|
+
|
|
61
|
+
* Reports can be downloaded using `download_redcap_report` with either a list of report IDs separated by commas or a
|
|
62
|
+
file with list of report IDs, one per line.
|
|
63
|
+
* Use the `--prefix` flag, to specify the prefix to be added for the filenames. Default is `redcap`.
|
|
64
|
+
|
|
65
|
+
An example call might look like this:
|
|
66
|
+
|
|
67
|
+
`download_redcap_report --file report_ids.csv --prefix StudyName report_data`
|
|
68
|
+
|
|
69
|
+
or
|
|
70
|
+
|
|
71
|
+
`download_redcap_report --id 32001,32004 --prefix StudyName report_data`
|
|
72
|
+
|
|
73
|
+
which will save all the reports for IDs listed in the `report_ids.csv` file in the `report_data` directory.
|
|
74
|
+
|
|
75
|
+
* The output report filenames will look like this:
|
|
76
|
+
```
|
|
77
|
+
report_data
|
|
78
|
+
├── StudyName__report_32001.csv
|
|
79
|
+
└── StudyName__report_32004.csv
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
### Splitting REDCap data into event files
|
|
83
|
+
|
|
84
|
+
* Use `split_redcap_data` to split the REDCap CSV file into:
|
|
85
|
+
* A file for each event
|
|
86
|
+
* A file for repeated instruments in events where they happen
|
|
87
|
+
|
|
88
|
+
* So, if your data has events 'scr', 'pre', and 'post', and 'pre' and 'post'
|
|
89
|
+
each have a repeated instrument called 'meds', you can expect the output files to like this:
|
|
90
|
+
```
|
|
91
|
+
├── redcap__scr.csv
|
|
92
|
+
├── redcap__pre.csv
|
|
93
|
+
├── redcap__pre__meds.csv
|
|
94
|
+
├── redcap__post.csv
|
|
95
|
+
├── redcap__post.csv
|
|
96
|
+
└── redcap__post__meds.csv
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
* In addition, if you don't like the whole _arm_1 appended to your event names
|
|
100
|
+
(who does like that?) or you're using events to denote arms and want all your
|
|
101
|
+
event's data together, you can use the event_map file for this. That file
|
|
102
|
+
should be a CSV file and contain the columns 'redcap_event' and 'filename_event'
|
|
103
|
+
|
|
104
|
+
* Example event maps might look like:
|
|
105
|
+
```
|
|
106
|
+
scr__all_arm_1,scr
|
|
107
|
+
pre__control_arm_1,pre
|
|
108
|
+
pre__intervention_arm_1,pre
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
An example call might look like this:
|
|
112
|
+
|
|
113
|
+
`split_redcap_data --event-map=event_map.csv --prefix StudyName --no-condense source_data/full_data.csv source_data`
|
|
114
|
+
|
|
115
|
+
where the split event files will be saved in the `source_data` directory as well with the prefix `StudyName` added to
|
|
116
|
+
them.
|
|
117
|
+
|
|
118
|
+
```
|
|
119
|
+
source_data
|
|
120
|
+
├── full_data.csv
|
|
121
|
+
├── StudyName__scr.csv
|
|
122
|
+
├── StudyName__pre.csv
|
|
123
|
+
├── StudyName__pre_meds.csv
|
|
124
|
+
├── StudyName__post.csv
|
|
125
|
+
└── StudyName__post_meds.csv
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
### Update records in REDCap
|
|
129
|
+
|
|
130
|
+
* Update the REDCap database with the minimum changes needed to make the system in sync.
|
|
131
|
+
* It is important that the updated data file has the same number of rows and columns as the original data file.
|
|
132
|
+
* This functionality is especially useful when updating the record information for Tracking purposes.
|
|
133
|
+
|
|
134
|
+
An example call might look like this:
|
|
135
|
+
|
|
136
|
+
`update_redcap_diff StudyName__scr.csv StudyName__scr_cache.csv`
|
|
137
|
+
|
|
138
|
+
where the `_cache.csv` file contains the changes made to the original data file.
|
|
139
|
+
|
|
140
|
+
## Credits
|
|
141
|
+
|
|
142
|
+
`redcap-toolbox` was written by Nate Vack <njvack@wisc.edu>, with features added by Nicholas
|
|
143
|
+
Vanhaute <nvanhaute@wisc.edu> and Stuti Shrivastava <sshrivastav6@wisc.edu>.
|
|
144
|
+
`redcap-toolbox` is copyright 2023 by the Boards of Regents of the University of Wisconsin System.
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
redcap_toolbox/__init__.py,sha256=lCVOluSc2Sm6jtqpoboZP3cSNa2tKmX3DkV1C5nMNJg,273
|
|
2
|
+
redcap_toolbox/download_redcap.py,sha256=49eE1gR6s1K11dw-oYkmg7jhpN1ZE5Sw1MSi-5Hpe64,1784
|
|
3
|
+
redcap_toolbox/download_redcap_report.py,sha256=LQd0a1XW9U4C8Oclo92f97LGKbU-HSoGnL_kxoZZBus,3203
|
|
4
|
+
redcap_toolbox/minchange.py,sha256=OqFVutzzpvpgv771mzs9krpU9FOZfWU4dOHGsqc2It8,3815
|
|
5
|
+
redcap_toolbox/split_redcap_data.py,sha256=9FDEw2SGH7g9vMulPwjnCSL3-f5o6c-c8211025R9c8,5361
|
|
6
|
+
redcap_toolbox/update_redcap_diff.py,sha256=kkAKZsfzkDtOsFFdTrE97_WZxEnpNTGqnJ0y-EgHjfU,2495
|
|
7
|
+
redcap_toolbox-2025.7.9.dist-info/METADATA,sha256=WywoN7tcEX7O7BsLUuMyknbyh0tAdnwnez8pCsqJU1I,5344
|
|
8
|
+
redcap_toolbox-2025.7.9.dist-info/WHEEL,sha256=tkmg4JIqwd9H8mL30xA7crRmoStyCtGp0VWshokd1Jc,105
|
|
9
|
+
redcap_toolbox-2025.7.9.dist-info/entry_points.txt,sha256=2E1RlSqV_WZjmAD6y9kY7at8o2sobmZ3CTcs5X-SnAo,258
|
|
10
|
+
redcap_toolbox-2025.7.9.dist-info/licenses/LICENSE,sha256=IzENQQtGRTcLpRjPl-4CgjxbkNlGuVIqxqBtATqwwsY,1081
|
|
11
|
+
redcap_toolbox-2025.7.9.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2024 Center for Healthy Minds
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|