cehrbert-data 0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. __init__.py +0 -0
  2. cehrbert_data/__init__.py +0 -0
  3. cehrbert_data/apps/__init__.py +0 -0
  4. cehrbert_data/apps/generate_concept_similarity_table.py +423 -0
  5. cehrbert_data/apps/generate_hierarchical_bert_training_data.py +238 -0
  6. cehrbert_data/apps/generate_included_concept_list.py +116 -0
  7. cehrbert_data/apps/generate_information_content.py +131 -0
  8. cehrbert_data/apps/generate_required_labs.py +112 -0
  9. cehrbert_data/apps/generate_training_data.py +337 -0
  10. cehrbert_data/cohorts/__init__.py +0 -0
  11. cehrbert_data/cohorts/atrial_fibrillation.py +45 -0
  12. cehrbert_data/cohorts/cabg.py +72 -0
  13. cehrbert_data/cohorts/coronary_artery_disease.py +84 -0
  14. cehrbert_data/cohorts/covid.py +43 -0
  15. cehrbert_data/cohorts/covid_inpatient.py +84 -0
  16. cehrbert_data/cohorts/death.py +46 -0
  17. cehrbert_data/cohorts/heart_failure.py +423 -0
  18. cehrbert_data/cohorts/ischemic_stroke.py +45 -0
  19. cehrbert_data/cohorts/last_visit_discharged_home.py +35 -0
  20. cehrbert_data/cohorts/query_builder.py +153 -0
  21. cehrbert_data/cohorts/spark_app_base.py +745 -0
  22. cehrbert_data/cohorts/type_two_diabietes.py +166 -0
  23. cehrbert_data/cohorts/ventilation.py +22 -0
  24. cehrbert_data/config/__init__.py +0 -0
  25. cehrbert_data/config/output_names.py +9 -0
  26. cehrbert_data/const/__init__.py +0 -0
  27. cehrbert_data/const/__pycache__/__init__.cpython-311.pyc +0 -0
  28. cehrbert_data/const/__pycache__/common.cpython-311.pyc +0 -0
  29. cehrbert_data/const/common.py +28 -0
  30. cehrbert_data/decorators/__init__.py +0 -0
  31. cehrbert_data/decorators/__pycache__/__init__.cpython-311.pyc +0 -0
  32. cehrbert_data/decorators/__pycache__/patient_event_decorator.cpython-311.pyc +0 -0
  33. cehrbert_data/decorators/patient_event_decorator.py +759 -0
  34. cehrbert_data/prediction_cohorts/__init__.py +0 -0
  35. cehrbert_data/prediction_cohorts/afib_ischemic_stroke.py +14 -0
  36. cehrbert_data/prediction_cohorts/cad_cabg_cohort.py +19 -0
  37. cehrbert_data/prediction_cohorts/cad_hf_cohort.py +14 -0
  38. cehrbert_data/prediction_cohorts/copd_readmission.py +76 -0
  39. cehrbert_data/prediction_cohorts/covid_death.py +14 -0
  40. cehrbert_data/prediction_cohorts/covid_ventilation.py +14 -0
  41. cehrbert_data/prediction_cohorts/discharge_home_death.py +18 -0
  42. cehrbert_data/prediction_cohorts/hf_readmission.py +82 -0
  43. cehrbert_data/prediction_cohorts/hospitalization.py +100 -0
  44. cehrbert_data/prediction_cohorts/hospitalization_mortality.py +77 -0
  45. cehrbert_data/prediction_cohorts/t2dm_hf_cohort.py +14 -0
  46. cehrbert_data/queries/__init__.py +0 -0
  47. cehrbert_data/queries/measurement_unit_stats_query.py +42 -0
  48. cehrbert_data/tools/__init__.py +0 -0
  49. cehrbert_data/tools/download_omop_tables.py +141 -0
  50. cehrbert_data/utils/__init__.py +0 -0
  51. cehrbert_data/utils/spark_parse_args.py +350 -0
  52. cehrbert_data/utils/spark_utils.py +1405 -0
  53. cehrbert_data-0.0.1.dist-info/LICENSE +21 -0
  54. cehrbert_data-0.0.1.dist-info/METADATA +116 -0
  55. cehrbert_data-0.0.1.dist-info/RECORD +57 -0
  56. cehrbert_data-0.0.1.dist-info/WHEEL +5 -0
  57. cehrbert_data-0.0.1.dist-info/top_level.txt +2 -0
@@ -0,0 +1,45 @@
1
+ from cehrbert_data.cohorts.query_builder import AncestorTableSpec, QueryBuilder, QuerySpec
2
+ from cehrbert_data.const.common import CONDITION_OCCURRENCE, PERSON, VISIT_OCCURRENCE
3
+
4
+ COHORT_QUERY_TEMPLATE = """
5
+ SELECT
6
+ co.person_id,
7
+ FIRST(DATE(vo.visit_start_date)) OVER (PARTITION BY co.person_id
8
+ ORDER BY DATE(vo.visit_start_date), vo.visit_occurrence_id) AS index_date,
9
+ FIRST(vo.visit_occurrence_id) OVER (PARTITION BY co.person_id
10
+ ORDER BY DATE(vo.visit_start_date), vo.visit_occurrence_id) AS visit_occurrence_id
11
+ FROM global_temp.condition_occurrence AS co
12
+ JOIN global_temp.visit_occurrence AS vo
13
+ ON co.visit_occurrence_id = vo.visit_occurrence_id
14
+ JOIN global_temp.{atrial_fibrillation_concepts} AS c
15
+ ON co.condition_concept_id = c.concept_id
16
+ """
17
+
18
+ ATRIAL_FIBRILLATION_CONCEPT_ID = [313217]
19
+
20
+ DEPENDENCY_LIST = [PERSON, VISIT_OCCURRENCE, CONDITION_OCCURRENCE]
21
+
22
+ DEFAULT_COHORT_NAME = "atrial_fibrillation"
23
+ ATRIAL_FIBRILLATION_CONCEPTS = "atrial_fibrillation_concepts"
24
+
25
+
26
+ def query_builder():
27
+ query = QuerySpec(
28
+ table_name=DEFAULT_COHORT_NAME,
29
+ query_template=COHORT_QUERY_TEMPLATE,
30
+ parameters={"atrial_fibrillation_concepts": ATRIAL_FIBRILLATION_CONCEPTS},
31
+ )
32
+
33
+ ancestor_table_specs = [
34
+ AncestorTableSpec(
35
+ table_name=ATRIAL_FIBRILLATION_CONCEPTS,
36
+ ancestor_concept_ids=ATRIAL_FIBRILLATION_CONCEPT_ID,
37
+ is_standard=True,
38
+ )
39
+ ]
40
+ return QueryBuilder(
41
+ cohort_name=DEFAULT_COHORT_NAME,
42
+ dependency_list=DEPENDENCY_LIST,
43
+ query=query,
44
+ ancestor_table_specs=ancestor_table_specs,
45
+ )
@@ -0,0 +1,72 @@
1
+ from cehrbert_data.cohorts.query_builder import AncestorTableSpec, QueryBuilder, QuerySpec
2
+ from cehrbert_data.const.common import PERSON, PROCEDURE_OCCURRENCE, VISIT_OCCURRENCE
3
+
4
+ COHORT_QUERY_TEMPLATE = """
5
+ SELECT DISTINCT
6
+ c.person_id,
7
+ c.index_date,
8
+ c.visit_occurrence_id
9
+ FROM
10
+ (
11
+ SELECT DISTINCT
12
+ vo.person_id,
13
+ FIRST(DATE(vo.visit_start_date)) OVER (PARTITION BY po.person_id
14
+ ORDER BY DATE(vo.visit_start_date), vo.visit_occurrence_id) AS index_date,
15
+ FIRST(vo.visit_occurrence_id) OVER (PARTITION BY po.person_id
16
+ ORDER BY DATE(vo.visit_start_date), vo.visit_occurrence_id) AS visit_occurrence_id
17
+ FROM global_temp.procedure_occurrence AS po
18
+ JOIN global_temp.visit_occurrence AS vo
19
+ ON po.visit_occurrence_id = vo.visit_occurrence_id
20
+ WHERE EXISTS (
21
+ SELECT 1
22
+ FROM global_temp.{cabg_concept_table} AS ie
23
+ WHERE po.procedure_concept_id = ie.concept_id
24
+ )
25
+ ) c
26
+ WHERE c.index_date >= '{date_lower_bound}'
27
+ """
28
+
29
+ DEFAULT_COHORT_NAME = "cabg"
30
+ DEPENDENCY_LIST = [PERSON, VISIT_OCCURRENCE, PROCEDURE_OCCURRENCE]
31
+ CABG_INCLUSION_TABLE = "CABG"
32
+ CABG_CONCEPTS = [
33
+ 43528001,
34
+ 43528003,
35
+ 43528004,
36
+ 43528002,
37
+ 4305852,
38
+ 4168831,
39
+ 2107250,
40
+ 2107216,
41
+ 2107222,
42
+ 2107231,
43
+ 4336464,
44
+ 4231998,
45
+ 4284104,
46
+ 2100873,
47
+ ]
48
+
49
+
50
+ def query_builder(spark_args):
51
+ query = QuerySpec(
52
+ table_name=DEFAULT_COHORT_NAME,
53
+ query_template=COHORT_QUERY_TEMPLATE,
54
+ parameters={
55
+ "cabg_concept_table": CABG_INCLUSION_TABLE,
56
+ "date_lower_bound": spark_args.date_lower_bound,
57
+ },
58
+ )
59
+
60
+ ancestor_table_specs = [
61
+ AncestorTableSpec(
62
+ table_name=CABG_INCLUSION_TABLE,
63
+ ancestor_concept_ids=CABG_CONCEPTS,
64
+ is_standard=True,
65
+ )
66
+ ]
67
+ return QueryBuilder(
68
+ cohort_name=DEFAULT_COHORT_NAME,
69
+ dependency_list=DEPENDENCY_LIST,
70
+ query=query,
71
+ ancestor_table_specs=ancestor_table_specs,
72
+ )
@@ -0,0 +1,84 @@
1
+ from cehrbert_data.cohorts.query_builder import AncestorTableSpec, QueryBuilder, QuerySpec
2
+ from cehrbert_data.const.common import CONDITION_OCCURRENCE, PERSON, PROCEDURE_OCCURRENCE, VISIT_OCCURRENCE
3
+
4
+ COHORT_QUERY_TEMPLATE = """
5
+ WITH prior_graft_stent AS (
6
+ SELECT
7
+ po.person_id,
8
+ po.procedure_date
9
+ FROM global_temp.procedure_occurrence AS po
10
+ WHERE EXISTS (
11
+ SELECT 1
12
+ FROM global_temp.{graft_stent_table} AS gs
13
+ WHERE po.procedure_concept_id = gs.concept_id
14
+ )
15
+ )
16
+ SELECT DISTINCT
17
+ c.person_id,
18
+ c.index_date,
19
+ c.visit_occurrence_id
20
+ FROM
21
+ (
22
+ SELECT DISTINCT
23
+ vo.person_id,
24
+ FIRST(DATE(vo.visit_start_date)) OVER (PARTITION BY co.person_id
25
+ ORDER BY DATE(vo.visit_start_date), vo.visit_occurrence_id) AS index_date,
26
+ FIRST(vo.visit_occurrence_id) OVER (PARTITION BY co.person_id
27
+ ORDER BY DATE(vo.visit_start_date), vo.visit_occurrence_id) AS visit_occurrence_id
28
+ FROM global_temp.condition_occurrence AS co
29
+ JOIN global_temp.visit_occurrence AS vo
30
+ ON co.visit_occurrence_id = vo.visit_occurrence_id
31
+ WHERE EXISTS (
32
+ SELECT 1
33
+ FROM global_temp.{cad_concept_table} AS ie
34
+ WHERE co.condition_concept_id = ie.concept_id
35
+ )
36
+ ) c
37
+ WHERE NOT EXISTS (
38
+ -- The patients who had a graft or stent procedures before the index date
39
+ -- need to be removed from the cohort
40
+ SELECT 1
41
+ FROM prior_graft_stent AS exclusion
42
+ WHERE exclusion.person_id = c.person_id
43
+ AND c.index_date > exclusion.procedure_date
44
+ ) AND c.index_date >= '{date_lower_bound}'
45
+ """
46
+
47
+ DEFAULT_COHORT_NAME = "coronary_artery_disease"
48
+ DEPENDENCY_LIST = [PERSON, VISIT_OCCURRENCE, CONDITION_OCCURRENCE, PROCEDURE_OCCURRENCE]
49
+ CAD_INCLUSION_TABLE = "CAD"
50
+ CAD_CONCEPTS = [317576]
51
+
52
+ PRIOR_PROCEDURE_TABLE = "graft_stent"
53
+ PRIOR_PROCEDURES = [4296227, 42537730, 762043, 44782770, 42537729]
54
+
55
+
56
+ def query_builder(spark_args):
57
+ query = QuerySpec(
58
+ table_name=DEFAULT_COHORT_NAME,
59
+ query_template=COHORT_QUERY_TEMPLATE,
60
+ parameters={
61
+ "cad_concept_table": CAD_INCLUSION_TABLE,
62
+ "graft_stent_table": PRIOR_PROCEDURE_TABLE,
63
+ "date_lower_bound": spark_args.date_lower_bound,
64
+ },
65
+ )
66
+
67
+ ancestor_table_specs = [
68
+ AncestorTableSpec(
69
+ table_name=CAD_INCLUSION_TABLE,
70
+ ancestor_concept_ids=CAD_CONCEPTS,
71
+ is_standard=True,
72
+ ),
73
+ AncestorTableSpec(
74
+ table_name=PRIOR_PROCEDURE_TABLE,
75
+ ancestor_concept_ids=PRIOR_PROCEDURES,
76
+ is_standard=True,
77
+ ),
78
+ ]
79
+ return QueryBuilder(
80
+ cohort_name=DEFAULT_COHORT_NAME,
81
+ dependency_list=DEPENDENCY_LIST,
82
+ query=query,
83
+ ancestor_table_specs=ancestor_table_specs,
84
+ )
@@ -0,0 +1,43 @@
1
+ from cehrbert_data.cohorts.query_builder import QueryBuilder, QuerySpec
2
+ from cehrbert_data.const.common import CONDITION_OCCURRENCE, MEASUREMENT, PERSON, VISIT_OCCURRENCE
3
+
4
+ COVID_COHORT_QUERY = """
5
+ SELECT DISTINCT
6
+ c.person_id,
7
+ DATE_ADD(FIRST(index_date) OVER (PARTITION BY person_id ORDER BY index_date, visit_occurrence_id), 1) AS index_date,
8
+ FIRST(visit_occurrence_id) OVER (PARTITION BY person_id ORDER BY index_date, visit_occurrence_id) AS visit_occurrence_id
9
+ FROM
10
+ (
11
+ SELECT DISTINCT
12
+ m.person_id,
13
+ FIRST(visit_start_date) OVER (PARTITION BY v.person_id ORDER BY visit_start_date, v.visit_occurrence_id) AS index_date,
14
+ FIRST(v.visit_occurrence_id) OVER (PARTITION BY v.person_id ORDER BY visit_start_date, v.visit_occurrence_id) AS visit_occurrence_id
15
+ FROM global_temp.measurement AS m
16
+ JOIN global_temp.visit_occurrence AS v
17
+ ON m.visit_occurrence_id = v.visit_occurrence_id
18
+ JOIN global_temp.concept AS c
19
+ ON m.value_as_concept_id = c.concept_id
20
+ WHERE m.measurement_concept_id IN (723475,723479,706178,723473,723474,586515,706177,706163,706180,706181)
21
+ AND c.concept_name IN ('Detected', 'Positve')
22
+
23
+ UNION
24
+
25
+ SELECT
26
+ co.person_id,
27
+ FIRST(visit_start_date) OVER (PARTITION BY v.person_id ORDER BY visit_start_date, v.visit_occurrence_id) AS index_date,
28
+ FIRST(v.visit_occurrence_id) OVER (PARTITION BY v.person_id ORDER BY visit_start_date, v.visit_occurrence_id) AS visit_occurrence_id
29
+ FROM global_temp.condition_occurrence AS co
30
+ JOIN global_temp.visit_occurrence AS v
31
+ ON co.visit_occurrence_id = v.visit_occurrence_id
32
+ WHERE co.condition_concept_id = 37311061
33
+ ) c
34
+ """
35
+
36
+ DEFAULT_COHORT_NAME = "covid19"
37
+ DEPENDENCY_LIST = [PERSON, VISIT_OCCURRENCE, MEASUREMENT, CONDITION_OCCURRENCE]
38
+
39
+
40
+ def query_builder():
41
+ query = QuerySpec(table_name=DEFAULT_COHORT_NAME, query_template=COVID_COHORT_QUERY, parameters={})
42
+
43
+ return QueryBuilder(cohort_name=DEFAULT_COHORT_NAME, dependency_list=DEPENDENCY_LIST, query=query)
@@ -0,0 +1,84 @@
1
+ from cehrbert_data.cohorts.query_builder import QueryBuilder, QuerySpec
2
+ from cehrbert_data.const.common import CONDITION_OCCURRENCE, MEASUREMENT, PERSON, VISIT_OCCURRENCE
3
+
4
+ COVID_COHORT_QUERY = """
5
+ WITH covid_positive AS
6
+ (
7
+
8
+ SELECT DISTINCT
9
+ ROW_NUMBER() OVER(ORDER BY c.person_id, c.index_date) AS test_row_number,
10
+ c.*
11
+ FROM
12
+ (
13
+ SELECT DISTINCT
14
+ m.person_id,
15
+ COALESCE(vo.visit_start_datetime, m.measurement_datetime) AS index_date,
16
+ vo.visit_occurrence_id,
17
+ vo.visit_concept_id
18
+ FROM global_temp.measurement AS m
19
+ LEFT JOIN global_temp.visit_occurrence AS vo
20
+ ON m.visit_occurrence_id = vo.visit_occurrence_id
21
+ WHERE measurement_concept_id IN (723475,723479,706178,723473,723474,586515,706177,706163,706180,706181)
22
+ AND value_source_value = 'Detected'
23
+
24
+ UNION
25
+
26
+ SELECT DISTINCT
27
+ co.person_id,
28
+ COALESCE(vo.visit_start_datetime, co.condition_start_datetime) AS index_date,
29
+ vo.visit_occurrence_id,
30
+ vo.visit_concept_id
31
+ FROM global_temp.condition_occurrence AS co
32
+ LEFT JOIN global_temp.visit_occurrence AS vo
33
+ ON co.visit_occurrence_id = vo.visit_occurrence_id
34
+ WHERE condition_concept_id = 37311061
35
+ ) c
36
+ ),
37
+
38
+ covid_test_with_no_visit AS
39
+ (
40
+ SELECT DISTINCT
41
+ c.test_row_number,
42
+ c.person_id,
43
+ FIRST_VALUE(vo.visit_start_datetime) OVER(PARTITION BY c.person_id ORDER BY vo.visit_start_datetime DESC) AS index_date,
44
+ FIRST_VALUE(vo.visit_occurrence_id) OVER(PARTITION BY c.person_id ORDER BY vo.visit_start_datetime DESC) AS visit_occurrence_id,
45
+ FIRST_VALUE(vo.visit_concept_id) OVER(PARTITION BY c.person_id ORDER BY vo.visit_start_datetime DESC) AS visit_concept_id
46
+ FROM covid_positive AS c
47
+ JOIN global_temp.visit_occurrence AS vo
48
+ ON c.person_id = vo.person_id AND c.index_date BETWEEN DATE_ADD(vo.visit_start_date, -7) AND vo.visit_start_date
49
+ WHERE c.visit_occurrence_id IS NULL
50
+ ),
51
+
52
+ all_covid_tests AS
53
+ (
54
+ SELECT DISTINCT
55
+ c.person_id,
56
+ COALESCE(c.index_date, cn.index_date) AS index_date,
57
+ COALESCE(c.visit_occurrence_id, cn.visit_occurrence_id) AS visit_occurrence_id,
58
+ COALESCE(c.visit_concept_id, cn.visit_concept_id) AS visit_concept_id
59
+ FROM covid_positive AS c
60
+ LEFT JOIN covid_test_with_no_visit AS cn
61
+ ON c.test_row_number = cn.test_row_number
62
+ )
63
+
64
+ SELECT DISTINCT
65
+ person_id,
66
+ FIRST_VALUE(vo.index_date) OVER(PARTITION BY vo.person_id ORDER BY vo.index_date) AS index_date,
67
+ FIRST_VALUE(vo.visit_occurrence_id) OVER(PARTITION BY vo.person_id ORDER BY vo.index_date) AS visit_occurrence_id
68
+ FROM
69
+ (
70
+ SELECT
71
+ co.*
72
+ FROM all_covid_tests AS co
73
+ WHERE visit_concept_id IN (262, 9203, 9201)
74
+ ) vo
75
+ """
76
+
77
+ DEFAULT_COHORT_NAME = "covid19"
78
+ DEPENDENCY_LIST = [PERSON, VISIT_OCCURRENCE, MEASUREMENT, CONDITION_OCCURRENCE]
79
+
80
+
81
+ def query_builder():
82
+ query = QuerySpec(table_name=DEFAULT_COHORT_NAME, query_template=COVID_COHORT_QUERY, parameters={})
83
+
84
+ return QueryBuilder(cohort_name=DEFAULT_COHORT_NAME, dependency_list=DEPENDENCY_LIST, query=query)
@@ -0,0 +1,46 @@
1
+ from cehrbert_data.cohorts.query_builder import QueryBuilder, QuerySpec, create_cohort_entry_query_spec
2
+ from cehrbert_data.const.common import DEATH, PERSON, VISIT_OCCURRENCE
3
+
4
+ DEATH_COHORT_QUERY = """
5
+ WITH max_death_date_cte AS
6
+ (
7
+ SELECT
8
+ person_id,
9
+ MAX(death_date) AS death_date
10
+ FROM global_temp.death
11
+ GROUP BY person_id
12
+ ),
13
+ last_visit_start_date AS
14
+ (
15
+ SELECT
16
+ person_id,
17
+ MAX(visit_start_date) AS last_visit_start_date
18
+ FROM global_temp.visit_occurrence
19
+ GROUP BY person_id
20
+ )
21
+
22
+ SELECT
23
+ d.person_id,
24
+ d.death_date AS index_date,
25
+ CAST(null AS INT) AS visit_occurrence_id
26
+ FROM max_death_date_cte AS d
27
+ JOIN last_visit_start_date AS v
28
+ ON d.person_id = v.person_id
29
+ AND v.last_visit_start_date <= d.death_date
30
+ """
31
+
32
+ DEFAULT_COHORT_NAME = "mortality"
33
+ DEPENDENCY_LIST = [PERSON, DEATH, VISIT_OCCURRENCE]
34
+
35
+
36
+ def query_builder():
37
+ query = QuerySpec(table_name=DEFAULT_COHORT_NAME, query_template=DEATH_COHORT_QUERY, parameters={})
38
+
39
+ entry_cohort_query = create_cohort_entry_query_spec(entry_query_template=DEATH_COHORT_QUERY, parameters={})
40
+
41
+ return QueryBuilder(
42
+ cohort_name=DEFAULT_COHORT_NAME,
43
+ dependency_list=DEPENDENCY_LIST,
44
+ query=query,
45
+ entry_cohort_query=entry_cohort_query,
46
+ )