closurizer 0.4.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

closurizer/cli.py

@@ -6,15 +6,32 @@ from closurizer.closurizer import add_closure
 @click.command()
 @click.option('--kg', required=True, help='KGX tar.gz archive')
 @click.option('--closure', required=True, help='TSV file of closure triples')
-@click.option('--output', '-o', required=True, help='file write kgx file with closure fields added')
-@click.option('--fields', multiple=True, help='fields to closurize')
+@click.option('--nodes-output', required=True, help='file write nodes kgx file with closure fields added')
+@click.option('--edges-output', required=True, help='file write edges kgx file with closure fields added')
+@click.option('--additional-node-constraints', required=False,
+              help='additional where clause constraints to apply to the generation of the denormalized nodes output')
+@click.option('--edge-fields', multiple=True, help='edge fields to expand with closure IDs, labels, etc')
+@click.option('--node-fields', multiple=True, help='node fields to expand with closure IDs, labels, etc')
 @click.option('--grouping-fields', multiple=True, help='fields to populate a single value grouping_key field')
+@click.option('--dry-run', is_flag=True, help='A dry run will not write the output file, but will print the SQL query')
 def main(kg: str,
          closure: str,
-         output: str,
-         fields: List[str] = None,
+         nodes_output: str,
+         edges_output: str,
+         additional_node_constraints: str = None,
+         dry_run: bool = False,
+         edge_fields: List[str] = None,
+         node_fields: List[str] = None,
          grouping_fields: List[str] = None):
-    add_closure(kg_archive=kg, closure_file=closure, fields=fields, output_file=output, grouping_fields=grouping_fields)
+    add_closure(kg_archive=kg,
+                closure_file=closure,
+                edge_fields=edge_fields,
+                node_fields=node_fields,
+                edges_output_file=edges_output,
+                nodes_output_file=nodes_output,
+                additional_node_constraints=additional_node_constraints,
+                dry_run=dry_run,
+                grouping_fields=grouping_fields)
 
 if __name__ == "__main__":
     main()

closurizer/closurizer.py

@@ -4,13 +4,13 @@ import os
 import tarfile
 import duckdb
 
-def columns(field):
+def edge_columns(field):
     column_text = f"""
        {field}.name as {field}_label, 
        {field}.category as {field}_category,
        {field}.namespace as {field}_namespace,
-       {field}_closure.closure as {field}_closure,
-       {field}_closure_label.closure_label as {field}_closure_label,    
+       list_aggregate({field}_closure.closure, 'string_agg', '|') as {field}_closure,
+       list_aggregate({field}_closure_label.closure_label,'string_agg', '|') as {field}_closure_label,    
     """
     if field in ['subject', 'object']:
         column_text += f"""
@@ -19,7 +19,7 @@ def columns(field):
         """
     return column_text
 
-def joins(field):
+def edge_joins(field):
     return f"""
     left outer join nodes as {field} on edges.{field} = {field}.id
     left outer join closure_id as {field}_closure on {field}.id = {field}_closure.id
@@ -31,6 +31,32 @@ def evidence_sum(evidence_fields):
     evidence_count_sum = "+".join([f"len(split({field}, '|'))" for field in evidence_fields])
     return f"{evidence_count_sum} as evidence_count,"
 
+
+def node_columns(predicate):
+    # strip the biolink predicate, if necessary to get the field name
+    field = predicate.replace('biolink:','')
+
+    return f"""
+    string_agg({field}_edges.object, '|') as {field},
+    string_agg({field}_edges.object_label, '|') as {field}_label,
+    count (distinct {field}_edges.object) as {field}_count,
+    list_aggregate(list_distinct(flatten(array_agg({field}_closure.closure))), 'string_agg', '|') as {field}_closure,
+    list_aggregate(list_distinct(flatten(array_agg({field}_closure_label.closure_label))), 'string_agg', '|') as {field}_closure_label,
+    """
+
+def node_joins(predicate):
+    # strip the biolink predicate, if necessary to get the field name
+    field = predicate.replace('biolink:','')
+    return f"""
+      left outer join denormalized_edges as {field}_edges 
+        on nodes.id = {field}_edges.subject 
+           and {field}_edges.predicate = 'biolink:{field}'
+      left outer join closure_id as {field}_closure
+        on {field}_edges.object = {field}_closure.id
+      left outer join closure_label as {field}_closure_label
+        on {field}_edges.object = {field}_closure_label.id
+    """
+
 def grouping_key(grouping_fields):
     fragments = []
     for field in grouping_fields:
@@ -39,12 +65,16 @@ def grouping_key(grouping_fields):
         else:
             fragments.append(field)
     grouping_key_fragments = ", ".join(fragments)
-    return f"concat_ws('🍪', {grouping_key_fragments}) as grouping_key"
+    return f"concat_ws('|', {grouping_key_fragments}) as grouping_key"
 
 def add_closure(kg_archive: str,
                 closure_file: str,
-                output_file: str,
-                fields: List[str] = ['subject', 'object'],
+                nodes_output_file: str,
+                edges_output_file: str,
+                node_fields: List[str] = None,
+                edge_fields: List[str] = ['subject', 'object'],
+                additional_node_constraints: str = None,
+                dry_run: bool  = False,
                 evidence_fields: List[str] = None,
                 grouping_fields: List[str] = None
                 ):
@@ -54,8 +84,8 @@ def add_closure(kg_archive: str,
 
     db = duckdb.connect(database='monarch-kg.duckdb')
 
-    if fields is None or len(fields) == 0:
-        fields = ['subject', 'object']
+    if edge_fields is None or len(edge_fields) == 0:
+        edge_fields = ['subject', 'object']
 
     if evidence_fields is None or len(evidence_fields) == 0:
         evidence_fields = ['has_evidence', 'publications']
@@ -63,65 +93,84 @@ def add_closure(kg_archive: str,
     if grouping_fields is None or len(grouping_fields) == 0:
         grouping_fields = ['subject', 'negated', 'predicate', 'object']
 
-    print(f"fields: {','.join(fields)}")
-    print(f"output_file: {output_file}")
 
-    tar = tarfile.open(f"{kg_archive}")
+    if not dry_run:
+        print(f"fields: {','.join(edge_fields)}")
+        print(f"output_file: {edges_output_file}")
+
+        tar = tarfile.open(f"{kg_archive}")
 
-    print("Loading node table...")
-    node_file_name = [member.name for member in tar.getmembers() if member.name.endswith('_nodes.tsv') ][0]
-    tar.extract(node_file_name,)
-    node_file = f"{node_file_name}"
-    print(f"node_file: {node_file}")
+        print("Loading node table...")
+        node_file_name = [member.name for member in tar.getmembers() if member.name.endswith('_nodes.tsv') ][0]
+        tar.extract(node_file_name,)
+        node_file = f"{node_file_name}"
+        print(f"node_file: {node_file}")
 
-    db.sql(f"""
-    create or replace table nodes as select *,  substr(id, 1, instr(id,':') -1) as namespace from read_csv('{node_file_name}', header=True, sep='\t', AUTO_DETECT=TRUE)
-    """)
+        db.sql(f"""
+        create or replace table nodes as select *,  substr(id, 1, instr(id,':') -1) as namespace from read_csv('{node_file_name}', header=True, sep='\t', AUTO_DETECT=TRUE)
+        """)
 
-    edge_file_name = [member.name for member in tar.getmembers() if member.name.endswith('_edges.tsv') ][0]
-    tar.extract(edge_file_name)
-    edge_file = f"{edge_file_name}"
-    print(f"edge_file: {edge_file}")
+        edge_file_name = [member.name for member in tar.getmembers() if member.name.endswith('_edges.tsv') ][0]
+        tar.extract(edge_file_name)
+        edge_file = f"{edge_file_name}"
+        print(f"edge_file: {edge_file}")
 
-    db.sql(f"""
-    create or replace table edges as select * from read_csv('{edge_file_name}', header=True, sep='\t', AUTO_DETECT=TRUE)
-    """)
+        db.sql(f"""
+        create or replace table edges as select * from read_csv('{edge_file_name}', header=True, sep='\t', AUTO_DETECT=TRUE)
+        """)
 
-    # Load the relation graph tsv in long format mapping a node to each of it's ancestors
-    db.sql(f"""
-    create or replace table closure as select * from read_csv('{closure_file}', sep='\t', names=['subject_id', 'predicate_id', 'object_id'], AUTO_DETECT=TRUE)
-    """)
+        # Load the relation graph tsv in long format mapping a node to each of it's ancestors
+        db.sql(f"""
+        create or replace table closure as select * from read_csv('{closure_file}', sep='\t', names=['subject_id', 'predicate_id', 'object_id'], AUTO_DETECT=TRUE)
+        """)
 
-    db.sql("""
-    create or replace table closure_id as select subject_id as id, array_agg(object_id) as closure from closure group by subject_id
-    """)
+        db.sql("""
+        create or replace table closure_id as select subject_id as id, array_agg(object_id) as closure from closure group by subject_id
+        """)
 
-    db.sql("""
-    create or replace table closure_label as select subject_id as id, array_agg(name) as closure_label from closure join nodes on object_id = id
-    group by subject_id
-    """)
+        db.sql("""
+        create or replace table closure_label as select subject_id as id, array_agg(name) as closure_label from closure join nodes on object_id = id
+        group by subject_id
+        """)
 
-    query = f"""
+    edges_query = f"""
     create or replace table denormalized_edges as
     select edges.*, 
-           {"".join([columns(field) for field in fields])}
+           {"".join([edge_columns(field) for field in edge_fields])}
            {evidence_sum(evidence_fields)}
            {grouping_key(grouping_fields)}  
     from edges
-        {"".join([joins(field) for field in fields])}
+        {"".join([edge_joins(field) for field in edge_fields])}
     """
 
-    print(query)
-    db.query(query)
+    print(edges_query)
 
-    db.query(f"""
-    -- write denormalized_edges as tsv
-    copy (select * from denormalized_edges) to '{output_file}' (header, delimiter '\t')
-    """)
-
-
-    # Clean up extracted node & edge files
-    if os.path.exists(f"{node_file}"):
-        os.remove(f"{node_file}")
-    if os.path.exists(f"{edge_file}"):
-        os.remove(f"{edge_file}")
+    nodes_query = f"""        
+    create or replace table denormalized_nodes as
+    select nodes.*, 
+        {"".join([node_columns(node_field) for node_field in node_fields])}
+    from nodes
+        {node_joins('has_phenotype')}
+    where {additional_node_constraints}
+    group by nodes.*
+    """
+    print(nodes_query)
+
+    if not dry_run:
+        db.query(edges_query)
+        db.query(f"""
+        -- write denormalized_edges as tsv
+        copy (select * from denormalized_edges) to '{edges_output_file}' (header, delimiter '\t')
+        """)
+        db.query(nodes_query)
+        db.query(f"""
+        -- write denormalized_nodes as tsv
+        copy (select * from denormalized_nodes) to '{nodes_output_file}' (header, delimiter '\t')
+        """)
+
+
+        # Clean up extracted node & edge files
+        if os.path.exists(f"{node_file}"):
+            os.remove(f"{node_file}")
+        if os.path.exists(f"{edge_file}"):
+            os.remove(f"{edge_file}")

{closurizer-0.4.0.dist-info → closurizer-0.5.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: closurizer
-Version: 0.4.0
+Version: 0.5.0
 Summary: Add closure expansion fields to kgx files following the Golr pattern
 Author: Kevin Schaper
 Author-email: kevin@tislab.org

closurizer-0.5.0.dist-info/RECORD

@@ -0,0 +1,6 @@
+closurizer/cli.py,sha256=AfK0Dy0lSmngUfzxKsT6VuH_YqjUVA1yU1Ko41Yil1w,1827
+closurizer/closurizer.py,sha256=u8Weefop4b6DPZsMc7kVxoASjCP-1pQlZBQWGVhet_8,6767
+closurizer-0.5.0.dist-info/METADATA,sha256=h8kByCLqn7B7KW311FfUAXHBAemimNC-kIKUfSOpCbQ,577
+closurizer-0.5.0.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
+closurizer-0.5.0.dist-info/entry_points.txt,sha256=MnAVu1lgP6DqDb3BZGNzVs2AnDMsp4sThi3ccWbONFo,50
+closurizer-0.5.0.dist-info/RECORD,,

closurizer-0.4.0.dist-info/RECORD

@@ -1,6 +0,0 @@
-closurizer/cli.py,sha256=Bh7kaVtM50oFSPhhIdUe02LFAQDBcYA4K9Thl8dGuuU,828
-closurizer/closurizer.py,sha256=U5h0NsimbazI2X3l-vplvS5eb4Fd-VPqv0wfYq7Zso4,4539
-closurizer-0.4.0.dist-info/METADATA,sha256=6x5ZnmcbLT6fmn9MU0UXlTLn5gYqjqTnoBwmghh3j6U,577
-closurizer-0.4.0.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
-closurizer-0.4.0.dist-info/entry_points.txt,sha256=MnAVu1lgP6DqDb3BZGNzVs2AnDMsp4sThi3ccWbONFo,50
-closurizer-0.4.0.dist-info/RECORD,,