closurizer 0.4.1__tar.gz → 0.5.0__tar.gz

{closurizer-0.4.1 → closurizer-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: closurizer
-Version: 0.4.1
+Version: 0.5.0
 Summary: Add closure expansion fields to kgx files following the Golr pattern
 Author: Kevin Schaper
 Author-email: kevin@tislab.org

closurizer-0.5.0/closurizer/cli.py

@@ -0,0 +1,37 @@
+import click
+from typing import List
+from closurizer.closurizer import add_closure
+
+
+@click.command()
+@click.option('--kg', required=True, help='KGX tar.gz archive')
+@click.option('--closure', required=True, help='TSV file of closure triples')
+@click.option('--nodes-output', required=True, help='file write nodes kgx file with closure fields added')
+@click.option('--edges-output', required=True, help='file write edges kgx file with closure fields added')
+@click.option('--additional-node-constraints', required=False,
+              help='additional where clause constraints to apply to the generation of the denormalized nodes output')
+@click.option('--edge-fields', multiple=True, help='edge fields to expand with closure IDs, labels, etc')
+@click.option('--node-fields', multiple=True, help='node fields to expand with closure IDs, labels, etc')
+@click.option('--grouping-fields', multiple=True, help='fields to populate a single value grouping_key field')
+@click.option('--dry-run', is_flag=True, help='A dry run will not write the output file, but will print the SQL query')
+def main(kg: str,
+         closure: str,
+         nodes_output: str,
+         edges_output: str,
+         additional_node_constraints: str = None,
+         dry_run: bool = False,
+         edge_fields: List[str] = None,
+         node_fields: List[str] = None,
+         grouping_fields: List[str] = None):
+    add_closure(kg_archive=kg,
+                closure_file=closure,
+                edge_fields=edge_fields,
+                node_fields=node_fields,
+                edges_output_file=edges_output,
+                nodes_output_file=nodes_output,
+                additional_node_constraints=additional_node_constraints,
+                dry_run=dry_run,
+                grouping_fields=grouping_fields)
+
+if __name__ == "__main__":
+    main()

{closurizer-0.4.1 → closurizer-0.5.0}/closurizer/closurizer.py

@@ -4,13 +4,13 @@ import os
 import tarfile
 import duckdb
 
-def columns(field):
+def edge_columns(field):
     column_text = f"""
        {field}.name as {field}_label, 
        {field}.category as {field}_category,
        {field}.namespace as {field}_namespace,
-       {field}_closure.closure as {field}_closure,
-       {field}_closure_label.closure_label as {field}_closure_label,    
+       list_aggregate({field}_closure.closure, 'string_agg', '|') as {field}_closure,
+       list_aggregate({field}_closure_label.closure_label,'string_agg', '|') as {field}_closure_label,    
     """
     if field in ['subject', 'object']:
         column_text += f"""
@@ -19,7 +19,7 @@ def columns(field):
         """
     return column_text
 
-def joins(field):
+def edge_joins(field):
     return f"""
     left outer join nodes as {field} on edges.{field} = {field}.id
     left outer join closure_id as {field}_closure on {field}.id = {field}_closure.id
@@ -31,6 +31,32 @@ def evidence_sum(evidence_fields):
     evidence_count_sum = "+".join([f"len(split({field}, '|'))" for field in evidence_fields])
     return f"{evidence_count_sum} as evidence_count,"
 
+
+def node_columns(predicate):
+    # strip the biolink predicate, if necessary to get the field name
+    field = predicate.replace('biolink:','')
+
+    return f"""
+    string_agg({field}_edges.object, '|') as {field},
+    string_agg({field}_edges.object_label, '|') as {field}_label,
+    count (distinct {field}_edges.object) as {field}_count,
+    list_aggregate(list_distinct(flatten(array_agg({field}_closure.closure))), 'string_agg', '|') as {field}_closure,
+    list_aggregate(list_distinct(flatten(array_agg({field}_closure_label.closure_label))), 'string_agg', '|') as {field}_closure_label,
+    """
+
+def node_joins(predicate):
+    # strip the biolink predicate, if necessary to get the field name
+    field = predicate.replace('biolink:','')
+    return f"""
+      left outer join denormalized_edges as {field}_edges 
+        on nodes.id = {field}_edges.subject 
+           and {field}_edges.predicate = 'biolink:{field}'
+      left outer join closure_id as {field}_closure
+        on {field}_edges.object = {field}_closure.id
+      left outer join closure_label as {field}_closure_label
+        on {field}_edges.object = {field}_closure_label.id
+    """
+
 def grouping_key(grouping_fields):
     fragments = []
     for field in grouping_fields:
@@ -39,12 +65,15 @@ def grouping_key(grouping_fields):
         else:
             fragments.append(field)
     grouping_key_fragments = ", ".join(fragments)
-    return f"concat_ws('🍪', {grouping_key_fragments}) as grouping_key"
+    return f"concat_ws('|', {grouping_key_fragments}) as grouping_key"
 
 def add_closure(kg_archive: str,
                 closure_file: str,
-                output_file: str,
-                fields: List[str] = ['subject', 'object'],
+                nodes_output_file: str,
+                edges_output_file: str,
+                node_fields: List[str] = None,
+                edge_fields: List[str] = ['subject', 'object'],
+                additional_node_constraints: str = None,
                 dry_run: bool  = False,
                 evidence_fields: List[str] = None,
                 grouping_fields: List[str] = None
@@ -55,8 +84,8 @@ def add_closure(kg_archive: str,
 
     db = duckdb.connect(database='monarch-kg.duckdb')
 
-    if fields is None or len(fields) == 0:
-        fields = ['subject', 'object']
+    if edge_fields is None or len(edge_fields) == 0:
+        edge_fields = ['subject', 'object']
 
     if evidence_fields is None or len(evidence_fields) == 0:
         evidence_fields = ['has_evidence', 'publications']
@@ -66,8 +95,8 @@ def add_closure(kg_archive: str,
 
 
     if not dry_run:
-        print(f"fields: {','.join(fields)}")
-        print(f"output_file: {output_file}")
+        print(f"fields: {','.join(edge_fields)}")
+        print(f"output_file: {edges_output_file}")
 
         tar = tarfile.open(f"{kg_archive}")
 
@@ -96,32 +125,49 @@ def add_closure(kg_archive: str,
         """)
 
         db.sql("""
-        create or replace table closure_id as select subject_id as id, string_agg(object_id, '|') as closure from closure group by subject_id
+        create or replace table closure_id as select subject_id as id, array_agg(object_id) as closure from closure group by subject_id
         """)
 
         db.sql("""
-        create or replace table closure_label as select subject_id as id, string_agg(name, '|') as closure_label from closure join nodes on object_id = id
+        create or replace table closure_label as select subject_id as id, array_agg(name) as closure_label from closure join nodes on object_id = id
         group by subject_id
         """)
 
-    query = f"""
+    edges_query = f"""
     create or replace table denormalized_edges as
     select edges.*, 
-           {"".join([columns(field) for field in fields])}
+           {"".join([edge_columns(field) for field in edge_fields])}
            {evidence_sum(evidence_fields)}
            {grouping_key(grouping_fields)}  
     from edges
-        {"".join([joins(field) for field in fields])}
+        {"".join([edge_joins(field) for field in edge_fields])}
     """
 
-    print(query)
+    print(edges_query)
+
+    nodes_query = f"""        
+    create or replace table denormalized_nodes as
+    select nodes.*, 
+        {"".join([node_columns(node_field) for node_field in node_fields])}
+    from nodes
+        {node_joins('has_phenotype')}
+    where {additional_node_constraints}
+    group by nodes.*
+    """
+    print(nodes_query)
 
     if not dry_run:
-        db.query(query)
+        db.query(edges_query)
         db.query(f"""
         -- write denormalized_edges as tsv
-        copy (select * from denormalized_edges) to '{output_file}' (header, delimiter '\t')
+        copy (select * from denormalized_edges) to '{edges_output_file}' (header, delimiter '\t')
         """)
+        db.query(nodes_query)
+        db.query(f"""
+        -- write denormalized_nodes as tsv
+        copy (select * from denormalized_nodes) to '{nodes_output_file}' (header, delimiter '\t')
+        """)
+
 
         # Clean up extracted node & edge files
         if os.path.exists(f"{node_file}"):

{closurizer-0.4.1 → closurizer-0.5.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "closurizer"
-version = "0.4.1"
+version = "0.5.0"
 description = "Add closure expansion fields to kgx files following the Golr pattern"
 authors = ["Kevin Schaper <kevin@tislab.org>"]
 

closurizer-0.4.1/closurizer/cli.py

@@ -1,22 +0,0 @@
-import click
-from typing import List
-from closurizer.closurizer import add_closure
-
-
-@click.command()
-@click.option('--kg', required=True, help='KGX tar.gz archive')
-@click.option('--closure', required=True, help='TSV file of closure triples')
-@click.option('--output', '-o', required=True, help='file write kgx file with closure fields added')
-@click.option('--fields', multiple=True, help='fields to closurize')
-@click.option('--grouping-fields', multiple=True, help='fields to populate a single value grouping_key field')
-@click.option('--dry-run', is_flag=True, help='A dry run will not write the output file, but will print the SQL query')
-def main(kg: str,
-         closure: str,
-         output: str,
-         dry_run: bool = False,
-         fields: List[str] = None,
-         grouping_fields: List[str] = None):
-    add_closure(kg_archive=kg, closure_file=closure, fields=fields, output_file=output, dry_run=dry_run, grouping_fields=grouping_fields)
-
-if __name__ == "__main__":
-    main()