closurizer 0.1.3__tar.gz → 0.5.1__tar.gz

{closurizer-0.1.3 → closurizer-0.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: closurizer
-Version: 0.1.3
+Version: 0.5.1
 Summary: Add closure expansion fields to kgx files following the Golr pattern
 Author: Kevin Schaper
 Author-email: kevin@tislab.org
@@ -8,6 +8,9 @@ Requires-Python: >=3.8,<4.0
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Requires-Dist: SQLAlchemy (>=1.4.37,<2.0.0)
 Requires-Dist: click (>=8,<9)
-Requires-Dist: petl (>=1.7.10,<2.0.0)
+Requires-Dist: duckdb (>=0.9.1,<0.10.0)

closurizer-0.5.1/closurizer/cli.py

@@ -0,0 +1,37 @@
+import click
+from typing import List
+from closurizer.closurizer import add_closure
+
+
+@click.command()
+@click.option('--kg', required=True, help='KGX tar.gz archive')
+@click.option('--closure', required=True, help='TSV file of closure triples')
+@click.option('--nodes-output', required=True, help='file write nodes kgx file with closure fields added')
+@click.option('--edges-output', required=True, help='file write edges kgx file with closure fields added')
+@click.option('--additional-node-constraints', required=False,
+              help='additional where clause constraints to apply to the generation of the denormalized nodes output')
+@click.option('--edge-fields', multiple=True, help='edge fields to expand with closure IDs, labels, etc')
+@click.option('--node-fields', multiple=True, help='node fields to expand with closure IDs, labels, etc')
+@click.option('--grouping-fields', multiple=True, help='fields to populate a single value grouping_key field')
+@click.option('--dry-run', is_flag=True, help='A dry run will not write the output file, but will print the SQL query')
+def main(kg: str,
+         closure: str,
+         nodes_output: str,
+         edges_output: str,
+         additional_node_constraints: str = None,
+         dry_run: bool = False,
+         edge_fields: List[str] = None,
+         node_fields: List[str] = None,
+         grouping_fields: List[str] = None):
+    add_closure(kg_archive=kg,
+                closure_file=closure,
+                edge_fields=edge_fields,
+                node_fields=node_fields,
+                edges_output_file=edges_output,
+                nodes_output_file=nodes_output,
+                additional_node_constraints=additional_node_constraints,
+                dry_run=dry_run,
+                grouping_fields=grouping_fields)
+
+if __name__ == "__main__":
+    main()

closurizer-0.5.1/closurizer/closurizer.py

@@ -0,0 +1,176 @@
+from typing import List
+
+import os
+import tarfile
+import duckdb
+
+def edge_columns(field):
+    column_text = f"""
+       {field}.name as {field}_label, 
+       {field}.category as {field}_category,
+       {field}.namespace as {field}_namespace,
+       list_aggregate({field}_closure.closure, 'string_agg', '|') as {field}_closure,
+       list_aggregate({field}_closure_label.closure_label,'string_agg', '|') as {field}_closure_label,    
+    """
+    if field in ['subject', 'object']:
+        column_text += f"""
+        {field}.in_taxon as {field}_taxon,
+        {field}.in_taxon_label as {field}_taxon_label,
+        """
+    return column_text
+
+def edge_joins(field):
+    return f"""
+    left outer join nodes as {field} on edges.{field} = {field}.id
+    left outer join closure_id as {field}_closure on {field}.id = {field}_closure.id
+    left outer join closure_label as {field}_closure_label on {field}.id = {field}_closure_label.id
+    """
+
+def evidence_sum(evidence_fields):
+    """ Sum together the length of each field after splitting on | """
+    evidence_count_sum = "+".join([f"ifnull(len(split({field}, '|')),0)" for field in evidence_fields])
+    return f"{evidence_count_sum} as evidence_count,"
+
+
+def node_columns(predicate):
+    # strip the biolink predicate, if necessary to get the field name
+    field = predicate.replace('biolink:','')
+
+    return f"""
+    string_agg({field}_edges.object, '|') as {field},
+    string_agg({field}_edges.object_label, '|') as {field}_label,
+    count (distinct {field}_edges.object) as {field}_count,
+    list_aggregate(list_distinct(flatten(array_agg({field}_closure.closure))), 'string_agg', '|') as {field}_closure,
+    list_aggregate(list_distinct(flatten(array_agg({field}_closure_label.closure_label))), 'string_agg', '|') as {field}_closure_label,
+    """
+
+def node_joins(predicate):
+    # strip the biolink predicate, if necessary to get the field name
+    field = predicate.replace('biolink:','')
+    return f"""
+      left outer join denormalized_edges as {field}_edges 
+        on nodes.id = {field}_edges.subject 
+           and {field}_edges.predicate = 'biolink:{field}'
+      left outer join closure_id as {field}_closure
+        on {field}_edges.object = {field}_closure.id
+      left outer join closure_label as {field}_closure_label
+        on {field}_edges.object = {field}_closure_label.id
+    """
+
+def grouping_key(grouping_fields):
+    fragments = []
+    for field in grouping_fields:
+        if field == 'negated':
+            fragments.append(f"coalesce({field}.replace('True','NOT'), '')")
+        else:
+            fragments.append(field)
+    grouping_key_fragments = ", ".join(fragments)
+    return f"concat_ws('|', {grouping_key_fragments}) as grouping_key"
+
+def add_closure(kg_archive: str,
+                closure_file: str,
+                nodes_output_file: str,
+                edges_output_file: str,
+                node_fields: List[str] = None,
+                edge_fields: List[str] = ['subject', 'object'],
+                additional_node_constraints: str = None,
+                dry_run: bool  = False,
+                evidence_fields: List[str] = None,
+                grouping_fields: List[str] = None
+                ):
+    print("Generating closure KG...")
+    print(f"kg_archive: {kg_archive}")
+    print(f"closure_file: {closure_file}")
+
+    db = duckdb.connect(database='monarch-kg.duckdb')
+
+    if edge_fields is None or len(edge_fields) == 0:
+        edge_fields = ['subject', 'object']
+
+    if evidence_fields is None or len(evidence_fields) == 0:
+        evidence_fields = ['has_evidence', 'publications']
+
+    if grouping_fields is None or len(grouping_fields) == 0:
+        grouping_fields = ['subject', 'negated', 'predicate', 'object']
+
+
+    if not dry_run:
+        print(f"fields: {','.join(edge_fields)}")
+        print(f"output_file: {edges_output_file}")
+
+        tar = tarfile.open(f"{kg_archive}")
+
+        print("Loading node table...")
+        node_file_name = [member.name for member in tar.getmembers() if member.name.endswith('_nodes.tsv') ][0]
+        tar.extract(node_file_name,)
+        node_file = f"{node_file_name}"
+        print(f"node_file: {node_file}")
+
+        db.sql(f"""
+        create or replace table nodes as select *,  substr(id, 1, instr(id,':') -1) as namespace from read_csv('{node_file_name}', header=True, sep='\t', AUTO_DETECT=TRUE)
+        """)
+
+        edge_file_name = [member.name for member in tar.getmembers() if member.name.endswith('_edges.tsv') ][0]
+        tar.extract(edge_file_name)
+        edge_file = f"{edge_file_name}"
+        print(f"edge_file: {edge_file}")
+
+        db.sql(f"""
+        create or replace table edges as select * from read_csv('{edge_file_name}', header=True, sep='\t', AUTO_DETECT=TRUE)
+        """)
+
+        # Load the relation graph tsv in long format mapping a node to each of it's ancestors
+        db.sql(f"""
+        create or replace table closure as select * from read_csv('{closure_file}', sep='\t', names=['subject_id', 'predicate_id', 'object_id'], AUTO_DETECT=TRUE)
+        """)
+
+        db.sql("""
+        create or replace table closure_id as select subject_id as id, array_agg(object_id) as closure from closure group by subject_id
+        """)
+
+        db.sql("""
+        create or replace table closure_label as select subject_id as id, array_agg(name) as closure_label from closure join nodes on object_id = id
+        group by subject_id
+        """)
+
+    edges_query = f"""
+    create or replace table denormalized_edges as
+    select edges.*, 
+           {"".join([edge_columns(field) for field in edge_fields])}
+           {evidence_sum(evidence_fields)}
+           {grouping_key(grouping_fields)}  
+    from edges
+        {"".join([edge_joins(field) for field in edge_fields])}
+    """
+
+    print(edges_query)
+
+    nodes_query = f"""        
+    create or replace table denormalized_nodes as
+    select nodes.*, 
+        {"".join([node_columns(node_field) for node_field in node_fields])}
+    from nodes
+        {node_joins('has_phenotype')}
+    where {additional_node_constraints}
+    group by nodes.*
+    """
+    print(nodes_query)
+
+    if not dry_run:
+        db.query(edges_query)
+        db.query(f"""
+        -- write denormalized_edges as tsv
+        copy (select * from denormalized_edges) to '{edges_output_file}' (header, delimiter '\t')
+        """)
+        db.query(nodes_query)
+        db.query(f"""
+        -- write denormalized_nodes as tsv
+        copy (select * from denormalized_nodes) to '{nodes_output_file}' (header, delimiter '\t')
+        """)
+
+
+        # Clean up extracted node & edge files
+        if os.path.exists(f"{node_file}"):
+            os.remove(f"{node_file}")
+        if os.path.exists(f"{edge_file}"):
+            os.remove(f"{edge_file}")

{closurizer-0.1.3 → closurizer-0.5.1}/pyproject.toml

@@ -1,14 +1,14 @@
 [tool.poetry]
 name = "closurizer"
-version = "0.1.3"
+version = "0.5.1"
 description = "Add closure expansion fields to kgx files following the Golr pattern"
 authors = ["Kevin Schaper <kevin@tislab.org>"]
 
 [tool.poetry.dependencies]
 python = "^3.8"
 click = "^8"
-petl = "^1.7.10"
 SQLAlchemy = "^1.4.37"
+duckdb = "^0.9.1"
 
 [tool.poetry.dev-dependencies]
 

closurizer-0.1.3/closurizer/cli.py

@@ -1,21 +0,0 @@
-import click
-from typing import List
-from closurizer import add_closure
-
-
-@click.command()
-@click.option('--nodes', help='KGX node file to provide closure labels')
-@click.option('--edges', help='KGX edge file to add closure fields to')
-@click.option('--closure', help='TSV file of closure triples')
-@click.option('--output', '-o', help='file write kgx file with closure fields added')
-@click.option('--fields', multiple=True, help='fields to closurize')
-def main(nodes: str,
-         edges: str,
-         closure: str,
-         output: str,
-         fields: List[str]):
-    add_closure(node_file=nodes, edge_file=edges, closure_file=closure, fields=fields, output_file=output)
-
-
-if __name__ == "__main__":
-    main()

closurizer-0.1.3/closurizer/closurizer.py

@@ -1,124 +0,0 @@
-from typing import List
-
-import petl as etl
-import sqlite3
-import os
-import tarfile
-
-
-def _string_agg(key, rows):
-    return [key, "|".join(row[1] for row in rows)]
-
-
-def add_closure(node_file: str,
-                edge_file: str,
-                kg_archive: str,
-                closure_file: str,
-                path: str,
-                fields: List[str],
-                output_file: str):
-
-    print("Generating closure KG...")
-    print(f"node_file: {node_file}")
-    print(f"edge_file: {edge_file}")
-    print(f"kg_archive: {kg_archive}")
-    print(f"closure_file: {closure_file}")
-    print(f"fields: {','.join(fields)}")
-    print(f"output_file: {output_file}")
-
-    tar = tarfile.open(f"{path}/{kg_archive}")
-    tar.extract(node_file, path=path)
-    tar.extract(edge_file, path=path)
-
-    db = "closurizer.db"
-
-    if os.path.exists(db):
-        os.remove(db)
-    sqlite = sqlite3.connect(db)
-
-    nodes = etl.fromtsv(node_file)
-    etl.todb(nodes, sqlite, "nodes", create=True)
-
-    edges = etl.fromtsv(edge_file)
-
-    for field in fields:
-        edges = etl.addfield(edges, f"{field}_namespace")
-        edges = etl.addfield(edges, f"{field}_category")
-        edges = etl.addfield(edges, f"{field}_closure")
-        edges = etl.addfield(edges, f"{field}_label")
-        edges = etl.addfield(edges, f"{field}_closure_label")
-
-    etl.todb(edges, sqlite, "edges", create=True)
-
-    closure_table = (etl
-                     .fromtsv(closure_file)
-                     .setheader(['id', 'predicate', 'ancestor'])
-                     .cutout('predicate')  # assume all predicates for now
-                     )
-
-    closure_id_table = etl.rowreduce(closure_table, key='id', reducer=_string_agg, header=['id', 'ancestors'])
-    etl.todb(closure_id_table, sqlite, "closure", create=True)
-
-    closure_label_table = (etl.leftjoin(closure_table,
-                                        etl.cut(nodes, ["id", "name"]),
-                                        lkey="ancestor",
-                                        rkey="id")
-                           .cutout("ancestor")
-                           .rename("name", "closure_label")
-                           .selectnotnone("closure_label")
-                           .rowreduce(key='id', reducer=_string_agg, header=['id', 'ancestor_labels']))
-    etl.todb(closure_label_table, sqlite, "closure_label", create=True)
-
-    cur = sqlite.cursor()
-
-    for field in fields:
-        etl.leftjoin(edges, closure_id_table, lkey=f"{field}", rkey="id")
-
-    for field in fields:
-
-        cur.execute(f"""
-        update edges 
-        set {field}_namespace = SUBSTR(nodes.id,1,INSTR(nodes.id,':') -1)
-        from nodes
-        where edges.{field} = nodes.id;
-        """)
-
-        cur.execute(f"""
-        update edges 
-        set {field}_category = nodes.category
-        from nodes
-        where edges.{field} = nodes.id;
-        """)
-
-        cur.execute(f"""
-        update edges
-        set {field}_closure = ancestors 
-        from closure
-        where edges.{field} = closure.id;
-        """)
-
-        cur.execute(f"""
-        update edges 
-        set {field}_label = nodes.name
-        from nodes
-        where edges.{field} = nodes.id;
-        """)
-
-        cur.execute(f"""
-        update edges
-        set {field}_closure_label = closure_label.ancestor_labels
-        from closure_label
-        where edges.{field} = closure_label.id;
-        """)
-
-    etl.fromdb(sqlite, 'select * from edges').totsv(f"{path}/{output_file}")
-
-    # Clean up the database
-    if os.path.exists(db):
-        os.remove(db)
-
-    # Clean up extracted node & edge files
-    if os.path.exists(f"{path}/{node_file}"):
-        os.remove(f"{path}/{node_file}")
-    if os.path.exists(f"{path}/{edge_file}"):
-        os.remove(f"{path}/{edge_file}")

closurizer-0.1.3/setup.py

@@ -1,34 +0,0 @@
-# -*- coding: utf-8 -*-
-from setuptools import setup
-
-packages = \
-['closurizer']
-
-package_data = \
-{'': ['*']}
-
-install_requires = \
-['SQLAlchemy>=1.4.37,<2.0.0', 'click>=8,<9', 'petl>=1.7.10,<2.0.0']
-
-entry_points = \
-{'console_scripts': ['closurizer = closurizer.cli:main']}
-
-setup_kwargs = {
-    'name': 'closurizer',
-    'version': '0.1.3',
-    'description': 'Add closure expansion fields to kgx files following the Golr pattern',
-    'long_description': None,
-    'author': 'Kevin Schaper',
-    'author_email': 'kevin@tislab.org',
-    'maintainer': None,
-    'maintainer_email': None,
-    'url': None,
-    'packages': packages,
-    'package_data': package_data,
-    'install_requires': install_requires,
-    'entry_points': entry_points,
-    'python_requires': '>=3.8,<4.0',
-}
-
-
-setup(**setup_kwargs)