linkml-store 0.2.10__tar.gz → 0.2.11__tar.gz

{linkml_store-0.2.10 → linkml_store-0.2.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: linkml-store
-Version: 0.2.10
+Version: 0.2.11
 Summary: linkml-store
 License: MIT
 Author: Author 1
@@ -24,6 +24,7 @@ Provides-Extra: map
 Provides-Extra: mongodb
 Provides-Extra: neo4j
 Provides-Extra: pyarrow
+Provides-Extra: pyreadr
 Provides-Extra: rdf
 Provides-Extra: renderer
 Provides-Extra: scipy
@@ -40,6 +41,7 @@ Requires-Dist: h5py ; extra == "h5py"
 Requires-Dist: jinja2 (>=3.1.4,<4.0.0)
 Requires-Dist: jsonlines (>=4.0.0,<5.0.0)
 Requires-Dist: jsonpatch (>=1.33)
+Requires-Dist: jsonpath-ng
 Requires-Dist: lightrdf ; extra == "rdf"
 Requires-Dist: linkml (>=1.8.0) ; extra == "validation"
 Requires-Dist: linkml-runtime (>=1.8.0)
@@ -56,6 +58,7 @@ Requires-Dist: py2neo ; extra == "neo4j"
 Requires-Dist: pyarrow ; extra == "pyarrow"
 Requires-Dist: pydantic (>=2.0.0,<3.0.0)
 Requires-Dist: pymongo (>=4.11,<5.0) ; extra == "mongodb"
+Requires-Dist: pyreadr ; extra == "pyreadr"
 Requires-Dist: pystow (>=0.5.4,<0.6.0)
 Requires-Dist: python-dotenv (>=1.0.1,<2.0.0)
 Requires-Dist: ruff (>=0.6.2) ; extra == "tests"

{linkml_store-0.2.10 → linkml_store-0.2.11}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "linkml-store"
-version = "0.2.10"
+version = "0.2.11"
 description = "linkml-store"
 authors = ["Author 1 <author@org.org>"]
 license = "MIT"
@@ -31,6 +31,7 @@ py2neo = { version="*", optional = true }
 networkx = { version="*", optional = true }
 #chromadb = { version="*", optional = true }
 pyarrow = { version="*", optional = true }
+pyreadr = { version="*", optional = true }
 h5py = { version="*", optional = true }
 scipy = { version="*", optional = true }
 scikit-learn = { version="*", optional = true }
@@ -49,6 +50,7 @@ fastapi = { version="*", optional = true }
 uvicorn = { version="*", optional = true }
 xmltodict = ">=0.13.0"
 jsonpatch = ">=1.33"
+jsonpath-ng = "*"
 python-dotenv = "^1.0.1"
 
 [tool.poetry.group.dev.dependencies]
@@ -90,6 +92,7 @@ neo4j = ["neo4j", "py2neo", "networkx"]
 #chromadb = ["chromadb"]
 h5py = ["h5py"]
 pyarrow = ["pyarrow"]
+pyreadr = ["pyreadr"]
 validation = ["linkml"]
 map = ["linkml_map"]
 renderer = ["linkml_renderer"]

{linkml_store-0.2.10 → linkml_store-0.2.11}/src/linkml_store/cli.py

@@ -25,6 +25,7 @@ from linkml_store.inference.inference_engine import ModelSerialization
 from linkml_store.utils.format_utils import Format, guess_format, load_objects, render_output, write_output
 from linkml_store.utils.object_utils import object_path_update
 from linkml_store.utils.pandas_utils import facet_summary_to_dataframe_unmelted
+from linkml_store.plotting.cli import plot_cli
 
 DEFAULT_LOCAL_CONF_PATH = Path("linkml.yaml")
 # global path is ~/.linkml.yaml in the user's home directory
@@ -205,9 +206,10 @@ def drop(ctx):
 @click.option("--format", "-f", type=format_choice, help="Input format")
 @click.option("--object", "-i", multiple=True, help="Input object as YAML")
 @click.option("--source-field", help="If provided, inject file path source as this field")
+@click.option("--glob-files/--no-glob-files", default=False, show_default=True, help="If true, use glob to find files")
 @json_select_query_option
 @click.pass_context
-def insert(ctx, files, replace, object, format, source_field, json_select_query):
+def insert(ctx, files, glob_files, replace, object, format, source_field, json_select_query):
     """Insert objects from files (JSON, YAML, TSV) into the specified collection.
 
     Using a configuration:
@@ -216,6 +218,11 @@ def insert(ctx, files, replace, object, format, source_field, json_select_query)
 
     Note: if you don't provide a schema this will be inferred, but it is
     usually better to provide an explicit schema
+
+    You can use --glob-files if the list of files is too long
+
+        linkml-store -C config.yaml -c genes insert "data/genes/*.json" --glob-files
+
     """
     settings = ctx.obj["settings"]
     collection = settings.collection
@@ -226,7 +233,15 @@ def insert(ctx, files, replace, object, format, source_field, json_select_query)
     load_objects_args = {}
     if json_select_query:
         load_objects_args["select_query"] = json_select_query
+    if glob_files:
+        import glob
+        new_files = []
+        for file_path in files:
+            new_files.extend(glob.glob(file_path))
+        logger.info(f"Found {len(new_files)} files matching glob pattern {files}")
+        files = new_files
     for file_path in files:
+        
         if format:
             objects = load_objects(file_path, format=format, **load_objects_args)
         else:
@@ -486,12 +501,14 @@ def fq(ctx, where, limit, columns, output_type, wide, output, **kwargs):
 
     Nested columns:
 
-        linkml-store -d phenopackets fq subject.timeAtLastEncounter.age
+        linkml-store -d phenopackets fq -S subject.timeAtLastEncounter.age
 
     Compound keys:
 
         linkml-store -d phenopackets fq subject.sex+subject.timeAtLastEncounter.age
 
+    (TODO: compound keys do not work on solr)
+
     """
     collection = ctx.obj["settings"].collection
     where_clause = yaml.safe_load(where) if where else None
@@ -948,5 +965,7 @@ def validate(ctx, output_type, output, collection_only, **kwargs):
         click.echo(output_data)
 
 
+cli.add_command(plot_cli, name="plot")
+
 if __name__ == "__main__":
     cli()

{linkml_store-0.2.10 → linkml_store-0.2.11}/src/linkml_store/utils/format_utils.py

@@ -44,6 +44,7 @@ class Format(Enum):
     DAT = "dat"
     MARKDOWN = "markdown"
     PKL = "pkl"
+    RDS = "rds"
     PYTHON = "python"
     PARQUET = "parquet"
     HDF5 = "hdf5"
@@ -195,6 +196,9 @@ def process_file(
         objs = xmltodict.parse(f.read())
     elif format == Format.PKL:
         objs = pd.read_pickle(f).to_dict(orient="records")
+    elif format == Format.RDS:
+        import pyreadr
+        objs = pyreadr.read_r(f)
     elif format == Format.XLSX:
         xls = pd.ExcelFile(f)
         objs = {sheet: clean_nested_structure(xls.parse(sheet).to_dict(orient="records")) for sheet in xls.sheet_names}
@@ -349,21 +353,25 @@ def load_objects(
             all_objects = process_file(f, format, expected_type, header_comment_token)
 
     logger.debug(f"Loaded {len(all_objects)} objects from {file_path}")
-    if select_query:
-        import jsonpath_ng as jp
-
-        path_expr = jp.parse(select_query)
-        new_objs = []
-        for obj in all_objects:
-            for match in path_expr.find(obj):
-                logging.debug(f"Match: {match.value}")
-                if isinstance(match.value, list):
-                    new_objs.extend(match.value)
-                else:
-                    new_objs.append(match.value)
-        all_objects = new_objs
+    all_objects = transform_objects(all_objects, select_query)
     return all_objects
 
+def transform_objects(all_objects: List[Dict[str, Any]], select_query: Optional[str]) -> List[Dict[str, Any]]:
+    if not select_query:
+        return all_objects
+    import jsonpath_ng as jp
+
+    path_expr = jp.parse(select_query)
+    new_objs = []
+    for obj in all_objects:
+        for match in path_expr.find(obj):
+            logging.debug(f"Match: {match.value}")
+            if isinstance(match.value, list):
+                new_objs.extend(match.value)
+            else:
+                new_objs.append(match.value)
+    all_objects = new_objs
+    return all_objects
 
 def write_output(
     data: Union[List[Dict[str, Any]], Dict[str, Any], pd.DataFrame],