zombie-squirrel 0.3.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

zombie_squirrel/__init__.py

@@ -1,7 +1,8 @@
 """Init package"""
-__version__ = "0.3.0"
+__version__ = "0.4.0"
 
 from zombie_squirrel.squirrels import (
     unique_project_names,
     unique_subject_ids,
+    asset_basics,
 )

zombie_squirrel/squirrels.py

@@ -91,7 +91,7 @@ def unique_subject_ids(force_update: bool = False) -> list[str]:
 def asset_basics(force_update: bool = False) -> pd.DataFrame:
     """Basic asset metadata.
 
-    _id, last_modified,
+    _id, _last_modified,
     modalities, project names, data_level, subject_id, acquisition_start and _end
     """
     df = rds_get_handle_empty(ACORN, NAMES["basics"])
@@ -107,6 +107,9 @@ def asset_basics(force_update: bool = False) -> pd.DataFrame:
 
     if df.empty or force_update:
         logging.info("Updating cache for asset basics")
+        df = pd.DataFrame(columns=["_id", "_last_modified", "modalities", "project_name",
+                                   "data_level", "subject_id",
+                                   "acquisition_start_time", "acquisition_end_time"])
         client = MetadataDbClient(
             host=API_GATEWAY_HOST,
             version="v2",
@@ -115,29 +118,49 @@ def asset_basics(force_update: bool = False) -> pd.DataFrame:
         # as large as DocDB. We'll also try to limit ourselves to only updating fields
         # that are necessary
         record_ids = client.retrieve_docdb_records(
-            filter_query={}, projection={"_id": 1, "last_modified": 1}, limit=0,
+            filter_query={}, projection={"_id": 1, "_last_modified": 1}, limit=0,
         )
         keep_ids = []
-        # Drop all _ids where last_modified matches cache
+        # Drop all _ids where _last_modified matches cache
         for record in record_ids:
             cached_row = df[df["_id"] == record["_id"]]
-            if cached_row.empty or cached_row["last_modified"].values[0] != record["last_modified"]:
+            if cached_row.empty or cached_row["_last_modified"].values[0] != record["_last_modified"]:
                 keep_ids.append(record["_id"])
 
         # Now batch by 100 IDs at a time to avoid overloading server, and fetch all the fields
         BATCH_SIZE = 100
         asset_records = []
         for i in range(0, len(keep_ids), BATCH_SIZE):
+            logging.info(f"Fetching asset basics batch {i // BATCH_SIZE + 1}...")
             batch_ids = keep_ids[i:i + BATCH_SIZE]
             batch_records = client.retrieve_docdb_records(
                 filter_query={"_id": {"$in": batch_ids}},
-                projection={field: 1 for field in FIELDS + ["_id", "last_modified"]},
+                projection={field: 1 for field in FIELDS + ["_id", "_last_modified"]},
                 limit=0,
             )
             asset_records.extend(batch_records)
+        
+        # Unwrap nested fields
+        records = []
+        for record in asset_records:
+
+            modalities = record.get("data_description", {}).get("modalities", [])
+            modality_abbreviations = [modality["abbreviation"] for modality in modalities if "abbreviation" in modality]
+            modality_abbreviations_str = ", ".join(modality_abbreviations)
+            flat_record = {
+                "_id": record["_id"],
+                "_last_modified": record.get("_last_modified", None),
+                "modalities": modality_abbreviations_str,
+                "project_name": record.get("data_description", {}).get("project_name", None),
+                "data_level": record.get("data_description", {}).get("data_level", None),
+                "subject_id": record.get("subject", {}).get("subject_id", None),
+                "acquisition_start_time": record.get("acquisition", {}).get("acquisition_start_time", None),
+                "acquisition_end_time": record.get("acquisition", {}).get("acquisition_end_time", None),
+            }
+            records.append(flat_record)
 
         # Combine new records with the old df and store in cache
-        new_df = pd.DataFrame(asset_records)
+        new_df = pd.DataFrame(records)
         df = pd.concat([df[df["_id"].isin(keep_ids) == False], new_df], ignore_index=True)
 
         ACORN.hide(NAMES["basics"], df)

{zombie_squirrel-0.3.0.dist-info → zombie_squirrel-0.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: zombie-squirrel
-Version: 0.3.0
+Version: 0.4.0
 Summary: Generated from aind-library-template
 Author: Allen Institute for Neural Dynamics
 License: MIT

zombie_squirrel-0.4.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+zombie_squirrel/__init__.py,sha256=8er6wgFVb0XMkMDsmLRvR_YeO1E_sL3KaOJN9VXXwOw,152
+zombie_squirrel/acorns.py,sha256=1mCnWCDFRnbHLddCCgiUG3RumuKUjMKVbyTVoYI0FB8,2188
+zombie_squirrel/squirrels.py,sha256=Ln8tsa51rK6d2rpOIktSAeHYX3sYMXr3o4njZzAujAo,6340
+zombie_squirrel/sync.py,sha256=jslTVIend5Z-sLJuNXKkhn-nqmKK_P0FAiRuFFYRnto,168
+zombie_squirrel/utils.py,sha256=74DSFK1Qbp8yQeUXpnli4kqx_QcAc8v4_6FZut0xZ8g,103
+zombie_squirrel-0.4.0.dist-info/licenses/LICENSE,sha256=U0Y7B3gZJHXpjJVLgTQjM8e_c8w4JJpLgGhIdsoFR1Y,1092
+zombie_squirrel-0.4.0.dist-info/METADATA,sha256=0Rv7O3SRGDe06_F4-Kefj9JxC2xMQG1m1l3BYrZyfUE,1382
+zombie_squirrel-0.4.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+zombie_squirrel-0.4.0.dist-info/top_level.txt,sha256=FmM0coe4AangURZLjM4JwwRv2B8H6oINYCoZLKLDCKA,16
+zombie_squirrel-0.4.0.dist-info/RECORD,,

zombie_squirrel-0.3.0.dist-info/RECORD

@@ -1,10 +0,0 @@
-zombie_squirrel/__init__.py,sha256=SU-iCf1qQAQUKc6VnAdEUcGBlcUwuDZ3CKpJXGSrzb0,134
-zombie_squirrel/acorns.py,sha256=1mCnWCDFRnbHLddCCgiUG3RumuKUjMKVbyTVoYI0FB8,2188
-zombie_squirrel/squirrels.py,sha256=BkS4l9cUB7ZXQV3ySa0HCgSLei_Kc47X28BJY69Bz54,4915
-zombie_squirrel/sync.py,sha256=jslTVIend5Z-sLJuNXKkhn-nqmKK_P0FAiRuFFYRnto,168
-zombie_squirrel/utils.py,sha256=74DSFK1Qbp8yQeUXpnli4kqx_QcAc8v4_6FZut0xZ8g,103
-zombie_squirrel-0.3.0.dist-info/licenses/LICENSE,sha256=U0Y7B3gZJHXpjJVLgTQjM8e_c8w4JJpLgGhIdsoFR1Y,1092
-zombie_squirrel-0.3.0.dist-info/METADATA,sha256=3YrkG4IV-36OekD-DEjPlxAbVOXvuHzjesCarDgJ2eQ,1382
-zombie_squirrel-0.3.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-zombie_squirrel-0.3.0.dist-info/top_level.txt,sha256=FmM0coe4AangURZLjM4JwwRv2B8H6oINYCoZLKLDCKA,16
-zombie_squirrel-0.3.0.dist-info/RECORD,,