zombie-squirrel 0.8.5__py3-none-any.whl → 0.8.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- zombie_squirrel/__init__.py +1 -1
- zombie_squirrel/acorns.py +11 -2
- zombie_squirrel/forest.py +16 -8
- {zombie_squirrel-0.8.5.dist-info → zombie_squirrel-0.8.6.dist-info}/METADATA +1 -1
- {zombie_squirrel-0.8.5.dist-info → zombie_squirrel-0.8.6.dist-info}/RECORD +8 -8
- {zombie_squirrel-0.8.5.dist-info → zombie_squirrel-0.8.6.dist-info}/WHEEL +0 -0
- {zombie_squirrel-0.8.5.dist-info → zombie_squirrel-0.8.6.dist-info}/licenses/LICENSE +0 -0
- {zombie_squirrel-0.8.5.dist-info → zombie_squirrel-0.8.6.dist-info}/top_level.txt +0 -0
zombie_squirrel/__init__.py
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
Provides functions to fetch and cache project names, subject IDs, and asset
|
|
4
4
|
metadata from the AIND metadata database with support for multiple backends."""
|
|
5
5
|
|
|
6
|
-
__version__ = "0.8.
|
|
6
|
+
__version__ = "0.8.6"
|
|
7
7
|
|
|
8
8
|
from zombie_squirrel.acorn_contents.asset_basics import asset_basics # noqa: F401
|
|
9
9
|
from zombie_squirrel.acorn_contents.raw_to_derived import raw_to_derived # noqa: F401
|
zombie_squirrel/acorns.py
CHANGED
|
@@ -9,6 +9,7 @@ from zombie_squirrel.forest import (
|
|
|
9
9
|
MemoryTree,
|
|
10
10
|
S3Tree,
|
|
11
11
|
)
|
|
12
|
+
from zombie_squirrel.utils import SquirrelMessage
|
|
12
13
|
|
|
13
14
|
# --- Backend setup ---------------------------------------------------
|
|
14
15
|
|
|
@@ -17,10 +18,18 @@ API_GATEWAY_HOST = "api.allenneuraldynamics.org"
|
|
|
17
18
|
forest_type = os.getenv("FOREST_TYPE", "memory").lower()
|
|
18
19
|
|
|
19
20
|
if forest_type == "s3": # pragma: no cover
|
|
20
|
-
logging.info(
|
|
21
|
+
logging.info(SquirrelMessage(
|
|
22
|
+
tree="S3Tree",
|
|
23
|
+
acorn="system",
|
|
24
|
+
message="Initializing S3 forest for caching"
|
|
25
|
+
).to_json())
|
|
21
26
|
TREE = S3Tree()
|
|
22
27
|
elif forest_type == "memory":
|
|
23
|
-
logging.info(
|
|
28
|
+
logging.info(SquirrelMessage(
|
|
29
|
+
tree="MemoryTree",
|
|
30
|
+
acorn="system",
|
|
31
|
+
message="Initializing in-memory forest for caching"
|
|
32
|
+
).to_json())
|
|
24
33
|
TREE = MemoryTree()
|
|
25
34
|
else:
|
|
26
35
|
raise ValueError(f"Unknown FOREST_TYPE: {forest_type}")
|
zombie_squirrel/forest.py
CHANGED
|
@@ -8,7 +8,7 @@ import boto3
|
|
|
8
8
|
import duckdb
|
|
9
9
|
import pandas as pd
|
|
10
10
|
|
|
11
|
-
from zombie_squirrel.utils import get_s3_cache_path, prefix_table_name
|
|
11
|
+
from zombie_squirrel.utils import get_s3_cache_path, prefix_table_name, SquirrelMessage
|
|
12
12
|
|
|
13
13
|
|
|
14
14
|
class Tree(ABC):
|
|
@@ -53,7 +53,11 @@ class S3Tree(Tree):
|
|
|
53
53
|
Key=s3_key,
|
|
54
54
|
Body=parquet_buffer.getvalue(),
|
|
55
55
|
)
|
|
56
|
-
logging.info(
|
|
56
|
+
logging.info(SquirrelMessage(
|
|
57
|
+
tree="S3Tree",
|
|
58
|
+
acorn=table_name,
|
|
59
|
+
message=f"Stored cache to s3://{self.bucket}/{s3_key}"
|
|
60
|
+
).to_json())
|
|
57
61
|
|
|
58
62
|
def scurry(self, table_name: str) -> pd.DataFrame:
|
|
59
63
|
"""Fetch DataFrame from S3 parquet file."""
|
|
@@ -68,14 +72,18 @@ class S3Tree(Tree):
|
|
|
68
72
|
)
|
|
69
73
|
"""
|
|
70
74
|
result = duckdb.query(query).to_df()
|
|
71
|
-
logging.info(
|
|
72
|
-
|
|
73
|
-
|
|
75
|
+
logging.info(SquirrelMessage(
|
|
76
|
+
tree="S3Tree",
|
|
77
|
+
acorn=table_name,
|
|
78
|
+
message=f"Retrieved cache from s3://{self.bucket}/{s3_key}"
|
|
79
|
+
).to_json())
|
|
74
80
|
return result
|
|
75
81
|
except Exception as e:
|
|
76
|
-
logging.warning(
|
|
77
|
-
|
|
78
|
-
|
|
82
|
+
logging.warning(SquirrelMessage(
|
|
83
|
+
tree="S3Tree",
|
|
84
|
+
acorn=table_name,
|
|
85
|
+
message=f"Error fetching from cache {s3_key}: {e}"
|
|
86
|
+
).to_json())
|
|
79
87
|
return pd.DataFrame()
|
|
80
88
|
|
|
81
89
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
zombie_squirrel/__init__.py,sha256=
|
|
2
|
-
zombie_squirrel/acorns.py,sha256=
|
|
3
|
-
zombie_squirrel/forest.py,sha256=
|
|
1
|
+
zombie_squirrel/__init__.py,sha256=xHHJ6bd_QrBfJVqejcucILL05-9V1SHbB3EFC4ebf70,693
|
|
2
|
+
zombie_squirrel/acorns.py,sha256=Sb5bYUGhvTvtWpRn7IuymBbFQ5-WhO6DOJf0hRZ-azI,1484
|
|
3
|
+
zombie_squirrel/forest.py,sha256=hsXBJrtAs4LnAabpIBp88ByDeDXfrK0AePqdkVMz-e8,3262
|
|
4
4
|
zombie_squirrel/sync.py,sha256=51ufe4n_3CvYrfTKW4KHngY4k-E9k0I_HGdWOIHY-bA,366
|
|
5
5
|
zombie_squirrel/utils.py,sha256=hyXoWstJ7rUYZVmazEtt5hXJ4SAMbfmlQMDUmBlfFZw,1280
|
|
6
6
|
zombie_squirrel/acorn_contents/__init__.py,sha256=N4Wun9kn44_nCwV4g6naEVzf76ozOgEoupAHLYpiGsI,224
|
|
@@ -9,8 +9,8 @@ zombie_squirrel/acorn_contents/raw_to_derived.py,sha256=7y2xA7GLC2hMj_Z7gz-uBOKi
|
|
|
9
9
|
zombie_squirrel/acorn_contents/source_data.py,sha256=I7A3mHqVO4C4880w82LNlBHZZ72uSHw3eCwrKtVCnSI,1901
|
|
10
10
|
zombie_squirrel/acorn_contents/unique_project_names.py,sha256=VAS0CZzsC9ObPM39KfrIjYPDIVcqu9H9wyZz8styznQ,1612
|
|
11
11
|
zombie_squirrel/acorn_contents/unique_subject_ids.py,sha256=H_eb3ucayNOdt4GtW26qckqvX6qGcRU59Q_D9NlnSbg,1585
|
|
12
|
-
zombie_squirrel-0.8.
|
|
13
|
-
zombie_squirrel-0.8.
|
|
14
|
-
zombie_squirrel-0.8.
|
|
15
|
-
zombie_squirrel-0.8.
|
|
16
|
-
zombie_squirrel-0.8.
|
|
12
|
+
zombie_squirrel-0.8.6.dist-info/licenses/LICENSE,sha256=U0Y7B3gZJHXpjJVLgTQjM8e_c8w4JJpLgGhIdsoFR1Y,1092
|
|
13
|
+
zombie_squirrel-0.8.6.dist-info/METADATA,sha256=dNdtuz95_8ZWfhRivvf8cUH4gtXGLO0d_6eaGfsHnt8,1897
|
|
14
|
+
zombie_squirrel-0.8.6.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
|
|
15
|
+
zombie_squirrel-0.8.6.dist-info/top_level.txt,sha256=FmM0coe4AangURZLjM4JwwRv2B8H6oINYCoZLKLDCKA,16
|
|
16
|
+
zombie_squirrel-0.8.6.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|