so-campaign-manager 0.0.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- so_campaign_manager-0.0.4.dist-info/METADATA +179 -0
- so_campaign_manager-0.0.4.dist-info/RECORD +44 -0
- so_campaign_manager-0.0.4.dist-info/WHEEL +5 -0
- so_campaign_manager-0.0.4.dist-info/entry_points.txt +2 -0
- so_campaign_manager-0.0.4.dist-info/licenses/LICENSE +24 -0
- so_campaign_manager-0.0.4.dist-info/top_level.txt +1 -0
- socm/__about__.py +34 -0
- socm/__init__.py +0 -0
- socm/__main__.py +35 -0
- socm/bookkeeper/__init__.py +1 -0
- socm/bookkeeper/bookkeeper.py +488 -0
- socm/configs/slurmise.toml +2 -0
- socm/core/__init__.py +1 -0
- socm/core/models.py +235 -0
- socm/enactor/__init__.py +3 -0
- socm/enactor/base.py +123 -0
- socm/enactor/dryrun_enactor.py +216 -0
- socm/enactor/rp_enactor.py +273 -0
- socm/execs/__init__.py +3 -0
- socm/execs/mapmaking.py +73 -0
- socm/planner/__init__.py +2 -0
- socm/planner/base.py +87 -0
- socm/planner/heft_planner.py +442 -0
- socm/resources/__init__.py +5 -0
- socm/resources/perlmutter.py +22 -0
- socm/resources/tiger.py +24 -0
- socm/resources/universe.py +18 -0
- socm/utils/__init__.py +0 -0
- socm/utils/misc.py +90 -0
- socm/utils/states.py +17 -0
- socm/workflows/__init__.py +41 -0
- socm/workflows/ml_mapmaking.py +111 -0
- socm/workflows/ml_null_tests/__init__.py +10 -0
- socm/workflows/ml_null_tests/base.py +117 -0
- socm/workflows/ml_null_tests/day_night_null_test.py +132 -0
- socm/workflows/ml_null_tests/direction_null_test.py +133 -0
- socm/workflows/ml_null_tests/elevation_null_test.py +118 -0
- socm/workflows/ml_null_tests/moon_close_null_test.py +165 -0
- socm/workflows/ml_null_tests/moonrise_set_null_test.py +151 -0
- socm/workflows/ml_null_tests/pwv_null_test.py +118 -0
- socm/workflows/ml_null_tests/sun_close_null_test.py +173 -0
- socm/workflows/ml_null_tests/time_null_test.py +76 -0
- socm/workflows/ml_null_tests/wafer_null_test.py +175 -0
- socm/workflows/sat_simulation.py +76 -0
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
from datetime import datetime, timedelta, timezone
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
from typing import Dict, List, Optional, Union
|
|
4
|
+
|
|
5
|
+
import astropy.units as u
|
|
6
|
+
import numpy as np
|
|
7
|
+
from astral import LocationInfo
|
|
8
|
+
from astropy.coordinates import AltAz, EarthLocation, SkyCoord, get_body
|
|
9
|
+
from astropy.time import Time
|
|
10
|
+
from pydantic import PrivateAttr
|
|
11
|
+
from sotodlib.core import Context
|
|
12
|
+
|
|
13
|
+
from socm.workflows.ml_null_tests import NullTestWorkflow
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class MoonCloseFarNullTestWorkflow(NullTestWorkflow):
|
|
17
|
+
"""
|
|
18
|
+
A workflow for day/night null tests.
|
|
19
|
+
|
|
20
|
+
A workflow for moon proximity-based null tests.
|
|
21
|
+
|
|
22
|
+
This workflow splits observations based on whether they were taken close to or far from the moon.
|
|
23
|
+
It creates time-interleaved splits with nsplits=2 as specified.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
chunk_nobs: Optional[int] = None
|
|
27
|
+
chunk_duration: Optional[timedelta] = None
|
|
28
|
+
nsplits: int = 2 # Fixed to 2 as specified in the issue
|
|
29
|
+
name: str = "moon_close_far_null_test_workflow"
|
|
30
|
+
sun_distance_threshold: float = (
|
|
31
|
+
10.0 # in degrees, threshold for close/far from the Moon
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
_field_view_radius_per_telescope: Dict[str, float] = PrivateAttr(
|
|
35
|
+
{"sat": 1.0, "act": 1.0, "lat": 1.0}
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
def _get_splits(
|
|
39
|
+
self, ctx: Context, obs_info: Dict[str, Dict[str, Union[float, str]]]
|
|
40
|
+
) -> Dict[str, List[List[str]]]:
|
|
41
|
+
"""
|
|
42
|
+
Distribute the observations across splits based on day/night.
|
|
43
|
+
|
|
44
|
+
Distribute the observations across splits based on proximity to the moon (close/far).
|
|
45
|
+
|
|
46
|
+
Groups observations by whether they were taken close to or far from the moon and then
|
|
47
|
+
creates time-interleaved splits for each with nsplits=2.
|
|
48
|
+
|
|
49
|
+
Args:
|
|
50
|
+
ctx: Context object
|
|
51
|
+
obs_info: Dictionary mapping obs_id to observation metadata
|
|
52
|
+
|
|
53
|
+
Returns:
|
|
54
|
+
Dict mapping 'day' and 'night' to list of splits, where each split is a list
|
|
55
|
+
Dict mapping 'close' and 'far' to list of splits, where each split is a list
|
|
56
|
+
of obs_ids.
|
|
57
|
+
"""
|
|
58
|
+
if self.chunk_nobs is None and self.chunk_duration is None:
|
|
59
|
+
raise ValueError("Either chunk_nobs or duration must be set.")
|
|
60
|
+
elif self.chunk_nobs is not None and self.chunk_duration is not None:
|
|
61
|
+
raise ValueError("Only one of chunk_nobs or duration can be set.")
|
|
62
|
+
elif self.chunk_nobs is None:
|
|
63
|
+
# Decide the chunk size based on the duration
|
|
64
|
+
raise NotImplementedError(
|
|
65
|
+
"Splitting by duration is not implemented yet. Please set chunk_nobs."
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
# Group observations by day/night
|
|
69
|
+
moon_position_splits = {"close": [], "far": []}
|
|
70
|
+
for obs_id, obs_meta in obs_info.items():
|
|
71
|
+
obs_time = datetime.fromtimestamp(
|
|
72
|
+
timestamp=obs_meta["start_time"], tz=timezone.utc
|
|
73
|
+
) # Assuming time is in ISO format
|
|
74
|
+
obs_time = Time(obs_meta["start_time"], format="unix", scale="utc")
|
|
75
|
+
|
|
76
|
+
city = LocationInfo(
|
|
77
|
+
"San Pedro de Atacama", "Chile", "America/Santiago", -22.91, -68.2
|
|
78
|
+
)
|
|
79
|
+
alt = 5190 # Altitude in meters
|
|
80
|
+
location = EarthLocation(
|
|
81
|
+
lat=city.latitude * u.deg, lon=city.longitude * u.deg, height=alt * u.m
|
|
82
|
+
)
|
|
83
|
+
altaz = AltAz(obstime=obs_time, location=location)
|
|
84
|
+
altaz_coord = SkyCoord(
|
|
85
|
+
az=obs_meta["az_center"] * u.deg,
|
|
86
|
+
alt=obs_meta["el_center"] * u.deg,
|
|
87
|
+
frame=altaz,
|
|
88
|
+
)
|
|
89
|
+
radec = altaz_coord.transform_to("icrs")
|
|
90
|
+
|
|
91
|
+
# Get Moon's position
|
|
92
|
+
moon = get_body("moon", obs_time, location=location)
|
|
93
|
+
|
|
94
|
+
# Compute angular separation
|
|
95
|
+
separation = radec.separation(moon)
|
|
96
|
+
|
|
97
|
+
if separation.deg <= (
|
|
98
|
+
self.sun_distance_threshold
|
|
99
|
+
+ self._field_view_radius_per_telescope[self.site]
|
|
100
|
+
):
|
|
101
|
+
moon_position_splits["close"].append(obs_id)
|
|
102
|
+
else:
|
|
103
|
+
moon_position_splits["far"].append(obs_id)
|
|
104
|
+
|
|
105
|
+
final_splits = {}
|
|
106
|
+
|
|
107
|
+
# For each direction, create time-interleaved splits
|
|
108
|
+
for moon_position, obs_infos in moon_position_splits.items():
|
|
109
|
+
if not obs_infos:
|
|
110
|
+
continue
|
|
111
|
+
|
|
112
|
+
# Sort by timestamp for time-based splitting
|
|
113
|
+
sorted_ids = sorted(obs_infos, key=lambda k: obs_info[k]["start_time"])
|
|
114
|
+
|
|
115
|
+
# Group in chunks based on chunk_nobs
|
|
116
|
+
num_chunks = self._get_num_chunks(len(sorted_ids))
|
|
117
|
+
obs_lists = np.array_split(sorted_ids, num_chunks) if num_chunks > 0 else []
|
|
118
|
+
|
|
119
|
+
# Create nsplits (=2) time-interleaved splits
|
|
120
|
+
splits = [[] for _ in range(self.nsplits)]
|
|
121
|
+
for i, obs_list in enumerate(obs_lists):
|
|
122
|
+
splits[i % self.nsplits] += obs_list.tolist()
|
|
123
|
+
|
|
124
|
+
final_splits[moon_position] = splits
|
|
125
|
+
|
|
126
|
+
return final_splits
|
|
127
|
+
|
|
128
|
+
@classmethod
|
|
129
|
+
def get_workflows(cls, desc=None) -> List[NullTestWorkflow]:
|
|
130
|
+
"""
|
|
131
|
+
Create a list of NullTestWorkflows instances from the provided descriptions.
|
|
132
|
+
|
|
133
|
+
Creates separate workflows for each direction split following the naming
|
|
134
|
+
convention: {setname} = direction_[rising,setting,middle]
|
|
135
|
+
"""
|
|
136
|
+
moon_position_workflow = cls(**desc)
|
|
137
|
+
|
|
138
|
+
workflows = []
|
|
139
|
+
for (
|
|
140
|
+
moon_position,
|
|
141
|
+
moon_position_splits,
|
|
142
|
+
) in moon_position_workflow._splits.items():
|
|
143
|
+
for split_idx, split in enumerate(moon_position_splits):
|
|
144
|
+
if not split:
|
|
145
|
+
continue
|
|
146
|
+
desc_copy = moon_position_workflow.model_dump(exclude_unset=True)
|
|
147
|
+
desc_copy["name"] = (
|
|
148
|
+
f"moon_{moon_position}_split_{split_idx + 1}_null_test_workflow"
|
|
149
|
+
)
|
|
150
|
+
# Follow the naming convention: direction_[rising,setting,middle]
|
|
151
|
+
desc_copy["output_dir"] = (
|
|
152
|
+
f"{moon_position_workflow.output_dir}/moon_{moon_position}_split_{split_idx + 1}"
|
|
153
|
+
)
|
|
154
|
+
desc_copy["datasize"] = 0
|
|
155
|
+
query_file = Path(desc_copy["output_dir"]) / "query.txt"
|
|
156
|
+
query_file.parent.mkdir(parents=True, exist_ok=True)
|
|
157
|
+
with open(query_file, "w") as f:
|
|
158
|
+
for oid in split:
|
|
159
|
+
f.write(f"{oid}\n")
|
|
160
|
+
desc_copy["query"] = f"file://{str(query_file.absolute())}"
|
|
161
|
+
desc_copy["chunk_nobs"] = 1
|
|
162
|
+
workflow = NullTestWorkflow(**desc_copy)
|
|
163
|
+
workflows.append(workflow)
|
|
164
|
+
|
|
165
|
+
return workflows
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
from datetime import datetime, timedelta, timezone
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
from typing import Dict, List, Optional, Union
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
from astral import LocationInfo
|
|
7
|
+
from astral.moon import moonrise, moonset
|
|
8
|
+
from sotodlib.core import Context
|
|
9
|
+
|
|
10
|
+
from socm.workflows.ml_null_tests import NullTestWorkflow
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class MoonRiseSetNullTestWorkflow(NullTestWorkflow):
|
|
14
|
+
"""
|
|
15
|
+
A workflow for moonrise/moonset null tests.
|
|
16
|
+
|
|
17
|
+
This workflow splits observations based on whether they were taken during the moonrise or moonset.
|
|
18
|
+
It creates time-interleaved splits with nsplits=2 as specified.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
chunk_nobs: Optional[int] = None
|
|
22
|
+
chunk_duration: Optional[timedelta] = None
|
|
23
|
+
nsplits: int = 2 # Fixed to 2 as specified in the issue
|
|
24
|
+
name: str = "moonset_null_test_workflow"
|
|
25
|
+
|
|
26
|
+
def _get_splits(
|
|
27
|
+
self, ctx: Context, obs_info: Dict[str, Dict[str, Union[float, str]]]
|
|
28
|
+
) -> Dict[str, List[List[str]]]:
|
|
29
|
+
"""
|
|
30
|
+
Distribute the observations across splits based on day/night.
|
|
31
|
+
|
|
32
|
+
Groups observations by whether they were taken during the day or night and then
|
|
33
|
+
creates time-interleaved splits for each with nsplits=2.
|
|
34
|
+
|
|
35
|
+
Args:
|
|
36
|
+
ctx: Context object
|
|
37
|
+
obs_info: Dictionary mapping obs_id to observation metadata
|
|
38
|
+
|
|
39
|
+
Returns:
|
|
40
|
+
Dict mapping 'day' and 'night' to list of splits, where each split is a list
|
|
41
|
+
of obs_ids
|
|
42
|
+
"""
|
|
43
|
+
if self.chunk_nobs is None and self.chunk_duration is None:
|
|
44
|
+
raise ValueError("Either chunk_nobs or duration must be set.")
|
|
45
|
+
elif self.chunk_nobs is not None and self.chunk_duration is not None:
|
|
46
|
+
raise ValueError("Only one of chunk_nobs or duration can be set.")
|
|
47
|
+
elif self.chunk_nobs is None:
|
|
48
|
+
# Decide the chunk size based on the duration
|
|
49
|
+
raise NotImplementedError(
|
|
50
|
+
"Splitting by duration is not implemented yet. Please set chunk_nobs."
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
# Group observations by day/night
|
|
54
|
+
moon_sky_splits = {"insky": [], "outsky": []}
|
|
55
|
+
for obs_id, obs_meta in obs_info.items():
|
|
56
|
+
obs_time = datetime.fromtimestamp(
|
|
57
|
+
timestamp=obs_meta["start_time"], tz=timezone.utc
|
|
58
|
+
) # Assuming time is in ISO format
|
|
59
|
+
|
|
60
|
+
# Determine if it's day or night using the sun position
|
|
61
|
+
city = LocationInfo(
|
|
62
|
+
"San Pedro de Atacama", "Chile", "America/Santiago", -22.91, -68.2
|
|
63
|
+
)
|
|
64
|
+
moon_rise = moonrise(city.observer, obs_time, timezone.utc)
|
|
65
|
+
moon_set = moonset(city.observer, obs_time, timezone.utc)
|
|
66
|
+
|
|
67
|
+
moon_times = []
|
|
68
|
+
if moon_set.hour < moon_rise.hour:
|
|
69
|
+
# Moon sets on a different day
|
|
70
|
+
start_of_day = obs_time.replace(
|
|
71
|
+
hour=0, minute=0, second=0, microsecond=0
|
|
72
|
+
)
|
|
73
|
+
end_of_day = obs_time.replace(
|
|
74
|
+
hour=23, minute=59, second=59, microsecond=999999
|
|
75
|
+
)
|
|
76
|
+
moon_times = [
|
|
77
|
+
{"start_time": start_of_day, "end_time": moon_set},
|
|
78
|
+
{"start_time": moon_rise, "end_time": end_of_day},
|
|
79
|
+
]
|
|
80
|
+
else:
|
|
81
|
+
# Moon sets on the same day
|
|
82
|
+
moon_times = [{"start_time": moon_rise, "end_time": moon_set}]
|
|
83
|
+
moon_in_sky = False
|
|
84
|
+
for mt in moon_times:
|
|
85
|
+
if mt["start_time"] <= obs_time <= mt["end_time"]:
|
|
86
|
+
moon_in_sky = True
|
|
87
|
+
break
|
|
88
|
+
if moon_in_sky:
|
|
89
|
+
moon_sky_splits["insky"].append(obs_id)
|
|
90
|
+
else:
|
|
91
|
+
moon_sky_splits["outsky"].append(obs_id)
|
|
92
|
+
|
|
93
|
+
final_splits = {}
|
|
94
|
+
|
|
95
|
+
# For each direction, create time-interleaved splits
|
|
96
|
+
for moon_sky, obs_infos in moon_sky_splits.items():
|
|
97
|
+
if not obs_infos:
|
|
98
|
+
continue
|
|
99
|
+
|
|
100
|
+
# Sort by timestamp for time-based splitting
|
|
101
|
+
sorted_ids = sorted(obs_infos, key=lambda k: obs_info[k]["start_time"])
|
|
102
|
+
|
|
103
|
+
# Group in chunks based on chunk_nobs
|
|
104
|
+
num_chunks = self._get_num_chunks(len(sorted_ids))
|
|
105
|
+
obs_lists = np.array_split(sorted_ids, num_chunks) if num_chunks > 0 else []
|
|
106
|
+
|
|
107
|
+
# Create nsplits (=2) time-interleaved splits
|
|
108
|
+
splits = [[] for _ in range(self.nsplits)]
|
|
109
|
+
for i, obs_list in enumerate(obs_lists):
|
|
110
|
+
splits[i % self.nsplits] += obs_list.tolist()
|
|
111
|
+
|
|
112
|
+
final_splits[moon_sky] = splits
|
|
113
|
+
|
|
114
|
+
return final_splits
|
|
115
|
+
|
|
116
|
+
@classmethod
|
|
117
|
+
def get_workflows(cls, desc=None) -> List[NullTestWorkflow]:
|
|
118
|
+
"""
|
|
119
|
+
Create a list of NullTestWorkflows instances from the provided descriptions.
|
|
120
|
+
|
|
121
|
+
Creates separate workflows for each direction split following the naming
|
|
122
|
+
convention: {setname} = direction_[rising,setting,middle]
|
|
123
|
+
"""
|
|
124
|
+
moon_sky_workflow = cls(**desc)
|
|
125
|
+
|
|
126
|
+
workflows = []
|
|
127
|
+
for moon_sky, moon_sky_splits in moon_sky_workflow._splits.items():
|
|
128
|
+
for split_idx, split in enumerate(moon_sky_splits):
|
|
129
|
+
if not split:
|
|
130
|
+
continue
|
|
131
|
+
desc_copy = moon_sky_workflow.model_dump(exclude_unset=True)
|
|
132
|
+
desc_copy["name"] = (
|
|
133
|
+
f"moon_{moon_sky}_split_{split_idx + 1}_null_test_workflow"
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
# Follow the naming convention: direction_[rising,setting,middle]
|
|
137
|
+
desc_copy["output_dir"] = (
|
|
138
|
+
f"{moon_sky_workflow.output_dir}/moon_{moon_sky}_split_{split_idx + 1}"
|
|
139
|
+
)
|
|
140
|
+
desc_copy["datasize"] = 0
|
|
141
|
+
query_file = Path(desc_copy["output_dir"]) / "query.txt"
|
|
142
|
+
query_file.parent.mkdir(parents=True, exist_ok=True)
|
|
143
|
+
with open(query_file, "w") as f:
|
|
144
|
+
for oid in split:
|
|
145
|
+
f.write(f"{oid}\n")
|
|
146
|
+
desc_copy["query"] = f"file://{str(query_file.absolute())}"
|
|
147
|
+
desc_copy["chunk_nobs"] = 1
|
|
148
|
+
workflow = NullTestWorkflow(**desc_copy)
|
|
149
|
+
workflows.append(workflow)
|
|
150
|
+
|
|
151
|
+
return workflows
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
from datetime import timedelta
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
from typing import Dict, List, Optional, Union
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
from sotodlib.core import Context
|
|
7
|
+
|
|
8
|
+
from socm.workflows.ml_null_tests import NullTestWorkflow
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class PWVNullTestWorkflow(NullTestWorkflow):
|
|
12
|
+
"""
|
|
13
|
+
A workflow for PWV (precipitable water vapor) null tests.
|
|
14
|
+
|
|
15
|
+
This workflow splits observations into two groups based on PWV levels ('high' and 'low')
|
|
16
|
+
and creates time-interleaved splits with nsplits=2 as specified.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
chunk_nobs: Optional[int] = None
|
|
20
|
+
chunk_duration: Optional[timedelta] = None
|
|
21
|
+
pwv_limit: float = 2.0 # Example limit for PWV, adjust as needed
|
|
22
|
+
nsplits: int = 2 # Fixed to 2 as specified in the issue
|
|
23
|
+
name: str = "pwv_null_test_workflow"
|
|
24
|
+
|
|
25
|
+
def _get_splits(
|
|
26
|
+
self, ctx: Context, obs_info: Dict[str, Dict[str, Union[float, str]]]
|
|
27
|
+
) -> Dict[str, List[List[str]]]:
|
|
28
|
+
"""
|
|
29
|
+
Distribute the observations across splits based on PWV values.
|
|
30
|
+
|
|
31
|
+
Groups observations by PWV level (high, low) and then
|
|
32
|
+
creates time-interleaved splits for each level with nsplits=2.
|
|
33
|
+
|
|
34
|
+
Args:
|
|
35
|
+
ctx: Context object
|
|
36
|
+
obs_info: Dictionary mapping obs_id to observation metadata
|
|
37
|
+
|
|
38
|
+
Returns:
|
|
39
|
+
Dict mapping PWV level to list of splits, where each split is a list
|
|
40
|
+
of obs_ids
|
|
41
|
+
"""
|
|
42
|
+
if self.chunk_nobs is None and self.chunk_duration is None:
|
|
43
|
+
raise ValueError("Either chunk_nobs or duration must be set.")
|
|
44
|
+
elif self.chunk_nobs is not None and self.chunk_duration is not None:
|
|
45
|
+
raise ValueError("Only one of chunk_nobs or duration can be set.")
|
|
46
|
+
elif self.chunk_nobs is None:
|
|
47
|
+
# Decide the chunk size based on the duration
|
|
48
|
+
raise NotImplementedError(
|
|
49
|
+
"Splitting by duration is not implemented yet. Please set chunk_nobs."
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
# Group observations by scan direction
|
|
53
|
+
pwv_splits = {"high": [], "low": []}
|
|
54
|
+
for obs_id, obs_meta in obs_info.items():
|
|
55
|
+
if obs_meta["pwv"] > self.pwv_limit:
|
|
56
|
+
pwv_splits["high"].append(obs_id)
|
|
57
|
+
else:
|
|
58
|
+
pwv_splits["low"].append(obs_id)
|
|
59
|
+
|
|
60
|
+
final_splits = {}
|
|
61
|
+
|
|
62
|
+
# For each pwv level, create time-interleaved splits
|
|
63
|
+
for pwv_level, pwv_obs_info in pwv_splits.items():
|
|
64
|
+
if not pwv_obs_info:
|
|
65
|
+
continue
|
|
66
|
+
|
|
67
|
+
# Sort by timestamp for time-based splitting
|
|
68
|
+
sorted_ids = sorted(pwv_obs_info, key=lambda k: obs_info[k]["start_time"])
|
|
69
|
+
|
|
70
|
+
# Group in chunks based on chunk_nobs
|
|
71
|
+
num_chunks = self._get_num_chunks(len(sorted_ids))
|
|
72
|
+
obs_lists = np.array_split(sorted_ids, num_chunks) if num_chunks > 0 else []
|
|
73
|
+
|
|
74
|
+
# Create nsplits (=2) time-interleaved splits
|
|
75
|
+
splits = [[] for _ in range(self.nsplits)]
|
|
76
|
+
for i, obs_list in enumerate(obs_lists):
|
|
77
|
+
splits[i % self.nsplits] += obs_list.tolist()
|
|
78
|
+
|
|
79
|
+
final_splits[pwv_level] = splits
|
|
80
|
+
|
|
81
|
+
return final_splits
|
|
82
|
+
|
|
83
|
+
@classmethod
|
|
84
|
+
def get_workflows(cls, desc=None) -> List[NullTestWorkflow]:
|
|
85
|
+
"""
|
|
86
|
+
Create a list of NullTestWorkflows instances from the provided descriptions.
|
|
87
|
+
|
|
88
|
+
Creates separate workflows for each PWV-based split following the naming
|
|
89
|
+
convention: {setname} = pwv_{pwv_level}_split_{split_idx + 1}_null_test_workflow
|
|
90
|
+
"""
|
|
91
|
+
pwv_workflow = cls(**desc)
|
|
92
|
+
|
|
93
|
+
workflows = []
|
|
94
|
+
for pwv_level, pwv_splits in pwv_workflow._splits.items():
|
|
95
|
+
for split_idx, split in enumerate(pwv_splits):
|
|
96
|
+
if not split:
|
|
97
|
+
continue
|
|
98
|
+
desc_copy = pwv_workflow.model_dump(exclude_unset=True)
|
|
99
|
+
desc_copy["name"] = (
|
|
100
|
+
f"pwv_{pwv_level}_split_{split_idx + 1}_null_test_workflow"
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
# Follow the naming convention: pwv_[high,low]
|
|
104
|
+
desc_copy["output_dir"] = (
|
|
105
|
+
f"{pwv_workflow.output_dir}/pwv_{pwv_level}_split_{split_idx + 1}"
|
|
106
|
+
)
|
|
107
|
+
desc_copy["datasize"] = 0
|
|
108
|
+
query_file = Path(desc_copy["output_dir"]) / "query.txt"
|
|
109
|
+
query_file.parent.mkdir(parents=True, exist_ok=True)
|
|
110
|
+
with open(query_file, "w") as f:
|
|
111
|
+
for oid in split:
|
|
112
|
+
f.write(f"{oid}\n")
|
|
113
|
+
desc_copy["query"] = f"file://{str(query_file.absolute())}"
|
|
114
|
+
desc_copy["chunk_nobs"] = 1
|
|
115
|
+
workflow = NullTestWorkflow(**desc_copy)
|
|
116
|
+
workflows.append(workflow)
|
|
117
|
+
|
|
118
|
+
return workflows
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
from datetime import datetime, timedelta, timezone
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
from typing import Dict, List, Optional, Union
|
|
4
|
+
|
|
5
|
+
import astropy.units as u
|
|
6
|
+
import numpy as np
|
|
7
|
+
from astral import LocationInfo
|
|
8
|
+
from astropy.coordinates import AltAz, EarthLocation, SkyCoord, get_sun
|
|
9
|
+
from astropy.time import Time
|
|
10
|
+
from pydantic import PrivateAttr
|
|
11
|
+
from sotodlib.core import Context
|
|
12
|
+
|
|
13
|
+
from socm.workflows.ml_null_tests import NullTestWorkflow
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class SunCloseFarNullTestWorkflow(NullTestWorkflow):
|
|
17
|
+
"""
|
|
18
|
+
A workflow for day/night null tests.
|
|
19
|
+
|
|
20
|
+
This workflow splits observations based on whether they were taken during the day or night.
|
|
21
|
+
A workflow for sun proximity-based null tests.
|
|
22
|
+
|
|
23
|
+
This workflow splits observations based on whether they are "close" or "far" from the Sun,
|
|
24
|
+
using a configurable sun distance threshold (in degrees). It creates time-interleaved splits
|
|
25
|
+
with nsplits=2 as specified.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
chunk_nobs: Optional[int] = None
|
|
29
|
+
chunk_duration: Optional[timedelta] = None
|
|
30
|
+
nsplits: int = 2 # Fixed to 2 as specified in the issue
|
|
31
|
+
name: str = "sun_close_far_null_test_workflow"
|
|
32
|
+
sun_distance_threshold: float = (
|
|
33
|
+
10.0 # in degrees, threshold for close/far from the Sun
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
_field_view_radius_per_telescope: Dict[str, float] = PrivateAttr(
|
|
37
|
+
{"sat": 1.0, "act": 1.0, "lat": 1.0}
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
def _get_splits(
|
|
41
|
+
self, ctx: Context, obs_info: Dict[str, Dict[str, Union[float, str]]]
|
|
42
|
+
) -> Dict[str, List[List[str]]]:
|
|
43
|
+
"""
|
|
44
|
+
Distribute the observations across splits based on day/night.
|
|
45
|
+
|
|
46
|
+
Groups observations by whether they were taken during the day or night and then
|
|
47
|
+
creates time-interleaved splits for each with nsplits=2.
|
|
48
|
+
|
|
49
|
+
Args:
|
|
50
|
+
ctx: Context object
|
|
51
|
+
obs_info: Dictionary mapping obs_id to observation metadata
|
|
52
|
+
|
|
53
|
+
Returns:
|
|
54
|
+
Distribute the observations across splits based on proximity to the sun.
|
|
55
|
+
|
|
56
|
+
Groups observations by whether they are 'close' or 'far' from the sun, according to
|
|
57
|
+
the sun_distance_threshold, and then creates time-interleaved splits for each group
|
|
58
|
+
with nsplits=2.
|
|
59
|
+
|
|
60
|
+
Args:
|
|
61
|
+
ctx: Context object
|
|
62
|
+
obs_info: Dictionary mapping obs_id to observation metadata
|
|
63
|
+
|
|
64
|
+
Returns:
|
|
65
|
+
Dict mapping 'close' and 'far' to list of splits, where each split is a list
|
|
66
|
+
of obs_ids
|
|
67
|
+
"""
|
|
68
|
+
if self.chunk_nobs is None and self.chunk_duration is None:
|
|
69
|
+
raise ValueError("Either chunk_nobs or duration must be set.")
|
|
70
|
+
elif self.chunk_nobs is not None and self.chunk_duration is not None:
|
|
71
|
+
raise ValueError("Only one of chunk_nobs or duration can be set.")
|
|
72
|
+
elif self.chunk_nobs is None:
|
|
73
|
+
# Decide the chunk size based on the duration
|
|
74
|
+
raise NotImplementedError(
|
|
75
|
+
"Splitting by duration is not implemented yet. Please set chunk_nobs."
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
# Group observations by day/night
|
|
79
|
+
sun_position_splits = {"close": [], "far": []}
|
|
80
|
+
for obs_id, obs_meta in obs_info.items():
|
|
81
|
+
obs_time = datetime.fromtimestamp(
|
|
82
|
+
timestamp=obs_meta["start_time"], tz=timezone.utc
|
|
83
|
+
) # Assuming time is in ISO format
|
|
84
|
+
obs_time = Time(obs_meta["start_time"], format="unix", scale="utc")
|
|
85
|
+
|
|
86
|
+
city = LocationInfo(
|
|
87
|
+
"San Pedro de Atacama", "Chile", "America/Santiago", -22.91, -68.2
|
|
88
|
+
)
|
|
89
|
+
alt = 5190 # Altitude in meters
|
|
90
|
+
location = EarthLocation(
|
|
91
|
+
lat=city.latitude * u.deg, lon=city.longitude * u.deg, height=alt * u.m
|
|
92
|
+
)
|
|
93
|
+
altaz = AltAz(obstime=obs_time, location=location)
|
|
94
|
+
altaz_coord = SkyCoord(
|
|
95
|
+
az=obs_meta["az_center"] * u.deg,
|
|
96
|
+
alt=obs_meta["el_center"] * u.deg,
|
|
97
|
+
frame=altaz,
|
|
98
|
+
)
|
|
99
|
+
radec = altaz_coord.transform_to("icrs")
|
|
100
|
+
|
|
101
|
+
# Get Sun's position
|
|
102
|
+
sun = get_sun(obs_time)
|
|
103
|
+
|
|
104
|
+
# Compute angular separation
|
|
105
|
+
separation = radec.separation(sun)
|
|
106
|
+
|
|
107
|
+
if separation.deg <= (
|
|
108
|
+
self.sun_distance_threshold
|
|
109
|
+
+ self._field_view_radius_per_telescope[self.site]
|
|
110
|
+
):
|
|
111
|
+
sun_position_splits["close"].append(obs_id)
|
|
112
|
+
else:
|
|
113
|
+
sun_position_splits["far"].append(obs_id)
|
|
114
|
+
|
|
115
|
+
final_splits = {}
|
|
116
|
+
|
|
117
|
+
# For each direction, create time-interleaved splits
|
|
118
|
+
for sun_position, obs_infos in sun_position_splits.items():
|
|
119
|
+
if not obs_infos:
|
|
120
|
+
continue
|
|
121
|
+
|
|
122
|
+
# Sort by timestamp for time-based splitting
|
|
123
|
+
sorted_ids = sorted(obs_infos, key=lambda k: obs_info[k]["start_time"])
|
|
124
|
+
|
|
125
|
+
# Group in chunks based on chunk_nobs
|
|
126
|
+
num_chunks = self._get_num_chunks(len(sorted_ids))
|
|
127
|
+
obs_lists = np.array_split(sorted_ids, num_chunks) if num_chunks > 0 else []
|
|
128
|
+
|
|
129
|
+
# Create nsplits (=2) time-interleaved splits
|
|
130
|
+
splits = [[] for _ in range(self.nsplits)]
|
|
131
|
+
for i, obs_list in enumerate(obs_lists):
|
|
132
|
+
splits[i % self.nsplits] += obs_list.tolist()
|
|
133
|
+
|
|
134
|
+
final_splits[sun_position] = splits
|
|
135
|
+
|
|
136
|
+
return final_splits
|
|
137
|
+
|
|
138
|
+
@classmethod
|
|
139
|
+
def get_workflows(cls, desc=None) -> List[NullTestWorkflow]:
|
|
140
|
+
"""
|
|
141
|
+
Create a list of NullTestWorkflows instances from the provided descriptions.
|
|
142
|
+
|
|
143
|
+
Creates separate workflows for each direction split following the naming
|
|
144
|
+
convention: {setname} = direction_[rising,setting,middle]
|
|
145
|
+
"""
|
|
146
|
+
sun_position_workflow = cls(**desc)
|
|
147
|
+
|
|
148
|
+
workflows = []
|
|
149
|
+
for sun_position, sun_position_splits in sun_position_workflow._splits.items():
|
|
150
|
+
for split_idx, split in enumerate(sun_position_splits):
|
|
151
|
+
if not split:
|
|
152
|
+
continue
|
|
153
|
+
desc_copy = sun_position_workflow.model_dump(exclude_unset=True)
|
|
154
|
+
desc_copy["name"] = (
|
|
155
|
+
f"sun_{sun_position}_split_{split_idx + 1}_null_test_workflow"
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
# Follow the naming convention: direction_[rising,setting,middle]
|
|
159
|
+
desc_copy["output_dir"] = (
|
|
160
|
+
f"{sun_position_workflow.output_dir}/sun_{sun_position}_split_{split_idx + 1}"
|
|
161
|
+
)
|
|
162
|
+
desc_copy["datasize"] = 0
|
|
163
|
+
query_file = Path(desc_copy["output_dir"]) / "query.txt"
|
|
164
|
+
query_file.parent.mkdir(parents=True, exist_ok=True)
|
|
165
|
+
with open(query_file, "w") as f:
|
|
166
|
+
for oid in split:
|
|
167
|
+
f.write(f"{oid}\n")
|
|
168
|
+
desc_copy["query"] = f"file://{str(query_file.absolute())}"
|
|
169
|
+
desc_copy["chunk_nobs"] = 1
|
|
170
|
+
workflow = NullTestWorkflow(**desc_copy)
|
|
171
|
+
workflows.append(workflow)
|
|
172
|
+
|
|
173
|
+
return workflows
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
from datetime import timedelta
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
from typing import Dict, List, Optional, Union
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
from sotodlib.core import Context
|
|
7
|
+
|
|
8
|
+
from socm.workflows.ml_null_tests import NullTestWorkflow
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class TimeNullTestWorkflow(NullTestWorkflow):
|
|
12
|
+
"""
|
|
13
|
+
A workflow for time null tests.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
chunk_nobs: Optional[int] = None
|
|
17
|
+
chunk_duration: Optional[timedelta] = None
|
|
18
|
+
nsplits: int = 8
|
|
19
|
+
name: str = "time_null_test_workflow"
|
|
20
|
+
|
|
21
|
+
def _get_splits(
|
|
22
|
+
self, ctx: Context, obs_info: Dict[str, Dict[str, Union[float, str]]]
|
|
23
|
+
) -> List[List[str]]:
|
|
24
|
+
"""
|
|
25
|
+
Distribute the observations across splits based on the context and observation IDs.
|
|
26
|
+
"""
|
|
27
|
+
if self.chunk_nobs is None and self.chunk_duration is None:
|
|
28
|
+
raise ValueError("Either chunk_nobs or duration must be set.")
|
|
29
|
+
elif self.chunk_nobs is not None and self.chunk_duration is not None:
|
|
30
|
+
raise ValueError("Only one of chunk_nobs or duration can be set.")
|
|
31
|
+
elif self.chunk_nobs is None:
|
|
32
|
+
# Decide the chunk size based on the duration. Each chunk needs to have the
|
|
33
|
+
# observations that their start times are just less than chunk_duration.
|
|
34
|
+
raise NotImplementedError(
|
|
35
|
+
"Splitting by duration is not implemented yet. Please set chunk_nobs."
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
sorted_ids = sorted(obs_info, key=lambda k: obs_info[k]["start_time"])
|
|
39
|
+
# Group in chunks of size self.chunk_nobs observations.
|
|
40
|
+
num_chunks = self._get_num_chunks(len(sorted_ids))
|
|
41
|
+
obs_lists = np.array_split(sorted_ids, num_chunks) if num_chunks > 0 else []
|
|
42
|
+
splits = [[] for _ in range(self.nsplits)]
|
|
43
|
+
for i, obs_list in enumerate(obs_lists):
|
|
44
|
+
splits[i % self.nsplits] += obs_list.tolist()
|
|
45
|
+
|
|
46
|
+
return splits
|
|
47
|
+
|
|
48
|
+
@classmethod
|
|
49
|
+
def get_workflows(cls, desc=None) -> List[NullTestWorkflow]:
|
|
50
|
+
"""
|
|
51
|
+
Create a list of NullTestWorkflows instances from the provided descriptions.
|
|
52
|
+
"""
|
|
53
|
+
|
|
54
|
+
time_workflow = cls(**desc)
|
|
55
|
+
|
|
56
|
+
workflows = []
|
|
57
|
+
for split in time_workflow._splits:
|
|
58
|
+
if not split:
|
|
59
|
+
continue
|
|
60
|
+
desc = time_workflow.model_dump(exclude_unset=True)
|
|
61
|
+
desc["name"] = f"mission_split_{len(workflows) + 1}_null_test_workflow"
|
|
62
|
+
desc["output_dir"] = (
|
|
63
|
+
f"{time_workflow.output_dir}/mission_split_{len(workflows) + 1}"
|
|
64
|
+
)
|
|
65
|
+
desc["datasize"] = 0
|
|
66
|
+
query_file = Path(desc["output_dir"]) / "query.txt"
|
|
67
|
+
query_file.parent.mkdir(parents=True, exist_ok=True)
|
|
68
|
+
with open(query_file, "w") as f:
|
|
69
|
+
for oid in split:
|
|
70
|
+
f.write(f"{oid}\n")
|
|
71
|
+
desc["query"] = f"file://{str(query_file.absolute())}"
|
|
72
|
+
desc["chunk_nobs"] = 1
|
|
73
|
+
workflow = NullTestWorkflow(**desc)
|
|
74
|
+
workflows.append(workflow)
|
|
75
|
+
|
|
76
|
+
return workflows
|