blue-sandbox 5.294.1__py3-none-any.whl → 5.318.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- blue_sandbox/.abcli/alias.sh +2 -0
- blue_sandbox/.abcli/tests/help.sh +0 -8
- blue_sandbox/README.py +6 -51
- blue_sandbox/__init__.py +1 -1
- blue_sandbox/config.env +3 -1
- blue_sandbox/env.py +5 -0
- blue_sandbox/help/functions.py +0 -10
- {blue_sandbox-5.294.1.dist-info → blue_sandbox-5.318.1.dist-info}/METADATA +2 -10
- blue_sandbox-5.318.1.dist-info/RECORD +28 -0
- blue_sandbox/.abcli/cemetery/inference/cloudwatch.sh +0 -40
- blue_sandbox/.abcli/cemetery/inference/inference.sh +0 -155
- blue_sandbox/.abcli/microsoft_building_damage_assessment/ingest/list.sh +0 -21
- blue_sandbox/.abcli/microsoft_building_damage_assessment/ingest.sh +0 -76
- blue_sandbox/.abcli/microsoft_building_damage_assessment/install.sh +0 -50
- blue_sandbox/.abcli/microsoft_building_damage_assessment/label.sh +0 -26
- blue_sandbox/.abcli/microsoft_building_damage_assessment/tensorboard.sh +0 -21
- blue_sandbox/.abcli/microsoft_building_damage_assessment/train.sh +0 -53
- blue_sandbox/.abcli/microsoft_building_damage_assessment.sh +0 -15
- blue_sandbox/.abcli/sagesemseg/cache_dataset.sh +0 -49
- blue_sandbox/.abcli/sagesemseg/consts.sh +0 -3
- blue_sandbox/.abcli/sagesemseg/train.sh +0 -48
- blue_sandbox/.abcli/sagesemseg/upload_dataset.sh +0 -47
- blue_sandbox/.abcli/tests/microsoft_building_damage_assessment_ingest.sh +0 -13
- blue_sandbox/.abcli/tests/microsoft_building_damage_assessment_ingest_list.sh +0 -19
- blue_sandbox/.abcli/tests/microsoft_building_damage_assessment_label.sh +0 -12
- blue_sandbox/.abcli/tests/microsoft_building_damage_assessment_train.sh +0 -20
- blue_sandbox/.abcli/tests/sagesemseg_train.sh +0 -29
- blue_sandbox/list.py +0 -37
- blue_sandbox/microsoft_building_damage_assessment/README.py +0 -45
- blue_sandbox/microsoft_building_damage_assessment/__init__.py +0 -0
- blue_sandbox/microsoft_building_damage_assessment/__main__.py +0 -82
- blue_sandbox/microsoft_building_damage_assessment/ingest.py +0 -31
- blue_sandbox/microsoft_building_damage_assessment/label.py +0 -38
- blue_sandbox/microsoft_building_damage_assessment/sas_token.py +0 -9
- blue_sandbox/microsoft_building_damage_assessment/train.py +0 -95
- blue_sandbox-5.294.1.dist-info/RECORD +0 -54
- {blue_sandbox-5.294.1.dist-info → blue_sandbox-5.318.1.dist-info}/LICENSE +0 -0
- {blue_sandbox-5.294.1.dist-info → blue_sandbox-5.318.1.dist-info}/WHEEL +0 -0
- {blue_sandbox-5.294.1.dist-info → blue_sandbox-5.318.1.dist-info}/top_level.txt +0 -0
@@ -1,49 +0,0 @@
|
|
1
|
-
#! /usr/bin/env bash
|
2
|
-
|
3
|
-
function runme() {
|
4
|
-
local options=$1
|
5
|
-
|
6
|
-
local script_full_name="${BASH_SOURCE[0]}"
|
7
|
-
local script_name=$(abcli_script_get name)
|
8
|
-
|
9
|
-
abcli_scripts source - sagesemseg/consts
|
10
|
-
|
11
|
-
if [ $(abcli_option_int "$options" help 0) == 1 ]; then
|
12
|
-
local options="$sagesemseg_cache_dataset_options,rm"
|
13
|
-
abcli_meta_script_show_usage $script_full_name "$EOP[$options]$EOPE" \
|
14
|
-
"cache dataset."
|
15
|
-
return
|
16
|
-
fi
|
17
|
-
|
18
|
-
local dataset_name=$(abcli_option "$options" dataset pascal-voc)
|
19
|
-
local do_rm=$(abcli_option "$options" rm 0)
|
20
|
-
local suffix=$(abcli_option "$options" suffix v1)
|
21
|
-
|
22
|
-
local dataset_object_name=$dataset_name-$suffix
|
23
|
-
|
24
|
-
local dataset_object_path=$ABCLI_OBJECT_ROOT/$dataset_object_name
|
25
|
-
mkdir -pv $dataset_object_path
|
26
|
-
|
27
|
-
if [[ -d "$dataset_object_path/$dataset_name" ]]; then
|
28
|
-
abcli_log "✅ $dataset_object_name/$dataset_name/"
|
29
|
-
return
|
30
|
-
fi
|
31
|
-
|
32
|
-
abcli_log "caching $dataset_object_name"
|
33
|
-
|
34
|
-
pushd $dataset_object_path >/dev/null
|
35
|
-
|
36
|
-
if [[ ! -f "$dataset_name.tgz" ]]; then
|
37
|
-
curl -O https://fast-ai-imagelocal.s3.amazonaws.com/pascal-voc.tgz
|
38
|
-
else
|
39
|
-
abcli_log "✅ $dataset_object_name/$dataset_name.tgz"
|
40
|
-
fi
|
41
|
-
|
42
|
-
tar -xvf $dataset_name.tgz
|
43
|
-
|
44
|
-
[[ "$do_rm" == 1 ]] && rm -v $dataset_name.tgz
|
45
|
-
|
46
|
-
popd >/dev/null
|
47
|
-
}
|
48
|
-
|
49
|
-
runme "$@"
|
@@ -1,48 +0,0 @@
|
|
1
|
-
#! /usr/bin/env bash
|
2
|
-
|
3
|
-
function runme() {
|
4
|
-
local options=$1
|
5
|
-
|
6
|
-
local script_full_name="${BASH_SOURCE[0]}"
|
7
|
-
local script_name=$(abcli_script_get name)
|
8
|
-
|
9
|
-
abcli_scripts source - sagesemseg/consts
|
10
|
-
|
11
|
-
if [ $(abcli_option_int "$options" help 0) == 1 ]; then
|
12
|
-
local options="dryrun,~upload"
|
13
|
-
local args="[--deploy 0]$ABCUL[--delete_endpoint 0]$ABCUL[--epochs 10]$ABCUL[--instance_type ml.p3.2xlarge]"
|
14
|
-
abcli_meta_script_show_usage $script_full_name "$EOP[$options]$ABCUL[test|<dataset-object-name>]$ABCUL[-|<model-object-name>]$ABCUL$args$EOPE" \
|
15
|
-
"<dataset-object-name> -train-> <model-object-name>."
|
16
|
-
return
|
17
|
-
fi
|
18
|
-
|
19
|
-
local do_dryrun=$(abcli_option_int "$options" dryrun 0)
|
20
|
-
local do_upload=$(abcli_not $do_dryrun)
|
21
|
-
|
22
|
-
local dataset_object_name=$(abcli_clarify_object "$2" test)
|
23
|
-
if [[ "$dataset_object_name" == "test" ]]; then
|
24
|
-
dataset_object_name="pascal-voc-v1-debug-v2"
|
25
|
-
do_upload=0
|
26
|
-
fi
|
27
|
-
|
28
|
-
do_upload=$(abcli_option_int "$options" upload $do_upload)
|
29
|
-
|
30
|
-
local model_object_name=$(abcli_clarify_object "$3" sagesemseg-model-test-$(abcli_string_timestamp))
|
31
|
-
|
32
|
-
abcli_log "$script_name: $dataset_object_name -> $model_object_name"
|
33
|
-
|
34
|
-
abcli_tags set \
|
35
|
-
$model_object_name \
|
36
|
-
trained_on.$dataset_object_name
|
37
|
-
|
38
|
-
abcli_eval dryrun=$do_dryrun \
|
39
|
-
python3 -m blue_sandbox.cemetry.sagesemseg train_model \
|
40
|
-
--dataset_object_name $dataset_object_name \
|
41
|
-
--model_object_name $model_object_name \
|
42
|
-
"${@:4}"
|
43
|
-
|
44
|
-
[[ "$do_upload" == 1 ]] &&
|
45
|
-
abcli_upload - $model_object_name
|
46
|
-
}
|
47
|
-
|
48
|
-
runme "$@"
|
@@ -1,47 +0,0 @@
|
|
1
|
-
#! /usr/bin/env bash
|
2
|
-
|
3
|
-
function runme() {
|
4
|
-
local options=$1
|
5
|
-
|
6
|
-
local script_full_name="${BASH_SOURCE[0]}"
|
7
|
-
local script_name=$(abcli_script_get name)
|
8
|
-
|
9
|
-
abcli_scripts source - sagesemseg/consts
|
10
|
-
|
11
|
-
if [ $(abcli_option_int "$options" help 0) == 1 ]; then
|
12
|
-
local cache_options=$sagesemseg_cache_dataset_options
|
13
|
-
local options="dryrun,suffix=<v1>"
|
14
|
-
local args="[--count <count>]"
|
15
|
-
abcli_meta_script_show_usage $script_full_name "$EOP[$cache_options]$ABCUL[$options]$ABCUL$args" \
|
16
|
-
"upload dataset to SageMaker for training."
|
17
|
-
return
|
18
|
-
fi
|
19
|
-
|
20
|
-
local cache_options=$1
|
21
|
-
local dataset_name=$(abcli_option "$cache_options" dataset pascal-voc)
|
22
|
-
local cache_suffix=$(abcli_option "$cache_options" suffix v1)
|
23
|
-
|
24
|
-
local dataset_object_name=$dataset_name
|
25
|
-
[[ ! -z "$cache_suffix" ]] && dataset_object_name=$dataset_name-$cache_suffix
|
26
|
-
|
27
|
-
[[ "$dataset_name" == "pascal-voc" ]] &&
|
28
|
-
abcli_scripts source dryrun=$do_dryrun \
|
29
|
-
sagesemseg/cache_dataset \
|
30
|
-
"$cache_options"
|
31
|
-
|
32
|
-
local options=$2
|
33
|
-
local do_dryrun=$(abcli_option_int "$options" dryrun 0)
|
34
|
-
local suffix=$(abcli_option "$options" suffix v1)
|
35
|
-
|
36
|
-
local object_name=$dataset_object_name-$suffix
|
37
|
-
|
38
|
-
abcli_log "$script_name: $dataset_object_name -> ☁️ / $object_name"
|
39
|
-
|
40
|
-
abcli_eval dryrun=$do_dryrun \
|
41
|
-
python3 -m blue_sandbox.cemetry.sagesemseg upload_dataset \
|
42
|
-
--dataset_object_name $dataset_object_name \
|
43
|
-
--object_name $object_name \
|
44
|
-
"${@:3}"
|
45
|
-
}
|
46
|
-
|
47
|
-
runme "$@"
|
@@ -1,13 +0,0 @@
|
|
1
|
-
#! /usr/bin/env bash
|
2
|
-
|
3
|
-
function test_blue_sandbox_microsoft_building_damage_assessment_ingest() {
|
4
|
-
local options=$1
|
5
|
-
|
6
|
-
abcli_eval ,$options \
|
7
|
-
blue_sandbox_microsoft_building_damage_assessment \
|
8
|
-
ingest \
|
9
|
-
event=Maui-Hawaii-fires-Aug-23,~upload,$options \
|
10
|
-
test_blue_sandbox_microsoft_building_damage_assessment_ingest-$(abcli_string_timestamp_short) \
|
11
|
-
--verbose 1 \
|
12
|
-
"${@:2}"
|
13
|
-
}
|
@@ -1,19 +0,0 @@
|
|
1
|
-
#! /usr/bin/env bash
|
2
|
-
|
3
|
-
function test_blue_sandbox_microsoft_building_damage_assessment_ingest_list() {
|
4
|
-
local options=$1
|
5
|
-
|
6
|
-
abcli_eval ,$options \
|
7
|
-
blue_sandbox_microsoft_building_damage_assessment_ingest list
|
8
|
-
[[ $? -ne 0 ]] && return 1
|
9
|
-
|
10
|
-
abcli_eval ,$options \
|
11
|
-
blue_sandbox_microsoft_building_damage_assessment_ingest list \
|
12
|
-
event=Maui-Hawaii-fires-Aug-23
|
13
|
-
[[ $? -ne 0 ]] && return 1
|
14
|
-
|
15
|
-
abcli_eval ,$options \
|
16
|
-
blue_sandbox_microsoft_building_damage_assessment_ingest list \
|
17
|
-
event=Maui-Hawaii-fires-Aug-23 \
|
18
|
-
04/
|
19
|
-
}
|
@@ -1,12 +0,0 @@
|
|
1
|
-
#! /usr/bin/env bash
|
2
|
-
|
3
|
-
function test_blue_sandbox_microsoft_building_damage_assessment_label() {
|
4
|
-
local options=$1
|
5
|
-
|
6
|
-
abcli_eval ,$options \
|
7
|
-
blue_sandbox_microsoft_building_damage_assessment \
|
8
|
-
label \
|
9
|
-
~upload,$options \
|
10
|
-
$DAMAGES_TEST_DATASET_OBJECT_NAME \
|
11
|
-
"${@:2}"
|
12
|
-
}
|
@@ -1,20 +0,0 @@
|
|
1
|
-
#! /usr/bin/env bash
|
2
|
-
|
3
|
-
function test_blue_sandbox_microsoft_building_damage_assessment_train() {
|
4
|
-
local options=$1
|
5
|
-
|
6
|
-
abcli_log_warning "disabled."
|
7
|
-
return 0
|
8
|
-
|
9
|
-
abcli_eval ,$options \
|
10
|
-
blue_sandbox_microsoft_building_damage_assessment \
|
11
|
-
train \
|
12
|
-
~upload,$options \
|
13
|
-
$DAMAGES_TEST_DATASET_OBJECT_NAME \
|
14
|
-
test_blue_sandbox_microsoft_building_damage_assessment_train-$(abcli_string_timestamp_short) \
|
15
|
-
--verbose 1 \
|
16
|
-
"${@:2}"
|
17
|
-
|
18
|
-
# ignore error - TODO: disable
|
19
|
-
return 0
|
20
|
-
}
|
@@ -1,29 +0,0 @@
|
|
1
|
-
#! /usr/bin/env bash
|
2
|
-
|
3
|
-
function test_blue_sandbox_cemetery_sagesemseg_train() {
|
4
|
-
local options=$1
|
5
|
-
local do_dryrun=$(abcli_option_int "$options" dryrun 0)
|
6
|
-
|
7
|
-
abcli_log_warning "🚧 may incur cost 💰, disabled."
|
8
|
-
do_dryrun=1
|
9
|
-
|
10
|
-
local dataset_object_name=dataset-$(abcli_string_timestamp)
|
11
|
-
|
12
|
-
abcli_eval dryrun=$do_dryrun \
|
13
|
-
roofai_dataset_ingest \
|
14
|
-
source=AIRS,target=sagemaker,$2 \
|
15
|
-
$dataset_object_name \
|
16
|
-
--test_count 0 \
|
17
|
-
--train_count 16 \
|
18
|
-
--val_count 16
|
19
|
-
|
20
|
-
local model_object_name=model-$(abcli_string_timestamp)
|
21
|
-
|
22
|
-
abcli_eval dryrun=$do_dryrun \
|
23
|
-
blue_sandbox_cemetery_sagesemseg_train \
|
24
|
-
,$3 \
|
25
|
-
$dataset_object_name \
|
26
|
-
$model_object_name \
|
27
|
-
--instance_type ml.g4dn.2xlarge \
|
28
|
-
"${@:4}"
|
29
|
-
}
|
blue_sandbox/list.py
DELETED
@@ -1,37 +0,0 @@
|
|
1
|
-
list_of_experiments = {
|
2
|
-
"palisades": {
|
3
|
-
"ICON": "🧑🏽🚒",
|
4
|
-
"title": "Geospatial AI for Post-Disaster Damage Assessment using Maxar Open Data.",
|
5
|
-
"url": "https://github.com/kamangir/palisades",
|
6
|
-
"marquee": "https://github.com/kamangir/assets/blob/main/palisades/predict-datacube-maxar_open_data-WildFires-LosAngeles-Jan-2025-11-031311102213-103001010B9A1B00-2025-01-20-x54yb0/640.gif?raw=true",
|
7
|
-
"status": "🎓",
|
8
|
-
},
|
9
|
-
"sagesemseg": {
|
10
|
-
"ICON": "🌀",
|
11
|
-
"title": "A SemSeg (Semantic Segmenter) trained and deployed on AWS Sagemaker.",
|
12
|
-
"url": "https://github.com/kamangir/blue-sandbox/blob/main/blue_sandbox/sagesemseg/README.md",
|
13
|
-
"marquee": "https://github.com/kamangir/assets/blob/main/blue-sandbox/sagesemseg-predict.png?raw=true",
|
14
|
-
"status": "⏸️",
|
15
|
-
},
|
16
|
-
"`@damages`": {
|
17
|
-
"ICON": "🌐",
|
18
|
-
"title": "Satellite imagery damage assessment workflow",
|
19
|
-
"url": "https://github.com/kamangir/blue-sandbox/blob/main/blue_sandbox/microsoft_building_damage_assessment/README.md",
|
20
|
-
"marquee": "https://github.com/kamangir/assets/raw/main/blue-sandbox/Maui-Hawaii-fires-Aug-23-ingest-2025-01-10-qqJqhm.png?raw=true",
|
21
|
-
"status": "⏸️",
|
22
|
-
},
|
23
|
-
"cemetery": {
|
24
|
-
"ICON": "🪦",
|
25
|
-
"title": "An AI cemetery.",
|
26
|
-
"url": "https://github.com/kamangir/blue-sandbox/blob/main/blue_sandbox/cemetery/README.md",
|
27
|
-
"marquee": "https://github.com/kamangir/assets/raw/main/blue-plugin/marquee.png?raw=true",
|
28
|
-
"status": "🪦",
|
29
|
-
},
|
30
|
-
"template": {
|
31
|
-
"ICON": "",
|
32
|
-
"title": "",
|
33
|
-
"url": "",
|
34
|
-
"marquee": "https://github.com/kamangir/assets/raw/main/blue-plugin/marquee.png?raw=true",
|
35
|
-
"status": "🎰",
|
36
|
-
},
|
37
|
-
}
|
@@ -1,45 +0,0 @@
|
|
1
|
-
from typing import Dict
|
2
|
-
|
3
|
-
from blue_sandbox.env import DAMAGES_TEST_DATASET_OBJECT_NAME
|
4
|
-
|
5
|
-
list_of_steps: Dict[str, Dict] = {
|
6
|
-
"ingest": {
|
7
|
-
"object_name": DAMAGES_TEST_DATASET_OBJECT_NAME,
|
8
|
-
"image_name": "Maui-Hawaii-fires-Aug-23-damage-2025-01-09-GgnjQC",
|
9
|
-
},
|
10
|
-
"label": {
|
11
|
-
"object_name": "",
|
12
|
-
"image_name": DAMAGES_TEST_DATASET_OBJECT_NAME,
|
13
|
-
},
|
14
|
-
"train": {
|
15
|
-
"object_name": "",
|
16
|
-
"image_name": "Maui-Hawaii-fires-Aug-23-model-2025-01-10-NQb8IS",
|
17
|
-
},
|
18
|
-
"predict": {"object_name": "", "image_name": ""},
|
19
|
-
"summarize": {"object_name": "", "image_name": ""},
|
20
|
-
}
|
21
|
-
|
22
|
-
items = (
|
23
|
-
[f"`{step}`" for step in list_of_steps]
|
24
|
-
+ [
|
25
|
-
(
|
26
|
-
"[`{}`](https://kamangir-public.s3.ca-central-1.amazonaws.com/{}.tar.gz)".format(
|
27
|
-
step["object_name"],
|
28
|
-
step["image_name"],
|
29
|
-
)
|
30
|
-
if step["object_name"]
|
31
|
-
else ""
|
32
|
-
)
|
33
|
-
for step in list_of_steps.values()
|
34
|
-
]
|
35
|
-
+ [
|
36
|
-
(
|
37
|
-
"".format(
|
38
|
-
step["image_name"],
|
39
|
-
)
|
40
|
-
if step["image_name"]
|
41
|
-
else ""
|
42
|
-
)
|
43
|
-
for step in list_of_steps.values()
|
44
|
-
]
|
45
|
-
)
|
File without changes
|
@@ -1,82 +0,0 @@
|
|
1
|
-
import argparse
|
2
|
-
|
3
|
-
from blueness import module
|
4
|
-
from blueness.argparse.generic import sys_exit
|
5
|
-
|
6
|
-
from blue_sandbox import NAME
|
7
|
-
from blue_sandbox.microsoft_building_damage_assessment.ingest import ingest
|
8
|
-
from blue_sandbox.microsoft_building_damage_assessment.label import label
|
9
|
-
from blue_sandbox.microsoft_building_damage_assessment.train import train
|
10
|
-
from blue_sandbox.microsoft_building_damage_assessment.sas_token import (
|
11
|
-
decode_token,
|
12
|
-
encode_token,
|
13
|
-
)
|
14
|
-
from blue_sandbox.help.microsoft_building_damage_assessment import list_of_events
|
15
|
-
from blue_sandbox.logger import logger
|
16
|
-
|
17
|
-
NAME = module.name(__file__, NAME)
|
18
|
-
|
19
|
-
parser = argparse.ArgumentParser(NAME)
|
20
|
-
parser.add_argument(
|
21
|
-
"task",
|
22
|
-
type=str,
|
23
|
-
help="decode_token | encode_token | ingest | label | train",
|
24
|
-
)
|
25
|
-
parser.add_argument(
|
26
|
-
"--object_name",
|
27
|
-
type=str,
|
28
|
-
)
|
29
|
-
parser.add_argument(
|
30
|
-
"--dataset_object_name",
|
31
|
-
type=str,
|
32
|
-
)
|
33
|
-
parser.add_argument(
|
34
|
-
"--model_object_name",
|
35
|
-
type=str,
|
36
|
-
)
|
37
|
-
parser.add_argument(
|
38
|
-
"--event_name",
|
39
|
-
type=str,
|
40
|
-
default=list_of_events[0],
|
41
|
-
help=" | ".join(list_of_events),
|
42
|
-
)
|
43
|
-
parser.add_argument(
|
44
|
-
"--verbose",
|
45
|
-
type=bool,
|
46
|
-
default=0,
|
47
|
-
help="0|1",
|
48
|
-
)
|
49
|
-
parser.add_argument(
|
50
|
-
"--token",
|
51
|
-
type=str,
|
52
|
-
)
|
53
|
-
args = parser.parse_args()
|
54
|
-
|
55
|
-
success = False
|
56
|
-
if args.task == "decode_token":
|
57
|
-
success = True
|
58
|
-
print(decode_token(args.token))
|
59
|
-
elif args.task == "encode_token":
|
60
|
-
success = True
|
61
|
-
print(encode_token(args.token))
|
62
|
-
elif args.task == "ingest":
|
63
|
-
success = ingest(
|
64
|
-
object_name=args.object_name,
|
65
|
-
event_name=args.event_name,
|
66
|
-
verbose=args.verbose == 1,
|
67
|
-
)
|
68
|
-
elif args.task == "label":
|
69
|
-
success = label(
|
70
|
-
object_name=args.object_name,
|
71
|
-
verbose=args.verbose == 1,
|
72
|
-
)
|
73
|
-
elif args.task == "train":
|
74
|
-
success = train(
|
75
|
-
dataset_object_name=args.dataset_object_name,
|
76
|
-
model_object_name=args.model_object_name,
|
77
|
-
verbose=args.verbose == 1,
|
78
|
-
)
|
79
|
-
else:
|
80
|
-
success = None
|
81
|
-
|
82
|
-
sys_exit(logger, NAME, args.task, success)
|
@@ -1,31 +0,0 @@
|
|
1
|
-
from blueness import module
|
2
|
-
|
3
|
-
from blue_objects import mlflow, metadata
|
4
|
-
|
5
|
-
from blue_sandbox import NAME
|
6
|
-
from blue_sandbox.logger import logger
|
7
|
-
|
8
|
-
|
9
|
-
NAME = module.name(__file__, NAME)
|
10
|
-
|
11
|
-
|
12
|
-
def ingest(
|
13
|
-
event_name: str,
|
14
|
-
object_name: str,
|
15
|
-
verbose: bool = False,
|
16
|
-
) -> bool:
|
17
|
-
logger.info(f"{NAME}.ingest({event_name}) -> {object_name}")
|
18
|
-
|
19
|
-
return all(
|
20
|
-
[
|
21
|
-
mlflow.set_tags(
|
22
|
-
object_name,
|
23
|
-
{"event": event_name},
|
24
|
-
),
|
25
|
-
metadata.post_to_object(
|
26
|
-
object_name,
|
27
|
-
"ingest",
|
28
|
-
{"event": event_name},
|
29
|
-
),
|
30
|
-
]
|
31
|
-
)
|
@@ -1,38 +0,0 @@
|
|
1
|
-
import os
|
2
|
-
|
3
|
-
from blueness import module
|
4
|
-
from blue_objects import metadata, file, objects
|
5
|
-
from blue_objects.env import abcli_path_git
|
6
|
-
|
7
|
-
from blue_sandbox import NAME
|
8
|
-
from blue_sandbox.logger import logger
|
9
|
-
|
10
|
-
|
11
|
-
NAME = module.name(__file__, NAME)
|
12
|
-
|
13
|
-
|
14
|
-
def label(
|
15
|
-
object_name: str,
|
16
|
-
verbose: bool = False,
|
17
|
-
) -> bool:
|
18
|
-
logger.info(f"{NAME}.label: {object_name}")
|
19
|
-
|
20
|
-
geojson_filename = os.path.join(
|
21
|
-
abcli_path_git,
|
22
|
-
"building-damage-assessment/data/demo/labels/Maui_Wildfires_August_0.geojson",
|
23
|
-
)
|
24
|
-
|
25
|
-
if not file.copy(
|
26
|
-
geojson_filename,
|
27
|
-
objects.path_of(
|
28
|
-
filename="label.geojson",
|
29
|
-
object_name=object_name,
|
30
|
-
),
|
31
|
-
):
|
32
|
-
return False
|
33
|
-
|
34
|
-
return metadata.post_to_object(
|
35
|
-
object_name,
|
36
|
-
"label",
|
37
|
-
{"geojson": geojson_filename},
|
38
|
-
)
|
@@ -1,95 +0,0 @@
|
|
1
|
-
import os
|
2
|
-
|
3
|
-
from blueness import module
|
4
|
-
from blue_objects import mlflow, metadata, file, objects, path
|
5
|
-
from blue_objects.env import abcli_path_git
|
6
|
-
|
7
|
-
from blue_sandbox import NAME
|
8
|
-
from blue_sandbox.env import ENCODED_BLOB_SAS_TOKEN
|
9
|
-
from blue_sandbox.microsoft_building_damage_assessment.sas_token import decode_token
|
10
|
-
from blue_sandbox.logger import logger
|
11
|
-
|
12
|
-
|
13
|
-
NAME = module.name(__file__, NAME)
|
14
|
-
|
15
|
-
|
16
|
-
# ... copy `configs/example_config.yml` and fill the first three sections.
|
17
|
-
def train(
|
18
|
-
dataset_object_name: str,
|
19
|
-
model_object_name: str,
|
20
|
-
verbose: bool = False,
|
21
|
-
) -> bool:
|
22
|
-
logger.info(f"{NAME}.train: {dataset_object_name} -> {model_object_name}")
|
23
|
-
|
24
|
-
config_filename = os.path.join(
|
25
|
-
abcli_path_git,
|
26
|
-
"building-damage-assessment/configs/example_config.yml",
|
27
|
-
)
|
28
|
-
success, config = file.load_yaml(config_filename)
|
29
|
-
if not success:
|
30
|
-
return False
|
31
|
-
|
32
|
-
config["experiment_dir"] = objects.object_path(model_object_name)
|
33
|
-
config["experiment_name"] = model_object_name
|
34
|
-
|
35
|
-
config["imagery"]["rgb_fn"] = config["imagery"]["raw_fn"] = objects.path_of(
|
36
|
-
filename="raw/maxar_lahaina_8_12_2023-visual.tif",
|
37
|
-
object_name=dataset_object_name,
|
38
|
-
)
|
39
|
-
|
40
|
-
config["inference"]["output_subdir"] = objects.path_of(
|
41
|
-
filename="outputs/",
|
42
|
-
object_name=model_object_name,
|
43
|
-
)
|
44
|
-
config["inference"]["checkpoint_fn"] = objects.path_of(
|
45
|
-
filename="checkpoints/last.ckpt",
|
46
|
-
object_name=model_object_name,
|
47
|
-
)
|
48
|
-
|
49
|
-
config["infrastructure"]["container_name"] = "sandbox"
|
50
|
-
config["infrastructure"][
|
51
|
-
"storage_account"
|
52
|
-
] = "https://kamangir.blob.core.windows.net/"
|
53
|
-
config["infrastructure"]["sas_token"] = decode_token(ENCODED_BLOB_SAS_TOKEN)
|
54
|
-
config["infrastructure"]["relative_path"] = dataset_object_name
|
55
|
-
|
56
|
-
config["labels"]["fn"] = objects.path_of(
|
57
|
-
filename="label.geojson",
|
58
|
-
object_name=dataset_object_name,
|
59
|
-
)
|
60
|
-
|
61
|
-
config["training"]["log_dir"] = objects.path_of(
|
62
|
-
filename="logs/",
|
63
|
-
object_name=model_object_name,
|
64
|
-
)
|
65
|
-
config["training"]["checkpoint_subdir"] = objects.path_of(
|
66
|
-
filename="checkpoints/",
|
67
|
-
object_name=model_object_name,
|
68
|
-
)
|
69
|
-
|
70
|
-
if not file.save_yaml(
|
71
|
-
objects.path_of(
|
72
|
-
filename="config.yml",
|
73
|
-
object_name=model_object_name,
|
74
|
-
),
|
75
|
-
config,
|
76
|
-
):
|
77
|
-
return False
|
78
|
-
|
79
|
-
return all(
|
80
|
-
[
|
81
|
-
mlflow.set_tags(
|
82
|
-
model_object_name,
|
83
|
-
{
|
84
|
-
"dataset": dataset_object_name,
|
85
|
-
},
|
86
|
-
),
|
87
|
-
metadata.post_to_object(
|
88
|
-
model_object_name,
|
89
|
-
"train",
|
90
|
-
{
|
91
|
-
"dataset": dataset_object_name,
|
92
|
-
},
|
93
|
-
),
|
94
|
-
]
|
95
|
-
)
|
@@ -1,54 +0,0 @@
|
|
1
|
-
blue_sandbox/README.py,sha256=oDwNZyY77GZnqSoyduGKpvKpnqRedSXJmq4Y61YPv9U,1734
|
2
|
-
blue_sandbox/__init__.py,sha256=KWV87147kAgo0v9f5k2N7g9eievY0rfpaGuuKzeS-bM,323
|
3
|
-
blue_sandbox/__main__.py,sha256=aPRHSpGpk-bDbzhHpfLNsd3y1gGEHpnhoTF-RBweNwc,361
|
4
|
-
blue_sandbox/config.env,sha256=V66pxIEv4ZSBK3XKVD7_Ubqld5EQTH5-HLfF_7j8UvU,307
|
5
|
-
blue_sandbox/env.py,sha256=Q1Fn4Kd0I9Pj6H2JDZ2Q17YuG6ipRTfiChTvvEu1ZR4,565
|
6
|
-
blue_sandbox/functions.py,sha256=U41kQFNPpfYV6KJpMnkqgqLkozqXiG4tgV6rj8IW1BU,7
|
7
|
-
blue_sandbox/host.py,sha256=uJpiM105rnm6ySF16zA7waWekGBdec-dlpoRRU_QqwU,210
|
8
|
-
blue_sandbox/list.py,sha256=Awd2JwveK4AhX37E6t8wVpczbrTidOopV8xGCAhF7-g,1789
|
9
|
-
blue_sandbox/logger.py,sha256=ZoFrTIfJJGNtZUm2d7lkQjdB2SPl_KBKDmHJOcIivPM,107
|
10
|
-
blue_sandbox/sample.env,sha256=qtSmtvFB1SITG4XaG5dT45YWvGZgOW1_DJZf-WUTUXo,42
|
11
|
-
blue_sandbox/urls.py,sha256=tIZ36ONJEoUBM-tFhOpkVhLwb1OgLegUeEaBDqW4USM,24
|
12
|
-
blue_sandbox/.abcli/abcli.sh,sha256=xsJ4IzuQsvLZog6U8VTBFVXsEi6ADe13L8rn47XtlbU,196
|
13
|
-
blue_sandbox/.abcli/actions.sh,sha256=vImEUI105GRcxs2mAKGMqcvoErtmOPZZ-7dfSmUUxvE,230
|
14
|
-
blue_sandbox/.abcli/aka.sh,sha256=RHDU_JbEEL2B0vvvRJ3NVSsRSEjSu09jNY85n7DLe-k,21
|
15
|
-
blue_sandbox/.abcli/alias.sh,sha256=14IXkg_mJuZ8-Z582dlre3_zmmOfz52US4U1uFF1P-0,158
|
16
|
-
blue_sandbox/.abcli/blue_sandbox.sh,sha256=PGRJOgNGlC3XL5cw4ecZH40LDuc_6NVazTKhCWtZ-3g,233
|
17
|
-
blue_sandbox/.abcli/browse.sh,sha256=f8qa4qDVts2Am6_ldDwNeJXzhBQTk9PUKl0-a9wW1ww,287
|
18
|
-
blue_sandbox/.abcli/install.sh,sha256=zvl0GsHBmfw62ORmkMlhug264N_Zr8nc3rlPGFoq7Mk,125
|
19
|
-
blue_sandbox/.abcli/microsoft_building_damage_assessment.sh,sha256=wIP7BvltFj3Gx5VmSgT6hY67K1BnubELef93HHwKOU0,470
|
20
|
-
blue_sandbox/.abcli/cemetery/inference/cloudwatch.sh,sha256=aUkJaPHSseTiRq2npdmCNlnDCgr33UdGcTiXQD1rLf4,1382
|
21
|
-
blue_sandbox/.abcli/cemetery/inference/inference.sh,sha256=NIdBmO4PIBzVapEQ4k4uK8WOBM7_lPpt9ghdueD-vAo,6107
|
22
|
-
blue_sandbox/.abcli/microsoft_building_damage_assessment/ingest.sh,sha256=w_EUK_i4iSBpSy22TeexfF9jEAqzNwZuvjhpRraGKhA,2662
|
23
|
-
blue_sandbox/.abcli/microsoft_building_damage_assessment/install.sh,sha256=M7MhMfGK0lKGG5CxegrhRgT7eL0Gzix6dY-8w6P0V8g,1509
|
24
|
-
blue_sandbox/.abcli/microsoft_building_damage_assessment/label.sh,sha256=RROjENj93FhCBOkbe8K05e1fR5QNny0syobkX14rwho,795
|
25
|
-
blue_sandbox/.abcli/microsoft_building_damage_assessment/tensorboard.sh,sha256=BHdnOvorV9gjO1ET64tbLOqSGviNFR96S67XHmJz2hs,746
|
26
|
-
blue_sandbox/.abcli/microsoft_building_damage_assessment/train.sh,sha256=XyTTFB2AD7p8XT0x1vvQLwndyscGN23QEDA80wS3cjk,1970
|
27
|
-
blue_sandbox/.abcli/microsoft_building_damage_assessment/ingest/list.sh,sha256=9bEjpbwO7Heqd-E2RTM-ck4Q39Eqsv4ged6e0WmxmSs,452
|
28
|
-
blue_sandbox/.abcli/sagesemseg/cache_dataset.sh,sha256=oAXW1AGer6tuLBQrdNqz7VyMIQtubMXDeUg4crkQiqM,1335
|
29
|
-
blue_sandbox/.abcli/sagesemseg/consts.sh,sha256=xKVE1hAzVPqz0SkY_0h7Fpgy1OEQOFsf3sQHfLHgnQ4,95
|
30
|
-
blue_sandbox/.abcli/sagesemseg/train.sh,sha256=4y7YjlB2k9FW6SbE2c4JRJ9L0qxI3yQcCaZWJvmNUxQ,1606
|
31
|
-
blue_sandbox/.abcli/sagesemseg/upload_dataset.sh,sha256=VZs7VKglLPpizVMjPcfMcL5TRJ7BRFPAOz8_nok2608,1542
|
32
|
-
blue_sandbox/.abcli/tests/README.sh,sha256=rmJM-BPnTcmpPbJ5GXsF8vd_a84JKryeKkZyVScUing,145
|
33
|
-
blue_sandbox/.abcli/tests/help.sh,sha256=QeHmHpFRye7Ht7fYLAzqBo8b9OTSLBR_0qn77qrvIQU,523
|
34
|
-
blue_sandbox/.abcli/tests/microsoft_building_damage_assessment_ingest.sh,sha256=oUUF8Kuyohxiqxo9FLY9EO0qlgkagntgfuFda6niHeM,426
|
35
|
-
blue_sandbox/.abcli/tests/microsoft_building_damage_assessment_ingest_list.sh,sha256=5P0H-FHV7YKt1fenXAs_0cSFhJYWZ9u3LzRpfSRSptw,577
|
36
|
-
blue_sandbox/.abcli/tests/microsoft_building_damage_assessment_label.sh,sha256=Envac5i9eGa5usDCO8MYfGFfyAZAT-f0KrrsAkaR91U,311
|
37
|
-
blue_sandbox/.abcli/tests/microsoft_building_damage_assessment_train.sh,sha256=0FlwyR1drn6wAKX9IHU_ma7WTmH1Iv8DoTNsdsyj3PA,533
|
38
|
-
blue_sandbox/.abcli/tests/sagesemseg_train.sh,sha256=Fz2yzxub7x4SgQAr1ctZIbX6YwJR8HHTEgXcou4dORw,799
|
39
|
-
blue_sandbox/.abcli/tests/version.sh,sha256=jF8zoJN1eKE3LfDeRVG9uHEosmEVJX6RtKfdioyeN-o,150
|
40
|
-
blue_sandbox/help/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
41
|
-
blue_sandbox/help/__main__.py,sha256=3Cqp5oISrZCOUApmwoQoCj_0sQgtkiEkm_ob3LFKzRE,234
|
42
|
-
blue_sandbox/help/functions.py,sha256=Ki0nUGyHKDo2t0WnIVzBwxEVJHyRmHh-hIRv3ohJK3c,431
|
43
|
-
blue_sandbox/microsoft_building_damage_assessment/README.py,sha256=qqe1wXoc4TjKS4o3gskwNCBuX8PTVduQEAaVHeovsrE,1275
|
44
|
-
blue_sandbox/microsoft_building_damage_assessment/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
45
|
-
blue_sandbox/microsoft_building_damage_assessment/__main__.py,sha256=RnyXyJdLmzLzccVmb6VZvD17Dwx-g1wByDugIisoPmQ,1995
|
46
|
-
blue_sandbox/microsoft_building_damage_assessment/ingest.py,sha256=tBchn4dWPIqArwAe8JjEM2xZZlFtrgBRDV_Ko26OEiM,639
|
47
|
-
blue_sandbox/microsoft_building_damage_assessment/label.py,sha256=mpRNOsF8ChxSLwOyvwGmXiD7aO2xfivJPZTC5qwnLaw,823
|
48
|
-
blue_sandbox/microsoft_building_damage_assessment/sas_token.py,sha256=t6oTXc6C52CH3tFbQo2H9YUB_7upAsfUwws43Iu9pHI,214
|
49
|
-
blue_sandbox/microsoft_building_damage_assessment/train.py,sha256=8DUt0S4LuKkRIi9XpASOg5skPaM3D2VaSBg5FEn8qog,2778
|
50
|
-
blue_sandbox-5.294.1.dist-info/LICENSE,sha256=ogEPNDSH0_dhiv_lT3ifVIdgIzHAqNA_SemnxUfPBJk,7048
|
51
|
-
blue_sandbox-5.294.1.dist-info/METADATA,sha256=u6jPecX03L6r16uPe-T2Rll1lhr7d04c-jtSWJONlRo,3785
|
52
|
-
blue_sandbox-5.294.1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
|
53
|
-
blue_sandbox-5.294.1.dist-info/top_level.txt,sha256=4D9Cb9QUCaqdYAmBiCwvtlaYBtUYVVxv0Sxcr_pzgS8,13
|
54
|
-
blue_sandbox-5.294.1.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|