blue-sandbox 5.47.1__py3-none-any.whl → 5.87.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- blue_sandbox/.abcli/alias.sh +1 -1
- blue_sandbox/.abcli/install.sh +1 -6
- blue_sandbox/.abcli/microsoft_building_damage_assessment/ingest.sh +2 -4
- blue_sandbox/.abcli/microsoft_building_damage_assessment/install.sh +49 -0
- blue_sandbox/.abcli/microsoft_building_damage_assessment/label.sh +26 -0
- blue_sandbox/.abcli/microsoft_building_damage_assessment/tensorboard.sh +21 -0
- blue_sandbox/.abcli/microsoft_building_damage_assessment/train.sh +53 -0
- blue_sandbox/.abcli/tests/help.sh +7 -3
- blue_sandbox/.abcli/tests/microsoft_building_damage_assessment_label.sh +12 -0
- blue_sandbox/.abcli/tests/microsoft_building_damage_assessment_train.sh +20 -0
- blue_sandbox/README.py +1 -1
- blue_sandbox/__init__.py +1 -1
- blue_sandbox/config.env +1 -1
- blue_sandbox/env.py +4 -2
- blue_sandbox/microsoft_building_damage_assessment/README.py +24 -13
- blue_sandbox/microsoft_building_damage_assessment/__main__.py +37 -2
- blue_sandbox/microsoft_building_damage_assessment/ingest.py +15 -3
- blue_sandbox/microsoft_building_damage_assessment/label.py +38 -0
- blue_sandbox/microsoft_building_damage_assessment/sas_token.py +9 -0
- blue_sandbox/microsoft_building_damage_assessment/train.py +95 -0
- blue_sandbox/sample.env +1 -0
- {blue_sandbox-5.47.1.dist-info → blue_sandbox-5.87.1.dist-info}/METADATA +11 -4
- blue_sandbox-5.87.1.dist-info/RECORD +46 -0
- blue_sandbox-5.47.1.dist-info/RECORD +0 -37
- {blue_sandbox-5.47.1.dist-info → blue_sandbox-5.87.1.dist-info}/LICENSE +0 -0
- {blue_sandbox-5.47.1.dist-info → blue_sandbox-5.87.1.dist-info}/WHEEL +0 -0
- {blue_sandbox-5.47.1.dist-info → blue_sandbox-5.87.1.dist-info}/top_level.txt +0 -0
blue_sandbox/.abcli/alias.sh
CHANGED
blue_sandbox/.abcli/install.sh
CHANGED
@@ -1,12 +1,7 @@
|
|
1
1
|
#! /usr/bin/env bash
|
2
2
|
|
3
3
|
function abcli_install_blue_sandbox() {
|
4
|
-
|
5
|
-
|
6
|
-
[[ "$abcli_is_mac" == true ]] &&
|
7
|
-
brew install wget
|
8
|
-
|
9
|
-
return 0
|
4
|
+
abcli_log "✅"
|
10
5
|
}
|
11
6
|
|
12
7
|
abcli_install_module blue_sandbox 1.1.1
|
@@ -26,8 +26,8 @@ function blue_sandbox_microsoft_building_damage_assessment_ingest() {
|
|
26
26
|
pushd $object_path >/dev/null
|
27
27
|
|
28
28
|
# https://github.com/microsoft/building-damage-assessment/blob/main/SATELLITE_WORKFLOW.md#1-image-acquisition-and-label-generation
|
29
|
-
mkdir -pv
|
30
|
-
cd
|
29
|
+
mkdir -pv raw
|
30
|
+
cd raw
|
31
31
|
|
32
32
|
if [[ "$do_download" == 1 ]]; then
|
33
33
|
# TODO: use $event_name and $count (options +=)
|
@@ -57,8 +57,6 @@ function blue_sandbox_microsoft_building_damage_assessment_ingest() {
|
|
57
57
|
rm -v maxar_lahaina_8_12_2023-visual.vrt
|
58
58
|
fi
|
59
59
|
|
60
|
-
cd ../../..
|
61
|
-
|
62
60
|
popd >/dev/null
|
63
61
|
|
64
62
|
abcli_eval dryrun=$do_dryrun \
|
@@ -0,0 +1,49 @@
|
|
1
|
+
#! /usr/bin/env bash
|
2
|
+
|
3
|
+
function blue_sandbox_microsoft_building_damage_assessment_install() {
|
4
|
+
local options=$1
|
5
|
+
local do_dryrun=$(abcli_option_int "$options" dryrun 0)
|
6
|
+
local recreate_env=$(abcli_option_int "$options" recreate_env 0)
|
7
|
+
|
8
|
+
if [[ -d "$abcli_path_git/building-damage-assessment" ]]; then
|
9
|
+
abcli_log "✅ building-damage-assessment"
|
10
|
+
else
|
11
|
+
abcli_git_clone https://github.com/microsoft/building-damage-assessment.git
|
12
|
+
[[ $? -ne 0 ]] && return 1
|
13
|
+
fi
|
14
|
+
|
15
|
+
[[ "$abcli_is_github_workflow" == true ]] &&
|
16
|
+
return 0
|
17
|
+
|
18
|
+
if [[ "$recreate_env" == 1 ]]; then
|
19
|
+
abcli_conda_rm name=bda
|
20
|
+
[[ $? -ne 0 ]] && return 1
|
21
|
+
fi
|
22
|
+
|
23
|
+
local exists=$(abcli_conda_exists name=bda)
|
24
|
+
if [[ "$exists" == 0 ]]; then
|
25
|
+
# https://github.com/microsoft/building-damage-assessment?tab=readme-ov-file#setup
|
26
|
+
conda env create \
|
27
|
+
-f $abcli_path_git/building-damage-assessment/environment.yml
|
28
|
+
[[ $? -ne 0 ]] && return 1
|
29
|
+
fi
|
30
|
+
|
31
|
+
conda activate bda
|
32
|
+
[[ $? -ne 0 ]] && return 1
|
33
|
+
|
34
|
+
pushd $abcli_path_git/blue-sandbox >/dev/null
|
35
|
+
pip3 install -e .
|
36
|
+
[[ $? -ne 0 ]] && return 1
|
37
|
+
popd >/dev/null
|
38
|
+
|
39
|
+
[[ "$abcli_is_mac" == true ]] &&
|
40
|
+
brew install wget
|
41
|
+
|
42
|
+
return 0
|
43
|
+
}
|
44
|
+
|
45
|
+
function abcli_install_microsoft_building_damage_assessment() {
|
46
|
+
blue_sandbox_microsoft_building_damage_assessment_install "$@"
|
47
|
+
}
|
48
|
+
|
49
|
+
abcli_install_module microsoft_building_damage_assessment 1.1.1
|
@@ -0,0 +1,26 @@
|
|
1
|
+
#! /usr/bin/env bash
|
2
|
+
|
3
|
+
function blue_sandbox_microsoft_building_damage_assessment_label() {
|
4
|
+
local options=$1
|
5
|
+
local do_dryrun=$(abcli_option_int "$options" dryrun 0)
|
6
|
+
local do_download=$(abcli_option_int "$options" download $(abcli_not $do_dryrun))
|
7
|
+
local do_upload=$(abcli_option_int "$options" upload $(abcli_not $do_dryrun))
|
8
|
+
|
9
|
+
local object_name=$(abcli_clarify_object $2 .)
|
10
|
+
[[ "$do_download" == 1 ]] &&
|
11
|
+
abcli_download - $object_name
|
12
|
+
|
13
|
+
abcli_log "labeling $object_name ..."
|
14
|
+
|
15
|
+
abcli_eval dryrun=$do_dryrun \
|
16
|
+
python3 -m blue_sandbox.microsoft_building_damage_assessment \
|
17
|
+
label \
|
18
|
+
--object_name $object_name \
|
19
|
+
"${@:3}"
|
20
|
+
local status="$?"
|
21
|
+
|
22
|
+
[[ "$do_upload" == 1 ]] &&
|
23
|
+
abcli_upload - $object_name
|
24
|
+
|
25
|
+
return $status
|
26
|
+
}
|
@@ -0,0 +1,21 @@
|
|
1
|
+
#! /usr/bin/env bash
|
2
|
+
|
3
|
+
function blue_sandbox_microsoft_building_damage_assessment_tensorboard() {
|
4
|
+
local options=$1
|
5
|
+
local do_dryrun=$(abcli_option_int "$options" dryrun 0)
|
6
|
+
local do_browse=$(abcli_option_int "$options" browse $(abcli_not $do_dryrun))
|
7
|
+
local do_download=$(abcli_option_int "$options" download $(abcli_not $do_dryrun))
|
8
|
+
local port=$(abcli_option "$options" port 8889)
|
9
|
+
|
10
|
+
local model_object_name=$(abcli_clarify_object $2 .)
|
11
|
+
[[ "$do_download" == 1 ]] &&
|
12
|
+
abcli_download - $model_object_name
|
13
|
+
|
14
|
+
[[ "$do_browse" == 1 ]] &&
|
15
|
+
abcli_browse http://localhost:$port/
|
16
|
+
|
17
|
+
abcli_eval ,$options \
|
18
|
+
tensorboard \
|
19
|
+
--logdir $ABCLI_OBJECT_ROOT/$model_object_name/logs/ \
|
20
|
+
--port $port
|
21
|
+
}
|
@@ -0,0 +1,53 @@
|
|
1
|
+
#! /usr/bin/env bash
|
2
|
+
|
3
|
+
function blue_sandbox_microsoft_building_damage_assessment_train() {
|
4
|
+
local options=$1
|
5
|
+
local do_dryrun=$(abcli_option_int "$options" dryrun 0)
|
6
|
+
local do_download=$(abcli_option_int "$options" download $(abcli_not $do_dryrun))
|
7
|
+
local do_upload=$(abcli_option_int "$options" upload $(abcli_not $do_dryrun))
|
8
|
+
|
9
|
+
local dataset_object_name=$(abcli_clarify_object $2 .)
|
10
|
+
[[ "$do_download" == 1 ]] &&
|
11
|
+
abcli_download - $dataset_object_name
|
12
|
+
|
13
|
+
local model_object_name=$(abcli_clarify_object $3 $dataset_object_name-model-$(abcli_string_timestamp_short))
|
14
|
+
|
15
|
+
abcli_log "training $dataset_object_name -> $model_object_name ..."
|
16
|
+
|
17
|
+
conda activate bda
|
18
|
+
[[ $? -ne 0 ]] && return 1
|
19
|
+
|
20
|
+
abcli_eval dryrun=$do_dryrun \
|
21
|
+
python3 -m blue_sandbox.microsoft_building_damage_assessment \
|
22
|
+
train \
|
23
|
+
--dataset_object_name $dataset_object_name \
|
24
|
+
--model_object_name $model_object_name \
|
25
|
+
"${@:4}"
|
26
|
+
[[ $? -ne 0 ]] && return 1
|
27
|
+
|
28
|
+
abcli_eval dryrun=$do_dryrun,path=$abcli_path_git/building-damage-assessment \
|
29
|
+
python3 project_setup.py \
|
30
|
+
--skip_health_check \
|
31
|
+
--config $ABCLI_OBJECT_ROOT/$model_object_name/config.yml
|
32
|
+
[[ $? -ne 0 ]] && return 1
|
33
|
+
|
34
|
+
abcli_eval dryrun=$do_dryrun,path=$abcli_path_git/building-damage-assessment \
|
35
|
+
python3 create_masks.py \
|
36
|
+
--config $ABCLI_OBJECT_ROOT/$model_object_name/config.yml
|
37
|
+
[[ $? -ne 0 ]] && return 1
|
38
|
+
|
39
|
+
abcli_eval dryrun=$do_dryrun,path=$abcli_path_git/building-damage-assessment \
|
40
|
+
python3 fine_tune.py \
|
41
|
+
--config $ABCLI_OBJECT_ROOT/$model_object_name/config.yml
|
42
|
+
[[ $? -ne 0 ]] && return 1
|
43
|
+
|
44
|
+
abcli_eval dryrun=$do_dryrun,path=$abcli_path_git/building-damage-assessment \
|
45
|
+
python3 inference.py \
|
46
|
+
--config $ABCLI_OBJECT_ROOT/$model_object_name/config.yml
|
47
|
+
[[ $? -ne 0 ]] && return 1
|
48
|
+
|
49
|
+
[[ "$do_upload" == 1 ]] &&
|
50
|
+
abcli_upload - $model_object_name
|
51
|
+
|
52
|
+
return 0
|
53
|
+
}
|
@@ -7,9 +7,13 @@ function test_blue_sandbox_help() {
|
|
7
7
|
for module in \
|
8
8
|
"@sandbox" \
|
9
9
|
\
|
10
|
-
"@
|
11
|
-
"@
|
12
|
-
"@
|
10
|
+
"@damages" \
|
11
|
+
"@damages ingest" \
|
12
|
+
"@damages ingest list" \
|
13
|
+
"@damages install" \
|
14
|
+
"@damages label" \
|
15
|
+
"@damages tensorboard" \
|
16
|
+
"@damages train" \
|
13
17
|
\
|
14
18
|
"blue_sandbox"; do
|
15
19
|
abcli_eval ,$options \
|
@@ -0,0 +1,12 @@
|
|
1
|
+
#! /usr/bin/env bash
|
2
|
+
|
3
|
+
function test_blue_sandbox_microsoft_building_damage_assessment_label() {
|
4
|
+
local options=$1
|
5
|
+
|
6
|
+
abcli_eval ,$options \
|
7
|
+
blue_sandbox_microsoft_building_damage_assessment \
|
8
|
+
label \
|
9
|
+
~upload,$options \
|
10
|
+
$DAMAGES_TEST_DATASET_OBJECT_NAME \
|
11
|
+
"${@:2}"
|
12
|
+
}
|
@@ -0,0 +1,20 @@
|
|
1
|
+
#! /usr/bin/env bash
|
2
|
+
|
3
|
+
function test_blue_sandbox_microsoft_building_damage_assessment_train() {
|
4
|
+
local options=$1
|
5
|
+
|
6
|
+
abcli_log_warning "disabled."
|
7
|
+
return 0
|
8
|
+
|
9
|
+
abcli_eval ,$options \
|
10
|
+
blue_sandbox_microsoft_building_damage_assessment \
|
11
|
+
train \
|
12
|
+
~upload,$options \
|
13
|
+
$DAMAGES_TEST_DATASET_OBJECT_NAME \
|
14
|
+
test_blue_sandbox_microsoft_building_damage_assessment_train-$(abcli_string_timestamp_short) \
|
15
|
+
--verbose 1 \
|
16
|
+
"${@:2}"
|
17
|
+
|
18
|
+
# ignore error - TODO: disable
|
19
|
+
return 0
|
20
|
+
}
|
blue_sandbox/README.py
CHANGED
@@ -11,7 +11,7 @@ from blue_sandbox.microsoft_building_damage_assessment import (
|
|
11
11
|
items = [
|
12
12
|
"{}[`{}`]({}) []({}) {}".format(
|
13
13
|
"🌐",
|
14
|
-
"`@
|
14
|
+
"`@damages`",
|
15
15
|
"https://github.com/kamangir/blue-sandbox/blob/main/blue_sandbox/microsoft_building_damage_assessment/README.md",
|
16
16
|
"https://github.com/microsoft/building-damage-assessment/raw/main/figures/damage.png",
|
17
17
|
"https://github.com/kamangir/blue-sandbox/blob/main/blue_sandbox/microsoft_building_damage_assessment/README.md",
|
blue_sandbox/__init__.py
CHANGED
blue_sandbox/config.env
CHANGED
@@ -1 +1 @@
|
|
1
|
-
|
1
|
+
DAMAGES_TEST_DATASET_OBJECT_NAME=Maui-Hawaii-fires-Aug-23-ingest-2025-01-10-qqJqhm
|
blue_sandbox/env.py
CHANGED
@@ -5,7 +5,9 @@ load_env(__name__)
|
|
5
5
|
load_config(__name__)
|
6
6
|
|
7
7
|
|
8
|
-
|
9
|
-
"
|
8
|
+
DAMAGES_TEST_DATASET_OBJECT_NAME = os.getenv(
|
9
|
+
"DAMAGES_TEST_DATASET_OBJECT_NAME",
|
10
10
|
"",
|
11
11
|
)
|
12
|
+
|
13
|
+
ENCODED_BLOB_SAS_TOKEN = os.getenv("ENCODED_BLOB_SAS_TOKEN", "")
|
@@ -1,11 +1,22 @@
|
|
1
1
|
from typing import Dict
|
2
2
|
|
3
|
+
from blue_sandbox.env import DAMAGES_TEST_DATASET_OBJECT_NAME
|
4
|
+
|
3
5
|
list_of_steps: Dict[str, Dict] = {
|
4
|
-
"ingest":
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
"
|
6
|
+
"ingest": {
|
7
|
+
"object_name": DAMAGES_TEST_DATASET_OBJECT_NAME,
|
8
|
+
"image_name": "Maui-Hawaii-fires-Aug-23-damage-2025-01-09-GgnjQC",
|
9
|
+
},
|
10
|
+
"label": {
|
11
|
+
"object_name": "",
|
12
|
+
"image_name": DAMAGES_TEST_DATASET_OBJECT_NAME,
|
13
|
+
},
|
14
|
+
"train": {
|
15
|
+
"object_name": "",
|
16
|
+
"image_name": "Maui-Hawaii-fires-Aug-23-model-2025-01-10-NQb8IS",
|
17
|
+
},
|
18
|
+
"predict": {"object_name": "", "image_name": ""},
|
19
|
+
"summarize": {"object_name": "", "image_name": ""},
|
9
20
|
}
|
10
21
|
|
11
22
|
items = (
|
@@ -13,22 +24,22 @@ items = (
|
|
13
24
|
+ [
|
14
25
|
(
|
15
26
|
"[`{}`](https://kamangir-public.s3.ca-central-1.amazonaws.com/{}.tar.gz)".format(
|
16
|
-
object_name,
|
17
|
-
|
27
|
+
step["object_name"],
|
28
|
+
step["image_name"],
|
18
29
|
)
|
19
|
-
if object_name
|
30
|
+
if step["object_name"]
|
20
31
|
else ""
|
21
32
|
)
|
22
|
-
for
|
33
|
+
for step in list_of_steps.values()
|
23
34
|
]
|
24
35
|
+ [
|
25
36
|
(
|
26
|
-
"
|
27
|
-
|
37
|
+
"".format(
|
38
|
+
step["image_name"],
|
28
39
|
)
|
29
|
-
if
|
40
|
+
if step["image_name"]
|
30
41
|
else ""
|
31
42
|
)
|
32
|
-
for
|
43
|
+
for step in list_of_steps.values()
|
33
44
|
]
|
34
45
|
)
|
@@ -5,6 +5,12 @@ from blueness.argparse.generic import sys_exit
|
|
5
5
|
|
6
6
|
from blue_sandbox import NAME
|
7
7
|
from blue_sandbox.microsoft_building_damage_assessment.ingest import ingest
|
8
|
+
from blue_sandbox.microsoft_building_damage_assessment.label import label
|
9
|
+
from blue_sandbox.microsoft_building_damage_assessment.train import train
|
10
|
+
from blue_sandbox.microsoft_building_damage_assessment.sas_token import (
|
11
|
+
decode_token,
|
12
|
+
encode_token,
|
13
|
+
)
|
8
14
|
from blue_sandbox.help.microsoft_building_damage_assessment import list_of_events
|
9
15
|
from blue_sandbox.logger import logger
|
10
16
|
|
@@ -14,12 +20,20 @@ parser = argparse.ArgumentParser(NAME)
|
|
14
20
|
parser.add_argument(
|
15
21
|
"task",
|
16
22
|
type=str,
|
17
|
-
help="ingest",
|
23
|
+
help="decode_token | encode_token | ingest | label | train",
|
18
24
|
)
|
19
25
|
parser.add_argument(
|
20
26
|
"--object_name",
|
21
27
|
type=str,
|
22
28
|
)
|
29
|
+
parser.add_argument(
|
30
|
+
"--dataset_object_name",
|
31
|
+
type=str,
|
32
|
+
)
|
33
|
+
parser.add_argument(
|
34
|
+
"--model_object_name",
|
35
|
+
type=str,
|
36
|
+
)
|
23
37
|
parser.add_argument(
|
24
38
|
"--event_name",
|
25
39
|
type=str,
|
@@ -32,15 +46,36 @@ parser.add_argument(
|
|
32
46
|
default=0,
|
33
47
|
help="0|1",
|
34
48
|
)
|
49
|
+
parser.add_argument(
|
50
|
+
"--token",
|
51
|
+
type=str,
|
52
|
+
)
|
35
53
|
args = parser.parse_args()
|
36
54
|
|
37
55
|
success = False
|
38
|
-
if args.task == "
|
56
|
+
if args.task == "decode_token":
|
57
|
+
success = True
|
58
|
+
print(decode_token(args.token))
|
59
|
+
elif args.task == "encode_token":
|
60
|
+
success = True
|
61
|
+
print(encode_token(args.token))
|
62
|
+
elif args.task == "ingest":
|
39
63
|
success = ingest(
|
40
64
|
object_name=args.object_name,
|
41
65
|
event_name=args.event_name,
|
42
66
|
verbose=args.verbose == 1,
|
43
67
|
)
|
68
|
+
elif args.task == "label":
|
69
|
+
success = label(
|
70
|
+
object_name=args.object_name,
|
71
|
+
verbose=args.verbose == 1,
|
72
|
+
)
|
73
|
+
elif args.task == "train":
|
74
|
+
success = train(
|
75
|
+
dataset_object_name=args.dataset_object_name,
|
76
|
+
model_object_name=args.model_object_name,
|
77
|
+
verbose=args.verbose == 1,
|
78
|
+
)
|
44
79
|
else:
|
45
80
|
success = None
|
46
81
|
|
@@ -1,5 +1,7 @@
|
|
1
1
|
from blueness import module
|
2
2
|
|
3
|
+
from blue_objects import mlflow, metadata
|
4
|
+
|
3
5
|
from blue_sandbox import NAME
|
4
6
|
from blue_sandbox.logger import logger
|
5
7
|
|
@@ -14,6 +16,16 @@ def ingest(
|
|
14
16
|
) -> bool:
|
15
17
|
logger.info(f"{NAME}.ingest({event_name}) -> {object_name}")
|
16
18
|
|
17
|
-
|
18
|
-
|
19
|
-
|
19
|
+
return all(
|
20
|
+
[
|
21
|
+
mlflow.set_tags(
|
22
|
+
object_name,
|
23
|
+
{"event": event_name},
|
24
|
+
),
|
25
|
+
metadata.post_to_object(
|
26
|
+
object_name,
|
27
|
+
"ingest",
|
28
|
+
{"event": event_name},
|
29
|
+
),
|
30
|
+
]
|
31
|
+
)
|
@@ -0,0 +1,38 @@
|
|
1
|
+
import os
|
2
|
+
|
3
|
+
from blueness import module
|
4
|
+
from blue_objects import metadata, file, objects
|
5
|
+
from blue_objects.env import abcli_path_git
|
6
|
+
|
7
|
+
from blue_sandbox import NAME
|
8
|
+
from blue_sandbox.logger import logger
|
9
|
+
|
10
|
+
|
11
|
+
NAME = module.name(__file__, NAME)
|
12
|
+
|
13
|
+
|
14
|
+
def label(
|
15
|
+
object_name: str,
|
16
|
+
verbose: bool = False,
|
17
|
+
) -> bool:
|
18
|
+
logger.info(f"{NAME}.label: {object_name}")
|
19
|
+
|
20
|
+
geojson_filename = os.path.join(
|
21
|
+
abcli_path_git,
|
22
|
+
"building-damage-assessment/data/demo/labels/Maui_Wildfires_August_0.geojson",
|
23
|
+
)
|
24
|
+
|
25
|
+
if not file.copy(
|
26
|
+
geojson_filename,
|
27
|
+
objects.path_of(
|
28
|
+
filename="label.geojson",
|
29
|
+
object_name=object_name,
|
30
|
+
),
|
31
|
+
):
|
32
|
+
return False
|
33
|
+
|
34
|
+
return metadata.post_to_object(
|
35
|
+
object_name,
|
36
|
+
"label",
|
37
|
+
{"geojson": geojson_filename},
|
38
|
+
)
|
@@ -0,0 +1,95 @@
|
|
1
|
+
import os
|
2
|
+
|
3
|
+
from blueness import module
|
4
|
+
from blue_objects import mlflow, metadata, file, objects, path
|
5
|
+
from blue_objects.env import abcli_path_git
|
6
|
+
|
7
|
+
from blue_sandbox import NAME
|
8
|
+
from blue_sandbox.env import ENCODED_BLOB_SAS_TOKEN
|
9
|
+
from blue_sandbox.microsoft_building_damage_assessment.sas_token import decode_token
|
10
|
+
from blue_sandbox.logger import logger
|
11
|
+
|
12
|
+
|
13
|
+
NAME = module.name(__file__, NAME)
|
14
|
+
|
15
|
+
|
16
|
+
# ... copy `configs/example_config.yml` and fill the first three sections.
|
17
|
+
def train(
|
18
|
+
dataset_object_name: str,
|
19
|
+
model_object_name: str,
|
20
|
+
verbose: bool = False,
|
21
|
+
) -> bool:
|
22
|
+
logger.info(f"{NAME}.train: {dataset_object_name} -> {model_object_name}")
|
23
|
+
|
24
|
+
config_filename = os.path.join(
|
25
|
+
abcli_path_git,
|
26
|
+
"building-damage-assessment/configs/example_config.yml",
|
27
|
+
)
|
28
|
+
success, config = file.load_yaml(config_filename)
|
29
|
+
if not success:
|
30
|
+
return False
|
31
|
+
|
32
|
+
config["experiment_dir"] = objects.object_path(model_object_name)
|
33
|
+
config["experiment_name"] = model_object_name
|
34
|
+
|
35
|
+
config["imagery"]["rgb_fn"] = config["imagery"]["raw_fn"] = objects.path_of(
|
36
|
+
filename="raw/maxar_lahaina_8_12_2023-visual.tif",
|
37
|
+
object_name=dataset_object_name,
|
38
|
+
)
|
39
|
+
|
40
|
+
config["inference"]["output_subdir"] = objects.path_of(
|
41
|
+
filename="outputs/",
|
42
|
+
object_name=model_object_name,
|
43
|
+
)
|
44
|
+
config["inference"]["checkpoint_fn"] = objects.path_of(
|
45
|
+
filename="checkpoints/last.ckpt",
|
46
|
+
object_name=model_object_name,
|
47
|
+
)
|
48
|
+
|
49
|
+
config["infrastructure"]["container_name"] = "sandbox"
|
50
|
+
config["infrastructure"][
|
51
|
+
"storage_account"
|
52
|
+
] = "https://kamangir.blob.core.windows.net/"
|
53
|
+
config["infrastructure"]["sas_token"] = decode_token(ENCODED_BLOB_SAS_TOKEN)
|
54
|
+
config["infrastructure"]["relative_path"] = dataset_object_name
|
55
|
+
|
56
|
+
config["labels"]["fn"] = objects.path_of(
|
57
|
+
filename="label.geojson",
|
58
|
+
object_name=dataset_object_name,
|
59
|
+
)
|
60
|
+
|
61
|
+
config["training"]["log_dir"] = objects.path_of(
|
62
|
+
filename="logs/",
|
63
|
+
object_name=model_object_name,
|
64
|
+
)
|
65
|
+
config["training"]["checkpoint_subdir"] = objects.path_of(
|
66
|
+
filename="checkpoints/",
|
67
|
+
object_name=model_object_name,
|
68
|
+
)
|
69
|
+
|
70
|
+
if not file.save_yaml(
|
71
|
+
objects.path_of(
|
72
|
+
filename="config.yml",
|
73
|
+
object_name=model_object_name,
|
74
|
+
),
|
75
|
+
config,
|
76
|
+
):
|
77
|
+
return False
|
78
|
+
|
79
|
+
return all(
|
80
|
+
[
|
81
|
+
mlflow.set_tags(
|
82
|
+
model_object_name,
|
83
|
+
{
|
84
|
+
"dataset": dataset_object_name,
|
85
|
+
},
|
86
|
+
),
|
87
|
+
metadata.post_to_object(
|
88
|
+
model_object_name,
|
89
|
+
"train",
|
90
|
+
{
|
91
|
+
"dataset": dataset_object_name,
|
92
|
+
},
|
93
|
+
),
|
94
|
+
]
|
95
|
+
)
|
blue_sandbox/sample.env
CHANGED
@@ -0,0 +1 @@
|
|
1
|
+
ENCODED_BLOB_SAS_TOKEN=value
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.2
|
2
2
|
Name: blue_sandbox
|
3
|
-
Version: 5.
|
3
|
+
Version: 5.87.1
|
4
4
|
Summary: 🌀 A sandbox for ideas and experiments.
|
5
5
|
Home-page: https://github.com/kamangir/blue-sandbox
|
6
6
|
Author: Arash Abadpour (Kamangir)
|
@@ -13,8 +13,12 @@ Classifier: Operating System :: OS Independent
|
|
13
13
|
Description-Content-Type: text/markdown
|
14
14
|
License-File: LICENSE
|
15
15
|
Requires-Dist: blueness
|
16
|
-
Requires-Dist: blue-options
|
17
16
|
Requires-Dist: abcli
|
17
|
+
Requires-Dist: blue-options
|
18
|
+
Requires-Dist: blue_objects
|
19
|
+
Requires-Dist: blue_geo
|
20
|
+
Requires-Dist: blueflow
|
21
|
+
Requires-Dist: blue_plugin
|
18
22
|
Requires-Dist: boto3
|
19
23
|
Requires-Dist: geojson
|
20
24
|
Requires-Dist: geopandas
|
@@ -27,6 +31,9 @@ Requires-Dist: pylint
|
|
27
31
|
Requires-Dist: pytest
|
28
32
|
Requires-Dist: python-dotenv[cli]
|
29
33
|
Requires-Dist: tqdm
|
34
|
+
Requires-Dist: azure-storage-blob
|
35
|
+
Requires-Dist: fiona
|
36
|
+
Requires-Dist: tensorboard
|
30
37
|
Dynamic: author
|
31
38
|
Dynamic: author-email
|
32
39
|
Dynamic: classifier
|
@@ -47,11 +54,11 @@ pip install blue-sandbox
|
|
47
54
|
|
48
55
|
| | | |
|
49
56
|
| --- | --- | --- |
|
50
|
-
| 🌐[``@
|
57
|
+
| 🌐[``@damages``](https://github.com/kamangir/blue-sandbox/blob/main/blue_sandbox/microsoft_building_damage_assessment/README.md) [](https://github.com/kamangir/blue-sandbox/blob/main/blue_sandbox/microsoft_building_damage_assessment/README.md) Satellite imagery damage assessment workflow | 🌀[`experiment 2`](#) [](#) description of experiment 2 ... | 🌀[`experiment 3`](#) [](#) description of experiment 3 ... |
|
51
58
|
|
52
59
|
---
|
53
60
|
|
54
61
|
|
55
62
|
[](https://github.com/kamangir/blue-sandbox/actions/workflows/pylint.yml) [](https://github.com/kamangir/blue-sandbox/actions/workflows/pytest.yml) [](https://github.com/kamangir/blue-sandbox/actions/workflows/bashtest.yml) [](https://pypi.org/project/blue-sandbox/) [](https://pypistats.org/packages/blue-sandbox)
|
56
63
|
|
57
|
-
built by 🌀 [`blue_options-4.
|
64
|
+
built by 🌀 [`blue_options-4.180.1`](https://github.com/kamangir/awesome-bash-cli), based on 🌀 [`blue_sandbox-5.87.1`](https://github.com/kamangir/blue-sandbox).
|
@@ -0,0 +1,46 @@
|
|
1
|
+
blue_sandbox/README.py,sha256=Q45yMxvgjPLPQAZCRzMQZx7Znxa0OAOaf3PZNvdaAyc,1563
|
2
|
+
blue_sandbox/__init__.py,sha256=EZzxJTfEvzoa0q1G0mM346fmyKOTcdzq4xXL-X5cBLI,322
|
3
|
+
blue_sandbox/__main__.py,sha256=aPRHSpGpk-bDbzhHpfLNsd3y1gGEHpnhoTF-RBweNwc,361
|
4
|
+
blue_sandbox/config.env,sha256=SHQeFdzivHKDimmTyHzVziCfAONN48DSDmc3WVG6gEw,82
|
5
|
+
blue_sandbox/env.py,sha256=dpRUbxT2vq-9s-G5MP3IlZBdC1hFvDb7mJxzU5lUd-Q,267
|
6
|
+
blue_sandbox/functions.py,sha256=U41kQFNPpfYV6KJpMnkqgqLkozqXiG4tgV6rj8IW1BU,7
|
7
|
+
blue_sandbox/host.py,sha256=uJpiM105rnm6ySF16zA7waWekGBdec-dlpoRRU_QqwU,210
|
8
|
+
blue_sandbox/logger.py,sha256=ZoFrTIfJJGNtZUm2d7lkQjdB2SPl_KBKDmHJOcIivPM,107
|
9
|
+
blue_sandbox/sample.env,sha256=ovfKDKAIeuqWBmMDKeAint_SKTnZuyDU1TCBZ-Y3Iqg,28
|
10
|
+
blue_sandbox/urls.py,sha256=tIZ36ONJEoUBM-tFhOpkVhLwb1OgLegUeEaBDqW4USM,24
|
11
|
+
blue_sandbox/.abcli/abcli.sh,sha256=xsJ4IzuQsvLZog6U8VTBFVXsEi6ADe13L8rn47XtlbU,196
|
12
|
+
blue_sandbox/.abcli/actions.sh,sha256=vImEUI105GRcxs2mAKGMqcvoErtmOPZZ-7dfSmUUxvE,230
|
13
|
+
blue_sandbox/.abcli/aka.sh,sha256=RHDU_JbEEL2B0vvvRJ3NVSsRSEjSu09jNY85n7DLe-k,21
|
14
|
+
blue_sandbox/.abcli/alias.sh,sha256=q8j-uG7c0ZLp5BZhBKTXyESjMa_kcEH5Mf8pX4EgC7A,118
|
15
|
+
blue_sandbox/.abcli/blue_sandbox.sh,sha256=PGRJOgNGlC3XL5cw4ecZH40LDuc_6NVazTKhCWtZ-3g,233
|
16
|
+
blue_sandbox/.abcli/browse.sh,sha256=f8qa4qDVts2Am6_ldDwNeJXzhBQTk9PUKl0-a9wW1ww,287
|
17
|
+
blue_sandbox/.abcli/install.sh,sha256=zvl0GsHBmfw62ORmkMlhug264N_Zr8nc3rlPGFoq7Mk,125
|
18
|
+
blue_sandbox/.abcli/microsoft_building_damage_assessment.sh,sha256=wIP7BvltFj3Gx5VmSgT6hY67K1BnubELef93HHwKOU0,470
|
19
|
+
blue_sandbox/.abcli/microsoft_building_damage_assessment/ingest.sh,sha256=w_EUK_i4iSBpSy22TeexfF9jEAqzNwZuvjhpRraGKhA,2662
|
20
|
+
blue_sandbox/.abcli/microsoft_building_damage_assessment/install.sh,sha256=UQTjsVf-GS_Q3bhTJzj6PMDZZzTNouMVh_3kEFSl5uo,1455
|
21
|
+
blue_sandbox/.abcli/microsoft_building_damage_assessment/label.sh,sha256=RROjENj93FhCBOkbe8K05e1fR5QNny0syobkX14rwho,795
|
22
|
+
blue_sandbox/.abcli/microsoft_building_damage_assessment/tensorboard.sh,sha256=BHdnOvorV9gjO1ET64tbLOqSGviNFR96S67XHmJz2hs,746
|
23
|
+
blue_sandbox/.abcli/microsoft_building_damage_assessment/train.sh,sha256=XyTTFB2AD7p8XT0x1vvQLwndyscGN23QEDA80wS3cjk,1970
|
24
|
+
blue_sandbox/.abcli/microsoft_building_damage_assessment/ingest/list.sh,sha256=9bEjpbwO7Heqd-E2RTM-ck4Q39Eqsv4ged6e0WmxmSs,452
|
25
|
+
blue_sandbox/.abcli/tests/README.sh,sha256=rmJM-BPnTcmpPbJ5GXsF8vd_a84JKryeKkZyVScUing,145
|
26
|
+
blue_sandbox/.abcli/tests/help.sh,sha256=QeHmHpFRye7Ht7fYLAzqBo8b9OTSLBR_0qn77qrvIQU,523
|
27
|
+
blue_sandbox/.abcli/tests/microsoft_building_damage_assessment_ingest.sh,sha256=oUUF8Kuyohxiqxo9FLY9EO0qlgkagntgfuFda6niHeM,426
|
28
|
+
blue_sandbox/.abcli/tests/microsoft_building_damage_assessment_ingest_list.sh,sha256=5P0H-FHV7YKt1fenXAs_0cSFhJYWZ9u3LzRpfSRSptw,577
|
29
|
+
blue_sandbox/.abcli/tests/microsoft_building_damage_assessment_label.sh,sha256=Envac5i9eGa5usDCO8MYfGFfyAZAT-f0KrrsAkaR91U,311
|
30
|
+
blue_sandbox/.abcli/tests/microsoft_building_damage_assessment_train.sh,sha256=0FlwyR1drn6wAKX9IHU_ma7WTmH1Iv8DoTNsdsyj3PA,533
|
31
|
+
blue_sandbox/.abcli/tests/version.sh,sha256=jF8zoJN1eKE3LfDeRVG9uHEosmEVJX6RtKfdioyeN-o,150
|
32
|
+
blue_sandbox/help/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
33
|
+
blue_sandbox/help/__main__.py,sha256=3Cqp5oISrZCOUApmwoQoCj_0sQgtkiEkm_ob3LFKzRE,234
|
34
|
+
blue_sandbox/help/functions.py,sha256=Ki0nUGyHKDo2t0WnIVzBwxEVJHyRmHh-hIRv3ohJK3c,431
|
35
|
+
blue_sandbox/microsoft_building_damage_assessment/README.py,sha256=qqe1wXoc4TjKS4o3gskwNCBuX8PTVduQEAaVHeovsrE,1275
|
36
|
+
blue_sandbox/microsoft_building_damage_assessment/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
37
|
+
blue_sandbox/microsoft_building_damage_assessment/__main__.py,sha256=RnyXyJdLmzLzccVmb6VZvD17Dwx-g1wByDugIisoPmQ,1995
|
38
|
+
blue_sandbox/microsoft_building_damage_assessment/ingest.py,sha256=tBchn4dWPIqArwAe8JjEM2xZZlFtrgBRDV_Ko26OEiM,639
|
39
|
+
blue_sandbox/microsoft_building_damage_assessment/label.py,sha256=mpRNOsF8ChxSLwOyvwGmXiD7aO2xfivJPZTC5qwnLaw,823
|
40
|
+
blue_sandbox/microsoft_building_damage_assessment/sas_token.py,sha256=t6oTXc6C52CH3tFbQo2H9YUB_7upAsfUwws43Iu9pHI,214
|
41
|
+
blue_sandbox/microsoft_building_damage_assessment/train.py,sha256=8DUt0S4LuKkRIi9XpASOg5skPaM3D2VaSBg5FEn8qog,2778
|
42
|
+
blue_sandbox-5.87.1.dist-info/LICENSE,sha256=ogEPNDSH0_dhiv_lT3ifVIdgIzHAqNA_SemnxUfPBJk,7048
|
43
|
+
blue_sandbox-5.87.1.dist-info/METADATA,sha256=9M7nPtpmiI0fX-oeF4ZwG31fT9TnSdndE7Vyt7iB_AU,2942
|
44
|
+
blue_sandbox-5.87.1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
|
45
|
+
blue_sandbox-5.87.1.dist-info/top_level.txt,sha256=4D9Cb9QUCaqdYAmBiCwvtlaYBtUYVVxv0Sxcr_pzgS8,13
|
46
|
+
blue_sandbox-5.87.1.dist-info/RECORD,,
|
@@ -1,37 +0,0 @@
|
|
1
|
-
blue_sandbox/README.py,sha256=gHxb5Czp9oyNZykiCKtI9Qnx08QJab8Ht9znKxHMkPs,1562
|
2
|
-
blue_sandbox/__init__.py,sha256=yAwWd5-dAV4y8R3MW_B3K5FX1Hf7ZNCSnuNJCfrS_co,322
|
3
|
-
blue_sandbox/__main__.py,sha256=aPRHSpGpk-bDbzhHpfLNsd3y1gGEHpnhoTF-RBweNwc,361
|
4
|
-
blue_sandbox/config.env,sha256=89ao50sGXYWiU-NMV_fUVXAXdTHKKfWV03BY3_p6EOE,25
|
5
|
-
blue_sandbox/env.py,sha256=ljEErQJNy0cdSvPpJ_ziMNvXK6Qlvi_8VBgTqjSm2uk,173
|
6
|
-
blue_sandbox/functions.py,sha256=U41kQFNPpfYV6KJpMnkqgqLkozqXiG4tgV6rj8IW1BU,7
|
7
|
-
blue_sandbox/host.py,sha256=uJpiM105rnm6ySF16zA7waWekGBdec-dlpoRRU_QqwU,210
|
8
|
-
blue_sandbox/logger.py,sha256=ZoFrTIfJJGNtZUm2d7lkQjdB2SPl_KBKDmHJOcIivPM,107
|
9
|
-
blue_sandbox/sample.env,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
10
|
-
blue_sandbox/urls.py,sha256=tIZ36ONJEoUBM-tFhOpkVhLwb1OgLegUeEaBDqW4USM,24
|
11
|
-
blue_sandbox/.abcli/abcli.sh,sha256=xsJ4IzuQsvLZog6U8VTBFVXsEi6ADe13L8rn47XtlbU,196
|
12
|
-
blue_sandbox/.abcli/actions.sh,sha256=vImEUI105GRcxs2mAKGMqcvoErtmOPZZ-7dfSmUUxvE,230
|
13
|
-
blue_sandbox/.abcli/aka.sh,sha256=RHDU_JbEEL2B0vvvRJ3NVSsRSEjSu09jNY85n7DLe-k,21
|
14
|
-
blue_sandbox/.abcli/alias.sh,sha256=A2Q2VBfKuAiWc0yRXeDj95lL70elSdDKZTKWPTK-590,117
|
15
|
-
blue_sandbox/.abcli/blue_sandbox.sh,sha256=PGRJOgNGlC3XL5cw4ecZH40LDuc_6NVazTKhCWtZ-3g,233
|
16
|
-
blue_sandbox/.abcli/browse.sh,sha256=f8qa4qDVts2Am6_ldDwNeJXzhBQTk9PUKl0-a9wW1ww,287
|
17
|
-
blue_sandbox/.abcli/install.sh,sha256=gRbmZfSMWd7RlmQaaXqQKuzuG9EXXA4at1R3KUfQ-qU,263
|
18
|
-
blue_sandbox/.abcli/microsoft_building_damage_assessment.sh,sha256=wIP7BvltFj3Gx5VmSgT6hY67K1BnubELef93HHwKOU0,470
|
19
|
-
blue_sandbox/.abcli/microsoft_building_damage_assessment/ingest.sh,sha256=7ctyhe8wJ4RDJTOsrQ26FSYvtSizgBysSCGzL7Ap8f8,2701
|
20
|
-
blue_sandbox/.abcli/microsoft_building_damage_assessment/ingest/list.sh,sha256=9bEjpbwO7Heqd-E2RTM-ck4Q39Eqsv4ged6e0WmxmSs,452
|
21
|
-
blue_sandbox/.abcli/tests/README.sh,sha256=rmJM-BPnTcmpPbJ5GXsF8vd_a84JKryeKkZyVScUing,145
|
22
|
-
blue_sandbox/.abcli/tests/help.sh,sha256=ULiOsZDZ-M2T7fFoCZk3xyBNmJchFeizV_x_WHVqCko,404
|
23
|
-
blue_sandbox/.abcli/tests/microsoft_building_damage_assessment_ingest.sh,sha256=oUUF8Kuyohxiqxo9FLY9EO0qlgkagntgfuFda6niHeM,426
|
24
|
-
blue_sandbox/.abcli/tests/microsoft_building_damage_assessment_ingest_list.sh,sha256=5P0H-FHV7YKt1fenXAs_0cSFhJYWZ9u3LzRpfSRSptw,577
|
25
|
-
blue_sandbox/.abcli/tests/version.sh,sha256=jF8zoJN1eKE3LfDeRVG9uHEosmEVJX6RtKfdioyeN-o,150
|
26
|
-
blue_sandbox/help/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
27
|
-
blue_sandbox/help/__main__.py,sha256=3Cqp5oISrZCOUApmwoQoCj_0sQgtkiEkm_ob3LFKzRE,234
|
28
|
-
blue_sandbox/help/functions.py,sha256=Ki0nUGyHKDo2t0WnIVzBwxEVJHyRmHh-hIRv3ohJK3c,431
|
29
|
-
blue_sandbox/microsoft_building_damage_assessment/README.py,sha256=AyZs8uYpZ7XSYs--5HkMHwgCfZZM1n_E-YWNZ6ncaO4,842
|
30
|
-
blue_sandbox/microsoft_building_damage_assessment/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
31
|
-
blue_sandbox/microsoft_building_damage_assessment/__main__.py,sha256=rjjr-csHJd2jgO2GDlnAbbQAO_zbaXwA7n6Ce0UX0lM,1009
|
32
|
-
blue_sandbox/microsoft_building_damage_assessment/ingest.py,sha256=F5GBHZ6s6QebC-w9e4QsRiQ6ZKxXjFbqckZqV8HDU1s,337
|
33
|
-
blue_sandbox-5.47.1.dist-info/LICENSE,sha256=ogEPNDSH0_dhiv_lT3ifVIdgIzHAqNA_SemnxUfPBJk,7048
|
34
|
-
blue_sandbox-5.47.1.dist-info/METADATA,sha256=bC0hgpyotWeXyJ5d4tZVHOBdGX52KfabS3PtxkD85uQ,2756
|
35
|
-
blue_sandbox-5.47.1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
|
36
|
-
blue_sandbox-5.47.1.dist-info/top_level.txt,sha256=4D9Cb9QUCaqdYAmBiCwvtlaYBtUYVVxv0Sxcr_pzgS8,13
|
37
|
-
blue_sandbox-5.47.1.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|