blue-sandbox 5.47.1__py3-none-any.whl → 5.87.1__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- blue_sandbox/.abcli/alias.sh +1 -1
- blue_sandbox/.abcli/install.sh +1 -6
- blue_sandbox/.abcli/microsoft_building_damage_assessment/ingest.sh +2 -4
- blue_sandbox/.abcli/microsoft_building_damage_assessment/install.sh +49 -0
- blue_sandbox/.abcli/microsoft_building_damage_assessment/label.sh +26 -0
- blue_sandbox/.abcli/microsoft_building_damage_assessment/tensorboard.sh +21 -0
- blue_sandbox/.abcli/microsoft_building_damage_assessment/train.sh +53 -0
- blue_sandbox/.abcli/tests/help.sh +7 -3
- blue_sandbox/.abcli/tests/microsoft_building_damage_assessment_label.sh +12 -0
- blue_sandbox/.abcli/tests/microsoft_building_damage_assessment_train.sh +20 -0
- blue_sandbox/README.py +1 -1
- blue_sandbox/__init__.py +1 -1
- blue_sandbox/config.env +1 -1
- blue_sandbox/env.py +4 -2
- blue_sandbox/microsoft_building_damage_assessment/README.py +24 -13
- blue_sandbox/microsoft_building_damage_assessment/__main__.py +37 -2
- blue_sandbox/microsoft_building_damage_assessment/ingest.py +15 -3
- blue_sandbox/microsoft_building_damage_assessment/label.py +38 -0
- blue_sandbox/microsoft_building_damage_assessment/sas_token.py +9 -0
- blue_sandbox/microsoft_building_damage_assessment/train.py +95 -0
- blue_sandbox/sample.env +1 -0
- {blue_sandbox-5.47.1.dist-info → blue_sandbox-5.87.1.dist-info}/METADATA +11 -4
- blue_sandbox-5.87.1.dist-info/RECORD +46 -0
- blue_sandbox-5.47.1.dist-info/RECORD +0 -37
- {blue_sandbox-5.47.1.dist-info → blue_sandbox-5.87.1.dist-info}/LICENSE +0 -0
- {blue_sandbox-5.47.1.dist-info → blue_sandbox-5.87.1.dist-info}/WHEEL +0 -0
- {blue_sandbox-5.47.1.dist-info → blue_sandbox-5.87.1.dist-info}/top_level.txt +0 -0
blue_sandbox/.abcli/alias.sh
CHANGED
blue_sandbox/.abcli/install.sh
CHANGED
@@ -1,12 +1,7 @@
|
|
1
1
|
#! /usr/bin/env bash
|
2
2
|
|
3
3
|
function abcli_install_blue_sandbox() {
|
4
|
-
|
5
|
-
|
6
|
-
[[ "$abcli_is_mac" == true ]] &&
|
7
|
-
brew install wget
|
8
|
-
|
9
|
-
return 0
|
4
|
+
abcli_log "✅"
|
10
5
|
}
|
11
6
|
|
12
7
|
abcli_install_module blue_sandbox 1.1.1
|
@@ -26,8 +26,8 @@ function blue_sandbox_microsoft_building_damage_assessment_ingest() {
|
|
26
26
|
pushd $object_path >/dev/null
|
27
27
|
|
28
28
|
# https://github.com/microsoft/building-damage-assessment/blob/main/SATELLITE_WORKFLOW.md#1-image-acquisition-and-label-generation
|
29
|
-
mkdir -pv
|
30
|
-
cd
|
29
|
+
mkdir -pv raw
|
30
|
+
cd raw
|
31
31
|
|
32
32
|
if [[ "$do_download" == 1 ]]; then
|
33
33
|
# TODO: use $event_name and $count (options +=)
|
@@ -57,8 +57,6 @@ function blue_sandbox_microsoft_building_damage_assessment_ingest() {
|
|
57
57
|
rm -v maxar_lahaina_8_12_2023-visual.vrt
|
58
58
|
fi
|
59
59
|
|
60
|
-
cd ../../..
|
61
|
-
|
62
60
|
popd >/dev/null
|
63
61
|
|
64
62
|
abcli_eval dryrun=$do_dryrun \
|
@@ -0,0 +1,49 @@
|
|
1
|
+
#! /usr/bin/env bash
|
2
|
+
|
3
|
+
function blue_sandbox_microsoft_building_damage_assessment_install() {
|
4
|
+
local options=$1
|
5
|
+
local do_dryrun=$(abcli_option_int "$options" dryrun 0)
|
6
|
+
local recreate_env=$(abcli_option_int "$options" recreate_env 0)
|
7
|
+
|
8
|
+
if [[ -d "$abcli_path_git/building-damage-assessment" ]]; then
|
9
|
+
abcli_log "✅ building-damage-assessment"
|
10
|
+
else
|
11
|
+
abcli_git_clone https://github.com/microsoft/building-damage-assessment.git
|
12
|
+
[[ $? -ne 0 ]] && return 1
|
13
|
+
fi
|
14
|
+
|
15
|
+
[[ "$abcli_is_github_workflow" == true ]] &&
|
16
|
+
return 0
|
17
|
+
|
18
|
+
if [[ "$recreate_env" == 1 ]]; then
|
19
|
+
abcli_conda_rm name=bda
|
20
|
+
[[ $? -ne 0 ]] && return 1
|
21
|
+
fi
|
22
|
+
|
23
|
+
local exists=$(abcli_conda_exists name=bda)
|
24
|
+
if [[ "$exists" == 0 ]]; then
|
25
|
+
# https://github.com/microsoft/building-damage-assessment?tab=readme-ov-file#setup
|
26
|
+
conda env create \
|
27
|
+
-f $abcli_path_git/building-damage-assessment/environment.yml
|
28
|
+
[[ $? -ne 0 ]] && return 1
|
29
|
+
fi
|
30
|
+
|
31
|
+
conda activate bda
|
32
|
+
[[ $? -ne 0 ]] && return 1
|
33
|
+
|
34
|
+
pushd $abcli_path_git/blue-sandbox >/dev/null
|
35
|
+
pip3 install -e .
|
36
|
+
[[ $? -ne 0 ]] && return 1
|
37
|
+
popd >/dev/null
|
38
|
+
|
39
|
+
[[ "$abcli_is_mac" == true ]] &&
|
40
|
+
brew install wget
|
41
|
+
|
42
|
+
return 0
|
43
|
+
}
|
44
|
+
|
45
|
+
function abcli_install_microsoft_building_damage_assessment() {
|
46
|
+
blue_sandbox_microsoft_building_damage_assessment_install "$@"
|
47
|
+
}
|
48
|
+
|
49
|
+
abcli_install_module microsoft_building_damage_assessment 1.1.1
|
@@ -0,0 +1,26 @@
|
|
1
|
+
#! /usr/bin/env bash
|
2
|
+
|
3
|
+
function blue_sandbox_microsoft_building_damage_assessment_label() {
|
4
|
+
local options=$1
|
5
|
+
local do_dryrun=$(abcli_option_int "$options" dryrun 0)
|
6
|
+
local do_download=$(abcli_option_int "$options" download $(abcli_not $do_dryrun))
|
7
|
+
local do_upload=$(abcli_option_int "$options" upload $(abcli_not $do_dryrun))
|
8
|
+
|
9
|
+
local object_name=$(abcli_clarify_object $2 .)
|
10
|
+
[[ "$do_download" == 1 ]] &&
|
11
|
+
abcli_download - $object_name
|
12
|
+
|
13
|
+
abcli_log "labeling $object_name ..."
|
14
|
+
|
15
|
+
abcli_eval dryrun=$do_dryrun \
|
16
|
+
python3 -m blue_sandbox.microsoft_building_damage_assessment \
|
17
|
+
label \
|
18
|
+
--object_name $object_name \
|
19
|
+
"${@:3}"
|
20
|
+
local status="$?"
|
21
|
+
|
22
|
+
[[ "$do_upload" == 1 ]] &&
|
23
|
+
abcli_upload - $object_name
|
24
|
+
|
25
|
+
return $status
|
26
|
+
}
|
@@ -0,0 +1,21 @@
|
|
1
|
+
#! /usr/bin/env bash
|
2
|
+
|
3
|
+
function blue_sandbox_microsoft_building_damage_assessment_tensorboard() {
|
4
|
+
local options=$1
|
5
|
+
local do_dryrun=$(abcli_option_int "$options" dryrun 0)
|
6
|
+
local do_browse=$(abcli_option_int "$options" browse $(abcli_not $do_dryrun))
|
7
|
+
local do_download=$(abcli_option_int "$options" download $(abcli_not $do_dryrun))
|
8
|
+
local port=$(abcli_option "$options" port 8889)
|
9
|
+
|
10
|
+
local model_object_name=$(abcli_clarify_object $2 .)
|
11
|
+
[[ "$do_download" == 1 ]] &&
|
12
|
+
abcli_download - $model_object_name
|
13
|
+
|
14
|
+
[[ "$do_browse" == 1 ]] &&
|
15
|
+
abcli_browse http://localhost:$port/
|
16
|
+
|
17
|
+
abcli_eval ,$options \
|
18
|
+
tensorboard \
|
19
|
+
--logdir $ABCLI_OBJECT_ROOT/$model_object_name/logs/ \
|
20
|
+
--port $port
|
21
|
+
}
|
@@ -0,0 +1,53 @@
|
|
1
|
+
#! /usr/bin/env bash
|
2
|
+
|
3
|
+
function blue_sandbox_microsoft_building_damage_assessment_train() {
|
4
|
+
local options=$1
|
5
|
+
local do_dryrun=$(abcli_option_int "$options" dryrun 0)
|
6
|
+
local do_download=$(abcli_option_int "$options" download $(abcli_not $do_dryrun))
|
7
|
+
local do_upload=$(abcli_option_int "$options" upload $(abcli_not $do_dryrun))
|
8
|
+
|
9
|
+
local dataset_object_name=$(abcli_clarify_object $2 .)
|
10
|
+
[[ "$do_download" == 1 ]] &&
|
11
|
+
abcli_download - $dataset_object_name
|
12
|
+
|
13
|
+
local model_object_name=$(abcli_clarify_object $3 $dataset_object_name-model-$(abcli_string_timestamp_short))
|
14
|
+
|
15
|
+
abcli_log "training $dataset_object_name -> $model_object_name ..."
|
16
|
+
|
17
|
+
conda activate bda
|
18
|
+
[[ $? -ne 0 ]] && return 1
|
19
|
+
|
20
|
+
abcli_eval dryrun=$do_dryrun \
|
21
|
+
python3 -m blue_sandbox.microsoft_building_damage_assessment \
|
22
|
+
train \
|
23
|
+
--dataset_object_name $dataset_object_name \
|
24
|
+
--model_object_name $model_object_name \
|
25
|
+
"${@:4}"
|
26
|
+
[[ $? -ne 0 ]] && return 1
|
27
|
+
|
28
|
+
abcli_eval dryrun=$do_dryrun,path=$abcli_path_git/building-damage-assessment \
|
29
|
+
python3 project_setup.py \
|
30
|
+
--skip_health_check \
|
31
|
+
--config $ABCLI_OBJECT_ROOT/$model_object_name/config.yml
|
32
|
+
[[ $? -ne 0 ]] && return 1
|
33
|
+
|
34
|
+
abcli_eval dryrun=$do_dryrun,path=$abcli_path_git/building-damage-assessment \
|
35
|
+
python3 create_masks.py \
|
36
|
+
--config $ABCLI_OBJECT_ROOT/$model_object_name/config.yml
|
37
|
+
[[ $? -ne 0 ]] && return 1
|
38
|
+
|
39
|
+
abcli_eval dryrun=$do_dryrun,path=$abcli_path_git/building-damage-assessment \
|
40
|
+
python3 fine_tune.py \
|
41
|
+
--config $ABCLI_OBJECT_ROOT/$model_object_name/config.yml
|
42
|
+
[[ $? -ne 0 ]] && return 1
|
43
|
+
|
44
|
+
abcli_eval dryrun=$do_dryrun,path=$abcli_path_git/building-damage-assessment \
|
45
|
+
python3 inference.py \
|
46
|
+
--config $ABCLI_OBJECT_ROOT/$model_object_name/config.yml
|
47
|
+
[[ $? -ne 0 ]] && return 1
|
48
|
+
|
49
|
+
[[ "$do_upload" == 1 ]] &&
|
50
|
+
abcli_upload - $model_object_name
|
51
|
+
|
52
|
+
return 0
|
53
|
+
}
|
@@ -7,9 +7,13 @@ function test_blue_sandbox_help() {
|
|
7
7
|
for module in \
|
8
8
|
"@sandbox" \
|
9
9
|
\
|
10
|
-
"@
|
11
|
-
"@
|
12
|
-
"@
|
10
|
+
"@damages" \
|
11
|
+
"@damages ingest" \
|
12
|
+
"@damages ingest list" \
|
13
|
+
"@damages install" \
|
14
|
+
"@damages label" \
|
15
|
+
"@damages tensorboard" \
|
16
|
+
"@damages train" \
|
13
17
|
\
|
14
18
|
"blue_sandbox"; do
|
15
19
|
abcli_eval ,$options \
|
@@ -0,0 +1,12 @@
|
|
1
|
+
#! /usr/bin/env bash
|
2
|
+
|
3
|
+
function test_blue_sandbox_microsoft_building_damage_assessment_label() {
|
4
|
+
local options=$1
|
5
|
+
|
6
|
+
abcli_eval ,$options \
|
7
|
+
blue_sandbox_microsoft_building_damage_assessment \
|
8
|
+
label \
|
9
|
+
~upload,$options \
|
10
|
+
$DAMAGES_TEST_DATASET_OBJECT_NAME \
|
11
|
+
"${@:2}"
|
12
|
+
}
|
@@ -0,0 +1,20 @@
|
|
1
|
+
#! /usr/bin/env bash
|
2
|
+
|
3
|
+
function test_blue_sandbox_microsoft_building_damage_assessment_train() {
|
4
|
+
local options=$1
|
5
|
+
|
6
|
+
abcli_log_warning "disabled."
|
7
|
+
return 0
|
8
|
+
|
9
|
+
abcli_eval ,$options \
|
10
|
+
blue_sandbox_microsoft_building_damage_assessment \
|
11
|
+
train \
|
12
|
+
~upload,$options \
|
13
|
+
$DAMAGES_TEST_DATASET_OBJECT_NAME \
|
14
|
+
test_blue_sandbox_microsoft_building_damage_assessment_train-$(abcli_string_timestamp_short) \
|
15
|
+
--verbose 1 \
|
16
|
+
"${@:2}"
|
17
|
+
|
18
|
+
# ignore error - TODO: disable
|
19
|
+
return 0
|
20
|
+
}
|
blue_sandbox/README.py
CHANGED
@@ -11,7 +11,7 @@ from blue_sandbox.microsoft_building_damage_assessment import (
|
|
11
11
|
items = [
|
12
12
|
"{}[`{}`]({}) [![image]({})]({}) {}".format(
|
13
13
|
"🌐",
|
14
|
-
"`@
|
14
|
+
"`@damages`",
|
15
15
|
"https://github.com/kamangir/blue-sandbox/blob/main/blue_sandbox/microsoft_building_damage_assessment/README.md",
|
16
16
|
"https://github.com/microsoft/building-damage-assessment/raw/main/figures/damage.png",
|
17
17
|
"https://github.com/kamangir/blue-sandbox/blob/main/blue_sandbox/microsoft_building_damage_assessment/README.md",
|
blue_sandbox/__init__.py
CHANGED
blue_sandbox/config.env
CHANGED
@@ -1 +1 @@
|
|
1
|
-
|
1
|
+
DAMAGES_TEST_DATASET_OBJECT_NAME=Maui-Hawaii-fires-Aug-23-ingest-2025-01-10-qqJqhm
|
blue_sandbox/env.py
CHANGED
@@ -5,7 +5,9 @@ load_env(__name__)
|
|
5
5
|
load_config(__name__)
|
6
6
|
|
7
7
|
|
8
|
-
|
9
|
-
"
|
8
|
+
DAMAGES_TEST_DATASET_OBJECT_NAME = os.getenv(
|
9
|
+
"DAMAGES_TEST_DATASET_OBJECT_NAME",
|
10
10
|
"",
|
11
11
|
)
|
12
|
+
|
13
|
+
ENCODED_BLOB_SAS_TOKEN = os.getenv("ENCODED_BLOB_SAS_TOKEN", "")
|
@@ -1,11 +1,22 @@
|
|
1
1
|
from typing import Dict
|
2
2
|
|
3
|
+
from blue_sandbox.env import DAMAGES_TEST_DATASET_OBJECT_NAME
|
4
|
+
|
3
5
|
list_of_steps: Dict[str, Dict] = {
|
4
|
-
"ingest":
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
"
|
6
|
+
"ingest": {
|
7
|
+
"object_name": DAMAGES_TEST_DATASET_OBJECT_NAME,
|
8
|
+
"image_name": "Maui-Hawaii-fires-Aug-23-damage-2025-01-09-GgnjQC",
|
9
|
+
},
|
10
|
+
"label": {
|
11
|
+
"object_name": "",
|
12
|
+
"image_name": DAMAGES_TEST_DATASET_OBJECT_NAME,
|
13
|
+
},
|
14
|
+
"train": {
|
15
|
+
"object_name": "",
|
16
|
+
"image_name": "Maui-Hawaii-fires-Aug-23-model-2025-01-10-NQb8IS",
|
17
|
+
},
|
18
|
+
"predict": {"object_name": "", "image_name": ""},
|
19
|
+
"summarize": {"object_name": "", "image_name": ""},
|
9
20
|
}
|
10
21
|
|
11
22
|
items = (
|
@@ -13,22 +24,22 @@ items = (
|
|
13
24
|
+ [
|
14
25
|
(
|
15
26
|
"[`{}`](https://kamangir-public.s3.ca-central-1.amazonaws.com/{}.tar.gz)".format(
|
16
|
-
object_name,
|
17
|
-
|
27
|
+
step["object_name"],
|
28
|
+
step["image_name"],
|
18
29
|
)
|
19
|
-
if object_name
|
30
|
+
if step["object_name"]
|
20
31
|
else ""
|
21
32
|
)
|
22
|
-
for
|
33
|
+
for step in list_of_steps.values()
|
23
34
|
]
|
24
35
|
+ [
|
25
36
|
(
|
26
|
-
"
|
27
|
-
|
37
|
+
"![image](https://github.com/kamangir/assets/blob/main/blue-sandbox/{}.png?raw=true)".format(
|
38
|
+
step["image_name"],
|
28
39
|
)
|
29
|
-
if
|
40
|
+
if step["image_name"]
|
30
41
|
else ""
|
31
42
|
)
|
32
|
-
for
|
43
|
+
for step in list_of_steps.values()
|
33
44
|
]
|
34
45
|
)
|
@@ -5,6 +5,12 @@ from blueness.argparse.generic import sys_exit
|
|
5
5
|
|
6
6
|
from blue_sandbox import NAME
|
7
7
|
from blue_sandbox.microsoft_building_damage_assessment.ingest import ingest
|
8
|
+
from blue_sandbox.microsoft_building_damage_assessment.label import label
|
9
|
+
from blue_sandbox.microsoft_building_damage_assessment.train import train
|
10
|
+
from blue_sandbox.microsoft_building_damage_assessment.sas_token import (
|
11
|
+
decode_token,
|
12
|
+
encode_token,
|
13
|
+
)
|
8
14
|
from blue_sandbox.help.microsoft_building_damage_assessment import list_of_events
|
9
15
|
from blue_sandbox.logger import logger
|
10
16
|
|
@@ -14,12 +20,20 @@ parser = argparse.ArgumentParser(NAME)
|
|
14
20
|
parser.add_argument(
|
15
21
|
"task",
|
16
22
|
type=str,
|
17
|
-
help="ingest",
|
23
|
+
help="decode_token | encode_token | ingest | label | train",
|
18
24
|
)
|
19
25
|
parser.add_argument(
|
20
26
|
"--object_name",
|
21
27
|
type=str,
|
22
28
|
)
|
29
|
+
parser.add_argument(
|
30
|
+
"--dataset_object_name",
|
31
|
+
type=str,
|
32
|
+
)
|
33
|
+
parser.add_argument(
|
34
|
+
"--model_object_name",
|
35
|
+
type=str,
|
36
|
+
)
|
23
37
|
parser.add_argument(
|
24
38
|
"--event_name",
|
25
39
|
type=str,
|
@@ -32,15 +46,36 @@ parser.add_argument(
|
|
32
46
|
default=0,
|
33
47
|
help="0|1",
|
34
48
|
)
|
49
|
+
parser.add_argument(
|
50
|
+
"--token",
|
51
|
+
type=str,
|
52
|
+
)
|
35
53
|
args = parser.parse_args()
|
36
54
|
|
37
55
|
success = False
|
38
|
-
if args.task == "
|
56
|
+
if args.task == "decode_token":
|
57
|
+
success = True
|
58
|
+
print(decode_token(args.token))
|
59
|
+
elif args.task == "encode_token":
|
60
|
+
success = True
|
61
|
+
print(encode_token(args.token))
|
62
|
+
elif args.task == "ingest":
|
39
63
|
success = ingest(
|
40
64
|
object_name=args.object_name,
|
41
65
|
event_name=args.event_name,
|
42
66
|
verbose=args.verbose == 1,
|
43
67
|
)
|
68
|
+
elif args.task == "label":
|
69
|
+
success = label(
|
70
|
+
object_name=args.object_name,
|
71
|
+
verbose=args.verbose == 1,
|
72
|
+
)
|
73
|
+
elif args.task == "train":
|
74
|
+
success = train(
|
75
|
+
dataset_object_name=args.dataset_object_name,
|
76
|
+
model_object_name=args.model_object_name,
|
77
|
+
verbose=args.verbose == 1,
|
78
|
+
)
|
44
79
|
else:
|
45
80
|
success = None
|
46
81
|
|
@@ -1,5 +1,7 @@
|
|
1
1
|
from blueness import module
|
2
2
|
|
3
|
+
from blue_objects import mlflow, metadata
|
4
|
+
|
3
5
|
from blue_sandbox import NAME
|
4
6
|
from blue_sandbox.logger import logger
|
5
7
|
|
@@ -14,6 +16,16 @@ def ingest(
|
|
14
16
|
) -> bool:
|
15
17
|
logger.info(f"{NAME}.ingest({event_name}) -> {object_name}")
|
16
18
|
|
17
|
-
|
18
|
-
|
19
|
-
|
19
|
+
return all(
|
20
|
+
[
|
21
|
+
mlflow.set_tags(
|
22
|
+
object_name,
|
23
|
+
{"event": event_name},
|
24
|
+
),
|
25
|
+
metadata.post_to_object(
|
26
|
+
object_name,
|
27
|
+
"ingest",
|
28
|
+
{"event": event_name},
|
29
|
+
),
|
30
|
+
]
|
31
|
+
)
|
@@ -0,0 +1,38 @@
|
|
1
|
+
import os
|
2
|
+
|
3
|
+
from blueness import module
|
4
|
+
from blue_objects import metadata, file, objects
|
5
|
+
from blue_objects.env import abcli_path_git
|
6
|
+
|
7
|
+
from blue_sandbox import NAME
|
8
|
+
from blue_sandbox.logger import logger
|
9
|
+
|
10
|
+
|
11
|
+
NAME = module.name(__file__, NAME)
|
12
|
+
|
13
|
+
|
14
|
+
def label(
|
15
|
+
object_name: str,
|
16
|
+
verbose: bool = False,
|
17
|
+
) -> bool:
|
18
|
+
logger.info(f"{NAME}.label: {object_name}")
|
19
|
+
|
20
|
+
geojson_filename = os.path.join(
|
21
|
+
abcli_path_git,
|
22
|
+
"building-damage-assessment/data/demo/labels/Maui_Wildfires_August_0.geojson",
|
23
|
+
)
|
24
|
+
|
25
|
+
if not file.copy(
|
26
|
+
geojson_filename,
|
27
|
+
objects.path_of(
|
28
|
+
filename="label.geojson",
|
29
|
+
object_name=object_name,
|
30
|
+
),
|
31
|
+
):
|
32
|
+
return False
|
33
|
+
|
34
|
+
return metadata.post_to_object(
|
35
|
+
object_name,
|
36
|
+
"label",
|
37
|
+
{"geojson": geojson_filename},
|
38
|
+
)
|
@@ -0,0 +1,95 @@
|
|
1
|
+
import os
|
2
|
+
|
3
|
+
from blueness import module
|
4
|
+
from blue_objects import mlflow, metadata, file, objects, path
|
5
|
+
from blue_objects.env import abcli_path_git
|
6
|
+
|
7
|
+
from blue_sandbox import NAME
|
8
|
+
from blue_sandbox.env import ENCODED_BLOB_SAS_TOKEN
|
9
|
+
from blue_sandbox.microsoft_building_damage_assessment.sas_token import decode_token
|
10
|
+
from blue_sandbox.logger import logger
|
11
|
+
|
12
|
+
|
13
|
+
NAME = module.name(__file__, NAME)
|
14
|
+
|
15
|
+
|
16
|
+
# ... copy `configs/example_config.yml` and fill the first three sections.
|
17
|
+
def train(
|
18
|
+
dataset_object_name: str,
|
19
|
+
model_object_name: str,
|
20
|
+
verbose: bool = False,
|
21
|
+
) -> bool:
|
22
|
+
logger.info(f"{NAME}.train: {dataset_object_name} -> {model_object_name}")
|
23
|
+
|
24
|
+
config_filename = os.path.join(
|
25
|
+
abcli_path_git,
|
26
|
+
"building-damage-assessment/configs/example_config.yml",
|
27
|
+
)
|
28
|
+
success, config = file.load_yaml(config_filename)
|
29
|
+
if not success:
|
30
|
+
return False
|
31
|
+
|
32
|
+
config["experiment_dir"] = objects.object_path(model_object_name)
|
33
|
+
config["experiment_name"] = model_object_name
|
34
|
+
|
35
|
+
config["imagery"]["rgb_fn"] = config["imagery"]["raw_fn"] = objects.path_of(
|
36
|
+
filename="raw/maxar_lahaina_8_12_2023-visual.tif",
|
37
|
+
object_name=dataset_object_name,
|
38
|
+
)
|
39
|
+
|
40
|
+
config["inference"]["output_subdir"] = objects.path_of(
|
41
|
+
filename="outputs/",
|
42
|
+
object_name=model_object_name,
|
43
|
+
)
|
44
|
+
config["inference"]["checkpoint_fn"] = objects.path_of(
|
45
|
+
filename="checkpoints/last.ckpt",
|
46
|
+
object_name=model_object_name,
|
47
|
+
)
|
48
|
+
|
49
|
+
config["infrastructure"]["container_name"] = "sandbox"
|
50
|
+
config["infrastructure"][
|
51
|
+
"storage_account"
|
52
|
+
] = "https://kamangir.blob.core.windows.net/"
|
53
|
+
config["infrastructure"]["sas_token"] = decode_token(ENCODED_BLOB_SAS_TOKEN)
|
54
|
+
config["infrastructure"]["relative_path"] = dataset_object_name
|
55
|
+
|
56
|
+
config["labels"]["fn"] = objects.path_of(
|
57
|
+
filename="label.geojson",
|
58
|
+
object_name=dataset_object_name,
|
59
|
+
)
|
60
|
+
|
61
|
+
config["training"]["log_dir"] = objects.path_of(
|
62
|
+
filename="logs/",
|
63
|
+
object_name=model_object_name,
|
64
|
+
)
|
65
|
+
config["training"]["checkpoint_subdir"] = objects.path_of(
|
66
|
+
filename="checkpoints/",
|
67
|
+
object_name=model_object_name,
|
68
|
+
)
|
69
|
+
|
70
|
+
if not file.save_yaml(
|
71
|
+
objects.path_of(
|
72
|
+
filename="config.yml",
|
73
|
+
object_name=model_object_name,
|
74
|
+
),
|
75
|
+
config,
|
76
|
+
):
|
77
|
+
return False
|
78
|
+
|
79
|
+
return all(
|
80
|
+
[
|
81
|
+
mlflow.set_tags(
|
82
|
+
model_object_name,
|
83
|
+
{
|
84
|
+
"dataset": dataset_object_name,
|
85
|
+
},
|
86
|
+
),
|
87
|
+
metadata.post_to_object(
|
88
|
+
model_object_name,
|
89
|
+
"train",
|
90
|
+
{
|
91
|
+
"dataset": dataset_object_name,
|
92
|
+
},
|
93
|
+
),
|
94
|
+
]
|
95
|
+
)
|
blue_sandbox/sample.env
CHANGED
@@ -0,0 +1 @@
|
|
1
|
+
ENCODED_BLOB_SAS_TOKEN=value
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.2
|
2
2
|
Name: blue_sandbox
|
3
|
-
Version: 5.
|
3
|
+
Version: 5.87.1
|
4
4
|
Summary: 🌀 A sandbox for ideas and experiments.
|
5
5
|
Home-page: https://github.com/kamangir/blue-sandbox
|
6
6
|
Author: Arash Abadpour (Kamangir)
|
@@ -13,8 +13,12 @@ Classifier: Operating System :: OS Independent
|
|
13
13
|
Description-Content-Type: text/markdown
|
14
14
|
License-File: LICENSE
|
15
15
|
Requires-Dist: blueness
|
16
|
-
Requires-Dist: blue-options
|
17
16
|
Requires-Dist: abcli
|
17
|
+
Requires-Dist: blue-options
|
18
|
+
Requires-Dist: blue_objects
|
19
|
+
Requires-Dist: blue_geo
|
20
|
+
Requires-Dist: blueflow
|
21
|
+
Requires-Dist: blue_plugin
|
18
22
|
Requires-Dist: boto3
|
19
23
|
Requires-Dist: geojson
|
20
24
|
Requires-Dist: geopandas
|
@@ -27,6 +31,9 @@ Requires-Dist: pylint
|
|
27
31
|
Requires-Dist: pytest
|
28
32
|
Requires-Dist: python-dotenv[cli]
|
29
33
|
Requires-Dist: tqdm
|
34
|
+
Requires-Dist: azure-storage-blob
|
35
|
+
Requires-Dist: fiona
|
36
|
+
Requires-Dist: tensorboard
|
30
37
|
Dynamic: author
|
31
38
|
Dynamic: author-email
|
32
39
|
Dynamic: classifier
|
@@ -47,11 +54,11 @@ pip install blue-sandbox
|
|
47
54
|
|
48
55
|
| | | |
|
49
56
|
| --- | --- | --- |
|
50
|
-
| 🌐[``@
|
57
|
+
| 🌐[``@damages``](https://github.com/kamangir/blue-sandbox/blob/main/blue_sandbox/microsoft_building_damage_assessment/README.md) [![image](https://github.com/microsoft/building-damage-assessment/raw/main/figures/damage.png)](https://github.com/kamangir/blue-sandbox/blob/main/blue_sandbox/microsoft_building_damage_assessment/README.md) Satellite imagery damage assessment workflow | 🌀[`experiment 2`](#) [![image](https://github.com/kamangir/assets/raw/main/blue-plugin/marquee.png?raw=true)](#) description of experiment 2 ... | 🌀[`experiment 3`](#) [![image](https://github.com/kamangir/assets/raw/main/blue-plugin/marquee.png?raw=true)](#) description of experiment 3 ... |
|
51
58
|
|
52
59
|
---
|
53
60
|
|
54
61
|
|
55
62
|
[![pylint](https://github.com/kamangir/blue-sandbox/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/blue-sandbox/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/blue-sandbox/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/blue-sandbox/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/blue-sandbox/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/blue-sandbox/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/blue-sandbox.svg)](https://pypi.org/project/blue-sandbox/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/blue-sandbox)](https://pypistats.org/packages/blue-sandbox)
|
56
63
|
|
57
|
-
built by 🌀 [`blue_options-4.
|
64
|
+
built by 🌀 [`blue_options-4.180.1`](https://github.com/kamangir/awesome-bash-cli), based on 🌀 [`blue_sandbox-5.87.1`](https://github.com/kamangir/blue-sandbox).
|
@@ -0,0 +1,46 @@
|
|
1
|
+
blue_sandbox/README.py,sha256=Q45yMxvgjPLPQAZCRzMQZx7Znxa0OAOaf3PZNvdaAyc,1563
|
2
|
+
blue_sandbox/__init__.py,sha256=EZzxJTfEvzoa0q1G0mM346fmyKOTcdzq4xXL-X5cBLI,322
|
3
|
+
blue_sandbox/__main__.py,sha256=aPRHSpGpk-bDbzhHpfLNsd3y1gGEHpnhoTF-RBweNwc,361
|
4
|
+
blue_sandbox/config.env,sha256=SHQeFdzivHKDimmTyHzVziCfAONN48DSDmc3WVG6gEw,82
|
5
|
+
blue_sandbox/env.py,sha256=dpRUbxT2vq-9s-G5MP3IlZBdC1hFvDb7mJxzU5lUd-Q,267
|
6
|
+
blue_sandbox/functions.py,sha256=U41kQFNPpfYV6KJpMnkqgqLkozqXiG4tgV6rj8IW1BU,7
|
7
|
+
blue_sandbox/host.py,sha256=uJpiM105rnm6ySF16zA7waWekGBdec-dlpoRRU_QqwU,210
|
8
|
+
blue_sandbox/logger.py,sha256=ZoFrTIfJJGNtZUm2d7lkQjdB2SPl_KBKDmHJOcIivPM,107
|
9
|
+
blue_sandbox/sample.env,sha256=ovfKDKAIeuqWBmMDKeAint_SKTnZuyDU1TCBZ-Y3Iqg,28
|
10
|
+
blue_sandbox/urls.py,sha256=tIZ36ONJEoUBM-tFhOpkVhLwb1OgLegUeEaBDqW4USM,24
|
11
|
+
blue_sandbox/.abcli/abcli.sh,sha256=xsJ4IzuQsvLZog6U8VTBFVXsEi6ADe13L8rn47XtlbU,196
|
12
|
+
blue_sandbox/.abcli/actions.sh,sha256=vImEUI105GRcxs2mAKGMqcvoErtmOPZZ-7dfSmUUxvE,230
|
13
|
+
blue_sandbox/.abcli/aka.sh,sha256=RHDU_JbEEL2B0vvvRJ3NVSsRSEjSu09jNY85n7DLe-k,21
|
14
|
+
blue_sandbox/.abcli/alias.sh,sha256=q8j-uG7c0ZLp5BZhBKTXyESjMa_kcEH5Mf8pX4EgC7A,118
|
15
|
+
blue_sandbox/.abcli/blue_sandbox.sh,sha256=PGRJOgNGlC3XL5cw4ecZH40LDuc_6NVazTKhCWtZ-3g,233
|
16
|
+
blue_sandbox/.abcli/browse.sh,sha256=f8qa4qDVts2Am6_ldDwNeJXzhBQTk9PUKl0-a9wW1ww,287
|
17
|
+
blue_sandbox/.abcli/install.sh,sha256=zvl0GsHBmfw62ORmkMlhug264N_Zr8nc3rlPGFoq7Mk,125
|
18
|
+
blue_sandbox/.abcli/microsoft_building_damage_assessment.sh,sha256=wIP7BvltFj3Gx5VmSgT6hY67K1BnubELef93HHwKOU0,470
|
19
|
+
blue_sandbox/.abcli/microsoft_building_damage_assessment/ingest.sh,sha256=w_EUK_i4iSBpSy22TeexfF9jEAqzNwZuvjhpRraGKhA,2662
|
20
|
+
blue_sandbox/.abcli/microsoft_building_damage_assessment/install.sh,sha256=UQTjsVf-GS_Q3bhTJzj6PMDZZzTNouMVh_3kEFSl5uo,1455
|
21
|
+
blue_sandbox/.abcli/microsoft_building_damage_assessment/label.sh,sha256=RROjENj93FhCBOkbe8K05e1fR5QNny0syobkX14rwho,795
|
22
|
+
blue_sandbox/.abcli/microsoft_building_damage_assessment/tensorboard.sh,sha256=BHdnOvorV9gjO1ET64tbLOqSGviNFR96S67XHmJz2hs,746
|
23
|
+
blue_sandbox/.abcli/microsoft_building_damage_assessment/train.sh,sha256=XyTTFB2AD7p8XT0x1vvQLwndyscGN23QEDA80wS3cjk,1970
|
24
|
+
blue_sandbox/.abcli/microsoft_building_damage_assessment/ingest/list.sh,sha256=9bEjpbwO7Heqd-E2RTM-ck4Q39Eqsv4ged6e0WmxmSs,452
|
25
|
+
blue_sandbox/.abcli/tests/README.sh,sha256=rmJM-BPnTcmpPbJ5GXsF8vd_a84JKryeKkZyVScUing,145
|
26
|
+
blue_sandbox/.abcli/tests/help.sh,sha256=QeHmHpFRye7Ht7fYLAzqBo8b9OTSLBR_0qn77qrvIQU,523
|
27
|
+
blue_sandbox/.abcli/tests/microsoft_building_damage_assessment_ingest.sh,sha256=oUUF8Kuyohxiqxo9FLY9EO0qlgkagntgfuFda6niHeM,426
|
28
|
+
blue_sandbox/.abcli/tests/microsoft_building_damage_assessment_ingest_list.sh,sha256=5P0H-FHV7YKt1fenXAs_0cSFhJYWZ9u3LzRpfSRSptw,577
|
29
|
+
blue_sandbox/.abcli/tests/microsoft_building_damage_assessment_label.sh,sha256=Envac5i9eGa5usDCO8MYfGFfyAZAT-f0KrrsAkaR91U,311
|
30
|
+
blue_sandbox/.abcli/tests/microsoft_building_damage_assessment_train.sh,sha256=0FlwyR1drn6wAKX9IHU_ma7WTmH1Iv8DoTNsdsyj3PA,533
|
31
|
+
blue_sandbox/.abcli/tests/version.sh,sha256=jF8zoJN1eKE3LfDeRVG9uHEosmEVJX6RtKfdioyeN-o,150
|
32
|
+
blue_sandbox/help/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
33
|
+
blue_sandbox/help/__main__.py,sha256=3Cqp5oISrZCOUApmwoQoCj_0sQgtkiEkm_ob3LFKzRE,234
|
34
|
+
blue_sandbox/help/functions.py,sha256=Ki0nUGyHKDo2t0WnIVzBwxEVJHyRmHh-hIRv3ohJK3c,431
|
35
|
+
blue_sandbox/microsoft_building_damage_assessment/README.py,sha256=qqe1wXoc4TjKS4o3gskwNCBuX8PTVduQEAaVHeovsrE,1275
|
36
|
+
blue_sandbox/microsoft_building_damage_assessment/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
37
|
+
blue_sandbox/microsoft_building_damage_assessment/__main__.py,sha256=RnyXyJdLmzLzccVmb6VZvD17Dwx-g1wByDugIisoPmQ,1995
|
38
|
+
blue_sandbox/microsoft_building_damage_assessment/ingest.py,sha256=tBchn4dWPIqArwAe8JjEM2xZZlFtrgBRDV_Ko26OEiM,639
|
39
|
+
blue_sandbox/microsoft_building_damage_assessment/label.py,sha256=mpRNOsF8ChxSLwOyvwGmXiD7aO2xfivJPZTC5qwnLaw,823
|
40
|
+
blue_sandbox/microsoft_building_damage_assessment/sas_token.py,sha256=t6oTXc6C52CH3tFbQo2H9YUB_7upAsfUwws43Iu9pHI,214
|
41
|
+
blue_sandbox/microsoft_building_damage_assessment/train.py,sha256=8DUt0S4LuKkRIi9XpASOg5skPaM3D2VaSBg5FEn8qog,2778
|
42
|
+
blue_sandbox-5.87.1.dist-info/LICENSE,sha256=ogEPNDSH0_dhiv_lT3ifVIdgIzHAqNA_SemnxUfPBJk,7048
|
43
|
+
blue_sandbox-5.87.1.dist-info/METADATA,sha256=9M7nPtpmiI0fX-oeF4ZwG31fT9TnSdndE7Vyt7iB_AU,2942
|
44
|
+
blue_sandbox-5.87.1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
|
45
|
+
blue_sandbox-5.87.1.dist-info/top_level.txt,sha256=4D9Cb9QUCaqdYAmBiCwvtlaYBtUYVVxv0Sxcr_pzgS8,13
|
46
|
+
blue_sandbox-5.87.1.dist-info/RECORD,,
|
@@ -1,37 +0,0 @@
|
|
1
|
-
blue_sandbox/README.py,sha256=gHxb5Czp9oyNZykiCKtI9Qnx08QJab8Ht9znKxHMkPs,1562
|
2
|
-
blue_sandbox/__init__.py,sha256=yAwWd5-dAV4y8R3MW_B3K5FX1Hf7ZNCSnuNJCfrS_co,322
|
3
|
-
blue_sandbox/__main__.py,sha256=aPRHSpGpk-bDbzhHpfLNsd3y1gGEHpnhoTF-RBweNwc,361
|
4
|
-
blue_sandbox/config.env,sha256=89ao50sGXYWiU-NMV_fUVXAXdTHKKfWV03BY3_p6EOE,25
|
5
|
-
blue_sandbox/env.py,sha256=ljEErQJNy0cdSvPpJ_ziMNvXK6Qlvi_8VBgTqjSm2uk,173
|
6
|
-
blue_sandbox/functions.py,sha256=U41kQFNPpfYV6KJpMnkqgqLkozqXiG4tgV6rj8IW1BU,7
|
7
|
-
blue_sandbox/host.py,sha256=uJpiM105rnm6ySF16zA7waWekGBdec-dlpoRRU_QqwU,210
|
8
|
-
blue_sandbox/logger.py,sha256=ZoFrTIfJJGNtZUm2d7lkQjdB2SPl_KBKDmHJOcIivPM,107
|
9
|
-
blue_sandbox/sample.env,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
10
|
-
blue_sandbox/urls.py,sha256=tIZ36ONJEoUBM-tFhOpkVhLwb1OgLegUeEaBDqW4USM,24
|
11
|
-
blue_sandbox/.abcli/abcli.sh,sha256=xsJ4IzuQsvLZog6U8VTBFVXsEi6ADe13L8rn47XtlbU,196
|
12
|
-
blue_sandbox/.abcli/actions.sh,sha256=vImEUI105GRcxs2mAKGMqcvoErtmOPZZ-7dfSmUUxvE,230
|
13
|
-
blue_sandbox/.abcli/aka.sh,sha256=RHDU_JbEEL2B0vvvRJ3NVSsRSEjSu09jNY85n7DLe-k,21
|
14
|
-
blue_sandbox/.abcli/alias.sh,sha256=A2Q2VBfKuAiWc0yRXeDj95lL70elSdDKZTKWPTK-590,117
|
15
|
-
blue_sandbox/.abcli/blue_sandbox.sh,sha256=PGRJOgNGlC3XL5cw4ecZH40LDuc_6NVazTKhCWtZ-3g,233
|
16
|
-
blue_sandbox/.abcli/browse.sh,sha256=f8qa4qDVts2Am6_ldDwNeJXzhBQTk9PUKl0-a9wW1ww,287
|
17
|
-
blue_sandbox/.abcli/install.sh,sha256=gRbmZfSMWd7RlmQaaXqQKuzuG9EXXA4at1R3KUfQ-qU,263
|
18
|
-
blue_sandbox/.abcli/microsoft_building_damage_assessment.sh,sha256=wIP7BvltFj3Gx5VmSgT6hY67K1BnubELef93HHwKOU0,470
|
19
|
-
blue_sandbox/.abcli/microsoft_building_damage_assessment/ingest.sh,sha256=7ctyhe8wJ4RDJTOsrQ26FSYvtSizgBysSCGzL7Ap8f8,2701
|
20
|
-
blue_sandbox/.abcli/microsoft_building_damage_assessment/ingest/list.sh,sha256=9bEjpbwO7Heqd-E2RTM-ck4Q39Eqsv4ged6e0WmxmSs,452
|
21
|
-
blue_sandbox/.abcli/tests/README.sh,sha256=rmJM-BPnTcmpPbJ5GXsF8vd_a84JKryeKkZyVScUing,145
|
22
|
-
blue_sandbox/.abcli/tests/help.sh,sha256=ULiOsZDZ-M2T7fFoCZk3xyBNmJchFeizV_x_WHVqCko,404
|
23
|
-
blue_sandbox/.abcli/tests/microsoft_building_damage_assessment_ingest.sh,sha256=oUUF8Kuyohxiqxo9FLY9EO0qlgkagntgfuFda6niHeM,426
|
24
|
-
blue_sandbox/.abcli/tests/microsoft_building_damage_assessment_ingest_list.sh,sha256=5P0H-FHV7YKt1fenXAs_0cSFhJYWZ9u3LzRpfSRSptw,577
|
25
|
-
blue_sandbox/.abcli/tests/version.sh,sha256=jF8zoJN1eKE3LfDeRVG9uHEosmEVJX6RtKfdioyeN-o,150
|
26
|
-
blue_sandbox/help/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
27
|
-
blue_sandbox/help/__main__.py,sha256=3Cqp5oISrZCOUApmwoQoCj_0sQgtkiEkm_ob3LFKzRE,234
|
28
|
-
blue_sandbox/help/functions.py,sha256=Ki0nUGyHKDo2t0WnIVzBwxEVJHyRmHh-hIRv3ohJK3c,431
|
29
|
-
blue_sandbox/microsoft_building_damage_assessment/README.py,sha256=AyZs8uYpZ7XSYs--5HkMHwgCfZZM1n_E-YWNZ6ncaO4,842
|
30
|
-
blue_sandbox/microsoft_building_damage_assessment/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
31
|
-
blue_sandbox/microsoft_building_damage_assessment/__main__.py,sha256=rjjr-csHJd2jgO2GDlnAbbQAO_zbaXwA7n6Ce0UX0lM,1009
|
32
|
-
blue_sandbox/microsoft_building_damage_assessment/ingest.py,sha256=F5GBHZ6s6QebC-w9e4QsRiQ6ZKxXjFbqckZqV8HDU1s,337
|
33
|
-
blue_sandbox-5.47.1.dist-info/LICENSE,sha256=ogEPNDSH0_dhiv_lT3ifVIdgIzHAqNA_SemnxUfPBJk,7048
|
34
|
-
blue_sandbox-5.47.1.dist-info/METADATA,sha256=bC0hgpyotWeXyJ5d4tZVHOBdGX52KfabS3PtxkD85uQ,2756
|
35
|
-
blue_sandbox-5.47.1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
|
36
|
-
blue_sandbox-5.47.1.dist-info/top_level.txt,sha256=4D9Cb9QUCaqdYAmBiCwvtlaYBtUYVVxv0Sxcr_pzgS8,13
|
37
|
-
blue_sandbox-5.47.1.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|