puml-for-markdown 1.2.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/README.md +181 -29
- package/bin/index.js +88 -71
- package/index.js +305 -206
- package/lib/cache.js +45 -0
- package/lib/network.js +38 -0
- package/package.json +19 -11
- package/.nvmrc +0 -1
- package/CONTRIBUTING.md +0 -1
- package/puml/c4_styles.puml +0 -37
- package/puml/constants.puml +0 -32
- package/puml/level_1_system_view.puml +0 -36
- package/puml/level_2_container_view.puml +0 -55
- package/puml/level_3_component_view_label_retrieval_job.puml +0 -40
- package/puml/level_3_component_view_pipeline.puml +0 -58
- package/puml/level_4_activity_diagram_export_bq_snapshot_job.puml +0 -13
- package/puml/level_4_activity_diagram_sampler_a.puml +0 -79
- package/puml/level_4_activity_diagram_sampler_b.puml +0 -32
- package/puml/level_4_erd.puml +0 -106
- package/puml/styles.puml +0 -22
package/package.json
CHANGED
|
@@ -1,22 +1,19 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "puml-for-markdown",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "2.0.0",
|
|
4
4
|
"description": "An application to add interactive PUML diagrams to your github markdown files. If running with default arguments, run in project root directory.",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"scripts": {
|
|
7
|
-
"prepare": "husky install"
|
|
7
|
+
"prepare": "husky install",
|
|
8
|
+
"test": "node --test"
|
|
8
9
|
},
|
|
9
10
|
"author": "danielyaa5",
|
|
10
|
-
"license": "
|
|
11
|
+
"license": "MIT",
|
|
11
12
|
"dependencies": {
|
|
12
|
-
"bent": "7.3.12",
|
|
13
|
-
"bluebird": "3.7.2",
|
|
14
13
|
"commander": "8.2.0",
|
|
15
|
-
"
|
|
16
|
-
"
|
|
17
|
-
"
|
|
18
|
-
"plantuml-encoder": "1.4.0",
|
|
19
|
-
"tinyurl": "1.1.7"
|
|
14
|
+
"ignore": "7.0.12",
|
|
15
|
+
"markdown-it": "15.0.3",
|
|
16
|
+
"plantuml-encoder": "1.4.0"
|
|
20
17
|
},
|
|
21
18
|
"bin": {
|
|
22
19
|
"puml-for-markdown": "./bin/index.js"
|
|
@@ -45,5 +42,16 @@
|
|
|
45
42
|
},
|
|
46
43
|
"devDependencies": {
|
|
47
44
|
"husky": "7.0.4"
|
|
48
|
-
}
|
|
45
|
+
},
|
|
46
|
+
"engines": {
|
|
47
|
+
"node": ">=22"
|
|
48
|
+
},
|
|
49
|
+
"files": [
|
|
50
|
+
"index.js",
|
|
51
|
+
"bin/",
|
|
52
|
+
"README.md",
|
|
53
|
+
"CHANGELOG.md",
|
|
54
|
+
"LICENSE.md",
|
|
55
|
+
"lib/"
|
|
56
|
+
]
|
|
49
57
|
}
|
package/.nvmrc
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
v17.3.1
|
package/CONTRIBUTING.md
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
TODO
|
package/puml/c4_styles.puml
DELETED
|
@@ -1,37 +0,0 @@
|
|
|
1
|
-
@startuml
|
|
2
|
-
|
|
3
|
-
'Icons
|
|
4
|
-
!define FA_5_URL https://raw.githubusercontent.com/tupadr3/plantuml-icon-font-sprites/e53fd61259c0f2449edfe304cad83f51d0fbb5c1/font-awesome-5
|
|
5
|
-
!include FA_5_URL/cloud.puml
|
|
6
|
-
!include FA_5_URL/youtube.puml
|
|
7
|
-
!include FA_5_URL/server.puml
|
|
8
|
-
|
|
9
|
-
!define MATERIAL_URL https://raw.githubusercontent.com/tupadr3/plantuml-icon-font-sprites/e53fd61259c0f2449edfe304cad83f51d0fbb5c1/material
|
|
10
|
-
!include MATERIAL_URL/zoom_in.puml
|
|
11
|
-
|
|
12
|
-
!define osaPuml https://raw.githubusercontent.com/Crashedmind/PlantUML-opensecurityarchitecture2-icons/master
|
|
13
|
-
!include osaPuml/Common.puml
|
|
14
|
-
!include osaPuml/User/all.puml
|
|
15
|
-
|
|
16
|
-
!include <office/Servers/database_server>
|
|
17
|
-
!include <office/Servers/file_server>
|
|
18
|
-
!include <office/Servers/application_server>
|
|
19
|
-
!include <office/Concepts/service_application>
|
|
20
|
-
!include <office/Concepts/firewall>
|
|
21
|
-
|
|
22
|
-
!define AWS_PUML https://raw.githubusercontent.com/awslabs/aws-icons-for-plantuml/main/dist
|
|
23
|
-
!include AWS_PUML/AWSCommon.puml
|
|
24
|
-
!include AWS_PUML/Robotics/RoboMaker.puml
|
|
25
|
-
|
|
26
|
-
'Common Styling
|
|
27
|
-
skinparam wrapWidth 250
|
|
28
|
-
|
|
29
|
-
UpdateElementStyle("System", $bgColor="#ffffff", $fontColor="#000000", $shadowing="true")
|
|
30
|
-
UpdateElementStyle("Person", $bgColor="#e0f1fb", $fontColor="#000000", $shape=RoundedBoxShape())
|
|
31
|
-
|
|
32
|
-
AddElementTag("clickableText", $fontColor="purple")
|
|
33
|
-
AddElementTag("button", $bgColor="#444444", $fontColor="#ffffff")
|
|
34
|
-
AddElementTag("primary", $bgColor="#ffdb58", $fontColor="#000000", $shadowing="false", $borderColor="Gray")
|
|
35
|
-
AddElementTag("highlightBoundary", $bgColor="#fce8ee")
|
|
36
|
-
|
|
37
|
-
@enduml
|
package/puml/constants.puml
DELETED
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
@startuml
|
|
2
|
-
' Tables
|
|
3
|
-
!global $TYPE_X_SAMPLE_POOL_TABLE = "type_x_sample_pool"
|
|
4
|
-
!global $TYPE_Y_SAMPLE_POOL_TABLE = "type_y_sample_pool"
|
|
5
|
-
!unquoted procedure $SAMPLE_POOL_TABLE_DESC()
|
|
6
|
-
A subquery from which to query samples, should be joined to any
|
|
7
|
-
columns that you want to include in the $DAILY_COLLECTED_SAMPLES_TABLE table
|
|
8
|
-
!endprocedure
|
|
9
|
-
|
|
10
|
-
!global $SAMPLE_LABELS_TABLE = "some_other_label_db"
|
|
11
|
-
!global $SAMPLE_LABELS_ID = "sampleLabelsDb"
|
|
12
|
-
!global $SAMPLE_LABELS_CONTAINER_DESC = $LABELED_DATASOURCE_DESC
|
|
13
|
-
!global $SAMPLE_LABELS_COMPONENT_DESC = "Stores all samples which have been labeled.\n"
|
|
14
|
-
!global $SAMPLE_LABELS_TYPE = "BigTable"
|
|
15
|
-
!global $SAMPLE_LABELS_SCHEMA = "https://docs.google.com/spreadsheets"
|
|
16
|
-
|
|
17
|
-
!global $DAILY_COLLECTED_SAMPLES_TABLE = "type_y_samples_{date}"
|
|
18
|
-
!global $DAILY_COLLECTED_SAMPLES_ID = "dailySamplesDb"
|
|
19
|
-
!global $DAILY_COLLECTED_SAMPLES_DESC = "A days worth of samples collected during single run of the pipeline.\n"
|
|
20
|
-
!global $DAILY_COLLECTED_SAMPLES_TYPE = "BigQueryDailyTable"
|
|
21
|
-
|
|
22
|
-
!global $LABELED_DATASOURCE_ID = "labeledDatasource"
|
|
23
|
-
!global $LABELED_DATASOURCE_NAME = "labeled_datasource_bucket"
|
|
24
|
-
!global $LABELED_DATASOURCE_TYPE = "GCS"
|
|
25
|
-
!global $LABELED_DATASOURCE_DESC = "Where the labelers write their labeling results"
|
|
26
|
-
|
|
27
|
-
'Columns
|
|
28
|
-
!global $SEND_FOR_LABELING_COLUMN = "send_for_labeling"
|
|
29
|
-
!global $CONTENT_ID_COLUMN = "some_id"
|
|
30
|
-
|
|
31
|
-
!global $SYSTEM_NAME = "My Project Name"
|
|
32
|
-
@enduml
|
|
@@ -1,36 +0,0 @@
|
|
|
1
|
-
@startuml
|
|
2
|
-
!include https://raw.githubusercontent.com/plantuml-stdlib/C4-PlantUML/master/C4_Context.puml
|
|
3
|
-
!include ./c4_styles.puml
|
|
4
|
-
!include ./constants.puml
|
|
5
|
-
|
|
6
|
-
left header
|
|
7
|
-
|
|
8
|
-
\t<font size=22 color=black>$SYSTEM_NAME - System View</font>
|
|
9
|
-
|
|
10
|
-
\t<font size=14 color=#484848>The goals of my project are described here</font>
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
end header
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
System(sysName, "Labeling System\n", "", $sprite="cloud", $tags="primary", $link="./level_2_container_view.puml")
|
|
19
|
-
|
|
20
|
-
Boundary(labelers, ' \n<font color=black size=16>Labelers</font>', 'Human Resources') {
|
|
21
|
-
Person(labelerB, "Labelers B", "Labeler Type 2\n", $tags="a")
|
|
22
|
-
Person(labelerA, "Labelers A", "Labeler Type 2\n", $tags="a")
|
|
23
|
-
}
|
|
24
|
-
Rel_U(sysName, labelerB," Trigger samples for labeling")
|
|
25
|
-
Rel_D(labelerB, sysName," Store labeling results ")
|
|
26
|
-
|
|
27
|
-
System(videoSamples, "Videos", "Users and internal customers", $sprite="youtube")
|
|
28
|
-
Rel_D(videoSamples, labelerA, " Send samples to labelers")
|
|
29
|
-
|
|
30
|
-
System(modelTrainingInference, "Model Training + Inference\n\n<font color=#e0575e><$RoboMaker></font>\n", "", $tags="secondary")
|
|
31
|
-
Rel_D(labelerA, modelTrainingInference," Send labels")
|
|
32
|
-
Rel_R(modelTrainingInference, sysName, "Send model scores")
|
|
33
|
-
Rel_L(sysName, modelTrainingInference, "Send labels")
|
|
34
|
-
|
|
35
|
-
HIDE_STEREOTYPE()
|
|
36
|
-
@enduml
|
|
@@ -1,55 +0,0 @@
|
|
|
1
|
-
@startuml
|
|
2
|
-
!include https://raw.githubusercontent.com/plantuml-stdlib/C4-PlantUML/master/C4_Container.puml
|
|
3
|
-
!include ./c4_styles.puml
|
|
4
|
-
!include ./constants.puml
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
left header
|
|
8
|
-
|
|
9
|
-
\t<font size=22 color=black>$SYSTEM_NAME - Container View</font>
|
|
10
|
-
|
|
11
|
-
\t<font size=14 color=black><U+25CF> Goal 1</font>
|
|
12
|
-
\t<font size=14 color=black><U+25CF> Goal 2</font>
|
|
13
|
-
\t<font size=14 color=black><U+25CF> Goal 3</font>
|
|
14
|
-
\t<font size=14 color=black><U+25CF> Goal 4</font>
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
end header
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
Person(labelerA, "Labeler A", "\t Labeler-A Description \t\n", $tags="a")
|
|
24
|
-
ContainerDb(bigQueryLabelDB, "LabelADb", "BigQuery", "A database where type A labels are stored")
|
|
25
|
-
ContainerDb(modelScoreDb, "ModelScoreDB", "BigQuery", "A database where model scores are stored")
|
|
26
|
-
Person(labelerB, "Labeler B", "\t Labeler-B Description \t\n", $tags="a")
|
|
27
|
-
ContainerDb($LABELED_DATASOURCE_ID, "$LABELED_DATASOURCE_NAME", "$LABELED_DATASOURCE_TYPE", $LABELED_DATASOURCE_DESC)
|
|
28
|
-
System_Boundary(sysName, ' \n<font color=black size=16>Label System FooBarBaz</font>') {
|
|
29
|
-
Container(samplingPipeline, "Pipeline", "some-etl-framework", "ETL job for sampling / labeling", $tags="primary", $link="./level_3_component_view_pipeline.puml")
|
|
30
|
-
Container(labelRetrievalJob, "LabelRetrievalJob", "some-etl-framework", "DAG for retrieving and storing label data from labelers.", $tags="primary", $link="./level_3_component_view_label_retrieval_job.puml")
|
|
31
|
-
ContainerDb($SAMPLE_LABELS_ID, $SAMPLE_LABELS_TABLE, $SAMPLE_LABELS_TYPE, $SAMPLE_LABELS_COMPONENT_DESC)
|
|
32
|
-
ContainerDb($DAILY_COLLECTED_SAMPLES_ID, $DAILY_COLLECTED_SAMPLES_TABLE, $DAILY_COLLECTED_SAMPLES_TYPE, $DAILY_COLLECTED_SAMPLES_DESC)
|
|
33
|
-
ContainerQueue(sampleQueue, "SampleQueueFoo", "SQS", "Samples which should be double reviewed")
|
|
34
|
-
Container(labelerBApi, "ReviewQueueAPI", "NodeJS", "API for pulling samples for review", $sprite="server")
|
|
35
|
-
Container(btBqSnapshotJob, "ExportBigQuerySnapshot", "some-etl-framework", "A job to take daily BQ snapshots\nof //$SAMPLE_LABELS_TABLE//", $tags="primary", $link="./level_4_activity_diagram_export_bq_snapshot_job.puml")
|
|
36
|
-
}
|
|
37
|
-
System(modelTrainingInference, "Model Training + Inference\n\n<font color=#e0575e><$RoboMaker></font>\n", "")
|
|
38
|
-
|
|
39
|
-
Rel_R(samplingPipeline, $DAILY_COLLECTED_SAMPLES_ID, " ")
|
|
40
|
-
Rel_D(samplingPipeline, sampleQueue, " ")
|
|
41
|
-
BiRel_D($SAMPLE_LABELS_ID, samplingPipeline, " ")
|
|
42
|
-
Rel_L(labelRetrievalJob, $SAMPLE_LABELS_ID, " ")
|
|
43
|
-
Rel_L($LABELED_DATASOURCE_ID, labelRetrievalJob, " ")
|
|
44
|
-
Rel_R(sampleQueue, labelerBApi, " ")
|
|
45
|
-
Rel_R(labelerBApi, labelerB, " ")
|
|
46
|
-
Rel_R(labelerB, $LABELED_DATASOURCE_ID, " ")
|
|
47
|
-
Rel_L(modelScoreDb, samplingPipeline, " ")
|
|
48
|
-
Rel_D(modelTrainingInference, modelScoreDb, " ")
|
|
49
|
-
Rel_D(bigQueryLabelDB, modelTrainingInference, " ")
|
|
50
|
-
Rel_L(labelerA, bigQueryLabelDB, " ")
|
|
51
|
-
Rel_L($SAMPLE_LABELS_ID, modelTrainingInference, " ")
|
|
52
|
-
Rel_D($SAMPLE_LABELS_ID, btBqSnapshotJob, " ")
|
|
53
|
-
|
|
54
|
-
HIDE_STEREOTYPE()
|
|
55
|
-
@enduml
|
|
@@ -1,40 +0,0 @@
|
|
|
1
|
-
@startuml
|
|
2
|
-
!include https://raw.githubusercontent.com/plantuml-stdlib/C4-PlantUML/master/C4_Component.puml
|
|
3
|
-
!include ./c4_styles.puml
|
|
4
|
-
!include ./constants.puml
|
|
5
|
-
|
|
6
|
-
LAYOUT_LANDSCAPE()
|
|
7
|
-
|
|
8
|
-
left header
|
|
9
|
-
|
|
10
|
-
\t<font size=22 color=black>Some Job - Component View</font>
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
end header
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
Component(scheduler, "Scheduler", "some-etl-framework", "Kicks off the job at some interval")
|
|
19
|
-
Component(extractTransform, "DatasourceExtractTransform", "Operator", "Pull data from labeler datasource and perform any transforms before storing the data")
|
|
20
|
-
Component(loader, "DatasourceLoader", "Operator", "Update //$SAMPLE_LABELS_TABLE// with new labels")
|
|
21
|
-
Component(someMicroService, "SomeMicroService", "SubDag", "Foo bar baz")
|
|
22
|
-
Component(performCalculations, "PerformCalculation", "Operator", "Trigger the calculation of xyz")
|
|
23
|
-
Component(calculatePrevalence, "CalculatePrevalence", "Operator", "Trigger the calculation prevalence and store it")
|
|
24
|
-
ComponentDb($SAMPLE_LABELS_ID, $SAMPLE_LABELS_TABLE, $SAMPLE_LABELS_TYPE, $SAMPLE_LABELS_COMPONENT_DESC)
|
|
25
|
-
|
|
26
|
-
Rel_R(scheduler, extractTransform, " ")
|
|
27
|
-
Rel_R(extractTransform, loader, " ")
|
|
28
|
-
Rel_R(loader, performCalculations, "When all labels retrieved")
|
|
29
|
-
Rel_U(loader, $SAMPLE_LABELS_ID, " ")
|
|
30
|
-
Rel_D(loader, "someMicroService", " ")
|
|
31
|
-
Rel_R(performCalculations, calculatePrevalence, " ")
|
|
32
|
-
|
|
33
|
-
center footer
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
end footer
|
|
38
|
-
|
|
39
|
-
HIDE_STEREOTYPE()
|
|
40
|
-
@enduml
|
|
@@ -1,58 +0,0 @@
|
|
|
1
|
-
@startuml
|
|
2
|
-
!include https://raw.githubusercontent.com/plantuml-stdlib/C4-PlantUML/master/C4_Component.puml
|
|
3
|
-
!include ./c4_styles.puml
|
|
4
|
-
!include ./constants.puml
|
|
5
|
-
|
|
6
|
-
LAYOUT_LANDSCAPE()
|
|
7
|
-
|
|
8
|
-
left header
|
|
9
|
-
|
|
10
|
-
\t<font size=22 color=black>The Pipeline - Component View</font>
|
|
11
|
-
|
|
12
|
-
\t<font size=14 color=black><U+25CF> Goal 1</font>
|
|
13
|
-
\t<font size=14 color=black><U+25CF> Goal 2</font>
|
|
14
|
-
\t<font size=14 color=black><U+25CF> Goal 3</font>
|
|
15
|
-
\t<font size=14 color=black><U+25CF> Goal 4</font>
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
end header
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
Person(labelerB, "Labeler B", " Labeler B Description \n", $tags="a")
|
|
25
|
-
ContainerDb($LABELED_DATASOURCE_ID, "$LABELED_DATASOURCE_NAME", "$LABELED_DATASOURCE_TYPE", $LABELED_DATASOURCE_DESC)
|
|
26
|
-
Boundary(pipeline, " \nEntire Pipeline Infrastructure\n") {
|
|
27
|
-
ComponentDb($SAMPLE_LABELS_ID, $SAMPLE_LABELS_TABLE, $SAMPLE_LABELS_TYPE, $SAMPLE_LABELS_COMPONENT_DESC)
|
|
28
|
-
Boundary(pipelineDag, "Pipeline DAG", $tags="highlightBoundary") {
|
|
29
|
-
Component(scheduler, "Scheduler", "some-etl-framework", "Kicks off pipeline at some interval")
|
|
30
|
-
Component(waitForDependentTables, "Wait For Dependent Tables", "worker", "Waits for dependent tables to be populated")
|
|
31
|
-
Component(prevalenceSampler, "Label Type A Sampler", "worker", "Queries for type A labels", $tags="primary", $link="./level_4_activity_diagram_sampler_a.puml")
|
|
32
|
-
Component(trainingSampler, "Label Type B Sampler", "worker", "Receives all sampling labels type B", $tags="primary", $link="./level_4_activity_diagram_sampler_b.puml")
|
|
33
|
-
Component(sendSamplesForLabeling, "SendSamplesForLabeling", "worker", "Send samples for labeling")
|
|
34
|
-
}
|
|
35
|
-
Container(reviewQueueApi, "ReviewQueueAPI", "NodeJS", "An API used for pulling labels", $sprite="server")
|
|
36
|
-
ComponentDb($DAILY_COLLECTED_SAMPLES_ID, $DAILY_COLLECTED_SAMPLES_TABLE, $DAILY_COLLECTED_SAMPLES_TYPE, $DAILY_COLLECTED_SAMPLES_DESC)
|
|
37
|
-
ComponentQueue(samplesForLabelingQueue, "SamplesForLabelingQueue", "SQS", "Queue for double review")
|
|
38
|
-
}
|
|
39
|
-
ContainerDb(modelScoreDb, "ModelScoreDB", "BigQuery", "A database where the inference score is stored")
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
Rel_R(scheduler, waitForDependentTables, " ")
|
|
44
|
-
Rel_R(waitForDependentTables, prevalenceSampler, "Query used to create table to sample from")
|
|
45
|
-
Rel_R(prevalenceSampler, trainingSampler, " ")
|
|
46
|
-
Rel_U(modelScoreDb, prevalenceSampler, " ")
|
|
47
|
-
Rel_D($SAMPLE_LABELS_ID, prevalenceSampler, "Check what samples have already been sent for labeling**\n** ")
|
|
48
|
-
Rel_R(trainingSampler, sendSamplesForLabeling, " ")
|
|
49
|
-
Rel_D(sendSamplesForLabeling, $SAMPLE_LABELS_ID, " ")
|
|
50
|
-
Rel_R(sendSamplesForLabeling, samplesForLabelingQueue, " ")
|
|
51
|
-
Rel_U(trainingSampler, $DAILY_COLLECTED_SAMPLES_ID, " ")
|
|
52
|
-
Rel_U(prevalenceSampler, $DAILY_COLLECTED_SAMPLES_ID, " ")
|
|
53
|
-
Rel_R(samplesForLabelingQueue, reviewQueueApi, " ")
|
|
54
|
-
Rel_R(reviewQueueApi, labelerB, " ")
|
|
55
|
-
Rel_U(labelerB, $LABELED_DATASOURCE_ID, " ")
|
|
56
|
-
|
|
57
|
-
HIDE_STEREOTYPE()
|
|
58
|
-
@enduml
|
|
@@ -1,79 +0,0 @@
|
|
|
1
|
-
@startuml
|
|
2
|
-
!include ./styles.puml
|
|
3
|
-
!include ./constants.puml
|
|
4
|
-
|
|
5
|
-
center header
|
|
6
|
-
|
|
7
|
-
\t<font size=22 color=black>Sampler A - Activity Diagram</font>
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
end header
|
|
11
|
-
|
|
12
|
-
start
|
|
13
|
-
#E4D4EF: <font size=14>Inputs</font>
|
|
14
|
-
----
|
|
15
|
-
|
|
16
|
-
*<U+00A0>""sample_padding_factor"": Description of // sample_padding_factor //
|
|
17
|
-
|
|
18
|
-
*<U+00A0>""sample_strategies"": Description of // sample_strategies //
|
|
19
|
-
|
|
20
|
-
*<U+00A0>""$TYPE_X_SAMPLE_POOL_TABLE"": $SAMPLE_POOL_TABLE_DESC()
|
|
21
|
-
]
|
|
22
|
-
-> \n;
|
|
23
|
-
|
|
24
|
-
partition "**operator** OperatorABC " {
|
|
25
|
-
:**REMOVE** samples of type x from ""$TYPE_X_SAMPLE_POOL_TABLE"" //i.e.//
|
|
26
|
-
|
|
27
|
-
""SELECT * FROM $TYPE_X_SAMPLE_POOL_TABLE EXCEPT DISTINCT (SELECT * FROM $DAILY_COLLECTED_SAMPLES_TABLE)"";
|
|
28
|
-
while(""**FOR EACH:** sampling_strategy"")
|
|
29
|
-
#E5E5E5: Create sub-query table ]
|
|
30
|
-
#E5E5E5: ""LIMIT"" = ""sampling_strategy.sample_size * sample_padding_factor""]
|
|
31
|
-
#E5E5E5: ""SELECT sampling_strategy.name AS strategy"" (to add strategy column)]
|
|
32
|
-
#LightBlue: <size: 11>**NEXT:** ""sampling_strategy"">
|
|
33
|
-
endwhile;
|
|
34
|
-
: ""UNION ALL"" strategy tables;
|
|
35
|
-
: **DEDUPLICATE** samples by adding the """strategies""" column. For each duplicate add the strategy to the
|
|
36
|
-
""strategies"" column in the distinct row. //i.e.//
|
|
37
|
-
|
|
38
|
-
""SELECT ARRAY_AGG(strategy) as strategies... FROM all_strategies_table GROUP BY uid"";
|
|
39
|
-
: **SAVE** subquery to ""distinct_type_x_sample_pool"" table;
|
|
40
|
-
}
|
|
41
|
-
-> \n;
|
|
42
|
-
-> \n;
|
|
43
|
-
#E4D4EF: <font size=14>Outputs</font>
|
|
44
|
-
----
|
|
45
|
-
|
|
46
|
-
*<U+00A0>""distinct_type_x_sample_pool"": Description of //distinct_type_x_sample_pool// output
|
|
47
|
-
|
|
48
|
-
]
|
|
49
|
-
-> \n;
|
|
50
|
-
-> \n;
|
|
51
|
-
partition "**operator** OperatorDEF\n<font size=10>Note: [[https://google.com More details on algorithm here]]</font> " {
|
|
52
|
-
:**SET** variable... ""extra_space **=** COUNT(*) of $DAILY_COLLECTED_SAMPLES_TABLE WHERE $SEND_FOR_LABELING_COLUMN = TRUE"";
|
|
53
|
-
#LightBlue:while(""**FOR EACH:** sampling_strategy"")
|
|
54
|
-
#E5E5E5: ""samples_for_strategy = SELECT * WHERE sampling_strategy.name IN strategies ORDER BY len(strategies) ASC""]
|
|
55
|
-
|
|
56
|
-
#E5E5E5: **""WHILE len(samples_for_strategy) > sampling_strategy.sample_size:""**
|
|
57
|
-
|
|
58
|
-
****<U+00A0> bla bla bla
|
|
59
|
-
****<U+00A0> code code code;
|
|
60
|
-
|
|
61
|
-
#E5E5E5: **""FOR sample IN samples_for_strategy""**
|
|
62
|
-
|
|
63
|
-
****<U+00A0> 1234
|
|
64
|
-
****<U+00A0> 5678]
|
|
65
|
-
#E5E5E5: **IS** ""len(strategy_samples)"" **less than** ""strategy.sample_size"" **?**]
|
|
66
|
-
split
|
|
67
|
-
<<true>>: **Yes** ]
|
|
68
|
-
<<true>>: ""extra_space += sampling_strategy.sample_size - len(strategy_samples)""]
|
|
69
|
-
split again
|
|
70
|
-
#FFFFFF: **No** ]
|
|
71
|
-
#FFFFFF: \t\t\t\t\t\t Do nothing \t\t\t\t\t\t]
|
|
72
|
-
end split
|
|
73
|
-
#LightBlue: <size: 11>**NEXT:** ""sampling_strategy"">
|
|
74
|
-
endwhile;
|
|
75
|
-
: **REALLOCATE** ""extra_space"" to increase the sample sizes of strategies whose size was reduced;
|
|
76
|
-
: **WRITE** output to ""$SAMPLE_LABELS_TABLE"" table;
|
|
77
|
-
}
|
|
78
|
-
stop
|
|
79
|
-
@enduml
|
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
@startuml
|
|
2
|
-
!include ./styles.puml
|
|
3
|
-
!include ./constants.puml
|
|
4
|
-
|
|
5
|
-
left header
|
|
6
|
-
|
|
7
|
-
\t<font size=22 color=black>Sampler B - Activity Diagram</font>
|
|
8
|
-
|
|
9
|
-
\t<font size=14 color=black>Description of goal </font>
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
end header
|
|
13
|
-
|
|
14
|
-
start
|
|
15
|
-
#E4D4EF: <font size=14>Inputs</font>
|
|
16
|
-
----
|
|
17
|
-
|
|
18
|
-
*<U+00A0>""$TYPE_Y_SAMPLE_POOL_TABLE"": These are samples of type y
|
|
19
|
-
|
|
20
|
-
*<U+00A0>""sample_size"": How many samples to collect
|
|
21
|
-
]
|
|
22
|
-
-> \n;
|
|
23
|
-
|
|
24
|
-
partition "**operator** OperatorFooBar " {
|
|
25
|
-
: **LIMIT** ""$TYPE_Y_SAMPLE_POOL_TABLE"" to ""sample_size"";
|
|
26
|
-
: **SET** ""field_a = 'a'"" on all samples;
|
|
27
|
-
:\t **SET** ""$SEND_FOR_LABELING_COLUMN = (sample IN $SAMPLE_LABELS_TABLE)"" \t;
|
|
28
|
-
:**WRITE** result to $DAILY_COLLECTED_SAMPLES_TABLE table;
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
stop
|
|
32
|
-
@enduml
|
package/puml/level_4_erd.puml
DELETED
|
@@ -1,106 +0,0 @@
|
|
|
1
|
-
@startuml
|
|
2
|
-
!include ./constants.puml
|
|
3
|
-
|
|
4
|
-
hide circle
|
|
5
|
-
skinparam linetype ortho
|
|
6
|
-
skinparam wrapWidth 350
|
|
7
|
-
|
|
8
|
-
left header
|
|
9
|
-
|
|
10
|
-
\t<font size=22 color=black>Entity Relation Diagram</font>
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
end header
|
|
14
|
-
|
|
15
|
-
!global $TABLE_1 = 'table 1'
|
|
16
|
-
!global $TABLE_1_TYPE = 'BigQuery'
|
|
17
|
-
!global $TABLE_1_DESC = 'Description for table 1'
|
|
18
|
-
|
|
19
|
-
!global $TABLE_2 = 'table 2'
|
|
20
|
-
!global $TABLE_2_TYPE = 'BigQuery'
|
|
21
|
-
!global $TABLE_2_DESC = 'Description for table 2'
|
|
22
|
-
|
|
23
|
-
!global $TABLE_3 = 'table 3'
|
|
24
|
-
!global $TABLE_3_TYPE = 'BigQuery'
|
|
25
|
-
!global $TABLE_3_DESC = 'Description for table 3'
|
|
26
|
-
|
|
27
|
-
!global $TABLE_4 = 'table 4'
|
|
28
|
-
!global $TABLE_4_TYPE = 'BigQuery'
|
|
29
|
-
!global $TABLE_4_DESC = 'Description for table 4'
|
|
30
|
-
|
|
31
|
-
entity "**$TABLE_1**" {
|
|
32
|
-
**$TABLE_1_TYPE**
|
|
33
|
-
|
|
34
|
-
$TABLE_1_DESC
|
|
35
|
-
..
|
|
36
|
-
+ ""id"": //string //
|
|
37
|
-
--
|
|
38
|
-
* ""some_id"": //string //: Description for some_id
|
|
39
|
-
* ""some_id_2"": //string// : Description for some_id_2
|
|
40
|
-
* ""config"": // JSON //: Description for config
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
entity "**$TABLE_2**" {
|
|
44
|
-
**$TABLE_2_TYPE**
|
|
45
|
-
|
|
46
|
-
$TABLE_2_DESC
|
|
47
|
-
..
|
|
48
|
-
+ ""id_1"": //string //
|
|
49
|
-
+ ""id_2"": //string //
|
|
50
|
-
--
|
|
51
|
-
* ""label"": //string //: Description for label
|
|
52
|
-
* ""scheduled_at"": //timestamp //: Description for scheduled_at
|
|
53
|
-
* ""foo_bar_ts"": //timestamp //: Description for foo_bar_ts
|
|
54
|
-
* ""baz_ts"": //timestamp //: Description for baz_ts
|
|
55
|
-
* ""abc_ts"": //timestamp //: Description for abc_ts
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
entity "**$TABLE_3**" {
|
|
59
|
-
**$TABLE_3_TYPE**
|
|
60
|
-
|
|
61
|
-
$TABLE_3_DESC
|
|
62
|
-
..
|
|
63
|
-
+ ""id"": //string //
|
|
64
|
-
--
|
|
65
|
-
* ""thingy_id"": //string //: Description for thingy_id
|
|
66
|
-
* ""config"": // JSON //: Configuration used
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
entity "**$DAILY_COLLECTED_SAMPLES_TABLE**" {
|
|
70
|
-
**$DAILY_COLLECTED_SAMPLES_TYPE**
|
|
71
|
-
|
|
72
|
-
$DAILY_COLLECTED_SAMPLES_DESC
|
|
73
|
-
..
|
|
74
|
-
+ ""$CONTENT_ID_COLUMN"": //string //
|
|
75
|
-
--
|
|
76
|
-
*""strategy"": //string //: Description of strategy
|
|
77
|
-
*""$SEND_FOR_LABELING_COLUMN"": //boolean //: Description of send for labeling
|
|
78
|
-
..
|
|
79
|
-
Metadata Columns
|
|
80
|
-
*""model_score"": //number //
|
|
81
|
-
*""num_views"": //number //
|
|
82
|
-
*""is_awesome"": //boolean //
|
|
83
|
-
.
|
|
84
|
-
.
|
|
85
|
-
.
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
entity "**$TABLE_4**" {
|
|
89
|
-
**$TABLE_4_TYPE**
|
|
90
|
-
|
|
91
|
-
$TABLE_4_DESC
|
|
92
|
-
..
|
|
93
|
-
+ ""id"": //string//
|
|
94
|
-
--
|
|
95
|
-
*""name"": //string //: Description of name
|
|
96
|
-
*""value"": //string //: Description of value
|
|
97
|
-
*""labeler_id"": //string //: Description of labeler_id
|
|
98
|
-
*""source"": //string //: Description of source
|
|
99
|
-
*""some_other_field"": //string //: Description of some_other_field
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
"**$TABLE_1**" }o-- "**$TABLE_2**"
|
|
103
|
-
"**$TABLE_2**" }o-- "**$TABLE_3**"
|
|
104
|
-
"**$TABLE_2**" }o-- "**$TABLE_4**"
|
|
105
|
-
|
|
106
|
-
@enduml
|
package/puml/styles.puml
DELETED
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
@startuml
|
|
2
|
-
<style>
|
|
3
|
-
activityDiagram {
|
|
4
|
-
diamond {
|
|
5
|
-
BackgroundColor LightBlue
|
|
6
|
-
}
|
|
7
|
-
|
|
8
|
-
.input {
|
|
9
|
-
MaximumWidth 500
|
|
10
|
-
BackgroundColor #E4D4EF
|
|
11
|
-
}
|
|
12
|
-
}
|
|
13
|
-
* {
|
|
14
|
-
}
|
|
15
|
-
|
|
16
|
-
.true {
|
|
17
|
-
FontColor White
|
|
18
|
-
BackgroundColor Black
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
</style>
|
|
22
|
-
@enduml
|