iotedgedev 3.3.4__zip → 3.3.6__zip

s/build/lib/iotedgedev/azurecli.py

@@ -6,6 +6,7 @@ import subprocess
 import sys
 from io import StringIO
 from threading import Thread, Timer
+from pathlib import Path
 
 from azure.cli.core import get_default_cli
 from fstrings import f
@@ -148,10 +149,13 @@ class AzureCli:
                         pass
                 else:
                     err = None
+                    err_list = []
                     try:
-                        err = stderr_queue.get_nowait()
+                        while True:
+                            err_list.append(stderr_queue.get_nowait())
                     except Empty:
                         pass
+                    err = "\n".join(err_list)
                     # Avoid empty sys.excepthook errors from underlying future
                     # There is already a uAMQP issue in work for this
                     # https://github.com/Azure/azure-uamqp-python/issues/30
@@ -465,9 +469,16 @@ class AzureCli:
                 self.output.prompt(
                     "Creating IoT Hub. Please wait as this could take a few minutes to complete...")
 
-                result = self.invoke_az_cli_outproc(["iot", "hub", "create", "--name", value, "--resource-group",
-                                                     resource_group, "--sku", sku, "--query", "[].{\"IoT Hub\":name}", "--out", "table"],
-                                                    f("Could not create the IoT Hub {value} in {resource_group} with sku {sku}."), stdout_io=io, stderr_io=error_io)
+                cmd = ["iot", "hub", "create", "--name", value, "--resource-group", resource_group, "--sku", sku, "--query", "[].{\"IoT Hub\":name}", "--out", "table"]
+                if sku == "F1":
+                    cmd = cmd + ["--partition-count", "2"]
+                    # the default partition-count is 4, but F1 allows only 2
+                    # from `az iot hub create --help`:
+                    #   The partition count is the number of partitions that the IoT Hub uses to distribute messages.
+                    #   The default value is 4.
+                    # In Azure Portal the default and only value is 2.
+
+                result = self.invoke_az_cli_outproc(cmd, f("Could not create the IoT Hub {value} in {resource_group} with sku {sku}."), stdout_io=io, stderr_io=error_io)
                 if not result and error_io.getvalue():
                     self.output.error(error_io.getvalue())
                     self.output.line()
@@ -540,3 +551,35 @@ class AzureCli:
                     return data["connectionString"]
 
         return ''
+
+    def get_ssh_key(self, ssh_key_file):
+        file_path = Path.home().joinpath(".ssh", ssh_key_file + ".pub")
+        if not os.path.exists(file_path):
+            self.output.status(f("No SSH key found at ~/.ssh/{ssh_key_file}.pub. Generating a new one..."))
+            os.system(f("ssh-keygen -t rsa -b 4096 -C \"iotedgedev-autogenerated\" -f ~/.ssh/{ssh_key_file} -q -N \"\""))
+
+        with open(file_path, 'r') as file:
+            ssh_key = file.read()
+            return ssh_key
+
+    def create_edge_vm(self, ssh_key_file, edge_device_id, iothub, resource_group):
+        self.output.status(f("\nCreating a new Virtual Machine in '{resource_group}' with ssh key file '{ssh_key_file}'..."))
+
+        connectionString = self.get_device_connection_string(edge_device_id, iothub, resource_group)
+        # Use IoTHub name for deployment - in this case it will create a new
+        # template if a new IoT Hub is created.
+        deploymentTemplateName = "edgeVM-" + iothub
+
+        ssh_key = self.get_ssh_key(ssh_key_file)
+
+        self.output.status(f("Creating Virtual Machine in '{resource_group}' with an incremental deployment template '{deploymentTemplateName}', this can take some time..."))
+        return self.invoke_az_cli_outproc(["deployment", "group", "create", "--name", deploymentTemplateName,
+                                           "--resource-group", resource_group,
+                                           "--template-uri", "https://raw.githubusercontent.com/Azure/iotedge-vm-deploy/master/edgeDeploy.json",
+                                           "--parameters", "dnsLabelPrefix=iotedgedev-vm",
+                                           "--parameters", "adminUsername=azureuser",
+                                           "--parameters", "authenticationType=sshPublicKey",
+                                           "--parameters", "adminPasswordOrKey=" + ssh_key,
+                                           "--parameters", "deviceConnectionString=" + connectionString,
+                                           "--output", "table"],
+                                          f("Failed to create the VM with deployment template '{deploymentTemplateName}'."))

s/build/lib/iotedgedev/cli.py

@@ -3,6 +3,11 @@ import socket
 import sys
 
 import click
+import copy
+import typing as t
+
+from click import Command
+from click import Group
 from fstrings import f
 
 from .azurecli import AzureCli
@@ -19,6 +24,22 @@ from .simulator import Simulator
 from .solution import Solution
 from .utility import Utility
 
+# Extend click.Group#add_command(cmd, name) to take deprecated=bool flag by monkey patching.
+def add_command_with_deprecation(self, cmd: Command, name: t.Optional[str] = None, deprecated = False, alt:  t.Optional[str] = None):
+    # Directly changing the deprecated flag of the cmd has side-effects on the reference source.
+    # To avoid this, copy the cmd object with deepcopy.
+    copied_command = copy.deepcopy(cmd)
+    copied_command.deprecated = deprecated
+    if copied_command.short_help is None:
+        copied_command.short_help = copied_command.help
+    if alt is not None:
+        alt_cmd = "[Use `iotedgedev " + alt + "` instead]"
+        copied_command.short_help += " " + alt_cmd  # use space for formatting short_help in `iotedgedev --help`
+        copied_command.help += "\n\n" + alt_cmd  # use newline for formatting help in `iotedgedev <command> --help`
+    self.add_command(copied_command, name)
+
+Group.add_command_with_deprecation = add_command_with_deprecation
+
 CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'], max_content_width=120)
 
 output = Output()
@@ -84,14 +105,16 @@ def new(name, module, template, edge_runtime_version, group_id):
     sol = Solution(output, utility)
     sol.create(name, module, template, edge_runtime_version, group_id)
 
-
-main.add_command(new)
+main.add_command_with_deprecation(new, deprecated=True, alt="solution new")
 
 
 @solution.command(context_settings=CONTEXT_SETTINGS,
                   help="Create a new IoT Edge solution and provision Azure resources",
                   # hack to prevent Click truncating help messages
                   short_help="Create a new IoT Edge solution and provision Azure resources")
+@click.argument("name",
+                required=False,
+                default=".")
 @click.option("--edge-runtime-version",
               "-er",
               default="1.2",
@@ -100,28 +123,22 @@ main.add_command(new)
               help="Specify the IoT Edge Runtime Version. Currently available 1.0, 1.1, 1.2")
 @add_module_options(envvars, init=True)
 @with_telemetry
-def init(module, template, group_id, edge_runtime_version):
+def init(name, module, template, group_id, edge_runtime_version):
     if edge_runtime_version is not None:
         if (str(edge_runtime_version) != "1.0" and str(edge_runtime_version) != "1.1" and str(edge_runtime_version) != "1.2"):
             output.info('-edge-runtime-version `{0}` is not valid. Currently supported versions are 1.0, 1.1, 1.2'.format(edge_runtime_version))
             sys.exit()
     utility = Utility(envvars, output)
 
-    if template == "java":
-        solcmd = "iotedgedev new . --module {0} --template {1} --edge-runtime-version {2} --group-id {3}".format(module, template, edge_runtime_version, group_id)
-    else:
-        solcmd = "iotedgedev new . --module {0} --template {1} --edge-runtime-version {2}".format(module, template, edge_runtime_version)
-    output.header(solcmd)
-    ret = utility.call_proc(solcmd.split())
+    sol = Solution(output, utility)
+    sol.create(name, module, template, edge_runtime_version, group_id)
 
-    if ret == 0:
-        azsetupcmd = "iotedgedev iothub setup --update-dotenv"
-        output.header(azsetupcmd)
-        # Had to use call_proc, because @click.invoke doesn't honor prompts
-        utility.call_proc(azsetupcmd.split())
+    azsetupcmd = "iotedgedev iothub setup --update-dotenv"
+    output.header(azsetupcmd)
+    utility.call_proc(azsetupcmd.split())
 
 
-main.add_command(init)
+main.add_command_with_deprecation(init, deprecated=True, alt="solution init")
 
 
 @solution.command(context_settings=CONTEXT_SETTINGS, help="Push, deploy, start, monitor")
@@ -144,7 +161,7 @@ def add(name, template, group_id):
     mod.add(name, template, group_id)
 
 
-main.add_command(add)
+main.add_command_with_deprecation(add, deprecated=True, alt="solution add")
 
 
 @solution.command(context_settings=CONTEXT_SETTINGS, help="Build the solution")
@@ -186,7 +203,7 @@ def build(ctx, push, do_deploy, template_file, platform):
         ctx.invoke(deploy)
 
 
-main.add_command(build)
+main.add_command_with_deprecation(build, deprecated=True, alt="solution build")
 
 
 @solution.command(context_settings=CONTEXT_SETTINGS, help="Push module images to container registry")
@@ -227,7 +244,7 @@ def push(ctx, do_deploy, no_build, template_file, platform):
         ctx.invoke(deploy)
 
 
-main.add_command(push)
+main.add_command_with_deprecation(push, deprecated=True, alt="solution push")
 
 
 @solution.command(context_settings=CONTEXT_SETTINGS, help="Deploy solution to IoT Edge device")
@@ -245,7 +262,7 @@ def deploy(manifest_file):
     edge.deploy(manifest_file)
 
 
-main.add_command(deploy)
+main.add_command_with_deprecation(deploy, deprecated=True, alt="solution deploy")
 
 
 @iothub.command(
@@ -340,7 +357,7 @@ def genconfig(template_file, platform, fail_on_validation_error):
     mod.build_push(template_file, platform, no_build=True, no_push=True, fail_on_validation_error=fail_on_validation_error)
 
 
-main.add_command(genconfig)
+main.add_command_with_deprecation(genconfig, deprecated=True, alt="solution genconfig")
 
 
 @simulator.command(context_settings=CONTEXT_SETTINGS,
@@ -363,7 +380,7 @@ def setup_simulator(gateway_host, iothub_connection_string):
     sim.setup(gateway_host, iothub_connection_string)
 
 
-main.add_command(setup_simulator)
+main.add_command_with_deprecation(setup_simulator, deprecated=True, alt="simulator setup")
 
 
 @simulator.command(context_settings=CONTEXT_SETTINGS,
@@ -438,7 +455,7 @@ def start_simulator(setup, solution, build, manifest_file, platform, verbose, in
         sim.start_single(inputs, port)
 
 
-main.add_command(start_simulator)
+main.add_command_with_deprecation(start_simulator, deprecated=True, alt="simulator start")
 
 
 @simulator.command(context_settings=CONTEXT_SETTINGS,
@@ -450,7 +467,7 @@ def stop_simulator():
     sim.stop()
 
 
-main.add_command(stop_simulator)
+main.add_command_with_deprecation(stop_simulator, deprecated=True, alt="simulator stop")
 
 
 @simulator.command(context_settings=CONTEXT_SETTINGS,
@@ -490,7 +507,7 @@ def monitor(timeout):
     ih.monitor_events(timeout)
 
 
-main.add_command(monitor)
+main.add_command_with_deprecation(monitor, deprecated=True, alt="iothub monitor")
 
 
 def ensure_azure_cli_iot_ext():
@@ -611,25 +628,31 @@ def validate_option(ctx, param, value):
 def list_edge_devices_and_set_default():
     if not azure_cli.list_edge_devices(envvars.IOTHUB_NAME):
         sys.exit()
-    return "iotedgedev-edgedevice"
+    output.info("Enter the IoT Edge Device Id (Creates a new Edge Device if not found) [iotedgedev-edgedevice]:")
+    return input() or "iotedgedev-edgedevice"
 
 
 def list_iot_hubs_and_set_default():
     if not azure_cli.list_iot_hubs(envvars.RESOURCE_GROUP_NAME):
         sys.exit()
 
+    subscription_rg_hash = hashlib.sha1((default_subscriptionId + envvars.RESOURCE_GROUP_NAME).encode('utf-8')).hexdigest()[:6]
+    iothub_name = "iotedgedev-iothub-" + subscription_rg_hash
+
     first_iothub = azure_cli.get_first_iothub(envvars.RESOURCE_GROUP_NAME)
     if first_iothub:
-        return first_iothub
-    else:
-        subscription_rg_hash = hashlib.sha1((default_subscriptionId + envvars.RESOURCE_GROUP_NAME).encode('utf-8')).hexdigest()[:6]
-        return "iotedgedev-iothub-" + subscription_rg_hash
+        iothub_name = first_iothub
+
+    output.info(f"Enter the IoT Hub Name (Creates a new IoT Hub if not found) [{iothub_name}]:")
+    return input() or iothub_name
 
 
 def list_resource_groups_and_set_default():
     if not azure_cli.list_resource_groups():
         sys.exit()
-    return "iotedgedev-rg"
+
+    output.info("Enter Resource Group Name (Creates a new Resource Group if not found) [iotedgedev-rg]:")
+    return input() or "iotedgedev-rg"
 
 
 def list_subscriptions_and_set_default():
@@ -646,14 +669,30 @@ def list_subscriptions_and_set_default():
     if not azure_cli.list_subscriptions():
         sys.exit()
     default_subscriptionId = azure_cli.get_default_subscription()
-    return default_subscriptionId
+    output.info(f"Select an Azure Subscription Name or Id [{default_subscriptionId}]:")
+    return input() or default_subscriptionId
 
 
-def header_and_default(header, default, default2=None):
+def header_and_default(header, prompt, default, options=None):
     output.header(header)
-    if default == '' and default2 is not None:
-        return default2
-    return default
+    output.info(f"{prompt} [{default}]:")
+    output.info(f"Available options: {options}")
+    return input() or default
+
+
+def handle_vm_create():
+    output.header("CREATE VIRTUAL MACHINE")
+    output.info("The virtual machine will be created with the IoT Edge runtime installed.\n"
+                "The deployment template can be found at: https://aka.ms/iotedge-vm-deploy \n"
+                "The name of the virtual machine will be a unique string generated by the template.\n"
+                "Details: os=ubuntu20_04-lts, size=Standard_DS1_v2, username=azureuser, allowSsh=true")
+    ssh_key_file = output.prompt_question("SSH Key file (must be located in ~/.ssh and will be created if it doesn't exist)", "id_rsa_iotedgedev")
+    if not azure_cli.create_edge_vm(ssh_key_file, envvars.EDGE_DEVICE_ID, envvars.IOTHUB_NAME, envvars.RESOURCE_GROUP_NAME):
+        raise click.BadParameter(f('Could not create Virtual Machine in {envvars.RESOURCE_GROUP_NAME}'))
+    else:
+        output.info("Virtual Machine created successfully. Check Azure Portal for details on how to connect.\n"
+                    "WARNING: Check the details of the Network Security Group, it is setup to allow SSH from all addresses.\n"
+                    "You will be able to connect using the SSH Key file: ~/.ssh/" + ssh_key_file)
 
 
 @iothub.command(context_settings=CONTEXT_SETTINGS,
@@ -677,51 +716,46 @@ def header_and_default(header, default, default2=None):
               help="Enter Azure Service Principal Credentials (username password tenant).")
 @click.option('--subscription',
               envvar=envvars.get_envvar_key_if_val("SUBSCRIPTION_ID"),
-              default=lambda: list_subscriptions_and_set_default(),
+              default=list_subscriptions_and_set_default,
               required=True,
               callback=validate_option,
-              prompt="Select an Azure Subscription Name or Id:",
               help="The Azure Subscription Name or Id.")
 @click.option('--resource-group-location',
               envvar=envvars.get_envvar_key_if_val("RESOURCE_GROUP_LOCATION"),
               required=True,
-              default=lambda: header_and_default('RESOURCE GROUP LOCATION', envvars.RESOURCE_GROUP_LOCATION, 'westus'),
-              type=click.Choice(['australiaeast', 'australiasoutheast', 'brazilsouth', 'canadacentral', 'canadaeast', 'centralindia', 'centralus', 'eastasia', 'eastus', 'eastus2',
-                                 'japanwest', 'japaneast', 'northeurope', 'northcentralus', 'southindia', 'uksouth', 'ukwest', 'westus', 'westeurope', 'southcentralus', 'westcentralus', 'westus2']),
+              default=lambda: header_and_default('RESOURCE GROUP LOCATION', 'Enter a Resource Group Location', envvars.RESOURCE_GROUP_LOCATION or 'westus',
+                                                 ['australiaeast', 'australiasoutheast', 'brazilsouth', 'canadacentral', 'canadaeast', 'centralindia', 'centralus', 'eastasia', 'eastus', 'eastus2',
+                                                  'japanwest', 'japaneast', 'northeurope', 'northcentralus', 'southindia', 'uksouth', 'ukwest', 'westus', 'westeurope', 'southcentralus', 'westcentralus', 'westus2']),
+              type=str,
               callback=validate_option,
-              prompt="Enter a Resource Group Location:",
               help="The Resource Group Location.")
 @click.option('--resource-group-name',
               envvar=envvars.get_envvar_key_if_val("RESOURCE_GROUP_NAME"),
               required=True,
-              default=lambda: list_resource_groups_and_set_default(),
+              default=list_resource_groups_and_set_default,
               type=str,
               callback=validate_option,
-              prompt="Enter Resource Group Name (Creates a new Resource Group if not found):",
               help="The Resource Group Name (Creates a new Resource Group if not found).")
 @click.option('--iothub-sku',
               envvar=envvars.get_envvar_key_if_val("IOTHUB_SKU"),
               required=True,
-              default=lambda: header_and_default('IOTHUB SKU', 'F1'),
-              type=click.Choice(['F1', 'S1', 'S2', 'S3']),
+              default=lambda: header_and_default('IOTHUB SKU', 'Enter IoT Hub SKU', 'F1', ['F1', 'S1', 'S2', 'S3']),
+              type=str,
               callback=validate_option,
-              prompt="Enter IoT Hub SKU (F1|S1|S2|S3):",
               help="The IoT Hub SKU.")
 @click.option('--iothub-name',
               envvar=envvars.get_envvar_key_if_val("IOTHUB_NAME"),
               required=True,
-              default=lambda: list_iot_hubs_and_set_default(),
+              default=list_iot_hubs_and_set_default,
               type=str,
               callback=validate_option,
-              prompt='Enter the IoT Hub Name (Creates a new IoT Hub if not found):',
               help='The IoT Hub Name (Creates a new IoT Hub if not found).')
 @click.option('--edge-device-id',
               envvar=envvars.get_envvar_key_if_val("EDGE_DEVICE_ID"),
               required=True,
-              default=lambda: list_edge_devices_and_set_default(),
+              default=list_edge_devices_and_set_default,
               type=str,
               callback=validate_option,
-              prompt='Enter the IoT Edge Device Id (Creates a new Edge Device if not found):',
               help='The IoT Edge Device Id (Creates a new Edge Device if not found).')
 @click.option('--update-dotenv',
               "-u",
@@ -732,6 +766,14 @@ def header_and_default(header, default, default2=None):
               is_flag=True,
               prompt='Update the .env file with connection strings?',
               help='If True, the current .env will be updated with the IoT Hub and Device connection strings.')
+@click.option('--create-edge-vm',
+              envvar=envvars.get_envvar_key_if_val("CREATE_EDGE_VM"),
+              required=True,
+              default=False,
+              show_default=True,
+              is_flag=True,
+              prompt='Create a new Virtual Machine with IoTEdge?',
+              help='If true, creates a new Virtual Machine with IoTEdge.')
 @with_telemetry
 def setup_iothub(credentials,
                  service_principal,
@@ -741,13 +783,17 @@ def setup_iothub(credentials,
                  iothub_sku,
                  iothub_name,
                  edge_device_id,
-                 update_dotenv):
+                 update_dotenv,
+                 create_edge_vm):
     if update_dotenv:
         if envvars.backup_dotenv():
             envvars.save_envvar("IOTHUB_CONNECTION_STRING", envvars.IOTHUB_CONNECTION_STRING)
             envvars.save_envvar("DEVICE_CONNECTION_STRING", envvars.DEVICE_CONNECTION_STRING)
             output.info("Updated current .env file")
 
+    if create_edge_vm:
+        handle_vm_create()
+
 
 @docker.command(context_settings=CONTEXT_SETTINGS,
                 help="Pull Edge runtime images from Microsoft Container Registry and push to your specified container registry. "
@@ -824,7 +870,7 @@ def log(show, save):
     dock.handle_logs_cmd(show, save)
 
 
-main.add_command(log)
+main.add_command_with_deprecation(log, deprecated=True, alt="docker log")
 
 if __name__ == "__main__":
     main()

s/build/lib/iotedgedev/connectionstring.py

@@ -2,8 +2,8 @@ from .utility import Utility
 
 
 class ConnectionString:
-    def __init__(self, value):
-        self.connection_string = value
+    def __init__(self, connection_string: str):
+        self.connection_string = connection_string
         self.data = dict()
 
         if self.connection_string:
@@ -22,25 +22,27 @@ class ConnectionString:
                 else:
                     # unexpected input of connection. Set to None and throw error
                     self.connection_string = None
-               
+
     def __getitem__(self, key):
         return self.data[key]
 
 
 class IoTHubConnectionString(ConnectionString):
-    def __init__(self, value):
-        ConnectionString.__init__(self, value)
+    def __init__(self, connection_string: str):
+        super().__init__(connection_string)
 
         if self.connection_string:
             self.shared_access_key_name = self["sharedaccesskeyname"]
 
 
 class DeviceConnectionString(ConnectionString):
-    def __init__(self, value):
-        ConnectionString.__init__(self, value)
+    def __init__(self, connection_string: str):
+        super().__init__(connection_string)
 
-        if self.connection_string:
+        if("deviceid" in self.data):
             self.device_id = self["deviceid"]
+        else:
+            self.device_id = None
 
 
 class IoTHubHost:

s/build/lib/iotedgedev/edge.py

@@ -14,7 +14,7 @@ class Edge:
         self.output.header("DEPLOYING CONFIGURATION")
 
         self.envvars.verify_envvar_has_val("IOTHUB_CONNECTION_STRING", self.envvars.IOTHUB_CONNECTION_INFO)
-        self.envvars.verify_envvar_has_val("DEVICE_CONNECTION_STRING", self.envvars.DEVICE_CONNECTION_INFO)
+        self.envvars.verify_envvar_has_val("EDGE_DEVICE_ID", self.envvars.DEVICE_CONNECTION_INFO.device_id)
         self.envvars.verify_envvar_has_val("DEPLOYMENT_CONFIG_FILE", self.envvars.DEPLOYMENT_CONFIG_FILE)
 
         if self.azure_cli.set_modules(config=manifest_file,

s/build/lib/iotedgedev/envvars.py

@@ -119,6 +119,7 @@ class EnvVars:
                 self.IOTHUB_NAME = self.get_envvar("IOTHUB_NAME")
                 self.IOTHUB_SKU = self.get_envvar("IOTHUB_SKU")
                 self.EDGE_DEVICE_ID = self.get_envvar("EDGE_DEVICE_ID")
+                self.CREATE_EDGE_VM = self.get_envvar("CREATE_EDGE_VM")
                 self.CREDENTIALS = self.get_envvar("CREDENTIALS")
                 self.UPDATE_DOTENV = self.get_envvar("UPDATE_DOTENV")
 

s/build/lib/iotedgedev/output.py

@@ -55,3 +55,7 @@ class Output:
 
     def confirm(self, text, default=False, abort=True):
         return click.confirm(text, default=default, abort=abort)
+
+    def prompt_question(self, text, default=""):
+        self.line()
+        return click.prompt(text, default=default)

s/docker/tool/build-docker.sh

@@ -15,7 +15,7 @@ fi
 
 # read IoTEdgeDev version from python __init__ file
 
-export VERSION=$(cat ../../iotedgedev/__init__.py | grep '__version__' | grep -oP "'\K[^']+")
+export VERSION=$(cat ../../iotedgedev/__init__.py | grep '__version__' | awk '{print $3}' | sed "s;';;g")
 
 IMAGE_NAME="$1"
 PLATFORM="$2"
@@ -27,7 +27,7 @@ if [ "$IMAGE_NAME" = "--help" ]; then
 fi
 
 PYTHON2="2.7.14" #TODO READ FROM deps.txt
-PYTHON3="3.6.6"
+PYTHON3="3.9.12"
 
 build_linux=1
 build_windows=1
@@ -165,4 +165,4 @@ fi
 
 if [ in_docker_folder = 0 ]; then
     cd original_folder
-fi
+fi

s/docker/tool/deps.txt

@@ -1,5 +1,5 @@
 PYTHON2_VERSION=2.7.14
-PYTHON3_VERSION=3.6.5
+PYTHON3_VERSION=3.9.12
 NODEJS_VERSION=8.11.1
 DOTNETCORESDK_VERSION=2.1.4
 DOCKER_VERSION=17.09.0

s/docker/tool/linux/Dockerfile.base

@@ -1,4 +1,4 @@
-FROM ubuntu:18.04
+FROM mcr.microsoft.com/mirror/docker/library/ubuntu:20.04
 ENV DEBIAN_FRONTEND noninteractive
 ENV DOTNETCORESDK_VERSION 2.1
 ENV DOCKER_COMPOSE_VERSION 1.22.0
@@ -9,7 +9,7 @@ RUN apt-get update && apt-get install -y apt-transport-https ca-certificates cur
     apt-get update && \
     apt-get install -y docker-ce
 RUN apt-get install git gnupg gnupg2 gnupg1 -y && \
-    apt-get install -y --no-install-recommends dialog apt-utils curl apt-transport-https python-pip python3-pip libltdl-dev && \ 
+    apt-get install -y --no-install-recommends dialog apt-utils curl apt-transport-https python3-pip libltdl-dev && \ 
     curl -sL https://deb.nodesource.com/setup_14.x | bash - && \
     apt-get install -y nodejs
 RUN apt-get install -y wget && \
@@ -27,7 +27,10 @@ RUN apt-get update && \
     npm i npm@latest -g && \
     npm i -g azure-iothub yo generator-azure-iot-edge-module && \
     apt-get install -y --no-install-recommends python-dev build-essential libssl-dev libffi-dev libxml2-dev libxslt1-dev zlib1g-dev sudo
-RUN python3 -m pip install --upgrade pip && \
+RUN apt install python3.9 -y && \
+    rm /usr/bin/python3 && \
+    ln -s /usr/bin/python3.9 /usr/bin/python3 && \
+    python3 -m pip install --upgrade pip && \
     pip3 install setuptools && \
     pip3 install cookiecutter
 RUN apt-get -y install openjdk-8-jdk
@@ -58,4 +61,4 @@ USER iotedgedev
 RUN az extension add --name azure-iot && \
     echo 'alias python=python3' >> ~/.bashrc && \
     echo 'alias pip=pip3' >> ~/.bashrc
-ENV DEBIAN_FRONTEND teletype
+ENV DEBIAN_FRONTEND teletype

s/docker/tool/windows/Dockerfile.base

@@ -1,4 +1,4 @@
-FROM microsoft/windowsservercore:1709 AS base
+FROM mcr.microsoft.com/windows/servercore:1709 AS base
 SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
 ARG PYTHON2_VERSION
 ARG PYTHON3_VERSION

s/docs/_Sidebar.md

@@ -1,13 +1,14 @@
 * [Home](home)
 * Dev Machine Setup
-    * [Quickstart with IoT Edge Dev Container](quickstart-with-iot-edge-dev-container)
-    * [Manual Setup](manual-dev-machine-setup)
+  * [Quickstart](quickstart)
+  * [Manual Setup](environment-setup/manual-dev-machine-setup)
 * [Azure Setup](azure-setup)
 * [Edge Device Setup](edge-device-setup)
 * [Command List](command-list)
-* [Python Virtual Environment Setup](python-virtual-environment-setup)
+* [Command Tips](command-tips)
+* [Python Virtual Environment Setup](environment-setup/python-virtual-environment-setup)
 * [Test Coverage](test-coverage)
 * [Troubleshooting](troubleshooting)
 * [Contributing](https://github.com/Azure/iotedgedev/blob/main/CONTRIBUTING.md)
 * Misc
-    * [Migration Guides for Public Preview Users](migration-guides)
+  * [Migration Guides for Public Preview Users](migration-guides)