iotedgedev 3.3.4__zip

s/build/lib/iotedgedev/utility.py

@@ -0,0 +1,179 @@
+import fnmatch
+import json
+import os
+import subprocess
+from base64 import b64decode, b64encode
+from hashlib import sha256
+from hmac import HMAC
+from time import time
+
+from urllib.parse import quote, urlencode
+
+from .constants import Constants
+
+
+class Utility:
+    def __init__(self, envvars, output):
+        self.envvars = envvars
+        self.output = output
+
+    def exe_proc(self, params, shell=False, cwd=None, suppress_out=False):
+        proc = subprocess.Popen(
+            params, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=shell, cwd=cwd)
+
+        stdout_data, stderr_data = proc.communicate()
+        if not suppress_out and stdout_data != "":
+            self.output.procout(self.decode(stdout_data))
+
+        if proc.returncode != 0:
+            raise Exception(self.decode(stderr_data))
+
+    def call_proc(self, params, shell=False, cwd=None):
+        try:
+            subprocess.check_call(params, shell=shell, cwd=cwd)
+            return 0
+        except KeyboardInterrupt as ki:
+            return 1
+        except Exception as e:
+            self.output.error("Error while executing command: {0}. {1}".format(' '.join(params), str(e)))
+            return 1
+
+    def check_dependency(self, params, description):
+        try:
+            self.exe_proc(params, shell=not self.envvars.is_posix(), suppress_out=True)
+        except FileNotFoundError:
+            raise FileNotFoundError("{0} is required by the Azure IoT Edge Dev Tool. For installation instructions, see https://aka.ms/iotedgedevwiki.".format(description))
+
+    def is_dir_empty(self, name):
+        if os.path.exists(name):
+            return len(os.listdir(name)) == 0
+        else:
+            return True
+
+    def ensure_dir(self, name):
+        if not os.path.exists(name):
+            os.makedirs(name)
+
+    def find_files(self, directory, pattern):
+        # find all files in directory that match the pattern.
+        for root, dirs, files in os.walk(directory):
+            for basename in files:
+                if fnmatch.fnmatch(basename, pattern):
+                    filename = os.path.join(root, basename)
+                    yield filename
+
+    def get_iot_hub_sas_token(self, uri, key, policy_name, expiry=3600):
+        ttl = time() + expiry
+        sign_key = "%s\n%d" % ((quote(uri)), int(ttl))
+        signature = b64encode(
+            HMAC(b64decode(key), sign_key.encode("utf-8"), sha256).digest())
+
+        rawtoken = {
+            "sr": uri,
+            "sig": signature,
+            "se": str(int(ttl))
+        }
+
+        if policy_name is not None:
+            rawtoken["skn"] = policy_name
+
+        return "SharedAccessSignature " + urlencode(rawtoken)
+
+    @staticmethod
+    def get_file_contents(file, expandvars=False):
+        with open(file, "r") as file:
+            content = file.read()
+            if expandvars:
+                return os.path.expandvars(content)
+            else:
+                return content
+
+    def decode(self, val):
+        return val.decode("utf-8").strip()
+
+    def get_config_files(self):
+        # config files are in root of solution
+
+        return [os.path.join(os.getcwd(), f) for f in os.listdir(os.getcwd()) if f.endswith("template.json")]
+
+    def get_bypass_modules(self):
+        return [module.strip()
+                for module in self.envvars.BYPASS_MODULES.split(",") if module]
+
+    def in_asterisk_list(self, item, asterisk_list):
+        return len(asterisk_list) > 0 and (asterisk_list[0] == "*" or item in asterisk_list)
+
+    def copy_from_template_dir(self, src_file, dest_dir, dest_file=None, replacements=None):
+        if dest_file is None:
+            dest_file = src_file
+
+        template_dir = os.path.join(os.path.split(__file__)[0], "template")
+        self.copy_template(os.path.join(template_dir, src_file), os.path.join(dest_dir, dest_file), replacements, expandvars=False)
+
+    def copy_template(self, src, dest=None, replacements=None, expandvars=True):
+        """Read file at src, replace the keys in replacements with their values, optionally expand environment variables, and save to dest"""
+        if dest is None:
+            dest = src
+
+        content = Utility.get_file_contents(src)
+
+        if replacements:
+            for key, value in replacements.items():
+                content = content.replace(key, value)
+
+        if expandvars:
+            content = os.path.expandvars(content)
+
+        with open(dest, "w") as dest_file:
+            dest_file.write(content)
+
+    def nested_set(self, dict_, key_path, value):
+        current = dict_
+        for key in key_path[:-1]:
+            if key not in current:
+                current[key] = {}
+            current = current.get(key)
+
+        current[key_path[-1]] = value
+
+    def del_key(self, dict_, key_path):
+        if not isinstance(dict_, dict):
+            return None
+
+        current = dict_
+        for key in key_path[:-1]:
+            current = current.get(key, None)
+
+            if not isinstance(current, dict):
+                return None
+
+        return current.pop(key_path[-1], None)
+
+    @staticmethod
+    def get_sha256_hash(val):
+        hash_object = sha256(val.encode('utf-8'))
+
+        return str(hash_object.hexdigest()).lower()
+
+    @staticmethod
+    def get_deployment_manifest_name(template_file, template_schema_ver, platform):
+        if "DEPLOYMENT_CONFIG_FILE" in os.environ:
+            return os.environ["DEPLOYMENT_CONFIG_FILE"]
+
+        if template_schema_ver is None:
+            if os.path.exists(template_file):
+                json_ = json.loads(Utility.get_file_contents(template_file, expandvars=True))
+                template_schema_ver = json_.get("$schema-template", "")
+            else:
+                template_schema_ver = Constants.deployment_template_schema_version
+
+        platform = "." + platform if template_schema_ver > "0.0.1" else ""
+        prefix = os.path.basename(template_file)
+        if prefix.endswith(Constants.deployment_template_suffix):
+            prefix = prefix[:-len(Constants.deployment_template_suffix)]
+        elif prefix.endswith(".json"):
+            prefix = prefix[:-len(".json")]
+        else:
+            prefix = "deployment"
+
+        return "{0}{1}.json".format(prefix, platform)

s/build/lib/iotedgedev/version.py

@@ -0,0 +1,4 @@
+import sys
+
+PY35 = sys.version_info >= (3, 5)
+PY3 = sys.version_info >= (3, 0)

s/build.sh

@@ -0,0 +1,150 @@
+#!/bin/bash
+
+# stop on error
+set -e
+
+function show_help
+{
+    echo "Usage:"
+    echo "build.sh local|test|prod none|minor|major imagename [windows|linux]"
+    echo "local: don't upload to pypi, test: uses pypitest, prod: uses pypi"
+    echo "none: don't bumpversion, minor: bumpversion minor --no-commit --no-tag, major: bumpversion major"
+    echo "imagename: localhost:5000/iotedgedev, jongacr.azurecr.io/iotedgedev, microsoft/iotedgedev"
+    echo "windows: builds only windows container, linux: builds only linux container. omit to build both."
+    echo "NOTES: 1. You must have .pypirc in repo root with pypi and pypitest sections. 2. You must have .env file in root with connection strings set."
+    
+    exit 1
+}
+
+MODE="$1"
+VERSION_BUMP="$2"
+IMAGE_NAME="$3"
+PLATFORM="$4"
+
+if [ -z "$MODE" ] || [ -z "$VERSION_BUMP" ] || [ -z "$IMAGE_NAME" ]; then
+    show_help
+fi
+
+echo -e "\n===== Setting up build environment"
+if [ "$MODE" = "local" ]; then
+    echo "Environment: $MODE"
+elif [ "$MODE" = "test" ]; then
+    echo "Environment: $MODE"
+elif [ "$MODE" = "prod" ]; then
+    echo "Environment: $MODE"
+else
+    echo "ERROR> Build mode parameter not known. must be 'local', 'prod' or 'test'"
+    exit 1
+fi
+
+if [ ! -z $PLATFORM ]; then
+    echo "Platform: $PLATFORM"
+fi
+
+if [ "$OSTYPE" = "msys" ]; then
+    echo -e "\n===== Checking pre-requisistes"
+    IS_ADMIN=$(powershell '([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator")')
+    if [ "$IS_ADMIN" = "False" ]; then        
+        echo "ERROR> Build script must be run as administrator"
+        exit 1
+    fi
+fi
+
+#TODO
+# check if running in administrator mode
+# make sure docker is in linux mode
+# make sure docker supports manifest option
+# stop and restart docker to make sure to avoid networking problem?
+# check that dockerhub exists and is accessible
+# check that pipy repo exists and is accessible
+# make sure there are no pending changes in GIT otherwise bumpversion will complain
+
+function run_tox {
+    echo -e "\n===== Preventive cleanup"
+    rm __pycache__ -rf
+    rm .pytest_cache -rf
+    rm .tox -rf
+    rm .pytest_cache -rf
+    rm tests/__pycache__ -rf
+
+    echo -e "\n===== Running Tox"
+    tox
+}
+
+function get_version
+{
+    VERSION=$(cat ./iotedgedev/__init__.py | grep '__version__' | grep -oP "'\K[^']+")
+    echo ${VERSION}
+}
+
+function run_bumpversion {
+
+    if [ "$VERSION_BUMP" = "none" ]; then
+        return
+    fi
+
+    echo -e "\n===== Bumping version"
+
+    if [ "$MODE" = "prod" ]; then
+        bumpversion $VERSION_BUMP
+    else
+        bumpversion $VERSION_BUMP --no-commit --no-tag --allow-dirty
+    fi
+}
+
+function run_build_wheel
+{
+    echo -e "\n===== Building Python Wheel"
+    python setup.py bdist_wheel 
+}
+
+function run_upload_pypi
+{
+    if [ "$MODE" != "local" ]; then
+        echo -e "\n===== Uploading to PyPi"
+        PYPI=$([ "$MODE" = "prod" ] && echo "pypi" || echo "pypitest")
+        twine upload -r ${PYPI} --config-file .pypirc dist/iotedgedev-$(get_version)-py3-none-any.whl
+    fi
+}
+
+function run_build_docker
+{
+    echo -e "\n===== Building Docker Containers"
+    ./docker/tool/build-docker.sh $IMAGE_NAME $PLATFORM
+}
+
+function run_push_docker 
+{
+    echo -e "\n===== Pushing Docker Containers"
+    ./docker/tool/push-docker.sh $IMAGE_NAME $PLATFORM
+}
+
+function run_push_git
+{
+    if [ "$MODE" = "prod" ]; then
+        echo 'Resetting __init__ file'
+        git checkout ./iotedgedev/__init__.py
+
+        echo -e "\n===== Pushing Tags to Git"
+        git push --tags && git push
+    fi
+}
+
+function set_analytics_key
+{
+    if [[ ${AIKEY//-/} =~ ^[[:xdigit:]]{32}$ ]]; then
+        echo 'Found AIKEY environment variable. Replacing __AIkey__'
+        sed -i "/__AIkey__/c __AIkey__ = '${AIKEY}'" ./iotedgedev/__init__.py
+    fi  
+}
+
+set_analytics_key
+run_bumpversion
+run_tox
+run_build_wheel
+run_build_docker
+run_upload_pypi
+run_push_docker
+run_push_git
+
+echo -e "\n===== All done"

s/cleanup.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+rm __pycache__ -rf
+rm .pytest_cache -rf
+rm .tox -rf
+rm .pytest_cache -rf
+rm tests/__pycache__ -rf

s/docker/tool/build-docker.sh

@@ -0,0 +1,168 @@
+#!/bin/bash
+
+# stop on error
+set -e
+
+# make sure we're in docker folder
+original_folder=$PWD
+
+if [ ! -z "$(echo $PWD | grep /docker/tool$)" ]; then 
+    in_docker_folder=1
+else
+    in_docker_folder=0
+    cd docker/tool
+fi
+
+# read IoTEdgeDev version from python __init__ file
+
+export VERSION=$(cat ../../iotedgedev/__init__.py | grep '__version__' | grep -oP "'\K[^']+")
+
+IMAGE_NAME="$1"
+PLATFORM="$2"
+
+if [ "$IMAGE_NAME" = "--help" ]; then    
+    echo "Usage:"
+    echo "build-docker.sh imagename [linux|windows]"
+    exit 1
+fi
+
+PYTHON2="2.7.14" #TODO READ FROM deps.txt
+PYTHON3="3.6.6"
+
+build_linux=1
+build_windows=1
+
+function switch_docker
+{
+    echo "===== Switching Docker engine"
+    echo "===== From: " $(docker version --format '{{.Server.Os}}')
+    /c/Program\ Files/Docker/Docker/DockerCli.exe -SwitchDaemon
+    echo "===== To: " $(docker version --format '{{.Server.Os}}')
+}
+
+function get_docker_mode
+{
+    echo $(docker version --format '{{.Server.Os}}')
+}
+
+function check_docker_expected_mode 
+{
+    local mode=$(get_docker_mode)    
+
+    if [ $mode != $PLATFORM ]; then
+        echo "===== ERROR: docker is not in expected mode: '$PLATFORM'"
+        exit 1
+    fi    
+}
+
+function build_linux
+{
+    echo "===== Building Linux Based images"
+
+    check_docker_expected_mode "linux"
+
+    cd linux
+
+    rm iotedgedev-$VERSION-py3-none-any.whl --force
+    
+    cp ../../../dist/iotedgedev-$VERSION-py3-none-any.whl iotedgedev-$VERSION-py3-none-any.whl
+
+    docker build \
+        -f Dockerfile.base \
+        -t iotedgedev-linux-base \
+        .
+
+    docker build \
+        -f Dockerfile \
+        --build-arg IOTEDGEDEV_VERSION=$VERSION \
+        -t $IMAGE_NAME:$VERSION-amd64 \
+        -t $IMAGE_NAME:latest-amd64 \
+        -t $IMAGE_NAME:$VERSION \
+        -t $IMAGE_NAME:latest \
+        .
+
+    rm iotedgedev-$VERSION-py3-none-any.whl --force
+
+    cd ..
+}
+
+function build_windows
+{
+    echo "===== Building Windows Based images"
+    
+    check_docker_expected_mode "windows"
+
+    cd windows
+
+    rm iotedgedev-$VERSION-py3-none-any.whl --force
+    
+    cp ../../../dist/iotedgedev-$VERSION-py3-none-any.whl iotedgedev-$VERSION-py3-none-any.whl
+
+    docker build \
+        -f Dockerfile.base \
+        -t iotedgedev-windows-base \
+        --build-arg PYTHON2_VERSION=$PYTHON2 \
+        --build-arg PYTHON3_VERSION=$PYTHON3 \
+        .
+        
+    docker build \
+        -f Dockerfile \
+        --build-arg IOTEDGEDEV_VERSION=$VERSION \
+        -t microsoft/iotedgedev:$VERSION-windows-amd64 \
+        -t microsoft/iotedgedev:latest-windows-amd64 \
+        -t mcr.microsoft.com/public/iotedge/iotedgedev:$VERSION-windows-amd64 \
+        -t mcr.microsoft.com/public/iotedge/iotedgedev:latest-windows-amd64 \
+        -t $IMAGE_NAME:$VERSION-windows-amd64 \
+        -t $IMAGE_NAME:latest-windows-amd64 \
+        .
+
+    rm iotedgedev-$VERSION-py3-none-any.whl --force
+
+    cd ..
+}
+
+
+
+if [ ! -z "$PLATFORM" ];  then
+    if [ "$PLATFORM" = "linux" ]; then
+        build_windows=0
+        echo "===== Building Linux image only"
+    elif [ "$PLATFORM" = "windows" ]; then
+        build_linux=0
+        echo "===== Building Windows image only"
+    else
+        echo "Unknown option: $PLATFORM"
+        echo "Use --help for help"
+        exit 1
+    fi
+else
+    echo "===== Building Windows and Linux images"
+fi
+
+mode=$(get_docker_mode)
+echo "===== Docker is in '$mode' container mode"
+if [ $mode = "windows" ]; then
+    # Docker is in Windows Container mode
+    if [ $build_windows = "1" ]; then
+        build_windows
+    fi
+    if [ $build_linux = "1" ]; then
+        switch_docker
+        build_linux
+        switch_docker
+    fi
+else
+    # Docker is in Linux Container mode
+    if [ $build_linux -eq "1" ]; then    
+        build_linux
+    fi
+    if [ $build_windows -eq "1" ]; then    
+        switch_docker
+        build_windows
+        switch_docker
+    fi
+fi
+
+if [ in_docker_folder = 0 ]; then
+    cd original_folder
+fi

s/docker/tool/deps.txt

@@ -0,0 +1,7 @@
+PYTHON2_VERSION=2.7.14
+PYTHON3_VERSION=3.6.5
+NODEJS_VERSION=8.11.1
+DOTNETCORESDK_VERSION=2.1.4
+DOCKER_VERSION=17.09.0
+DOCKER_COMPOSE_VERSION 1.22.0
+

s/docker/tool/linux/Dockerfile

@@ -0,0 +1,7 @@
+FROM iotedgedev-linux-base
+ARG IOTEDGEDEV_VERSION
+COPY iotedgedev-$IOTEDGEDEV_VERSION-py3-none-any.whl dist/iotedgedev-$IOTEDGEDEV_VERSION-py3-none-any.whl
+RUN sudo -H pip3 install dist/iotedgedev-$IOTEDGEDEV_VERSION-py3-none-any.whl && \
+    sudo rm -rf dist/
+ENV LC_ALL C.UTF-8
+ENV LANG C.UTF-8

s/docker/tool/linux/Dockerfile.base

@@ -0,0 +1,61 @@
+FROM ubuntu:18.04
+ENV DEBIAN_FRONTEND noninteractive
+ENV DOTNETCORESDK_VERSION 2.1
+ENV DOCKER_COMPOSE_VERSION 1.22.0
+ENV MAVEN_VERSION=3.5.4
+RUN apt-get update && apt-get install -y apt-transport-https ca-certificates curl software-properties-common && \
+    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \
+    add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" && \
+    apt-get update && \
+    apt-get install -y docker-ce
+RUN apt-get install git gnupg gnupg2 gnupg1 -y && \
+    apt-get install -y --no-install-recommends dialog apt-utils curl apt-transport-https python-pip python3-pip libltdl-dev && \ 
+    curl -sL https://deb.nodesource.com/setup_14.x | bash - && \
+    apt-get install -y nodejs
+RUN apt-get install -y wget && \
+    wget -qO- https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.asc.gpg && \
+    mv microsoft.asc.gpg /etc/apt/trusted.gpg.d/ && \
+    wget -q https://packages.microsoft.com/config/ubuntu/18.04/prod.list && \
+    mv prod.list /etc/apt/sources.list.d/microsoft-prod.list && \
+    apt-get install -y apt-transport-https && \
+    apt-get update && \
+    apt-get install -y dotnet-sdk-$DOTNETCORESDK_VERSION
+RUN curl -L https://github.com/docker/compose/releases/download/$DOCKER_COMPOSE_VERSION/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose && \
+    chmod +x /usr/local/bin/docker-compose
+RUN curl -sL https://aka.ms/InstallAzureCLIDeb | bash
+RUN apt-get update && \
+    npm i npm@latest -g && \
+    npm i -g azure-iothub yo generator-azure-iot-edge-module && \
+    apt-get install -y --no-install-recommends python-dev build-essential libssl-dev libffi-dev libxml2-dev libxslt1-dev zlib1g-dev sudo
+RUN python3 -m pip install --upgrade pip && \
+    pip3 install setuptools && \
+    pip3 install cookiecutter
+RUN apt-get -y install openjdk-8-jdk
+RUN apt-get update && \
+    apt-get install -y ca-certificates-java && \
+    update-ca-certificates -f && \
+    rm -rf /var/cache/oracle-jdk8-installer
+ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
+ENV PATH $JAVA_HOME/bin:$PATH
+RUN mkdir -p /usr/share/maven /usr/share/maven/ref && \
+    curl -fsSL -o /tmp/apache-maven.tar.gz https://www.apache.org/dist/maven/maven-3/$MAVEN_VERSION/binaries/apache-maven-$MAVEN_VERSION-bin.tar.gz && \
+    tar -xzf /tmp/apache-maven.tar.gz -C /usr/share/maven --strip-components=1 && \
+    rm -f /tmp/apache-maven.tar.gz && \
+    ln -s /usr/share/maven/bin/mvn /usr/bin/mvn
+RUN apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+WORKDIR /home/iotedge
+COPY install-dev.sh /scripts/install-dev.sh
+RUN sed -i 's/\r//' /scripts/install-dev.sh
+
+# Switch to a non-root user because Yeoman does not work with root users
+# https://github.com/Azure/iotedgedev/issues/312
+RUN useradd -m iotedgedev && \
+    echo 'iotedgedev ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers && \
+    chown -R iotedgedev:iotedgedev /home/iotedge
+USER iotedgedev
+# Azure CLI extensions are installed per user
+RUN az extension add --name azure-iot && \
+    echo 'alias python=python3' >> ~/.bashrc && \
+    echo 'alias pip=pip3' >> ~/.bashrc
+ENV DEBIAN_FRONTEND teletype

s/docker/tool/linux/install-dev.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+# This script will be executed INSIDE the container
+
+cd /home/iotedge/tool
+pip install -r requirements_dev.txt
+pip install -e .
+cd /home/iotedge

s/docker/tool/linux/run.ps1

@@ -0,0 +1,5 @@
+# Get IoTEdgeDev source folder
+$source_folder = Get-Location | Split-Path | Split-Path | Split-Path
+
+# Run Docker Container
+docker run -it -v //var//run//docker.sock://var//run//docker.sock -v ${source_folder}:/home/iotedge/tool microsoft/iotedgedev:latest-linux

s/docker/tool/linux/run.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+folder=$(echo $PWD | cut -d/ -f-6 | sed 's,/,//,g')
+
+winpty docker run -it -v //var//run//docker.sock://var//run//docker.sock -v $folder://home//iotedge//tool microsoft/iotedgedev:latest-linux

s/docker/tool/push-docker.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# stop on error
+set -e
+
+function show_help
+{
+    echo "Usage:"
+    echo "push-docker.sh [<version>]"
+    echo ""
+    echo "version: version to push. Automatically detected if not specified"
+    exit 1
+}
+
+# iotedgetoolscontainerregistry.azure.io is the ACR that has a webhook to publish to MCR
+# only this ACR should be used
+ACR_LOGIN_SERVER="iotedgetoolscontainerregistry.azurecr.io"
+IMAGE_NAME="iotedgedev"
+VERSION="$1"
+
+if [ "$VERSION" = "--help" ]; then    
+    show_help
+fi
+
+if [ -z "$VERSION" ]; then
+    echo -e "\n===== Detecting version"
+    VERSION=$(grep '__version__' ../../iotedgedev/__init__.py | grep -oP "'\K[^']+")
+    echo "Detected version $VERSION"
+fi
+
+echo -e "\n===== Pushing Docker images"
+docker push $ACR_LOGIN_SERVER/public/iotedge/$IMAGE_NAME:$VERSION-amd64
+docker push $ACR_LOGIN_SERVER/public/iotedge/$IMAGE_NAME:$VERSION
+docker push $ACR_LOGIN_SERVER/public/iotedge/$IMAGE_NAME:latest-amd64
+docker push $ACR_LOGIN_SERVER/public/iotedge/$IMAGE_NAME:latest

s/docker/tool/windows/Dockerfile

@@ -0,0 +1,4 @@
+FROM iotedgedev-windows-base
+ARG IOTEDGEDEV_VERSION
+COPY iotedgedev-$IOTEDGEDEV_VERSION-py3-none-any.whl dist/iotedgedev-latest-py3-none-any.whl
+RUN pip install dist/iotedgedev-latest-py3-none-any.whl