From c864ea60c9829d38d8bc75181d532174205804e9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C3=ABl=20Arnauts?= <michael.arnauts@gmail.com>
Date: Mon, 2 Jan 2017 22:04:09 +0100
Subject: [PATCH] Improve development workflow in docker (#5079)
* Allow bower install of frontend components as root. Needed for frontend development in docker since everything runs as root in the docker image.
* Improve development workflow in docker
* Use LANG=C.UTF-8 in tox. Fixes installation of libraries with UTF-8 in it's readme.
* Install mysqlclient psycopg2 uvloop after requirements_all.txt again, but with a --no-cache-dir this time. Allows bootstrap_frontend to be executed in a different path like the other scripts.
---
Dockerfile | 2 +-
script/bootstrap | 6 +--
script/bootstrap_frontend | 16 +++++++-
script/bootstrap_server | 6 +++
script/build_frontend | 20 +++++++---
script/build_python_openzwave | 8 +++-
script/dev_docker | 8 +++-
script/dev_openzwave_docker | 4 +-
script/fingerprint_frontend.py | 2 +-
script/lint | 4 +-
script/lint_docker | 7 +++-
script/release | 6 ++-
script/server | 6 +--
script/setup | 8 +++-
script/test | 4 +-
script/test_docker | 9 +++--
script/update | 5 ++-
script/update_mdi.py | 2 +-
tox.ini | 2 +-
virtualization/Docker/Dockerfile.dev | 56 +++++++++++++++++++++++++++
virtualization/Docker/Dockerfile.test | 32 ---------------
21 files changed, 142 insertions(+), 71 deletions(-)
create mode 100644 virtualization/Docker/Dockerfile.dev
delete mode 100644 virtualization/Docker/Dockerfile.test
diff --git a/Dockerfile b/Dockerfile
index 634edb8af59..1141149d9fd 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -23,7 +23,7 @@ RUN script/build_python_openzwave && \
COPY requirements_all.txt requirements_all.txt
RUN pip3 install --no-cache-dir -r requirements_all.txt && \
- pip3 install mysqlclient psycopg2 uvloop
+ pip3 install --no-cache-dir mysqlclient psycopg2 uvloop
# Copy source
COPY . .
diff --git a/script/bootstrap b/script/bootstrap
index f4cb6753fe8..4e77ba60ed5 100755
--- a/script/bootstrap
+++ b/script/bootstrap
@@ -1,9 +1,9 @@
#!/bin/sh
+# Resolve all dependencies that the application requires to run.
-# script/bootstrap: Resolve all dependencies that the application requires to
-# run.
+# Stop on errors
+set -e
cd "$(dirname "$0")/.."
-
script/bootstrap_server
script/bootstrap_frontend
diff --git a/script/bootstrap_frontend b/script/bootstrap_frontend
index 7062ecf3db5..296f56c8f88 100755
--- a/script/bootstrap_frontend
+++ b/script/bootstrap_frontend
@@ -1,7 +1,21 @@
+#!/bin/sh
+# Resolve all frontend dependencies that the application requires to run.
+
+# Stop on errors
+set -e
+
+cd "$(dirname "$0")/.."
+
echo "Bootstrapping frontend..."
+
git submodule update
cd homeassistant/components/frontend/www_static/home-assistant-polymer
+
+# Install node modules
npm install
-./node_modules/.bin/bower install
+
+# Install bower web components. Allow to download the components as root since the user in docker is root.
+./node_modules/.bin/bower install --allow-root
+
npm run setup_js_dev
cd ../../../../..
diff --git a/script/bootstrap_server b/script/bootstrap_server
index f71abda0e65..ccb7d1a8464 100755
--- a/script/bootstrap_server
+++ b/script/bootstrap_server
@@ -1,3 +1,9 @@
+#!/bin/sh
+# Resolve all server dependencies that the application requires to run.
+
+# Stop on errors
+set -e
+
cd "$(dirname "$0")/.."
echo "Installing dependencies..."
diff --git a/script/build_frontend b/script/build_frontend
index a00f89f1eea..0cb976d4fa5 100755
--- a/script/build_frontend
+++ b/script/build_frontend
@@ -1,22 +1,30 @@
+#!/bin/sh
# Builds the frontend for production
+# Stop on errors
+set -e
+
cd "$(dirname "$0")/.."
-cd homeassistant/components/frontend/www_static
-rm -rf core.js* frontend.html* webcomponents-lite.min.js* panels
-cd home-assistant-polymer
+# Clean up
+rm -rf homeassistant/components/frontend/www_static/core.js* \
+ homeassistant/components/frontend/www_static/frontend.html* \
+ homeassistant/components/frontend/www_static/webcomponents-lite.min.js* \
+ homeassistant/components/frontend/www_static/panels
+cd homeassistant/components/frontend/www_static/home-assistant-polymer
npm run clean
-npm run frontend_prod
+# Build frontend
+npm run frontend_prod
cp bower_components/webcomponentsjs/webcomponents-lite.min.js ..
cp -r build/* ..
BUILD_DEV=0 node script/gen-service-worker.js
cp build/service_worker.js ..
-
cd ..
+# Pack frontend
gzip -f -k -9 *.html *.js ./panels/*.html
+cd ../../../..
# Generate the MD5 hash of the new frontend
-cd ../../../..
script/fingerprint_frontend.py
diff --git a/script/build_python_openzwave b/script/build_python_openzwave
index d4e3e07b769..db82fe08d8b 100755
--- a/script/build_python_openzwave
+++ b/script/build_python_openzwave
@@ -1,7 +1,11 @@
-# Sets up and builds python open zwave to be used with Home Assistant
+#!/bin/sh
+# Sets up and builds python open zwave to be used with Home Assistant.
# Dependencies that need to be installed:
# apt-get install cython3 libudev-dev python3-sphinx python3-setuptools
+# Stop on errors
+set -e
+
cd "$(dirname "$0")/.."
if [ ! -d build ]; then
@@ -15,7 +19,7 @@ if [ -d python-openzwave ]; then
git pull --recurse-submodules=yes
git submodule update --init --recursive
else
- git clone --recursive --depth 1 https://github.com/OpenZWave/python-openzwave.git
+ git clone --branch python3 --recursive --depth 1 https://github.com/OpenZWave/python-openzwave.git
cd python-openzwave
fi
diff --git a/script/dev_docker b/script/dev_docker
index b63afaa36da..73c4ee60d0a 100755
--- a/script/dev_docker
+++ b/script/dev_docker
@@ -1,11 +1,15 @@
-# Build and run Home Assinstant in Docker
+#!/bin/sh
+# Build and run Home Assinstant in Docker.
# Optional: pass in a timezone as first argument
# If not given will attempt to mount /etc/localtime
+# Stop on errors
+set -e
+
cd "$(dirname "$0")/.."
-docker build -t home-assistant-dev .
+docker build -t home-assistant-dev -f virtualization/Docker/Dockerfile.dev .
if [ $# -gt 0 ]
then
diff --git a/script/dev_openzwave_docker b/script/dev_openzwave_docker
index 387c38ef6da..7304995f3e1 100755
--- a/script/dev_openzwave_docker
+++ b/script/dev_openzwave_docker
@@ -1,5 +1,5 @@
-# Open a docker that can be used to debug/dev python-openzwave
-# Pass in a command line argument to build
+#!/bin/sh
+# Open a docker that can be used to debug/dev python-openzwave. Pass in a command line argument to build
cd "$(dirname "$0")/.."
diff --git a/script/fingerprint_frontend.py b/script/fingerprint_frontend.py
index e04f3eefe6a..23b959cdc37 100755
--- a/script/fingerprint_frontend.py
+++ b/script/fingerprint_frontend.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python3
-
"""Generate a file with all md5 hashes of the assets."""
+
from collections import OrderedDict
import glob
import hashlib
diff --git a/script/lint b/script/lint
index d607aa87bb6..624ff0e5093 100755
--- a/script/lint
+++ b/script/lint
@@ -1,7 +1,5 @@
#!/bin/sh
-#
-# NOTE: all testing is now driven through tox. The tox command below
-# performs roughly what this test did in the past.
+# Execute lint to spot code mistakes.
if [ "$1" = "--changed" ]; then
export files="`git diff upstream/dev --name-only | grep -e '\.py$'`"
diff --git a/script/lint_docker b/script/lint_docker
index 61f4e4be96a..dca877d49ff 100755
--- a/script/lint_docker
+++ b/script/lint_docker
@@ -1,5 +1,8 @@
-#!/bin/bash
+#!/bin/sh
+# Execute lint in a docker container to spot code mistakes.
+
+# Stop on errors
set -e
-docker build -t home-assistant-test -f virtualization/Docker/Dockerfile.test .
+docker build -t home-assistant-test -f virtualization/Docker/Dockerfile.dev .
docker run --rm -it home-assistant-test tox -e lint
diff --git a/script/release b/script/release
index 43af9b92cb1..3bcddcfef76 100755
--- a/script/release
+++ b/script/release
@@ -1,4 +1,8 @@
-# Pushes a new version to PyPi
+#!/bin/sh
+# Pushes a new version to PyPi.
+
+# Stop on errors
+set -e
cd "$(dirname "$0")/.."
diff --git a/script/server b/script/server
index 0904bfd728e..4d246c8be44 100755
--- a/script/server
+++ b/script/server
@@ -1,8 +1,8 @@
#!/bin/sh
+# Launch the application and any extra required processes locally.
-# script/server: Launch the application and any extra required processes
-# locally.
+# Stop on errors
+set -e
cd "$(dirname "$0")/.."
-
python3 -m homeassistant -c config
diff --git a/script/setup b/script/setup
index 443dee7889f..3dbfc1d2c97 100755
--- a/script/setup
+++ b/script/setup
@@ -1,6 +1,10 @@
-#!/usr/bin/env sh
-cd "$(dirname "$0")/.."
+#!/bin/sh
+# Setups the repository.
+
+# Stop on errors
+set -e
+cd "$(dirname "$0")/.."
git submodule init
script/bootstrap
python3 setup.py develop
diff --git a/script/test b/script/test
index dac5c43d2de..7aca00421b3 100755
--- a/script/test
+++ b/script/test
@@ -1,6 +1,4 @@
#!/bin/sh
-#
-# NOTE: all testing is now driven through tox. The tox command below
-# performs roughly what this test did in the past.
+# Excutes the tests with tox.
tox -e py34
diff --git a/script/test_docker b/script/test_docker
index ab2296cf5fc..78b4247857b 100755
--- a/script/test_docker
+++ b/script/test_docker
@@ -1,5 +1,8 @@
-#!/bin/bash
+#!/bin/sh
+# Excutes the tests with tox in a docker container.
+
+# Stop on errors
set -e
-docker build -t home-assistant-test -f virtualization/Docker/Dockerfile.test .
-docker run --rm -it home-assistant-test tox -e py34
+docker build -t home-assistant-test -f virtualization/Docker/Dockerfile.dev .
+docker run --rm -it home-assistant-test tox -e py35
diff --git a/script/update b/script/update
index 9f8b2530a7e..7866cd7a18d 100755
--- a/script/update
+++ b/script/update
@@ -1,8 +1,9 @@
#!/bin/sh
+# Update application to run for its current checkout.
-# script/update: Update application to run for its current checkout.
+# Stop on errors
+set -e
cd "$(dirname "$0")/.."
-
git pull
git submodule update
diff --git a/script/update_mdi.py b/script/update_mdi.py
index af9ee26c949..f9a0a8aca9f 100755
--- a/script/update_mdi.py
+++ b/script/update_mdi.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python3
-
"""Download the latest Polymer v1 iconset for materialdesignicons.com."""
+
import gzip
import os
import re
diff --git a/tox.ini b/tox.ini
index 1cf402468b5..a5f6ea45f53 100644
--- a/tox.ini
+++ b/tox.ini
@@ -8,7 +8,7 @@ setenv =
; which get read in from setup.py. If we don't force our locale to a
; utf8 one, tox's env is reset. And the install of these 2 packages
; fail.
- LANG=en_US.UTF-8
+ LANG=C.UTF-8
PYTHONPATH = {toxinidir}:{toxinidir}/homeassistant
commands =
py.test --timeout=30 --duration=10 --cov --cov-report= {posargs}
diff --git a/virtualization/Docker/Dockerfile.dev b/virtualization/Docker/Dockerfile.dev
new file mode 100644
index 00000000000..3b5eb493f82
--- /dev/null
+++ b/virtualization/Docker/Dockerfile.dev
@@ -0,0 +1,56 @@
+# Dockerfile for development
+# Based on the production Dockerfile, but with development additions.
+# Keep this file as close as possible to the production Dockerfile, so the environments match.
+
+FROM python:3.5
+MAINTAINER Paulus Schoutsen <Paulus@PaulusSchoutsen.nl>
+
+VOLUME /config
+
+RUN mkdir -p /usr/src/app
+WORKDIR /usr/src/app
+
+RUN pip3 install --no-cache-dir colorlog cython
+
+# For the nmap tracker, bluetooth tracker, Z-Wave, tellstick
+RUN echo "deb http://download.telldus.com/debian/ stable main" >> /etc/apt/sources.list.d/telldus.list && \
+ wget -qO - http://download.telldus.se/debian/telldus-public.key | apt-key add - && \
+ apt-get update && \
+ apt-get install -y --no-install-recommends nmap net-tools cython3 libudev-dev sudo libglib2.0-dev bluetooth libbluetooth-dev \
+ libtelldus-core2 && \
+ apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+COPY script/build_python_openzwave script/build_python_openzwave
+RUN script/build_python_openzwave && \
+ mkdir -p /usr/local/share/python-openzwave && \
+ ln -sf /usr/src/app/build/python-openzwave/openzwave/config /usr/local/share/python-openzwave/config
+
+COPY requirements_all.txt requirements_all.txt
+RUN pip3 install --no-cache-dir -r requirements_all.txt && \
+ pip3 install --no-cache-dir mysqlclient psycopg2 uvloop
+
+# BEGIN: Development additions
+
+# Install nodejs
+RUN curl -sL https://deb.nodesource.com/setup_7.x | sudo -E bash - && \
+ apt-get install -y nodejs
+
+# Install tox
+RUN pip3 install --no-cache-dir tox
+
+# Copy over everything required to run tox
+COPY requirements_test.txt .
+COPY setup.cfg .
+COPY setup.py .
+COPY tox.ini .
+COPY homeassistant/const.py homeassistant/const.py
+
+# Get all dependencies
+RUN tox -e py35 --notest
+
+# END: Development additions
+
+# Copy source
+COPY . .
+
+CMD [ "python", "-m", "homeassistant", "--config", "/config" ]
diff --git a/virtualization/Docker/Dockerfile.test b/virtualization/Docker/Dockerfile.test
deleted file mode 100644
index 651f19e4720..00000000000
--- a/virtualization/Docker/Dockerfile.test
+++ /dev/null
@@ -1,32 +0,0 @@
-FROM python:3.4
-MAINTAINER Paulus Schoutsen <Paulus@PaulusSchoutsen.nl>
-
-VOLUME /config
-
-RUN mkdir -p /usr/src/app
-WORKDIR /usr/src/app
-
-RUN pip3 install --no-cache-dir colorlog cython
-
-# For the nmap tracker, bluetooth tracker, Z-Wave
-RUN apt-get update && \
- apt-get install -y --no-install-recommends nmap net-tools cython3 libudev-dev sudo libglib2.0-dev locales-all bluetooth libbluetooth-dev && \
- apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-
-RUN pip3 install --no-cache-dir tox
-
-# Copy over everything required to run tox
-COPY requirements_all.txt requirements_all.txt
-COPY requirements_test.txt requirements_test.txt
-COPY setup.cfg setup.cfg
-COPY setup.py setup.py
-COPY tox.ini tox.ini
-COPY homeassistant/const.py homeassistant/const.py
-
-# Get deps
-RUN tox --notest
-
-# Copy source and run tests
-COPY . .
-
-CMD [ "tox" ]
--
GitLab