From bcc1194d48006faff46c3e712763a235e28938c3 Mon Sep 17 00:00:00 2001 From: Rik Veenboer Date: Fri, 5 Jun 2015 14:09:36 +0100 Subject: [PATCH] Merge branch 'master' of https://github.com/Boukefalos/docker-deployment --- .gitignore | 9 +-- README.md | 88 ++++++++++++++++++++++++++ bin/test/attach | 1 + bin/test/config | 14 ++++ bin/test/enter | 2 + bin/test/pidenter | 1 + bin/test/setup | 51 +++++++++++++++ bin/test/test | 7 ++ images/webmo/build | 12 ++++ images/webmo/gaussian.int | 17 +++++ images/webmo/globals.int | 52 +++++++++++++++ todo.txt | 130 ++++++++++++++++++++++++++++++++++++++ 12 files changed, 376 insertions(+), 8 deletions(-) create mode 100644 README.md create mode 100644 bin/test/attach create mode 100644 bin/test/config create mode 100644 bin/test/enter create mode 100644 bin/test/pidenter create mode 100644 bin/test/setup create mode 100644 bin/test/test create mode 100644 images/webmo/build create mode 100644 images/webmo/gaussian.int create mode 100644 images/webmo/globals.int create mode 100644 todo.txt diff --git a/.gitignore b/.gitignore index 1196748..9948f63 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1 @@ -/tmp.txt -/todo.txt -/test/ -maestro.yaml -setup -README.md -/build/Dockerfile -/images/phpmyadmin/Dockerfile +/build/Dockerfile \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..41ea0a2 --- /dev/null +++ b/README.md @@ -0,0 +1,88 @@ + +## Introduction +This project bundles a set of scripts to simplify the deployment of docker containers. + +The basic idea is adapted from the excellent [Ubuntu base image](https://github.com/phusion/baseimage-docker): + +- install software using bash scripts +- use a tailored init script to ensure correct startup and shutdown behavior +- use [runit](http://smarden.org/runit/) to supervise services +- provide a preconfigured SSH server +- provide a consistent way of managing environment variables + +## Installation +Make sure Docker is [installed](https://docs.docker.com/installation/) correctly, in most cases, the following command should suffice: +``` +curl http://get.docker.io | sudo sh +``` +Install [maestro](https://github.com/signalfuse/maestro-ng): +``` +pip install --user --upgrade git+git://github.com/signalfuse/maestro-ng +``` + +Install nsenter +``` +docker run --rm -v /usr/local/bin:/target jpetazzo/nsenter +``` +Then change /etc/default/docker to read: +``` +DOCKER_OPTS="-e lxc -r=false" +``` + +And make sure to restart the docker daemon. This enables LXC related tricks such as +```docker attach``` and to allows to modify networking configuration using ```--lxc-conf``` in ```docker run```. It also disables the automatic restart of previously running containers. + +## Utility scripts + +### In $DOCKER_HOME/bin + +#### app +#### attach +#### build +#### clean +#### killall +#### make +#### purge +#### run +#### ssh +#### stopall + +### In $DOCKER_HOME/images/<image>/ +#### app + +#### run + +#### ssh + + + +Images are build using Makefiles in order to provide reusable pieces of functionality. + +## Build files + +### Boot +These scripts are executed by the init script before runit starts any services. Numeric prefixes are used to enforce a specific order of execution. These scripts are used to prepare the container with live parameters from the host. It is more convenient to make service specific modifications in the respective runit scripts than to create separate boot scripts. + +### Script +These scripts are executed during the build and install the software. Numeric prefixes are used to enforce a specific order of execution. The following convention is recommended: + +- 00_ for essential modifications to the base image without which other software might fail to install. +- 01_ for essential modifications regarding the functioning of the base image. +- 02_ for system wide modifications or the installation of very general system components. +- 03_ for the installation of image specific software. +- 99_ to clean the image from unnecessary junk. + +### Runit +These scripts are executed by runit to start services. Runit requires services to run in the foreground and correct parameters must be passed to prevent the service from daemonising. These scripts are also used to initialise files and directories on host volumes. + +### Make +The Makefiles link (symbolic link) the required files (installation script, boot script and runit script) for each separate component into the build directory. These files are quite general since the aforementioned scripts for each separate component are typically named after the build target. Dependencies on other component are defined here. + +## Current limitations +- This approach does not benefit from the caching mechanisms used by Docker. The Dockerfile is generated at build time and installation scripts are added dynamically. +- There is no mechanism to keep track of software installed during the build. It would be desirable to have a *developer mode* that keeps useful software (e.g., vim, curl, wget) after the build or a *production mode* that removes this software to obtain as small an image as possible. + +## Todo +- Make hardcoded network interface (eth0) configurable +- Update python pid to port based (kill -9 $(lsof -i udp:5353 -t)) +- Option to use cache and force re-execute scripts by adding $RANDOM \ No newline at end of file diff --git a/bin/test/attach b/bin/test/attach new file mode 100644 index 0000000..ee4900b --- /dev/null +++ b/bin/test/attach @@ -0,0 +1 @@ +lxc-attach -n `docker ps --no-trunc | grep $1: | cut -d' ' -f1` diff --git a/bin/test/config b/bin/test/config new file mode 100644 index 0000000..60c39ac --- /dev/null +++ b/bin/test/config @@ -0,0 +1,14 @@ +#!/bin/sh +set -e + +command_test() { + echo "$@" + echo "$1" +} + +command_remove() { + cat bla | xargs echo +} + +command_test bla +command_test bla da ra \ No newline at end of file diff --git a/bin/test/enter b/bin/test/enter new file mode 100644 index 0000000..8773e67 --- /dev/null +++ b/bin/test/enter @@ -0,0 +1,2 @@ +PID=`docker inspect -f "{{ .State.Pid }}" $1` +nsenter --target $PID --mount --uts --ipc --net --pid -- /bin/bash -l diff --git a/bin/test/pidenter b/bin/test/pidenter new file mode 100644 index 0000000..1cf5ba4 --- /dev/null +++ b/bin/test/pidenter @@ -0,0 +1 @@ +nsenter --target $1 --mount --uts --ipc --net --pid -- /bin/bash -l diff --git a/bin/test/setup b/bin/test/setup new file mode 100644 index 0000000..fb97865 --- /dev/null +++ b/bin/test/setup @@ -0,0 +1,51 @@ +#!/bin/sh +set -e +# +# This script is meant for quick & easy install via: +# 'curl -sL https://raw.githubusercontent.com/Boukefalos/docker-deployment/master/setup | sh' +# or: +# 'wget -qO- https://raw.githubusercontent.com/Boukefalos/docker-deployment/master/setup | sh' + +home=/opt/docker +url='https://get.docker.io/' + +command_exists() { + command -v "$@" > /dev/null 2>&1 +} + +if ! command_exists docker; then + echo Docker should be installed! + exit 0 +fi + +user="$(id -un 2>/dev/null || true)" + +sh_c='sh -c' +if [ "$user" != 'root' ]; then + if command_exists sudo; then + sh_c='sudo sh -c' + elif command_exists su; then + sh_c='su -c' + else + echo >&2 'Error: this installer needs the ability to run commands as root.' + echo >&2 'We are unable to find either "sudo" or "su" available to make this happen.' + exit 1 + fi +fi + +curl='' +if command_exists curl; then + curl='curl -sL' +elif command_exists wget; then + curl='wget -qO-' +elif command_exists busybox && busybox --list-modules | grep -q wget; then + curl='busybox wget -qO-' +fi + +if [ -z "$DOCKER_HOME" ]; then + echo "DOCKER_HOME has not been set, defaulting to $home" + export DOCKER_HOME=$home + $sh_c "echo DOCKER_HOME=$home >> /etc/environment" +fi + +mkdir -p $DOCKER_HOME \ No newline at end of file diff --git a/bin/test/test b/bin/test/test new file mode 100644 index 0000000..eee9344 --- /dev/null +++ b/bin/test/test @@ -0,0 +1,7 @@ +docker run -t -i -n=false \ + --lxc-conf="lxc.network.type = veth" \ + --lxc-conf="lxc.network.flags = up" \ + --lxc-conf="lxc.network.link = br0" \ + --lxc-conf="lxc.network.ipv4 = 10.0.2.123" \ + --lxc-conf="lxc.network.ipv4.gateway=10.0.2.15" \ + hamachi /bin/bash diff --git a/images/webmo/build b/images/webmo/build new file mode 100644 index 0000000..892c36c --- /dev/null +++ b/images/webmo/build @@ -0,0 +1,12 @@ +/opt/docker/bin/build lighttpd +DIR=`dirname $0` +DIR=`readlink -e $DIR` +BASE=`basename $DIR` +IFS='/' read -ra ADDR <<< "$DIR" +CONTAINER=${ADDR[-2]} +echo $DIR +cp "/media/mercury/Software/Install/Science/Gaussian/Gaussian 09 Unix.tgz" ${DIR}/../gaussian.tgz +cp "/media/mercury/Software/Install/Science/WebMO/WebMO.14.0.006.tar.gz" ${DIR}/../webmo.tgz +docker build -t $CONTAINER ${DIR}/../ +rm ${DIR}/../gaussian.tgz +rm ${DIR}/../webmo.tgz \ No newline at end of file diff --git a/images/webmo/gaussian.int b/images/webmo/gaussian.int new file mode 100644 index 0000000..8059eda --- /dev/null +++ b/images/webmo/gaussian.int @@ -0,0 +1,17 @@ +interfaceDescription{'gaussian'}="Ab initio and semi-empirical calculations" +interfaceDescription{'gaussian_admin'}="Edit preferences specific to Gaussian" +interfaceDescription{'gaussian_name'}="Gaussian" +gaussianVersion="g09" +gaussianBase="/opt/g09" +gaussianRoot="/opt" +GAUSS_EXEDIR="/opt/g09/bsd:/opt/g09/private:/opt/g09" +GMAIN="/opt/g09/bsd:/opt/g09/private:/opt/g09" +LD_LIBRARY_PATH="/opt/g09/bsd:/opt/g09/private:/opt/g09" +gaussianBasis="/opt/g09/basis" +nodesMin="1" +nodesMax="1" +nodesDefault="1" +ppnMin="1" +ppnMax="1" +ppnDefault="1" +nodeTypeDefault="" diff --git a/images/webmo/globals.int b/images/webmo/globals.int new file mode 100644 index 0000000..0fd46dd --- /dev/null +++ b/images/webmo/globals.int @@ -0,0 +1,52 @@ +catPath="/bin/cat" +cgiBase="/opt/webmo/source/cgi-bin" +chmodPath="/bin/chmod" +chownPath="/bin/chown" +cpOptions="-r" +cpPath="/bin/cp" +externalAuthentication="" +externalBatchQueue="" +host="" +htmlBase="/opt/webmo/source" +killPath="/bin/kill" +license="1207-1902-5611" +loginMsg="" +loginMsgTimestamp="" +mkdirPath="/bin/mkdir" +mvPath="/bin/mv" +ncores="1" +owner="Computational Chemistry on the WWW" +perlPath="/usr/bin/perl" +psOptions="-ef" +psPath="/bin/ps" +psRegExp="(\$pid, \$ppid, \$cpu_time) = /.{8}(.{6})(.{6}).{3}.{6}.{2}(.{16}).*/" +rcpPath="" +rmOptions="-rf" +rmPath="/bin/rm" +rshPath="" +scpOptions="" +scpPath="" +sendmailOptions="-t" +sendmailPath="" +servers=",0" +sshOptions="" +sshPath="" +sudoEnabled="" +sudoPath="/usr/bin/sudo" +systemScratch="/tmp" +tarOptions="cf" +tarPath="/bin/tar" +unamePath="/bin/uname" +uniqueId="39" +unzipPath="" +url_cgiBase="/cgi-bin" +url_htmlBase="" +userBase="/host/var/webmo" +userStorageEnabled="" +version="14.0.006" +webserver="" +webserverProtocol="" +whereisPath="/usr/bin/whereis" +zcatPath="/bin/zcat" +zipOptions="-r -q" +zipPath="" diff --git a/todo.txt b/todo.txt new file mode 100644 index 0000000..8bb894a --- /dev/null +++ b/todo.txt @@ -0,0 +1,130 @@ +ftp: + RUN apt-get install -y vsftpd + ENV FTP_CONFIG /etc/vsftpd.conf + +mpd: + RUN apt-get install mpd -y + ENV MPD_CONFIG /etc/mpd.conf + +redis: + RUN apt-get install redis-server + + ENV REDIS_CONFIG /etc/redis/redis.conf + ENV REDIS_LOG /host/var/log/redis/redis.log + ENV REDIS_DATA /host/var/lib/redis + + RUN sed -i "s,^\(daemonize\s*\).*$,\1no," $REDIS_CONFIG + RUN sed -i "s,^\(logfile\s*\).*$,\1$REDIS_LOG," $REDIS_CONFIG + RUN sed -i "s,^\(dir\s*\).*$,\1$REDIS_DATA," $REDIS_CONFIG + +shipyard: + RUN cd /opt && \ + git clone https://github.com/shipyard/shipyard.git + + RUN curl https://raw.github.com/pypa/pip/master/contrib/get-pip.py | python + + RUN cd /opt/shipyard && \ + pip install -r requirements.txt || true + + ENV SHIPYARD_CONFIG /opt/shipyard/shipyard/settings.py + ENV SHIPYARD_DB /host/var/lib/shipyard/shipyard.db + + RUN sed -i "s;^\(\s*'NAME':\s*\).*$;\1'$SHIPYARD_DB',;" $SHIPYARD_CONFIG + + mkdir -p `dirname $SHIPYARD_DB` + touch $SHIPYARD_DB + + /opt/shipyard/manage.py syncdb --noinput + /opt/shipyard/manage.py migrate + /opt/shipyard/manage.py update_admin_user --username=admin --password=kreeft + /opt/shipyard/manage.py runserver 0.0.0.0:8000 + +minidlna: + RUN apt-get install -y gettext libavutil-dev libavcodec-dev libavformat-dev libjpeg-dev libsqlite3-dev libexif-dev libid3tag0-dev libogg-dev libvorbis-dev libFLAC-dev + + ENV MINIDLNA_VERSION 1.1.3 + ENV MINIDLNA_CONFIG /host/etc/minidlna.conf + ENV MINIDLNA_LOG /host/var/log/minidlna.log + ENV MINIDLNA_DATA /host/var/lib/minidlna + + RUN cd /opt && \ + wget http://downloads.sourceforge.net/project/minidlna/minidlna/$MINIDLNA_VERSION/minidlna-$MINIDLNA_VERSION.tar.gz && \ + tar xzf minidlna-$MINIDLNA_VERSION.tar.gz + RUN cd /opt/minidlna-$MINIDLNA_VERSION && \ + ./configure && \ + make && \ + make install + + mkdir -p $MINIDLNA_DATA + /usr/local/sbin/minidlnad -d -f $MINIDLNA_CONFIG >> $MINIDLNA_LOG + + /opt/docker/bin/run -t $CONTAINER " + -p 8200:8200 + -p 8200:8200/udp + -v /opt/docker/images/$CONTAINER/fs/bin:/host/bin + -v /opt/docker/images/$CONTAINER/fs/var:/host/var + -v /opt/docker/images/$CONTAINER/fs/etc:/host/etc + -v /media/neptune/Music:/host/srv/audio + -v /media/helios/Photographs:/host/srv/pictures + -v /media/neptune/Video:/host/srv/video" $1 + # -p 1900:1900/udp + /opt/docker/bin/run -t $CONTAINER " + -v /opt/docker/images/$CONTAINER/fs/bin:/host/bin + -v /opt/docker/images/$CONTAINER/fs/var:/host/var + -v /opt/docker/images/$CONTAINER/fs/etc:/host/etc + -v /media/neptune/Music:/host/srv/audio + -v /media/helios/Photographs:/host/srv/pictures + -v /media/neptune/Video:/host/srv/video" \ + -d -n=false \ + -lxc-conf="lxc.network.type = veth" \ + -lxc-conf="lxc.network.flags = up" \ + -lxc-conf="lxc.network.link = eth0" \ + -lxc-conf="lxc.network.ipv4 = 192.168.1.151" \ + -lxc-conf="lxc.network.ipv4.gateway=192.168.1.1" $1 + +shipyard: + RUN apt-get install -y multiarch-support binutils-multiarch + RUN dpkg --add-architecture i386 + RUN apt-get update + RUN apt-get install -y libX11-6:i386 libgdk-pixbuf2.0-0:i386 \ + libgtk2.0-0:i386 libsqlite3-0:i386 + + #RUN apt-get install -y ia32-libs + #RUN apt-get install -y ia32-libs-gtk + +syncovery: + RUN cd /opt && \ + wget http://www.superflexible.com/SuperFlexibleLinux.tar.gz && \ + tar xzf SuperFlexibleLinux.tar.gz && \ + rm SuperFlexibleLinux.tar.gz + RUN chmod +x /opt/SuperFlexibleSynchronizer + + RUN apt-get install -y openssh-server net-tools + /opt/SuperFlexibleSynchronizer + +webmo: + RUN apt-get install -y csh + + ADD gaussian.tgz /opt + + ENV GAUSS_EXEDIR /opt/g09 + RUN cd $GAUSS_EXEDIR && \ + ./bsd/install + + ADD webmo.tgz /opt + RUN mv /opt/WebMO.install /opt/webmo + RUN ln -s /host/etc/webmo/globals.int /opt/webmo/cgi-bin/interfaces/ + RUN ln -s /host/etc/webmo/gaussian.int /opt/webmo/cgi-bin/interfaces/ + RUN ln -s /opt/webmo/scripts/secure.pl /opt/webmo/cgi-bin/ + RUN sed -i 's,$webserverProtocol://${webserver},,' /opt/webmo/cgi-bin/redirect.cgi + RUN usermod -G root www-data + +pyopenssl: + install gcc libpython-dev libffi-dev libssl-dev + pip install git+git://github.com/pyca/pyopenssl.git + +misc: + /opt/init --skip-runit sleep infinity + + ARGS="-v /root/.config/pulse:/root/.config/pulse" + ARGS="-v /tmp/.X11-unix:/tmp/.X11-unix -v /dev/shm:/dev/shm -v /run/user/${UID}/pulse:/run/user/${UID}/pulse" \ No newline at end of file