Merge branch 'master' of https://github.com/Boukefalos/docker-deployment

2015-06-05 14:09:36 +01:00
parent 4c74bdab15
commit bcc1194d48
12 changed files with 376 additions and 8 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -1,8 +1 @@
-/tmp.txt
-/todo.txt
-/test/
-maestro.yaml
-setup
-README.md
 /build/Dockerfile
-/images/phpmyadmin/Dockerfile
--- a/README.md
+++ b/README.md
@@ -0,0 +1,88 @@
+
+## Introduction
+This project bundles a set of scripts to simplify the deployment of docker containers.
+
+The basic idea is adapted from the excellent [Ubuntu base image](https://github.com/phusion/baseimage-docker):
+
+- install software using bash scripts
+- use a tailored init script to ensure correct startup and shutdown behavior
+- use [runit](http://smarden.org/runit/) to supervise services
+- provide a preconfigured SSH server
+- provide a consistent way of managing environment variables
+
+## Installation
+Make sure Docker is [installed](https://docs.docker.com/installation/) correctly, in most cases, the following command should suffice:
+```
+curl http://get.docker.io | sudo sh
+``` 
+Install [maestro](https://github.com/signalfuse/maestro-ng):
+```
+pip install --user --upgrade git+git://github.com/signalfuse/maestro-ng
+``` 
+
+Install nsenter
+```
+docker run --rm -v /usr/local/bin:/target jpetazzo/nsenter
+```
+Then change /etc/default/docker to read:
+```
+DOCKER_OPTS="-e lxc -r=false"
+```
+
+And make sure to restart the docker daemon. This enables LXC related tricks such as 
+```docker attach``` and to allows to modify networking configuration using ```--lxc-conf``` in ```docker run```. It also disables the automatic restart of previously running containers.
+
+## Utility scripts
+
+### In $DOCKER_HOME/bin
+
+#### app
+#### attach
+#### build
+#### clean
+#### killall
+#### make
+#### purge
+#### run
+#### ssh
+#### stopall
+
+### In $DOCKER_HOME/images/&lt;image&gt;/
+#### app
+
+#### run
+
+#### ssh
+
+
+
+Images are build using Makefiles in order to provide reusable pieces of functionality.
+
+## Build files
+
+### Boot
+These scripts are executed by the init script before runit starts any services. Numeric prefixes are used to enforce a specific order of execution. These scripts are used to prepare the container with live parameters from the host. It is more convenient to make service specific modifications in the respective runit scripts than to create separate boot scripts.
+
+### Script
+These scripts are executed during the build and install the software. Numeric prefixes are used to enforce a specific order of execution. The following convention is recommended:
+
+- 00_ for essential modifications to the base image without which other software might fail to install.
+- 01_ for essential modifications regarding the functioning of the base image.
+- 02_ for system wide modifications or the installation of very general system components.
+- 03_ for the installation of image specific software.
+- 99_ to clean the image from unnecessary junk.
+
+### Runit
+These scripts are executed by runit to start services. Runit requires services to run in the foreground and correct parameters must be passed to prevent the service from daemonising. These scripts are also used to initialise files and directories on host volumes.
+
+### Make
+The Makefiles link (symbolic link) the required files (installation script, boot script and runit script) for each separate component into the build directory. These files are quite general since the aforementioned scripts for each separate component are typically named after the build target. Dependencies on other component are defined here.
+
+## Current limitations
+- This approach does not benefit from the caching mechanisms used by Docker. The Dockerfile is generated at build time and installation scripts are added dynamically.
+- There is no mechanism to keep track of software installed during the build. It would be desirable to have a *developer mode* that keeps useful software (e.g., vim, curl, wget) after the build or a *production mode* that removes this software to obtain as small an image as possible.
+
+## Todo
+- Make hardcoded network interface (eth0) configurable
+- Update python pid to port based (kill -9 $(lsof -i udp:5353 -t))
+- Option to use cache and force re-execute scripts by adding $RANDOM
--- a/bin/test/attach
+++ b/bin/test/attach
@@ -0,0 +1 @@
+lxc-attach -n `docker ps --no-trunc | grep $1: | cut -d' ' -f1`
--- a/bin/test/config
+++ b/bin/test/config
@@ -0,0 +1,14 @@
+#!/bin/sh
+set -e
+
+command_test() {
+	echo "$@"
+	echo "$1"
+}
+
+command_remove() {
+    cat bla | xargs echo
+}
+
+command_test bla
+command_test bla da ra
--- a/bin/test/enter
+++ b/bin/test/enter
@@ -0,0 +1,2 @@
+PID=`docker inspect -f "{{ .State.Pid }}" $1`
+nsenter --target $PID --mount --uts --ipc --net --pid -- /bin/bash -l
--- a/bin/test/pidenter
+++ b/bin/test/pidenter
@@ -0,0 +1 @@
+nsenter --target $1 --mount --uts --ipc --net --pid -- /bin/bash -l
--- a/bin/test/setup
+++ b/bin/test/setup
@@ -0,0 +1,51 @@
+#!/bin/sh
+set -e
+#
+# This script is meant for quick & easy install via:
+#   'curl -sL https://raw.githubusercontent.com/Boukefalos/docker-deployment/master/setup | sh'
+# or:
+#   'wget -qO- https://raw.githubusercontent.com/Boukefalos/docker-deployment/master/setup | sh'
+
+home=/opt/docker
+url='https://get.docker.io/'
+
+command_exists() {
+	command -v "$@" > /dev/null 2>&1
+}
+
+if ! command_exists docker; then
+    echo Docker should be installed!
+    exit 0
+fi
+
+user="$(id -un 2>/dev/null || true)"
+
+sh_c='sh -c'
+if [ "$user" != 'root' ]; then
+	if command_exists sudo; then
+		sh_c='sudo sh -c'
+	elif command_exists su; then
+		sh_c='su -c'
+	else
+		echo >&2 'Error: this installer needs the ability to run commands as root.'
+		echo >&2 'We are unable to find either "sudo" or "su" available to make this happen.'
+		exit 1
+	fi
+fi
+
+curl=''
+if command_exists curl; then
+	curl='curl -sL'
+elif command_exists wget; then
+	curl='wget -qO-'
+elif command_exists busybox && busybox --list-modules | grep -q wget; then
+	curl='busybox wget -qO-'
+fi
+
+if [ -z "$DOCKER_HOME" ]; then
+    echo "DOCKER_HOME has not been set, defaulting to $home"
+    export DOCKER_HOME=$home
+    $sh_c "echo DOCKER_HOME=$home >> /etc/environment"
+fi
+
+mkdir -p $DOCKER_HOME
--- a/bin/test/test
+++ b/bin/test/test
@@ -0,0 +1,7 @@
+docker run -t -i -n=false \
+  --lxc-conf="lxc.network.type = veth" \
+  --lxc-conf="lxc.network.flags = up" \
+  --lxc-conf="lxc.network.link = br0" \
+  --lxc-conf="lxc.network.ipv4 = 10.0.2.123" \
+  --lxc-conf="lxc.network.ipv4.gateway=10.0.2.15" \
+  hamachi /bin/bash
--- a/images/webmo/build
+++ b/images/webmo/build
@@ -0,0 +1,12 @@
+/opt/docker/bin/build lighttpd
+DIR=`dirname $0`
+DIR=`readlink -e $DIR`
+BASE=`basename $DIR`
+IFS='/' read -ra ADDR <<< "$DIR"
+CONTAINER=${ADDR[-2]}
+echo $DIR
+cp "/media/mercury/Software/Install/Science/Gaussian/Gaussian 09 Unix.tgz" ${DIR}/../gaussian.tgz
+cp "/media/mercury/Software/Install/Science/WebMO/WebMO.14.0.006.tar.gz" ${DIR}/../webmo.tgz
+docker build -t $CONTAINER ${DIR}/../
+rm ${DIR}/../gaussian.tgz
+rm ${DIR}/../webmo.tgz
--- a/images/webmo/gaussian.int
+++ b/images/webmo/gaussian.int
@@ -0,0 +1,17 @@
+interfaceDescription{'gaussian'}="Ab initio and semi-empirical calculations"
+interfaceDescription{'gaussian_admin'}="Edit preferences specific to Gaussian"
+interfaceDescription{'gaussian_name'}="Gaussian"
+gaussianVersion="g09"
+gaussianBase="/opt/g09"
+gaussianRoot="/opt"
+GAUSS_EXEDIR="/opt/g09/bsd:/opt/g09/private:/opt/g09"
+GMAIN="/opt/g09/bsd:/opt/g09/private:/opt/g09"
+LD_LIBRARY_PATH="/opt/g09/bsd:/opt/g09/private:/opt/g09"
+gaussianBasis="/opt/g09/basis"
+nodesMin="1"
+nodesMax="1"
+nodesDefault="1"
+ppnMin="1"
+ppnMax="1"
+ppnDefault="1"
+nodeTypeDefault=""
--- a/images/webmo/globals.int
+++ b/images/webmo/globals.int
@@ -0,0 +1,52 @@
+catPath="/bin/cat"
+cgiBase="/opt/webmo/source/cgi-bin"
+chmodPath="/bin/chmod"
+chownPath="/bin/chown"
+cpOptions="-r"
+cpPath="/bin/cp"
+externalAuthentication=""
+externalBatchQueue=""
+host=""
+htmlBase="/opt/webmo/source"
+killPath="/bin/kill"
+license="1207-1902-5611"
+loginMsg=""
+loginMsgTimestamp=""
+mkdirPath="/bin/mkdir"
+mvPath="/bin/mv"
+ncores="1"
+owner="Computational Chemistry on the WWW"
+perlPath="/usr/bin/perl"
+psOptions="-ef"
+psPath="/bin/ps"
+psRegExp="(\$pid, \$ppid, \$cpu_time) = /.{8}(.{6})(.{6}).{3}.{6}.{2}(.{16}).*/"
+rcpPath=""
+rmOptions="-rf"
+rmPath="/bin/rm"
+rshPath=""
+scpOptions=""
+scpPath=""
+sendmailOptions="-t"
+sendmailPath=""
+servers=",0"
+sshOptions=""
+sshPath=""
+sudoEnabled=""
+sudoPath="/usr/bin/sudo"
+systemScratch="/tmp"
+tarOptions="cf"
+tarPath="/bin/tar"
+unamePath="/bin/uname"
+uniqueId="39"
+unzipPath=""
+url_cgiBase="/cgi-bin"
+url_htmlBase=""
+userBase="/host/var/webmo"
+userStorageEnabled=""
+version="14.0.006"
+webserver=""
+webserverProtocol=""
+whereisPath="/usr/bin/whereis"
+zcatPath="/bin/zcat"
+zipOptions="-r -q"
+zipPath=""
--- a/todo.txt
+++ b/todo.txt
@@ -0,0 +1,130 @@
+ftp:
+    RUN apt-get install -y vsftpd
+    ENV FTP_CONFIG /etc/vsftpd.conf
+
+mpd:
+    RUN apt-get install mpd -y 
+    ENV MPD_CONFIG /etc/mpd.conf
+
+redis:
+    RUN apt-get install redis-server
+
+    ENV REDIS_CONFIG /etc/redis/redis.conf
+    ENV REDIS_LOG /host/var/log/redis/redis.log
+    ENV REDIS_DATA /host/var/lib/redis
+
+    RUN sed -i "s,^\(daemonize\s*\).*$,\1no," $REDIS_CONFIG
+    RUN sed -i "s,^\(logfile\s*\).*$,\1$REDIS_LOG," $REDIS_CONFIG
+    RUN sed -i "s,^\(dir\s*\).*$,\1$REDIS_DATA," $REDIS_CONFIG
+    
+shipyard:
+    RUN cd /opt && \
+        git clone https://github.com/shipyard/shipyard.git
+
+    RUN curl https://raw.github.com/pypa/pip/master/contrib/get-pip.py | python
+
+    RUN cd /opt/shipyard && \
+        pip install -r requirements.txt || true
+        
+    ENV SHIPYARD_CONFIG /opt/shipyard/shipyard/settings.py
+    ENV SHIPYARD_DB /host/var/lib/shipyard/shipyard.db
+
+    RUN sed -i "s;^\(\s*'NAME':\s*\).*$;\1'$SHIPYARD_DB',;" $SHIPYARD_CONFIG
+    
+    mkdir -p `dirname $SHIPYARD_DB`
+    touch $SHIPYARD_DB
+
+    /opt/shipyard/manage.py syncdb --noinput
+    /opt/shipyard/manage.py migrate
+    /opt/shipyard/manage.py update_admin_user --username=admin --password=kreeft
+    /opt/shipyard/manage.py runserver 0.0.0.0:8000
+    
+minidlna:
+    RUN apt-get install -y gettext libavutil-dev libavcodec-dev libavformat-dev libjpeg-dev libsqlite3-dev libexif-dev libid3tag0-dev libogg-dev libvorbis-dev libFLAC-dev
+
+    ENV MINIDLNA_VERSION 1.1.3
+    ENV MINIDLNA_CONFIG /host/etc/minidlna.conf
+    ENV MINIDLNA_LOG /host/var/log/minidlna.log
+    ENV MINIDLNA_DATA /host/var/lib/minidlna
+
+    RUN cd /opt && \
+        wget http://downloads.sourceforge.net/project/minidlna/minidlna/$MINIDLNA_VERSION/minidlna-$MINIDLNA_VERSION.tar.gz && \
+        tar xzf minidlna-$MINIDLNA_VERSION.tar.gz
+    RUN cd /opt/minidlna-$MINIDLNA_VERSION && \
+        ./configure && \
+        make && \
+        make install
+            
+    mkdir -p $MINIDLNA_DATA
+    /usr/local/sbin/minidlnad -d -f $MINIDLNA_CONFIG >> $MINIDLNA_LOG
+    
+    /opt/docker/bin/run -t $CONTAINER "
+        -p 8200:8200
+        -p 8200:8200/udp
+        -v /opt/docker/images/$CONTAINER/fs/bin:/host/bin
+        -v /opt/docker/images/$CONTAINER/fs/var:/host/var
+        -v /opt/docker/images/$CONTAINER/fs/etc:/host/etc
+        -v /media/neptune/Music:/host/srv/audio
+        -v /media/helios/Photographs:/host/srv/pictures
+        -v /media/neptune/Video:/host/srv/video" $1
+        # -p 1900:1900/udp
+    /opt/docker/bin/run -t $CONTAINER "
+        -v /opt/docker/images/$CONTAINER/fs/bin:/host/bin
+        -v /opt/docker/images/$CONTAINER/fs/var:/host/var
+        -v /opt/docker/images/$CONTAINER/fs/etc:/host/etc
+        -v /media/neptune/Music:/host/srv/audio
+        -v /media/helios/Photographs:/host/srv/pictures
+        -v /media/neptune/Video:/host/srv/video" \
+        -d -n=false \  
+        -lxc-conf="lxc.network.type = veth" \
+        -lxc-conf="lxc.network.flags = up" \
+        -lxc-conf="lxc.network.link = eth0" \
+        -lxc-conf="lxc.network.ipv4 = 192.168.1.151" \
+        -lxc-conf="lxc.network.ipv4.gateway=192.168.1.1" $1
+    
+shipyard:
+    RUN apt-get install -y multiarch-support binutils-multiarch
+    RUN dpkg --add-architecture i386
+    RUN apt-get update
+    RUN apt-get install -y libX11-6:i386 libgdk-pixbuf2.0-0:i386 \
+                           libgtk2.0-0:i386 libsqlite3-0:i386
+
+    #RUN apt-get install -y ia32-libs
+    #RUN apt-get install -y ia32-libs-gtk
+
+syncovery:
+    RUN cd /opt && \
+        wget http://www.superflexible.com/SuperFlexibleLinux.tar.gz && \
+        tar xzf SuperFlexibleLinux.tar.gz && \
+        rm SuperFlexibleLinux.tar.gz
+    RUN chmod +x /opt/SuperFlexibleSynchronizer
+
+    RUN apt-get install -y openssh-server net-tools
+    /opt/SuperFlexibleSynchronizer
+    
+webmo:
+    RUN apt-get install -y csh
+
+    ADD gaussian.tgz /opt
+
+    ENV GAUSS_EXEDIR /opt/g09
+    RUN cd $GAUSS_EXEDIR && \
+        ./bsd/install
+
+    ADD webmo.tgz /opt
+    RUN mv /opt/WebMO.install /opt/webmo
+    RUN ln -s /host/etc/webmo/globals.int /opt/webmo/cgi-bin/interfaces/
+    RUN ln -s /host/etc/webmo/gaussian.int /opt/webmo/cgi-bin/interfaces/
+    RUN ln -s /opt/webmo/scripts/secure.pl /opt/webmo/cgi-bin/
+    RUN sed -i 's,$webserverProtocol://${webserver},,' /opt/webmo/cgi-bin/redirect.cgi
+    RUN usermod -G root www-data
+    
+pyopenssl:
+    install gcc libpython-dev libffi-dev libssl-dev
+    pip install git+git://github.com/pyca/pyopenssl.git
+
+misc:
+    /opt/init --skip-runit sleep infinity
+
+    ARGS="-v /root/.config/pulse:/root/.config/pulse"
+    ARGS="-v /tmp/.X11-unix:/tmp/.X11-unix -v /dev/shm:/dev/shm -v /run/user/${UID}/pulse:/run/user/${UID}/pulse"
				`@@ -0,0 +1 @@`
				lxc-attach -n `docker ps --no-trunc \| grep $1: \| cut -d' ' -f1`
				`@@ -0,0 +1 @@`
				`nsenter --target $1 --mount --uts --ipc --net --pid -- /bin/bash -l`