From beab5e25730f5353630c44ddb1fad362e0a5c619 Mon Sep 17 00:00:00 2001 From: Benedikt Heine Date: Mon, 5 Nov 2018 20:06:47 +0100 Subject: [PATCH 01/21] Remove superfluous packages from production image We don't need git, pip and the dev libraries for rdd in the production image. --- Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index ca1e78b..e90548a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -107,13 +107,11 @@ RUN export DEBIAN_FRONTEND=noninteractive \ && apt-get install --yes --no-install-recommends \ collectd \ expect \ - git \ libcairo2 \ - librrd-dev \ + librrd8 \ memcached \ nginx \ python3-ldap \ - python3-pip \ redis \ sqlite3 \ && apt-get clean \ From fb64f84d11fe08ba5b9d678aaea4425286a311a4 Mon Sep 17 00:00:00 2001 From: Benedikt Heine Date: Mon, 5 Nov 2018 20:54:21 +0100 Subject: [PATCH 02/21] Move ARGs in front of actual usage --- Dockerfile | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/Dockerfile b/Dockerfile index e90548a..1ef0de7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,34 +35,32 @@ RUN python3 -m pip install --upgrade virtualenv virtualenv-tools \ rrdtool ARG version=1.1.4 -ARG whisper_version=${version} -ARG carbon_version=${version} -ARG graphite_version=${version} - -ARG whisper_repo=https://github.com/graphite-project/whisper.git -ARG carbon_repo=https://github.com/graphite-project/carbon.git -ARG graphite_repo=https://github.com/graphite-project/graphite-web.git - -ARG statsd_version=v0.8.0 - -ARG statsd_repo=https://github.com/etsy/statsd.git # install whisper -RUN git clone -b ${whisper_version} --depth 1 ${whisper_repo} /usr/local/src/whisper -WORKDIR /usr/local/src/whisper -RUN . /opt/graphite/bin/activate && python3 ./setup.py install +ARG whisper_version=${version} +ARG whisper_repo=https://github.com/graphite-project/whisper.git +RUN git clone -b ${whisper_version} --depth 1 ${whisper_repo} /usr/local/src/whisper \ + && cd /usr/local/src/whisper \ + && . /opt/graphite/bin/activate \ + && python3 ./setup.py install # install carbon -RUN git clone -b ${carbon_version} --depth 1 ${carbon_repo} /usr/local/src/carbon -WORKDIR /usr/local/src/carbon -RUN . /opt/graphite/bin/activate && pip3 install -r requirements.txt \ - && python3 ./setup.py install +ARG carbon_version=${version} +ARG carbon_repo=https://github.com/graphite-project/carbon.git +RUN . /opt/graphite/bin/activate \ + && git clone -b ${carbon_version} --depth 1 ${carbon_repo} /usr/local/src/carbon \ + && cd /usr/local/src/carbon \ + && pip3 install -r requirements.txt \ + && python3 ./setup.py install # install graphite -RUN git clone -b ${graphite_version} --depth 1 ${graphite_repo} /usr/local/src/graphite-web -WORKDIR /usr/local/src/graphite-web -RUN . /opt/graphite/bin/activate && pip3 install -r requirements.txt \ - && python3 ./setup.py install +ARG graphite_version=${version} +ARG graphite_repo=https://github.com/graphite-project/graphite-web.git +RUN . /opt/graphite/bin/activate \ + && git clone -b ${graphite_version} --depth 1 ${graphite_repo} /usr/local/src/graphite-web \ + && cd /usr/local/src/graphite-web \ + && pip3 install -r requirements.txt \ + && python3 ./setup.py install # fixing RRD support (see https://github.com/graphite-project/docker-graphite-statsd/issues/63) RUN sed -i \ @@ -75,7 +73,9 @@ RUN wget https://nodejs.org/download/release/v6.14.4/node-v6.14.4-linux-x64.tar. tar -xvpzf node-v6.14.4-linux-x64.tar.gz && rm node-v6.14.4-linux-x64.tar.gz && mv node-v6.14.4-linux-x64 nodejs # install statsd -RUN git clone -b ${statsd_version} ${statsd_repo} /opt/statsd +ARG statsd_version=v0.8.0 +ARG statsd_repo=https://github.com/etsy/statsd.git +RUN git clone -b ${statsd_version} --depth 1 ${statsd_repo} /opt/statsd # config graphite COPY conf/opt/graphite/conf/*.conf /opt/graphite/conf/ From e7657c3728637d6d1072dc6ddf759be60e01bbc5 Mon Sep 17 00:00:00 2001 From: Benedikt Heine Date: Mon, 5 Nov 2018 20:59:43 +0100 Subject: [PATCH 03/21] Remove timezone ARG The timezone argument can get seeded alternatively via the TZ env variable or the special GRAPHITE_TIME_ZONE env. --- Dockerfile | 9 --------- 1 file changed, 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1ef0de7..c7a0a04 100644 --- a/Dockerfile +++ b/Dockerfile @@ -92,15 +92,6 @@ LABEL maintainer="Denys Zhdanov " ENV STATSD_INTERFACE udp -# choose a timezone at build-time -# use `--build-arg CONTAINER_TIMEZONE=Europe/Brussels` in `docker build` -ARG CONTAINER_TIMEZONE - -RUN if [ ! -z "${CONTAINER_TIMEZONE}" ]; \ - then ln -sf /usr/share/zoneinfo/$CONTAINER_TIMEZONE /etc/localtime && \ - dpkg-reconfigure -f noninteractive tzdata; \ - fi - RUN export DEBIAN_FRONTEND=noninteractive \ && apt-get update --fix-missing \ && apt-get -y upgrade \ From 480e3c385aa8f5e31e63a5c78cf281104cbf4b48 Mon Sep 17 00:00:00 2001 From: Benedikt Heine Date: Mon, 5 Nov 2018 21:07:49 +0100 Subject: [PATCH 04/21] Make scripts runnable on normal bourne shell (not bash) --- conf/etc/my_init.d/01_conf_init.sh | 2 +- conf/etc/service/carbon-aggregator/run | 4 ++-- conf/etc/service/carbon/run | 4 ++-- conf/etc/service/collectd/run | 6 +++--- conf/etc/service/graphite/run | 6 +++--- conf/etc/service/nginx/run | 2 +- conf/etc/service/redis/run | 6 +++--- conf/etc/service/statsd/run | 3 +-- conf/usr/local/bin/manage.sh | 2 +- 9 files changed, 17 insertions(+), 18 deletions(-) diff --git a/conf/etc/my_init.d/01_conf_init.sh b/conf/etc/my_init.d/01_conf_init.sh index f533bd5..d69a298 100755 --- a/conf/etc/my_init.d/01_conf_init.sh +++ b/conf/etc/my_init.d/01_conf_init.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh conf_dir=/etc/graphite-statsd/conf diff --git a/conf/etc/service/carbon-aggregator/run b/conf/etc/service/carbon-aggregator/run index 2021f7f..74d2e1e 100755 --- a/conf/etc/service/carbon-aggregator/run +++ b/conf/etc/service/carbon-aggregator/run @@ -1,5 +1,5 @@ -#!/bin/bash +#!/bin/sh rm -f /opt/graphite/storage/carbon-aggregator-a.pid -source /opt/graphite/bin/activate +. /opt/graphite/bin/activate exec python3 /opt/graphite/bin/carbon-aggregator.py start --debug 2>&1 >> /var/log/carbon-aggregator.log diff --git a/conf/etc/service/carbon/run b/conf/etc/service/carbon/run index 81be778..3319203 100755 --- a/conf/etc/service/carbon/run +++ b/conf/etc/service/carbon/run @@ -1,5 +1,5 @@ -#!/bin/bash +#!/bin/sh rm -f /opt/graphite/storage/carbon-cache-a.pid -source /opt/graphite/bin/activate +. /opt/graphite/bin/activate exec python3 /opt/graphite/bin/carbon-cache.py start --debug 2>&1 >> /var/log/carbon.log diff --git a/conf/etc/service/collectd/run b/conf/etc/service/collectd/run index 57db6a3..554e9ce 100755 --- a/conf/etc/service/collectd/run +++ b/conf/etc/service/collectd/run @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh -[[ -n ${COLLECTD} ]] || exit 1 -exec /usr/sbin/collectd -f -C /etc/collectd/collectd.conf \ No newline at end of file +[ -n "${COLLECTD}" ] || exit 1 +exec /usr/sbin/collectd -f -C /etc/collectd/collectd.conf diff --git a/conf/etc/service/graphite/run b/conf/etc/service/graphite/run index 5f153c7..f3771b9 100755 --- a/conf/etc/service/graphite/run +++ b/conf/etc/service/graphite/run @@ -1,7 +1,7 @@ -#!/bin/bash +#!/bin/sh sv start nginx || exit 1 -source /opt/graphite/bin/activate +. /opt/graphite/bin/activate export GRAPHITE_WSGI_PROCESSES=${GRAPHITE_WSGI_PROCESSES:-4} export GRAPHITE_WSGI_THREADS=${GRAPHITE_WSGI_THREADS:-2} export GRAPHITE_WSGI_REQUEST_TIMEOUT=${GRAPHITE_WSGI_REQUEST_TIMEOUT:-65} @@ -15,4 +15,4 @@ exec /opt/graphite/bin/gunicorn wsgi --preload --pythonpath=/opt/graphite/webapp --max-requests=${GRAPHITE_WSGI_MAX_REQUESTS} \ --timeout=${GRAPHITE_WSGI_REQUEST_TIMEOUT} \ --bind=0.0.0.0:8080 \ - --log-file=/var/log/gunicorn.log \ No newline at end of file + --log-file=/var/log/gunicorn.log diff --git a/conf/etc/service/nginx/run b/conf/etc/service/nginx/run index 7691c03..fdeca60 100755 --- a/conf/etc/service/nginx/run +++ b/conf/etc/service/nginx/run @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh mkdir -p /var/log/nginx exec /usr/sbin/nginx -c /etc/nginx/nginx.conf diff --git a/conf/etc/service/redis/run b/conf/etc/service/redis/run index 2aff14f..ebabb4e 100755 --- a/conf/etc/service/redis/run +++ b/conf/etc/service/redis/run @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh -[[ -n ${REDIS_TAGDB} ]] || exit 1 -exec /usr/bin/redis-server /etc/redis/redis.conf \ No newline at end of file +[ -n "${REDIS_TAGDB}" ] || exit 1 +exec /usr/bin/redis-server /etc/redis/redis.conf diff --git a/conf/etc/service/statsd/run b/conf/etc/service/statsd/run index 00e555e..dd74d4d 100755 --- a/conf/etc/service/statsd/run +++ b/conf/etc/service/statsd/run @@ -1,4 +1,3 @@ -#!/bin/bash +#!/bin/sh exec /opt/nodejs/bin/node /opt/statsd/stats.js /opt/statsd/config_$STATSD_INTERFACE.js >> /var/log/statsd.log 2>&1 - diff --git a/conf/usr/local/bin/manage.sh b/conf/usr/local/bin/manage.sh index aaf52e0..48d68f8 100755 --- a/conf/usr/local/bin/manage.sh +++ b/conf/usr/local/bin/manage.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh # Django 1.11 PYTHONPATH=/opt/graphite/webapp /opt/graphite/bin/django-admin.py makemigrations --settings=graphite.settings PYTHONPATH=/opt/graphite/webapp /opt/graphite/bin/django-admin.py migrate auth --settings=graphite.settings From 14154a04ad763717e38bfaba0e311dad20bfb490 Mon Sep 17 00:00:00 2001 From: Benedikt Heine Date: Mon, 5 Nov 2018 21:19:29 +0100 Subject: [PATCH 05/21] Move expect script into entrypoint This script shall only run during initialisation --- Dockerfile | 2 -- conf/etc/my_init.d/01_conf_init.sh | 16 ++++++++++++++- conf/usr/local/bin/django_admin_init.exp | 26 ------------------------ conf/usr/local/bin/manage.sh | 11 ---------- 4 files changed, 15 insertions(+), 40 deletions(-) delete mode 100755 conf/usr/local/bin/django_admin_init.exp delete mode 100755 conf/usr/local/bin/manage.sh diff --git a/Dockerfile b/Dockerfile index c7a0a04..6d578f0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -97,7 +97,6 @@ RUN export DEBIAN_FRONTEND=noninteractive \ && apt-get -y upgrade \ && apt-get install --yes --no-install-recommends \ collectd \ - expect \ libcairo2 \ librrd8 \ memcached \ @@ -120,7 +119,6 @@ COPY conf /etc/graphite-statsd/conf/ # copy /opt from build image COPY --from=build /opt /opt -RUN /usr/local/bin/django_admin_init.exp # defaults EXPOSE 80 2003-2004 2023-2024 8080 8125 8125/udp 8126 diff --git a/conf/etc/my_init.d/01_conf_init.sh b/conf/etc/my_init.d/01_conf_init.sh index d69a298..6cad1bd 100755 --- a/conf/etc/my_init.d/01_conf_init.sh +++ b/conf/etc/my_init.d/01_conf_init.sh @@ -26,7 +26,21 @@ fi if folder_empty /opt/graphite/storage; then mkdir -p /opt/graphite/storage/whisper - /usr/local/bin/django_admin_init.exp + + export PYTHONPATH=/opt/graphite/webapp + export DJANGO_SETTINGS_MODULE=graphite.settings + + /opt/graphite/bin/django-admin.py makemigrations + /opt/graphite/bin/django-admin.py migrate auth + /opt/graphite/bin/django-admin.py migrate --run-syncdb + /opt/graphite/bin/django-admin.py createsuperuser + --noinput \ + --email root.graphite@mailinator.com \ + --username root << ENDINPUT +root +root +ENDINPUT + fi if folder_empty /opt/graphite/webapp/graphite/functions/custom; then diff --git a/conf/usr/local/bin/django_admin_init.exp b/conf/usr/local/bin/django_admin_init.exp deleted file mode 100755 index e28b512..0000000 --- a/conf/usr/local/bin/django_admin_init.exp +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env expect - -set timeout -1 -spawn /usr/local/bin/manage.sh - -#expect "Would you like to create one now" { -# send "yes\r" -#} - -expect "Username (leave blank to use 'root'):" { - send "root\r" -} - -expect "Email address:" { - send "root.graphite@mailinator.com\r" -} - -expect "Password:" { - send "root\r" -} - -expect "Password *:" { - send "root\r" -} - -expect "Superuser created successfully" diff --git a/conf/usr/local/bin/manage.sh b/conf/usr/local/bin/manage.sh deleted file mode 100755 index 48d68f8..0000000 --- a/conf/usr/local/bin/manage.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh -# Django 1.11 -PYTHONPATH=/opt/graphite/webapp /opt/graphite/bin/django-admin.py makemigrations --settings=graphite.settings -PYTHONPATH=/opt/graphite/webapp /opt/graphite/bin/django-admin.py migrate auth --settings=graphite.settings -PYTHONPATH=/opt/graphite/webapp /opt/graphite/bin/django-admin.py migrate --run-syncdb --settings=graphite.settings -PYTHONPATH=/opt/graphite/webapp /opt/graphite/bin/django-admin.py createsuperuser --settings=graphite.settings - -# Django 1.8 -# PYTHONPATH=/opt/graphite/webapp django-admin.py syncdb --settings=graphite.settings -# Django 1.6 -# PYTHONPATH=/opt/graphite/webapp django-admin.py update_users --settings=graphite.settings From 8d74450796ff70e8bca3b2751a5cc5ec2a138c3b Mon Sep 17 00:00:00 2001 From: Benedikt Heine Date: Mon, 5 Nov 2018 21:35:30 +0100 Subject: [PATCH 06/21] Move Dockerfile to alpine --- Dockerfile | 56 +++++++++++++++++---------------------- conf/etc/nginx/nginx.conf | 2 +- 2 files changed, 26 insertions(+), 32 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6d578f0..281bbb5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,30 +1,26 @@ -FROM phusion/baseimage:0.11 as build +FROM alpine:3.8 as build LABEL maintainer="Denys Zhdanov " -RUN export DEBIAN_FRONTEND=noninteractive \ - && apt-get -y update \ - && apt-get -y upgrade \ - && apt-get -y install \ +RUN true \ + && apk add --update \ + alpine-sdk \ + cairo \ git \ - libcairo2-dev \ libffi-dev \ - librrd-dev \ + librrd \ nginx \ - pkg-config \ - python3-cairo \ + pkgconfig \ + py3-cairo \ + py3-pip \ + py3-pyldap \ + py3-virtualenv \ + py-rrd \ python3-dev \ - python3-ldap \ - python3-pip \ - python3-rrdtool \ - sqlite3 \ + rrdtool-dev \ + sqlite \ wget \ - && rm -rf /var/lib/apt/lists/* - -# fix python dependencies (LTS Django) -RUN python3 -m pip install --upgrade virtualenv virtualenv-tools \ && virtualenv /opt/graphite \ && . /opt/graphite/bin/activate \ - && python3 -m pip install --upgrade pip \ && pip3 install \ django==1.11.15 \ django-statsd-mozilla \ @@ -87,28 +83,26 @@ RUN mkdir -p /var/log/graphite/ \ # config statsd COPY conf/opt/statsd/config_*.js /opt/statsd/ -FROM phusion/baseimage:0.11 as production +FROM alpine:3.8 as production LABEL maintainer="Denys Zhdanov " ENV STATSD_INTERFACE udp -RUN export DEBIAN_FRONTEND=noninteractive \ - && apt-get update --fix-missing \ - && apt-get -y upgrade \ - && apt-get install --yes --no-install-recommends \ +RUN true \ + && apk add --update \ + cairo \ collectd \ - libcairo2 \ - librrd8 \ + collectd-disk \ + collectd-nginx \ + findutils \ + librrd \ memcached \ nginx \ - python3-ldap \ + py3-pyldap \ redis \ - sqlite3 \ - && apt-get clean \ - && apt-get autoremove --yes \ + sqlite \ && rm -rf \ - /var/lib/apt/lists/* \ - /etc/nginx/sites-enabled/default \ + /etc/nginx/conf.d/default.conf \ && mkdir -p \ /var/log/carbon \ /var/log/graphite diff --git a/conf/etc/nginx/nginx.conf b/conf/etc/nginx/nginx.conf index bc50cd1..80e0c16 100644 --- a/conf/etc/nginx/nginx.conf +++ b/conf/etc/nginx/nginx.conf @@ -1,4 +1,4 @@ -user www-data; +user nginx; worker_processes 4; pid /run/nginx.pid; daemon off; From acbe27058053b53800e14c44fb645fa2595b7d9a Mon Sep 17 00:00:00 2001 From: Benedikt Heine Date: Mon, 5 Nov 2018 21:54:57 +0100 Subject: [PATCH 07/21] Make NodeJS version an ARG --- Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 281bbb5..1ad3b11 100644 --- a/Dockerfile +++ b/Dockerfile @@ -64,9 +64,10 @@ RUN sed -i \ /opt/graphite/webapp/graphite/readers/rrd.py # installing nodejs 6 -WORKDIR /opt -RUN wget https://nodejs.org/download/release/v6.14.4/node-v6.14.4-linux-x64.tar.gz && \ - tar -xvpzf node-v6.14.4-linux-x64.tar.gz && rm node-v6.14.4-linux-x64.tar.gz && mv node-v6.14.4-linux-x64 nodejs +ARG NODEJS_VERSION=6.14.4 +RUN wget -q -O - https://nodejs.org/download/release/v${NODEJS_VERSION}/node-v${NODEJS_VERSION}-linux-x64.tar.gz \ + | tar xz -C /opt \ + && mv /opt/node-v${NODEJS_VERSION}-linux-x64 /opt/nodejs # install statsd ARG statsd_version=v0.8.0 From 9092b00ed236c4b9d4864fa2f0731486987078eb Mon Sep 17 00:00:00 2001 From: Benedikt Heine Date: Mon, 5 Nov 2018 22:25:27 +0100 Subject: [PATCH 08/21] Implement runit functionality --- Dockerfile | 3 ++- conf/{etc/my_init.d/01_conf_init.sh => entrypoint} | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) rename conf/{etc/my_init.d/01_conf_init.sh => entrypoint} (98%) diff --git a/Dockerfile b/Dockerfile index 1ad3b11..1dbd355 100644 --- a/Dockerfile +++ b/Dockerfile @@ -101,6 +101,7 @@ RUN true \ nginx \ py3-pyldap \ redis \ + runit \ sqlite \ && rm -rf \ /etc/nginx/conf.d/default.conf \ @@ -119,4 +120,4 @@ COPY --from=build /opt /opt EXPOSE 80 2003-2004 2023-2024 8080 8125 8125/udp 8126 VOLUME ["/opt/graphite/conf", "/opt/graphite/storage", "/opt/graphite/webapp/graphite/functions/custom", "/etc/nginx", "/opt/statsd", "/etc/logrotate.d", "/var/log", "/var/lib/redis"] -CMD ["/sbin/my_init"] +ENTRYPOINT ["/entrypoint"] diff --git a/conf/etc/my_init.d/01_conf_init.sh b/conf/entrypoint similarity index 98% rename from conf/etc/my_init.d/01_conf_init.sh rename to conf/entrypoint index 6cad1bd..614456c 100755 --- a/conf/etc/my_init.d/01_conf_init.sh +++ b/conf/entrypoint @@ -54,3 +54,5 @@ if folder_empty /opt/statsd; then git clone -b v0.8.0 https://github.com/etsy/statsd.git /opt/statsd && \ cp $conf_dir/opt/statsd/config_*.js /opt/statsd/ fi + +runsvdir -P /etc/service From 3ec3b1fa00f07a95c5fe6b2aa332702c06a117f4 Mon Sep 17 00:00:00 2001 From: Benedikt Heine Date: Mon, 5 Nov 2018 22:43:28 +0100 Subject: [PATCH 09/21] Add base packages into another base image This improves build time, as the base image already contains most of the packages for the dev installation --- Dockerfile | 51 +++++++++++++++++++++++++-------------------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1dbd355..063273f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,34 @@ -FROM alpine:3.8 as build +FROM alpine:3.8 as base LABEL maintainer="Denys Zhdanov " RUN true \ && apk add --update \ - alpine-sdk \ cairo \ - git \ - libffi-dev \ + collectd \ + collectd-disk \ + collectd-nginx \ + findutils \ librrd \ + memcached \ nginx \ + py3-pyldap \ + redis \ + runit \ + sqlite \ + && rm -rf \ + /etc/nginx/conf.d/default.conf \ + && mkdir -p \ + /var/log/carbon \ + /var/log/graphite + +FROM base as build +LABEL maintainer="Denys Zhdanov " + +RUN true \ + && apk add --update \ + alpine-sdk \ + git \ + libffi-dev \ pkgconfig \ py3-cairo \ py3-pip \ @@ -17,7 +37,6 @@ RUN true \ py-rrd \ python3-dev \ rrdtool-dev \ - sqlite \ wget \ && virtualenv /opt/graphite \ && . /opt/graphite/bin/activate \ @@ -84,31 +103,11 @@ RUN mkdir -p /var/log/graphite/ \ # config statsd COPY conf/opt/statsd/config_*.js /opt/statsd/ -FROM alpine:3.8 as production +FROM base as production LABEL maintainer="Denys Zhdanov " ENV STATSD_INTERFACE udp -RUN true \ - && apk add --update \ - cairo \ - collectd \ - collectd-disk \ - collectd-nginx \ - findutils \ - librrd \ - memcached \ - nginx \ - py3-pyldap \ - redis \ - runit \ - sqlite \ - && rm -rf \ - /etc/nginx/conf.d/default.conf \ - && mkdir -p \ - /var/log/carbon \ - /var/log/graphite - COPY conf / COPY conf /etc/graphite-statsd/conf/ From 72dce466db58420b133fb1e038c734dec1a1e45b Mon Sep 17 00:00:00 2001 From: Benedikt Heine Date: Tue, 6 Nov 2018 00:15:59 +0100 Subject: [PATCH 10/21] Kick out external NodeJS and use statsd from master Even though statsd didn't get updated for a long time, the patches on current master should suffice to make it run on a current NodeJS. --- Dockerfile | 20 +++++++++----------- conf/etc/service/statsd/run | 2 +- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index 063273f..2275d7a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,6 +11,7 @@ RUN true \ librrd \ memcached \ nginx \ + nodejs \ py3-pyldap \ redis \ runit \ @@ -77,22 +78,19 @@ RUN . /opt/graphite/bin/activate \ && pip3 install -r requirements.txt \ && python3 ./setup.py install +# install statsd (as we have to use this ugly way) +ARG statsd_version=8d5363cb109cc6363661a1d5813e0b96787c4411 +ARG statsd_repo=https://github.com/etsy/statsd.git +RUN git init /opt/statsd \ + && git -C /opt/statsd remote add origin "${statsd_repo}" \ + && git -C /opt/statsd fetch origin "${statsd_version}" \ + && git -C /opt/statsd checkout "${statsd_version}" + # fixing RRD support (see https://github.com/graphite-project/docker-graphite-statsd/issues/63) RUN sed -i \ 's/return os.path.realpath(fs_path)/return os.path.realpath(fs_path).decode("utf-8")/' \ /opt/graphite/webapp/graphite/readers/rrd.py -# installing nodejs 6 -ARG NODEJS_VERSION=6.14.4 -RUN wget -q -O - https://nodejs.org/download/release/v${NODEJS_VERSION}/node-v${NODEJS_VERSION}-linux-x64.tar.gz \ - | tar xz -C /opt \ - && mv /opt/node-v${NODEJS_VERSION}-linux-x64 /opt/nodejs - -# install statsd -ARG statsd_version=v0.8.0 -ARG statsd_repo=https://github.com/etsy/statsd.git -RUN git clone -b ${statsd_version} --depth 1 ${statsd_repo} /opt/statsd - # config graphite COPY conf/opt/graphite/conf/*.conf /opt/graphite/conf/ COPY conf/opt/graphite/webapp/graphite/local_settings.py /opt/graphite/webapp/graphite/local_settings.py diff --git a/conf/etc/service/statsd/run b/conf/etc/service/statsd/run index dd74d4d..3779fea 100755 --- a/conf/etc/service/statsd/run +++ b/conf/etc/service/statsd/run @@ -1,3 +1,3 @@ #!/bin/sh -exec /opt/nodejs/bin/node /opt/statsd/stats.js /opt/statsd/config_$STATSD_INTERFACE.js >> /var/log/statsd.log 2>&1 +exec node /opt/statsd/stats.js /opt/statsd/config_$STATSD_INTERFACE.js >> /var/log/statsd.log 2>&1 From feb025f448c97d38ddc937e8c1ae45562400eb6a Mon Sep 17 00:00:00 2001 From: Benedikt Heine Date: Tue, 6 Nov 2018 00:18:29 +0100 Subject: [PATCH 11/21] Kick out reclone of statsd This is absolutely not necessary. If someone really wants to edit and save the config, mount the file. --- conf/entrypoint | 8 -------- 1 file changed, 8 deletions(-) diff --git a/conf/entrypoint b/conf/entrypoint index 614456c..d6ddd38 100755 --- a/conf/entrypoint +++ b/conf/entrypoint @@ -47,12 +47,4 @@ if folder_empty /opt/graphite/webapp/graphite/functions/custom; then touch /opt/graphite/webapp/graphite/functions/custom/__init__.py fi -# auto setup statsd with default config if /opt/statsd is missing -# needed for the use case when a docker host volume is mounted at an of the following: -# - /opt/statsd -if folder_empty /opt/statsd; then - git clone -b v0.8.0 https://github.com/etsy/statsd.git /opt/statsd && \ - cp $conf_dir/opt/statsd/config_*.js /opt/statsd/ -fi - runsvdir -P /etc/service From 3d28b51cb51baf67f9f8eb3e65653fc0ffed19d5 Mon Sep 17 00:00:00 2001 From: Benedikt Heine Date: Wed, 7 Nov 2018 18:43:30 +0100 Subject: [PATCH 12/21] Use --no-cache and not save the pkglists --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2275d7a..0b87608 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM alpine:3.8 as base LABEL maintainer="Denys Zhdanov " RUN true \ - && apk add --update \ + && apk add --no-cache \ cairo \ collectd \ collectd-disk \ From b8cae163b24148baecacd396ce9295a9eae446f3 Mon Sep 17 00:00:00 2001 From: Benedikt Heine Date: Wed, 7 Nov 2018 18:56:51 +0100 Subject: [PATCH 13/21] Only copy necessary default conf files --- Dockerfile | 4 +++- conf/entrypoint | 6 ++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0b87608..c30ee57 100644 --- a/Dockerfile +++ b/Dockerfile @@ -91,6 +91,9 @@ RUN sed -i \ 's/return os.path.realpath(fs_path)/return os.path.realpath(fs_path).decode("utf-8")/' \ /opt/graphite/webapp/graphite/readers/rrd.py +COPY conf/opt/graphite/conf/ /opt/defaultconf/graphite/ +COPY conf/opt/graphite/webapp/graphite/local_settings.py /opt/defaultconf/graphite/local_settings.py + # config graphite COPY conf/opt/graphite/conf/*.conf /opt/graphite/conf/ COPY conf/opt/graphite/webapp/graphite/local_settings.py /opt/graphite/webapp/graphite/local_settings.py @@ -107,7 +110,6 @@ LABEL maintainer="Denys Zhdanov " ENV STATSD_INTERFACE udp COPY conf / -COPY conf /etc/graphite-statsd/conf/ # copy /opt from build image COPY --from=build /opt /opt diff --git a/conf/entrypoint b/conf/entrypoint index d6ddd38..afb5717 100755 --- a/conf/entrypoint +++ b/conf/entrypoint @@ -1,7 +1,5 @@ #!/bin/sh -conf_dir=/etc/graphite-statsd/conf - # auto setup graphite with default configs if /opt/graphite/conf is missing # needed for the use case when a docker host volume is mounted at an of the following: # - /opt/graphite/conf @@ -17,11 +15,11 @@ if folder_empty /var/log/graphite; then fi if folder_empty /opt/graphite/conf; then - cp -R $conf_dir/opt/graphite/conf/*.conf /opt/graphite/conf/ + cp /opt/defaultconf/graphite/conf/*.conf /opt/graphite/conf/ fi if folder_empty /opt/graphite/webapp/graphite; then - cp $conf_dir/opt/graphite/webapp/graphite/local_settings.py /opt/graphite/webapp/graphite/local_settings.py + cp /opt/defaultconf/graphite/local_settings.py /opt/graphite/webapp/graphite/local_settings.py fi if folder_empty /opt/graphite/storage; then From 67fd60c7596f33750566a7767b0d5521d56be43c Mon Sep 17 00:00:00 2001 From: Benedikt Heine Date: Thu, 8 Nov 2018 14:02:13 +0100 Subject: [PATCH 14/21] Separate statsd config properly --- Dockerfile | 4 ++-- README.md | 10 +++++----- conf/entrypoint | 4 ++++ conf/etc/service/statsd/run | 2 +- conf/opt/statsd/{config_tcp.js => config/tcp.js} | 0 conf/opt/statsd/{config_udp.js => config/udp.js} | 0 6 files changed, 12 insertions(+), 8 deletions(-) rename conf/opt/statsd/{config_tcp.js => config/tcp.js} (100%) rename conf/opt/statsd/{config_udp.js => config/udp.js} (100%) diff --git a/Dockerfile b/Dockerfile index c30ee57..e2374ca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -102,7 +102,7 @@ RUN mkdir -p /var/log/graphite/ \ && PYTHONPATH=/opt/graphite/webapp /opt/graphite/bin/django-admin.py collectstatic --noinput --settings=graphite.settings # config statsd -COPY conf/opt/statsd/config_*.js /opt/statsd/ +COPY conf/opt/statsd/config/ /opt/defaultconf/statsd/config/ FROM base as production LABEL maintainer="Denys Zhdanov " @@ -117,6 +117,6 @@ COPY --from=build /opt /opt # defaults EXPOSE 80 2003-2004 2023-2024 8080 8125 8125/udp 8126 -VOLUME ["/opt/graphite/conf", "/opt/graphite/storage", "/opt/graphite/webapp/graphite/functions/custom", "/etc/nginx", "/opt/statsd", "/etc/logrotate.d", "/var/log", "/var/lib/redis"] +VOLUME ["/opt/graphite/conf", "/opt/graphite/storage", "/opt/graphite/webapp/graphite/functions/custom", "/etc/nginx", "/opt/statsd/config", "/etc/logrotate.d", "/var/log", "/var/lib/redis"] ENTRYPOINT ["/entrypoint"] diff --git a/README.md b/README.md index cf577c2..f3704a4 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ Host | Container | Service 2024 | 2024 | [carbon aggregator - pickle](http://graphite.readthedocs.io/en/latest/carbon-daemons.html#carbon-aggregator-py) 8080 | 8080 | Graphite internal gunicorn port (without Nginx proxying). 8125 | 8125 | [statsd](https://github.com/etsy/statsd/blob/master/docs/server.md) -8126 | 8126 | [statsd admin](https://github.com/etsy/statsd/blob/v0.7.2/docs/admin_interface.md) +8126 | 8126 | [statsd admin](https://github.com/etsy/statsd/blob/master/docs/admin_interface.md) By default, statsd listens on the UDP port 8125. If you want it to listen on the TCP port 8125 instead, you can set the environment variable `STATSD_INTERFACE` to `tcp` when running the container. @@ -63,7 +63,7 @@ DOCKER ASSIGNED | /opt/graphite/conf | graphite config DOCKER ASSIGNED | /opt/graphite/storage | graphite stats storage DOCKER ASSIGNED | /opt/graphite/webapp/graphite/functions/custom | graphite custom functions dir DOCKER ASSIGNED | /etc/nginx | nginx config -DOCKER ASSIGNED | /opt/statsd | statsd config +DOCKER ASSIGNED | /opt/statsd/config | statsd config DOCKER ASSIGNED | /etc/logrotate.d | logrotate config DOCKER ASSIGNED | /var/log | log files DOCKER ASSIGNED | /var/lib/redis | Redis TagDB data (optional) @@ -179,7 +179,7 @@ be sure to delete the old whisper files under `/opt/graphite/storage/whisper/`. --- **Important:** Ensure your Statsd flush interval is at least as long as the highest-resolution retention. -For example, if `/opt/statsd/config.js` looks like this. +For example, if `/opt/statsd/config/udp.js` looks like this. ``` flushInterval: 10000 @@ -218,12 +218,12 @@ docker run -d\ --restart=always\ -v /path/to/graphite/configs:/opt/graphite/conf\ -v /path/to/graphite/data:/opt/graphite/storage\ - -v /path/to/statsd:/opt/statsd\ + -v /path/to/statsd_config:/opt/statsd/config\ graphiteapp/graphite-statsd ``` **Note**: The container will initialize properly if you mount empty volumes at - `/opt/graphite/conf`, `/opt/graphite/storage`, or `/opt/statsd`. + `/opt/graphite/conf`, `/opt/graphite/storage`, or `/opt/statsd/config`. ## Memcached config diff --git a/conf/entrypoint b/conf/entrypoint index afb5717..7204aa8 100755 --- a/conf/entrypoint +++ b/conf/entrypoint @@ -14,6 +14,10 @@ if folder_empty /var/log/graphite; then touch /var/log/syslog fi +if folder_empty /opt/statsd/config/; then + cp /opt/defaultconf/statsd/config/*.js /opt/statsd/config/ +fi + if folder_empty /opt/graphite/conf; then cp /opt/defaultconf/graphite/conf/*.conf /opt/graphite/conf/ fi diff --git a/conf/etc/service/statsd/run b/conf/etc/service/statsd/run index 3779fea..7ac0811 100755 --- a/conf/etc/service/statsd/run +++ b/conf/etc/service/statsd/run @@ -1,3 +1,3 @@ #!/bin/sh -exec node /opt/statsd/stats.js /opt/statsd/config_$STATSD_INTERFACE.js >> /var/log/statsd.log 2>&1 +exec node /opt/statsd/stats.js /opt/statsd/config/$STATSD_INTERFACE.js >> /var/log/statsd.log 2>&1 diff --git a/conf/opt/statsd/config_tcp.js b/conf/opt/statsd/config/tcp.js similarity index 100% rename from conf/opt/statsd/config_tcp.js rename to conf/opt/statsd/config/tcp.js diff --git a/conf/opt/statsd/config_udp.js b/conf/opt/statsd/config/udp.js similarity index 100% rename from conf/opt/statsd/config_udp.js rename to conf/opt/statsd/config/udp.js From eaaf1df8bf7c893ce15477871b7e017df4b0422b Mon Sep 17 00:00:00 2001 From: Benedikt Heine Date: Thu, 8 Nov 2018 14:04:57 +0100 Subject: [PATCH 15/21] Use tabs to indent, so we can use HERE documents HERE documents have got the ability to remove leading tabs, when used with a trailing dash (<<-). When using spaces, the indentation wouldn't be possible. http://pubs.opengroup.org/onlinepubs/009695399/utilities/xcu_chap02.html#tag_02_07_04 --- conf/entrypoint | 41 ++++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/conf/entrypoint b/conf/entrypoint index 7204aa8..176ffce 100755 --- a/conf/entrypoint +++ b/conf/entrypoint @@ -10,8 +10,8 @@ function folder_empty() { } if folder_empty /var/log/graphite; then - mkdir -p /var/log/graphite - touch /var/log/syslog + mkdir -p /var/log/graphite + touch /var/log/syslog fi if folder_empty /opt/statsd/config/; then @@ -19,34 +19,33 @@ if folder_empty /opt/statsd/config/; then fi if folder_empty /opt/graphite/conf; then - cp /opt/defaultconf/graphite/conf/*.conf /opt/graphite/conf/ + cp /opt/defaultconf/graphite/conf/*.conf /opt/graphite/conf/ fi if folder_empty /opt/graphite/webapp/graphite; then - cp /opt/defaultconf/graphite/local_settings.py /opt/graphite/webapp/graphite/local_settings.py + cp /opt/defaultconf/graphite/local_settings.py /opt/graphite/webapp/graphite/local_settings.py fi if folder_empty /opt/graphite/storage; then - mkdir -p /opt/graphite/storage/whisper - - export PYTHONPATH=/opt/graphite/webapp - export DJANGO_SETTINGS_MODULE=graphite.settings - - /opt/graphite/bin/django-admin.py makemigrations - /opt/graphite/bin/django-admin.py migrate auth - /opt/graphite/bin/django-admin.py migrate --run-syncdb - /opt/graphite/bin/django-admin.py createsuperuser - --noinput \ - --email root.graphite@mailinator.com \ - --username root << ENDINPUT -root -root -ENDINPUT - + mkdir -p /opt/graphite/storage/whisper + + export PYTHONPATH=/opt/graphite/webapp + export DJANGO_SETTINGS_MODULE=graphite.settings + + /opt/graphite/bin/django-admin.py makemigrations + /opt/graphite/bin/django-admin.py migrate auth + /opt/graphite/bin/django-admin.py migrate --run-syncdb + /opt/graphite/bin/django-admin.py createsuperuser + --noinput \ + --email root.graphite@mailinator.com \ + --username root <<- ENDINPUT + root + root + ENDINPUT fi if folder_empty /opt/graphite/webapp/graphite/functions/custom; then - touch /opt/graphite/webapp/graphite/functions/custom/__init__.py + touch /opt/graphite/webapp/graphite/functions/custom/__init__.py fi runsvdir -P /etc/service From 5eca109e4ad74c65efb3f5b3cd7f97f4dc98a362 Mon Sep 17 00:00:00 2001 From: Benedikt Heine Date: Thu, 8 Nov 2018 14:23:38 +0100 Subject: [PATCH 16/21] Activate virtualenv in entrypoint --- conf/entrypoint | 2 ++ conf/etc/service/carbon-aggregator/run | 1 - conf/etc/service/carbon/run | 1 - conf/etc/service/graphite/run | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conf/entrypoint b/conf/entrypoint index 176ffce..5331e7b 100755 --- a/conf/entrypoint +++ b/conf/entrypoint @@ -1,5 +1,7 @@ #!/bin/sh +. /opt/graphite/bin/activate + # auto setup graphite with default configs if /opt/graphite/conf is missing # needed for the use case when a docker host volume is mounted at an of the following: # - /opt/graphite/conf diff --git a/conf/etc/service/carbon-aggregator/run b/conf/etc/service/carbon-aggregator/run index 74d2e1e..6925c81 100755 --- a/conf/etc/service/carbon-aggregator/run +++ b/conf/etc/service/carbon-aggregator/run @@ -1,5 +1,4 @@ #!/bin/sh rm -f /opt/graphite/storage/carbon-aggregator-a.pid -. /opt/graphite/bin/activate exec python3 /opt/graphite/bin/carbon-aggregator.py start --debug 2>&1 >> /var/log/carbon-aggregator.log diff --git a/conf/etc/service/carbon/run b/conf/etc/service/carbon/run index 3319203..e11d104 100755 --- a/conf/etc/service/carbon/run +++ b/conf/etc/service/carbon/run @@ -1,5 +1,4 @@ #!/bin/sh rm -f /opt/graphite/storage/carbon-cache-a.pid -. /opt/graphite/bin/activate exec python3 /opt/graphite/bin/carbon-cache.py start --debug 2>&1 >> /var/log/carbon.log diff --git a/conf/etc/service/graphite/run b/conf/etc/service/graphite/run index f3771b9..d181835 100755 --- a/conf/etc/service/graphite/run +++ b/conf/etc/service/graphite/run @@ -1,7 +1,7 @@ #!/bin/sh sv start nginx || exit 1 -. /opt/graphite/bin/activate + export GRAPHITE_WSGI_PROCESSES=${GRAPHITE_WSGI_PROCESSES:-4} export GRAPHITE_WSGI_THREADS=${GRAPHITE_WSGI_THREADS:-2} export GRAPHITE_WSGI_REQUEST_TIMEOUT=${GRAPHITE_WSGI_REQUEST_TIMEOUT:-65} From d44637d1c751b00cc68ed82cac6c211e465d2187 Mon Sep 17 00:00:00 2001 From: Benedikt Heine Date: Thu, 8 Nov 2018 14:28:02 +0100 Subject: [PATCH 17/21] Remove ugly trap --- conf/etc/service/graphite/run | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conf/etc/service/graphite/run b/conf/etc/service/graphite/run index d181835..cd9c89e 100755 --- a/conf/etc/service/graphite/run +++ b/conf/etc/service/graphite/run @@ -7,7 +7,8 @@ export GRAPHITE_WSGI_THREADS=${GRAPHITE_WSGI_THREADS:-2} export GRAPHITE_WSGI_REQUEST_TIMEOUT=${GRAPHITE_WSGI_REQUEST_TIMEOUT:-65} export GRAPHITE_WSGI_REQUEST_LINE=${GRAPHITE_WSGI_REQUEST_LINE:-0} export GRAPHITE_WSGI_MAX_REQUESTS=${GRAPHITE_WSGI_MAX_REQUESTS:-1000} -export PYTHONPATH=/opt/graphite/webapp && \ +export PYTHONPATH=/opt/graphite/webapp + exec /opt/graphite/bin/gunicorn wsgi --preload --pythonpath=/opt/graphite/webapp/graphite \ --workers=${GRAPHITE_WSGI_PROCESSES} \ --threads=${GRAPHITE_WSGI_THREADS} \ From 9da44c577803a8250433fe65f7ed9408e75c243e Mon Sep 17 00:00:00 2001 From: Benedikt Heine Date: Thu, 8 Nov 2018 14:35:26 +0100 Subject: [PATCH 18/21] Do not fail script, when service isn't activated --- conf/etc/service/collectd/run | 2 +- conf/etc/service/redis/run | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/etc/service/collectd/run b/conf/etc/service/collectd/run index 554e9ce..311aaa7 100755 --- a/conf/etc/service/collectd/run +++ b/conf/etc/service/collectd/run @@ -1,4 +1,4 @@ #!/bin/sh -[ -n "${COLLECTD}" ] || exit 1 +[ -n "${COLLECTD}" ] || exit 0 exec /usr/sbin/collectd -f -C /etc/collectd/collectd.conf diff --git a/conf/etc/service/redis/run b/conf/etc/service/redis/run index ebabb4e..b63003f 100755 --- a/conf/etc/service/redis/run +++ b/conf/etc/service/redis/run @@ -1,4 +1,4 @@ #!/bin/sh -[ -n "${REDIS_TAGDB}" ] || exit 1 +[ -n "${REDIS_TAGDB}" ] || exit 0 exec /usr/bin/redis-server /etc/redis/redis.conf From 624faa0875319eb7e187497bd920effce450bf2b Mon Sep 17 00:00:00 2001 From: Benedikt Heine Date: Thu, 8 Nov 2018 14:47:54 +0100 Subject: [PATCH 19/21] Get rid of unnecessary collectd error message --- conf/entrypoint | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/conf/entrypoint b/conf/entrypoint index 5331e7b..aa3eadb 100755 --- a/conf/entrypoint +++ b/conf/entrypoint @@ -28,6 +28,11 @@ if folder_empty /opt/graphite/webapp/graphite; then cp /opt/defaultconf/graphite/local_settings.py /opt/graphite/webapp/graphite/local_settings.py fi +[ -d /etc/collectd/collectd.conf.d ] || mkdir -p /etc/collectd/collectd.conf.d +if folder_empty /etc/collectd/collectd.conf.d; then + touch /etc/collectd/collectd.conf.d/do_not_spill_warning_about_folder_is_empty.conf +fi + if folder_empty /opt/graphite/storage; then mkdir -p /opt/graphite/storage/whisper From 7ec10386a917fd722ccb40514b20b046fd7ae329 Mon Sep 17 00:00:00 2001 From: Benedikt Heine Date: Thu, 8 Nov 2018 14:50:02 +0100 Subject: [PATCH 20/21] Use tee to paste log into docker logfiles --- conf/etc/service/carbon-aggregator/run | 2 +- conf/etc/service/carbon/run | 2 +- conf/etc/service/statsd/run | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conf/etc/service/carbon-aggregator/run b/conf/etc/service/carbon-aggregator/run index 6925c81..4219cfb 100755 --- a/conf/etc/service/carbon-aggregator/run +++ b/conf/etc/service/carbon-aggregator/run @@ -1,4 +1,4 @@ #!/bin/sh rm -f /opt/graphite/storage/carbon-aggregator-a.pid -exec python3 /opt/graphite/bin/carbon-aggregator.py start --debug 2>&1 >> /var/log/carbon-aggregator.log +exec python3 /opt/graphite/bin/carbon-aggregator.py start --debug 2>&1 | tee -a /var/log/carbon-aggregator.log diff --git a/conf/etc/service/carbon/run b/conf/etc/service/carbon/run index e11d104..c079873 100755 --- a/conf/etc/service/carbon/run +++ b/conf/etc/service/carbon/run @@ -1,4 +1,4 @@ #!/bin/sh rm -f /opt/graphite/storage/carbon-cache-a.pid -exec python3 /opt/graphite/bin/carbon-cache.py start --debug 2>&1 >> /var/log/carbon.log +exec python3 /opt/graphite/bin/carbon-cache.py start --debug 2>&1 | tee -a /var/log/carbon.log diff --git a/conf/etc/service/statsd/run b/conf/etc/service/statsd/run index 7ac0811..a765258 100755 --- a/conf/etc/service/statsd/run +++ b/conf/etc/service/statsd/run @@ -1,3 +1,3 @@ #!/bin/sh -exec node /opt/statsd/stats.js /opt/statsd/config/$STATSD_INTERFACE.js >> /var/log/statsd.log 2>&1 +exec node /opt/statsd/stats.js /opt/statsd/config/$STATSD_INTERFACE.js 2>&1 | tee -a /var/log/statsd.log From 9838dfd7bba3cf301cedc602de82d686f5a43ddb Mon Sep 17 00:00:00 2001 From: Benedikt Heine Date: Thu, 8 Nov 2018 14:57:08 +0100 Subject: [PATCH 21/21] Move folder setup commands into service run files This allows faster startup time and uses paralell initialisation. --- conf/entrypoint | 55 ++------------------------------- conf/etc/service/collectd/run | 6 ++++ conf/etc/service/graphite/run | 37 +++++++++++++++++++++- conf/etc/service/statsd/run | 4 +++ conf/usr/local/bin/folder_empty | 3 ++ 5 files changed, 52 insertions(+), 53 deletions(-) create mode 100755 conf/usr/local/bin/folder_empty diff --git a/conf/entrypoint b/conf/entrypoint index aa3eadb..93a1181 100755 --- a/conf/entrypoint +++ b/conf/entrypoint @@ -2,57 +2,8 @@ . /opt/graphite/bin/activate -# auto setup graphite with default configs if /opt/graphite/conf is missing -# needed for the use case when a docker host volume is mounted at an of the following: -# - /opt/graphite/conf -# - /opt/graphite/storage - -function folder_empty() { - [ -z "$(find "${1}" -mindepth 1 -not -name "lost+found" -print -quit)" ] -} - -if folder_empty /var/log/graphite; then - mkdir -p /var/log/graphite - touch /var/log/syslog -fi - -if folder_empty /opt/statsd/config/; then - cp /opt/defaultconf/statsd/config/*.js /opt/statsd/config/ -fi - -if folder_empty /opt/graphite/conf; then - cp /opt/defaultconf/graphite/conf/*.conf /opt/graphite/conf/ -fi - -if folder_empty /opt/graphite/webapp/graphite; then - cp /opt/defaultconf/graphite/local_settings.py /opt/graphite/webapp/graphite/local_settings.py -fi - -[ -d /etc/collectd/collectd.conf.d ] || mkdir -p /etc/collectd/collectd.conf.d -if folder_empty /etc/collectd/collectd.conf.d; then - touch /etc/collectd/collectd.conf.d/do_not_spill_warning_about_folder_is_empty.conf -fi - -if folder_empty /opt/graphite/storage; then - mkdir -p /opt/graphite/storage/whisper - - export PYTHONPATH=/opt/graphite/webapp - export DJANGO_SETTINGS_MODULE=graphite.settings - - /opt/graphite/bin/django-admin.py makemigrations - /opt/graphite/bin/django-admin.py migrate auth - /opt/graphite/bin/django-admin.py migrate --run-syncdb - /opt/graphite/bin/django-admin.py createsuperuser - --noinput \ - --email root.graphite@mailinator.com \ - --username root <<- ENDINPUT - root - root - ENDINPUT -fi - -if folder_empty /opt/graphite/webapp/graphite/functions/custom; then - touch /opt/graphite/webapp/graphite/functions/custom/__init__.py -fi +# Ensure /usr/local/bin is in PATH. On some +# weird platforms,this might not be the case +PATH="${PATH}:/usr/local/bin" runsvdir -P /etc/service diff --git a/conf/etc/service/collectd/run b/conf/etc/service/collectd/run index 311aaa7..a4af505 100755 --- a/conf/etc/service/collectd/run +++ b/conf/etc/service/collectd/run @@ -1,4 +1,10 @@ #!/bin/sh [ -n "${COLLECTD}" ] || exit 0 + +[ -d /etc/collectd/collectd.conf.d ] || mkdir -p /etc/collectd/collectd.conf.d +if folder_empty /etc/collectd/collectd.conf.d; then + touch /etc/collectd/collectd.conf.d/do_not_spill_warning_about_folder_is_empty.conf +fi + exec /usr/sbin/collectd -f -C /etc/collectd/collectd.conf diff --git a/conf/etc/service/graphite/run b/conf/etc/service/graphite/run index cd9c89e..acd53be 100755 --- a/conf/etc/service/graphite/run +++ b/conf/etc/service/graphite/run @@ -1,6 +1,39 @@ #!/bin/sh -sv start nginx || exit 1 +if folder_empty /var/log/graphite; then + mkdir -p /var/log/graphite + touch /var/log/syslog +fi + +if folder_empty /opt/graphite/conf; then + cp /opt/defaultconf/graphite/conf/*.conf /opt/graphite/conf/ +fi + +if folder_empty /opt/graphite/webapp/graphite; then + cp /opt/defaultconf/graphite/local_settings.py /opt/graphite/webapp/graphite/local_settings.py +fi + +if folder_empty /opt/graphite/storage; then + mkdir -p /opt/graphite/storage/whisper + + export PYTHONPATH=/opt/graphite/webapp + export DJANGO_SETTINGS_MODULE=graphite.settings + + /opt/graphite/bin/django-admin.py makemigrations + /opt/graphite/bin/django-admin.py migrate auth + /opt/graphite/bin/django-admin.py migrate --run-syncdb + /opt/graphite/bin/django-admin.py createsuperuser + --noinput \ + --email root.graphite@mailinator.com \ + --username root <<- ENDINPUT + root + root + ENDINPUT +fi + +if folder_empty /opt/graphite/webapp/graphite/functions/custom; then + touch /opt/graphite/webapp/graphite/functions/custom/__init__.py +fi export GRAPHITE_WSGI_PROCESSES=${GRAPHITE_WSGI_PROCESSES:-4} export GRAPHITE_WSGI_THREADS=${GRAPHITE_WSGI_THREADS:-2} @@ -9,6 +42,8 @@ export GRAPHITE_WSGI_REQUEST_LINE=${GRAPHITE_WSGI_REQUEST_LINE:-0} export GRAPHITE_WSGI_MAX_REQUESTS=${GRAPHITE_WSGI_MAX_REQUESTS:-1000} export PYTHONPATH=/opt/graphite/webapp +sv start nginx || exit 1 + exec /opt/graphite/bin/gunicorn wsgi --preload --pythonpath=/opt/graphite/webapp/graphite \ --workers=${GRAPHITE_WSGI_PROCESSES} \ --threads=${GRAPHITE_WSGI_THREADS} \ diff --git a/conf/etc/service/statsd/run b/conf/etc/service/statsd/run index a765258..571530f 100755 --- a/conf/etc/service/statsd/run +++ b/conf/etc/service/statsd/run @@ -1,3 +1,7 @@ #!/bin/sh +if folder_empty /opt/statsd/config/; then + cp /opt/defaultconf/statsd/config/*.js /opt/statsd/config/ +fi + exec node /opt/statsd/stats.js /opt/statsd/config/$STATSD_INTERFACE.js 2>&1 | tee -a /var/log/statsd.log diff --git a/conf/usr/local/bin/folder_empty b/conf/usr/local/bin/folder_empty new file mode 100755 index 0000000..d88d138 --- /dev/null +++ b/conf/usr/local/bin/folder_empty @@ -0,0 +1,3 @@ +#!/bin/sh + +[ -z "$(find "${1}" -mindepth 1 -not -name "lost+found" -print -quit)" ]