Enable queue running for dev environments to update with current code without needing to be restarted

.gitlab-ci.yml

@@ -3,21 +3,21 @@ image: docker:latest
 stages:
 - test
 - build
+- build-test
 
 variables:
-  BRANCH: plus
-  VERSION: 7.1-fpm-${BRANCH}
+  BRANCH: mp
+  VERSION: 7.4-fpm-${BRANCH}
   CACHETAG: build-${BRANCH}
-  DOCKER_REGISTRY: registry.leenooks.net
-  DOCKER_HOST: tcp://${DOCKER_REGISTRY}-leenooks-ci-docker:2375
+  DOCKER_HOST: tcp://docker:2375
 
 services:
-- ${DOCKER_REGISTRY}/leenooks/ci-docker:dind
+- docker:dind
 
 before_script:
 - docker info
 - docker version
-- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN ${DOCKER_REGISTRY}
+- echo "$CI_JOB_TOKEN" | docker login -u "$CI_REGISTRY_USER" "$CI_REGISTRY" --password-stdin
 
 test:
   stage: test
@@ -29,14 +29,29 @@ test:
   only:
   - debug
 
-build:
+x86_64:build:
   stage: build
   script:
-  - docker pull ${CI_REGISTRY_IMAGE}:${CACHETAG} || true
+  - if [ -f init ]; then chmod 500 init; fi
+  - ([ -z "$REFRESH" ] && docker pull ${CI_REGISTRY_IMAGE}:${CACHETAG}) || echo "true"
   - docker build --cache-from ${CI_REGISTRY_IMAGE}:${CACHETAG} -t ${CI_REGISTRY_IMAGE}:${VERSION} -t ${CI_REGISTRY_IMAGE}:${CACHETAG} .
   - docker push ${CI_REGISTRY_IMAGE}:${VERSION}
   - docker push ${CI_REGISTRY_IMAGE}:${CACHETAG}
   tags:
   - docker
+  - x86_64
   only:
-  - plus
+  - mysql
+
+x86_64:build-test:
+  stage: build-test
+  script:
+  - if [ -f init ]; then chmod 500 init; fi
+  - ([ -z "$REFRESH" ] && docker pull ${CI_REGISTRY_IMAGE}:${CACHETAG}) || echo "true"
+  - docker build --file Dockerfile.phptest --cache-from ${CI_REGISTRY_IMAGE}:${CACHETAG} -t ${CI_REGISTRY_IMAGE}:${VERSION}-test . 
+  - docker push ${CI_REGISTRY_IMAGE}:${VERSION}-test
+  tags:
+  - docker
+  - x86_64
+  only:
+  - mysql

Dockerfile

@@ -1,23 +1,42 @@
 # NAME leenooks/php
-# VERSION 7.1-fpm-plus
+# VERSION 7.4-fpm-mp
 
-FROM php:7.1-fpm
-RUN echo "deb http://deb.debian.org/debian jessie non-free" >> /etc/apt/sources.list.d/non-free.list && apt-get update \
-	&& apt-get install -y pkg-config libbz2-dev libgmp-dev libpng-dev libjpeg-dev libfreetype6-dev libsnmp-dev snmp-mibs-downloader libmagickwand-dev --no-install-recommends \
-	&& download-mibs \
-	&& docker-php-ext-configure gd --with-freetype-dir=/usr/include/freetype2 --with-jpeg-dir=/usr/include/ \
-	&& docker-php-ext-install -j$(nproc) pdo_mysql bz2 gettext sockets gmp gd pcntl snmp exif \
-	&& pecl install imagick && docker-php-ext-enable imagick \
-	&& rm -rf /var/lib/apt/lists/* /tmp/*
+FROM php:7.4-fpm
 
-RUN apt-get update && apt-get install -y openssh-server && rm -rf /var/lib/apt/lists/* \
-	&& useradd -c "Hosting Admin User" -u 1000 -g users -G www-data -d /var/www/html -M lamp
+RUN apt-get update && apt-get install -y openssh-server unzip git msmtp nginx wait-for-it \
+	&& rm /etc/nginx/sites-enabled/default \
+	&& apt-get -y autoremove \
+	&& apt-get clean \
+	&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
-EXPOSE 9000/tcp 22/tcp
+RUN useradd -c "Hosting Admin User" -u 1000 -g users -G www-data -d /var/www/html -M lamp
+RUN sed -e 's/^expose_php = On/expose_php = Off/' /usr/local/etc/php/php.ini-production > /usr/local/etc/php/php.ini
+COPY www.conf /usr/local/etc/php-fpm.d/
+COPY nginx-app.conf /etc/nginx/conf.d/
 
-COPY sshd_config.patch /tmp
+COPY sshd_config.patch /tmp/
 RUN (cd / && patch -p0 ) < /tmp/sshd_config.patch && rm /tmp/sshd_config.patch
 
-COPY start /usr/local/sbin
-ENTRYPOINT [ "/usr/local/sbin/start" ]
-CMD ["php-fpm"]
+COPY msmtprc /etc/
+RUN sed -i -e 's#^;sendmail_path =#sendmail_path = "/usr/bin/msmtp -t"#' /usr/local/etc/php/php.ini
+
+EXPOSE 22/tcp
+
+# Add composer
+RUN curl https://getcomposer.org/installer|php -- --install-dir=/usr/local/bin --filename=composer
+
+# Mysql/Postgress/LDAP
+RUN apt-get update && apt-get install -y openssh-server libpq5 libpq-dev unzip git libldap2-dev \
+	&& docker-php-ext-install -j$(nproc) pdo_mysql pdo_pgsql ldap gettext \
+	&& apt-get -y purge libpq-dev libldap2-dev libx11-6 dbus ncurses-term systemd \
+	&& apt-get -y autoremove \
+	&& apt-get clean \
+	&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+# Enable phpredis
+RUN pecl install -o -f igbinary && pecl install -o -f redis && docker-php-ext-enable redis igbinary && rm -rf /tmp/*
+
+COPY init /sbin/
+RUN chmod 550 /sbin/init && chown 0:0 /sbin/init
+ENTRYPOINT [ "/sbin/init" ]
+CMD [ "php-fpm" ]

Dockerfile.phptest

@@ -0,0 +1,12 @@
+# NAME leenooks/php
+# VERSION 7.4-fpm-test
+
+FROM registry.leenooks.net/leenooks/php:7.4-fpm-mp
+
+RUN pecl install xdebug \
+	&& docker-php-ext-enable xdebug
+
+RUN apt-get update && apt-get install -y npm \
+        && apt-get -y autoremove \
+        && apt-get clean \
+        && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*

init

@@ -0,0 +1,176 @@
+#!/bin/bash
+
+set -e
+role=${CONTAINER_ROLE:-app}
+env=${APP_ENV:-live}
+php=${PHP_DIR:-/var/www/html}
+composer=${COMPOSER_DIR:-/var/www/.composer}
+NO_NGINX=${NO_NGINX:-TRUE}
+SSH_START=${SSH_START:-FALSE}
+
+function mp() {
+	set +e
+	mountpoint -q $1
+	local mp=$?
+	set -e
+	echo ${mp}
+}
+
+function nginx_start() {
+	# Start NGINX
+	if [ -x /usr/sbin/nginx -a "${NO_NGINX}" != "TRUE" ]; then
+		echo "* Starting NGINX..."
+		start-stop-daemon --start --pidfile /var/run/nginx.pid --exec /usr/sbin/nginx -- -g 'daemon on; master_process on;'
+	fi
+}
+
+# General Setup
+if [ -x /usr/sbin/sshd -a "${SSH_START}" = "TRUE" ]; then
+	[ ! -d /var/run/sshd ] && mkdir /var/run/sshd
+	start-stop-daemon --start --pidfile /var/run/sshd.pid --exec /usr/sbin/sshd -- -p 22
+fi
+
+# Laravel Specific
+if [ "${role}" = "app" -a -e artisan ]; then
+	if [ ! -e ${php}/.env ]; then
+		echo "! ERROR: NO .env file..."
+		exec /bin/bash
+	fi
+
+	mp=$(mp ${php})
+
+	# Only adjust perms if this is an external mountpoint
+	if [ ${mp} -eq 0 -o -n "${FORCE_PERMS}" ] ; then
+		if [ -n "${FORCE_PERMS}" -o "${env}" != "dev" -a -z "${SKIP_PERM}" ]; then
+			echo "* Setting Permissions..."
+			# Make sure our permissions are appropraite
+			find ${php} -type f -exec chmod 640 {} \;
+			find ${php} -type d -exec chmod 750 {} \;
+			chmod o+rx ${php}
+			chmod -R o+rx ${php}/public
+			chown -R lamp:www-data ${php}
+			chown -R www-data:www-data ${php}/storage ${php}/bootstrap ${php}/composer.*
+			[ -e ${php}/vendor ] && chown -R www-data:www-data ${php}/vendor
+		fi
+	fi
+
+	if [ "${env}" != "local" -a -r "artisan" ]; then
+		# See if we need to refresh our dependancies
+		if [[ -r composer.json && ( -e .composer.refresh || ! -d vendor ) ]]; then
+			rm -f ${php}/bootstrap/cache/*.php
+			if [ "${env}" != "dev" ]; then
+				NODEV="--no-dev"
+			fi
+
+			mp=$(mp ${composer})
+
+			if [ ${mp} -eq 0 -o -n "${FORCE_PERMS}" ] ; then
+				[ -n "${FORCE_PERMS}" -o "${env}" != "dev" -a -z "${SKIP_PERM}" ] && chown -R www-data:www-data ${composer}
+				[ ! -d ${php}/vendor ] && mkdir -m 750 ${php}/vendor && chown www-data:www-data ${php}/vendor
+				[ -n "${FORCE_PERMS}" -o "${env}" != "dev" -a -z "${SKIP_PERM}" ] && chmod g+w ${php}
+			fi
+
+			su www-data -s /bin/sh -c "composer install --optimize-autoloader ${NODEV}" && ( test -e .composer.refresh && rm -f .composer.refresh )
+			[ -n "${FORCE_PERMS}" -o "${env}" != "dev" -a -z "${SKIP_PERM}" ] && [ ${mp} -eq 0 ] && chmod g-w ${php}
+		fi
+
+		if [ -e .lumen ]; then
+			echo "* Lumen detected..."
+		else
+			echo "* Caching configuration..."
+			su www-data -s /bin/sh -c "(php artisan config:cache && php artisan route:cache && php artisan view:cache)"
+		fi
+
+		if [ -r .migrate ]; then
+			echo "* Running migration..."
+			# If DB_HOST not set, source the env file
+			[ -z "${DB_HOST}" -a -r .env ] && . .env
+
+			if [ -n "${DB_HOST}" -a -n "${DB_PORT}" ]; then
+				while ! wait-for-it -h ${DB_HOST} -p ${DB_PORT} -t 5 -q; do
+					echo "? Waiting for database at ${DB_HOST}:${DB_PORT}"
+					sleep 1;
+				done
+				echo "- DB is active on ${DB_HOST}:${DB_PORT}"
+			fi
+
+			su www-data -s /bin/sh -c "php artisan migrate" && rm -f .migrate
+		fi
+
+		# If passport is installed
+		if [ -d ${php}/vendor/laravel/passport ]; then
+			echo "* Generating OAUTH keys ..."
+			su www-data -s /bin/sh -c "php artisan passport:keys"
+		fi
+	fi
+
+	nginx_start
+
+	exec /usr/local/bin/docker-php-entrypoint "$@"
+
+elif [ "$role" = "queue" -a -e artisan ]; then
+	if [ ! -e ${php}/.env ]; then
+		echo "! ERROR: NO .env file..."
+		exec /bin/bash
+	fi
+
+	QUEUE_CMD=work
+	if [ "${env}" == "dev" ]; then
+		QUEUE_CMD=listen
+	fi
+
+	if [ -e .lumen ]; then
+		echo "* Lumen detected..."
+	else
+		# We only check for non mount points, in case this container has the app inside
+		mp=$(mp ${php})
+		if [ ${mp} -eq 1 ]; then
+			echo "* Caching configuration..."
+			su www-data -s /bin/sh -c "(php artisan config:cache && php artisan route:cache && php artisan view:cache)"
+		fi
+	fi
+
+	echo "* Running the queue..."
+	# We'll delay starting in case the app is caching
+	sleep 15
+
+	su www-data -s /bin/sh -c "
+	while true; do
+		php ${PHP_OPTIONS} artisan queue:${QUEUE_CMD} --verbose --tries=${WORK_TRIES:-1} --timeout=${WORK_TIMEOUT:-90} ${WORK_QUEUES:+--queue=${WORK_QUEUES}} ${WORK_MEMORY:+--memory=${WORK_MEMORY}} ${WORK_ONCE:+--once}
+	done
+	"
+
+elif [ "$role" = "scheduler" -a -e artisan ]; then
+	if [ ! -e ${php}/.env ]; then
+		echo "! ERROR: NO .env file..."
+		exec /bin/bash
+	fi
+
+	if [ -e .lumen ]; then
+		echo "* Lumen detected..."
+	else
+		# We only check for non mount points, in case this container has the app inside
+		mp=$(mp ${php})
+		if [ ${mp} -eq 1 ]; then
+			echo "* Caching configuration..."
+			su www-data -s /bin/sh -c "(php artisan config:cache && php artisan route:cache && php artisan view:cache)"
+		fi
+	fi
+
+	echo "* Running the scheduler..."
+	# We'll delay starting in case the app is caching
+	sleep 15
+
+	su www-data -s /bin/sh -c "
+	while true; do
+		(php ${PHP_OPTIONS} artisan schedule:run --verbose --no-interaction &)
+		sleep 60
+	done
+	"
+
+else
+	nginx_start
+
+	echo "? NO container role \"${role}\", AND/OR no laravel install, just starting php-fpm"
+	exec /usr/local/bin/docker-php-entrypoint "$@"
+fi

msmtprc

@@ -0,0 +1,18 @@
+# A system wide configuration file is optional.
+# If it exists, it usually defines a default account.
+# This allows msmtp to be used like /usr/sbin/sendmail.
+account default
+
+# The SMTP smarthost
+host smtp
+
+# Envelope-from address
+#from user@example.com
+domain example.com
+
+# Construct envelope-from addresses of the form "user@oursite.example"
+#auto_from on
+#maildomain example.com
+
+# Syslog logging with facility LOG_MAIL instead of the default LOG_USER
+syslog LOG_MAIL

nginx-app.conf

@@ -0,0 +1,34 @@
+server {
+	listen			80 default_server;
+	listen			[::]:80 default_server;
+
+	access_log		off;
+	client_max_body_size	10m;
+	fastcgi_buffers		16 16k;
+	fastcgi_buffer_size	16k;
+	index			index.php index.html;
+	root			/var/www/html/public;
+	server_tokens		off;
+
+	set $my_https "off";
+	if ($http_x_forwarded_proto = "https") {
+		set $my_https "on";
+	}
+
+	location / {
+		try_files $uri $uri/ /index.php?$query_string;
+	}
+
+	location ~ \.php$ {
+		try_files $uri =404;
+		fastcgi_split_path_info ^(.+\.php)(/.+)$;
+		fastcgi_pass 127.0.0.1:9000;
+		fastcgi_index index.php;
+		include fastcgi_params;
+		fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
+		fastcgi_param SERVER_NAME $host;
+		fastcgi_param PATH_INFO $fastcgi_path_info;
+		fastcgi_param HTTPS $my_https;
+		fastcgi_param PHP_ADMIN_VALUE "sendmail_path=/usr/sbin/sendmail -i -t";
+	}
+}

sshd_config.patch

@@ -1,20 +1,18 @@
---- /etc/ssh/sshd_config.orig    2017-12-13 10:12:21.098005827 +0000
-+++ /etc/ssh/sshd_config 2017-12-13 10:14:11.461687661 +0000
-@@ -25,7 +25,7 @@
+--- /etc/ssh/sshd_config.orig   2018-02-27 08:33:29.613104521 +0000
++++ /etc/ssh/sshd_config        2018-02-27 08:34:43.413485512 +0000
+@@ -30,6 +30,7 @@
  
- # Authentication:
- LoginGraceTime 120
--PermitRootLogin without-password
+ #LoginGraceTime 2m
+ #PermitRootLogin prohibit-password
 +PermitRootLogin no
- StrictModes yes
+ #StrictModes yes
+ #MaxAuthTries 6
+ #MaxSessions 10
+@@ -54,6 +55,7 @@
  
- RSAAuthentication yes
-@@ -49,7 +49,7 @@
- ChallengeResponseAuthentication no
- 
- # Change to no to disable tunnelled clear text passwords
--#PasswordAuthentication yes
+ # To disable tunneled clear text passwords, change to no here!
+ #PasswordAuthentication yes
 +PasswordAuthentication no
+ #PermitEmptyPasswords no
  
- # Kerberos options
- #KerberosAuthentication no
+ # Change to yes to enable challenge-response passwords (beware issues with

start

@@ -1,8 +0,0 @@
-#!/bin/sh
-
-if [  -x /usr/sbin/sshd -a "${SSH_START}" = "TRUE" ]; then
-	[ ! -d /var/run/sshd ] && mkdir /var/run/sshd
-	start-stop-daemon --start --pidfile /var/run/sshd.pid --exec /usr/sbin/sshd -- -p 22
-fi
-
-exec /usr/local/bin/docker-php-entrypoint "$@"

www.conf

@@ -0,0 +1,9 @@
+[www]
+group = www-data
+listen = 127.0.0.1:9000
+pm = dynamic
+pm.max_children = 25
+pm.max_spare_servers = 10
+pm.min_spare_servers = 5
+pm.start_servers = 10
+user = www-data