D2943.id7502.diff
No OneTemporary
Actions

Size

12 KB

Referenced Files

None

Subscribers

None

D2943.id7502.diff
View Options

	diff --git a/PORTS b/PORTS
	--- a/PORTS
	+++ b/PORTS
	@@ -41,6 +41,8 @@
	43080 Hauk
	44080 Hound
	# 45080 should be reserved for OpenGrok to compare with Hound
	+ 46080 Airflow - HTTP
	+ 46555 Airflow - Flower
	47080 Jenkins HTTP - Test
	50000 Jenkins controller's port for JNLP-based Jenkins agents - CD
	52000 Jenkins controller's port for JNLP-based Jenkins agents - Test
	diff --git a/_modules/credentials.py b/_modules/credentials.py
	--- a/_modules/credentials.py
	+++ b/_modules/credentials.py
	@@ -117,6 +117,14 @@
	return get_password(key, prefix)


	+def get_dsn(host, key, prefix=None):
	+ if _are_credentials_hidden():
	+ return "credential for " + key
	+
	+ secret = read_secret(key, prefix)
	+ return f"{secret['username']}:{secret['password']}@{host}"
	+
	+
	# -------------------------------------------------------------
	# Helpers for Sentry credentials
	# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
	diff --git a/pillar/credentials/vault.sls b/pillar/credentials/vault.sls
	--- a/pillar/credentials/vault.sls
	+++ b/pillar/credentials/vault.sls
	@@ -103,6 +103,10 @@
	# Format: ops/secrets/nasqueron/service/<...>
	#

	+ - ops/secrets/nasqueron/airflow/admin_account
	+ - ops/secrets/nasqueron/airflow/fernet
	+ - ops/secrets/dbserver/cluster-A/users/airflow
	+
	- ops/secrets/nasqueron/rabbitmq/white-rabbit/erlang-cookie
	- ops/secrets/nasqueron/rabbitmq/white-rabbit/root

	@@ -163,6 +167,15 @@

	paas-docker-dev:

	+ #
	+ # Credentials used by Nasqueron services
	+ # Format: ops/secrets/nasqueron/service/<...>
	+ #
	+
	+ - ops/secrets/nasqueron/airflow/admin_account
	+ - ops/secrets/nasqueron/airflow/fernet
	+ - ops/secrets/dbserver/cluster-A/users/airflow
	+
	#
	# Credentials used by projects hosted by Nasqueron
	# Format: <project name>.<service>.<type>
	diff --git a/pillar/paas/docker/dwellers/airflow.sls b/pillar/paas/docker/dwellers/airflow.sls
	new file mode 100644
	--- /dev/null
	+++ b/pillar/paas/docker/dwellers/airflow.sls
	@@ -0,0 +1,68 @@
	+# -------------------------------------------------------------
	+# Salt — Provision Docker engine
	+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
	+# Project: Nasqueron
	+# License: Trivial work, not eligible to copyright
	+# Service: Airfllow
	+# -------------------------------------------------------------
	+
	+docker_images:
	+ - apache/airflow:2.5.2
	+ - redis
	+
	+airflow_default_container_args: &airflow
	+ realm: nasqueron
	+ network: airflow
	+
	+docker_containers:
	+ redis:
	+ airflow_redis: *airflow
	+
	+ airflow:
	+ airflow_web:
	+ <<: *airflow
	+ command: webserver
	+ command_port: 8080
	+ app_port: 46080
	+ healthcheck: ["CMD", "curl", "--fail", "http://localhost:8080/health"]
	+
	+ airflow_scheduler:
	+ <<: *airflow
	+ command: scheduler
	+ healthcheck: ["CMD", "curl", "--fail", "http://localhost:8974/health"]
	+
	+ airflow_worker:
	+ <<: *airflow
	+ command: celery worker
	+ healthcheck:
	+ - CMD-SHELL
	+ - celery --app airflow.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}"
	+
	+ airflow_triggerer:
	+ <<: *airflow
	+ command: triggerer
	+ healthcheck:
	+ - CMD-SHELL
	+ - airflow jobs check --job-type TriggererJob --hostname "$${HOSTNAME}"
	+
	+ airflow_flower:
	+ <<: *airflow
	+ command: celery flower
	+ command_port: 5555
	+ app_port: 46555
	+ healthcheck: ["CMD", "curl", "--fail", "http://localhost:5555/"]
	+
	+docker_networks:
	+ airflow:
	+ subnet: 172.18.4.0/24
	+
	+airflow_realms:
	+ nasqueron:
	+ network: airflow
	+ services:
	+ redis: airflow_redis
	+ postgresql: 172.27.27.8 # db-A-001.nasqueron.drake
	+ credentials:
	+ admin_account: nasqueron/airflow/admin_account
	+ fernet_key: nasqueron/airflow/fernet
	+ postgresql: dbserver/cluster-A/users/airflow
	diff --git a/roles/paas-docker/containers/airflow.sls b/roles/paas-docker/containers/airflow.sls
	new file mode 100644
	--- /dev/null
	+++ b/roles/paas-docker/containers/airflow.sls
	@@ -0,0 +1,117 @@
	+# -------------------------------------------------------------
	+# Salt — Provision Docker engine
	+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
	+# Project: Nasqueron
	+# License: Trivial work, not eligible to copyright
	+# -------------------------------------------------------------
	+
	+{% set has_selinux = salt['grains.get']('selinux:enabled', False) %}
	+
	+# -------------------------------------------------------------
	+# Data directory
	+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
	+
	+{% for realm, realm_args in pillar['airflow_realms'].items() %}
	+
	+/srv/airflow/{{ realm }}:
	+ file.directory:
	+ - user: 50000
	+ - group: 0
	+ - makedirs: True
	+
	+{% for subdir in ["dags", "logs", "plugins"] %}
	+/srv/airflow/{{ realm }}/{{ subdir }}:
	+ file.directory:
	+ - user: 50000
	+ - group: 0
	+{% endfor %}
	+
	+/srv/airflow/{{ realm }}/bin/airflow:
	+ file.managed:
	+ - source: salt://roles/paas-docker/containers/files/airflow/airflow.sh.jinja
	+ - makedirs: True
	+ - mode: 755
	+ - template: jinja
	+ - context:
	+ realm: {{ realm }}
	+ network: {{ realm_args.network }}
	+ credentials: {{ realm_args.credentials }}
	+ services: {{ realm_args.services }}
	+
	+{% if has_selinux %}
	+selinux_context_{{ realm }}_airflow_data:
	+ selinux.fcontext_policy_present:
	+ - name: /srv/airflow/{{ realm }}
	+ - sel_type: container_file_t
	+
	+selinux_context_{{ realm }}_airflow_data_applied:
	+ selinux.fcontext_policy_applied:
	+ - name: /srv/airflow/{{ realm }}
	+{% endif %}
	+
	+# -------------------------------------------------------------
	+# Service initialization
	+#
	+# The first time Airflow is installed for a realm,
	+# it needs to run database migrations.
	+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
	+
	+airflow_init_{{ realm }}:
	+ cmd.run:
	+ - name: \|
	+ airflow {{ realm }} upgrade
	+ touch /srv/airflow/{{ realm }}/.initialized
	+ - environment:
	+ - _AIRFLOW_WWW_USER: {{ realm_args["credentials"]["admin_account"] }}
	+ - creates: /srv/airflow/{{ realm }}/.initialized
	+
	+{% endfor %}
	+
	+# -------------------------------------------------------------
	+# Containers
	+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
	+
	+{% for instance, container in pillar['docker_containers']['airflow'].items() %}
	+
	+{% set realm = container["realm"] %}
	+{% set realm_args = pillar["airflow_realms"][realm] %}
	+
	+{% set postgresql_dsn = salt["credentials.get_dsn"](realm_args["services"]["postgresql"], realm_args["credentials"]["postgresql"]) %}
	+
	+{{ instance }}:
	+ docker_container.running:
	+ - detach: True
	+ - interactive: True
	+ - image: apache/airflow:2.5.2
	+ - command: {{ container["command"] }}
	+ - binds:
	+ - /srv/airflow/{{ realm }}/dags:/opt/airflow/dags
	+ - /srv/airflow/{{ realm }}/logs:/opt/airflow/logs
	+ - /srv/airflow/{{ realm }}/plugins:/opt/airflow/plugins
	+ - environment:
	+ - AIRFLOW__CORE__EXECUTOR: CeleryExecutor
	+ - AIRFLOW__CORE__FERNET_KEY: {{ salt["credentials.get_password"](realm_args["credentials"]["fernet_key"]) }}
	+ - AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "true"
	+ - AIRFLOW__CORE__LOAD_EXAMPLES: "false"
	+
	+ - AIRFLOW__API__AUTH_BACKENDS: airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session
	+
	+ - AIRFLOW__CELERY__BROKER_URL: redis://:@{{ realm_args["services"]["redis"] }}:6379/0
	+ - AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://{{ postgresql_dsn }}/airflow
	+
	+ - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://{{ postgresql_dsn }}/airflow
	+
	+ - AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK: "true"
	+ {% if "app_port" in container %}
	+ - ports:
	+ - {{ container['command_port'] }}
	+ - port_bindings:
	+ - 127.0.0.1:{{ container['app_port'] }}:{{ container['command_port'] }}
	+ {% endif %}
	+ - healthcheck:
	+ Test: {{ container['healthcheck'] }}
	+ Interval: 30000000000
	+ - networks:
	+ - {{ container['network'] }}
	+
	+{% endfor %}
	diff --git a/roles/paas-docker/containers/files/airflow/airflow.sh.jinja b/roles/paas-docker/containers/files/airflow/airflow.sh.jinja
	new file mode 100644
	--- /dev/null
	+++ b/roles/paas-docker/containers/files/airflow/airflow.sh.jinja
	@@ -0,0 +1,51 @@
	+#!/bin/sh
	+
	+# -------------------------------------------------------------
	+# PaaS Docker
	+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
	+# Project: Nasqueron
	+# License: Trivial work, not eligible to copyright
	+# Description: Wrapper for airflow command (local instance)
	+# Source file: roles/paas-docker/containers/files/sentry/airflow.sh.jinja
	+# Realm: {{ realm }}
	+# -------------------------------------------------------------
	+#
	+# <auto-generated>
	+# This file is managed by our rOPS SaltStack repository.
	+#
	+# Changes to this file may cause incorrect behavior
	+# and will be lost if the state is redeployed.
	+# </auto-generated>
	+
	+set -e
	+
	+COMMAND=$1
	+shift
	+
	+EXTRA_ARGS=""
	+
	+if [ -n "$_AIRFLOW_WWW_USER" ]; then
	+ # Used by initialization to create a default first user
	+ user="$(credential $_AIRFLOW_WWW_USER username)"
	+ password="$(credential $_AIRFLOW_WWW_USER)"
	+
	+ EXTRA_ARGS="$EXTRA_ARGS \
	+ -e _AIRFLOW_WWW_USER_CREATE=true \
	+ -e _AIRFLOW_WWW_USER_USERNAME=$user \
	+ -e _AIRFLOW_WWW_USER_PASSWORD=$password"
	+fi
	+
	+if [ "$COMMAND" = "upgrade" ]; then
	+ EXTRA_ARGS="$EXTRA_ARGS \
	+ -e _AIRFLOW_DB_UPGRADE=true"
	+ COMMAND=version
	+fi
	+
	+dsn="$(credential {{ credentials.postgresql }} username):$(credential {{ credentials.postgresql }})@{{ services.postgresql }}"
	+docker run -it --rm \
	+ --network {{ network }} \
	+ -e _PIP_ADDITIONAL_REQUIREMENTS="" \
	+ -e CONNECTION_CHECK_MAX_COUNT=0 \
	+ -e AIRFLOW__DATABASE__SQL_ALCHEMY_CONN="postgresql+psycopg2://$dsn/airflow" \
	+ $EXTRA_ARGS \
	+ apache/airflow:2.5.2 bash -c "airflow $COMMAND $@"
	diff --git a/roles/paas-docker/wrappers/files/sentry.sh b/roles/paas-docker/wrappers/files/run-by-realm.sh.jinja
	rename from roles/paas-docker/wrappers/files/sentry.sh
	rename to roles/paas-docker/wrappers/files/run-by-realm.sh.jinja
	--- a/roles/paas-docker/wrappers/files/sentry.sh
	+++ b/roles/paas-docker/wrappers/files/run-by-realm.sh.jinja
	@@ -4,9 +4,11 @@
	# PaaS Docker
	# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
	# Project: Nasqueron
	-# Created: 2018-11-11
	# License: Trivial work, not eligible to copyright
	-# Source file: roles/paas-docker/wrappers/files/sentry.sh
	+# Source file: roles/paas-docker/wrappers/files/run-by-realm.sh
	+# Description: Redirect to a specific realm wrapper,
	+# customized with correct volumes and environment.
	+# Command: {{ service }}
	# -------------------------------------------------------------
	#
	# <auto-generated>
	@@ -24,16 +26,16 @@
	REALM=$1
	shift

	-if [ ! -d "/srv/sentry/$REALM" ]; then
	+if [ ! -d "/srv/{{ service }}/$REALM" ]; then
	echo "Realm doesn't exist: $REALM" 1>&2;
	exit 2
	fi

	-DOCKER_RUN_SCRIPT=/srv/sentry/$REALM/bin/sentry
	+DOCKER_RUN_SCRIPT=/srv/{{ service }}/$REALM/bin/{{ service }}

	if [ ! -f "$DOCKER_RUN_SCRIPT" ]; then
	echo "File doesn't exist: $DOCKER_RUN_SCRIPT" 1>&2;
	- echo "You can generate it running 'deploy-container sentry' command on the Salt primary server. 1>&2;"
	+ echo "You can generate it running 'deploy-container {{ service }}' command on the Salt primary server. 1>&2;"
	exit 4
	fi

	diff --git a/roles/paas-docker/wrappers/init.sls b/roles/paas-docker/wrappers/init.sls
	--- a/roles/paas-docker/wrappers/init.sls
	+++ b/roles/paas-docker/wrappers/init.sls
	@@ -12,13 +12,23 @@
	# Wrapper binaries
	# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

	-{% for command in ['certbot', 'jenkins', 'phpbb', 'mysql', 'sentry', 'openfire', 'geoipupdate'] %}
	+{% for command in ['certbot', 'jenkins', 'phpbb', 'mysql', 'openfire', 'geoipupdate'] %}
	{{ dirs.bin }}/{{ command }}:
	file.managed:
	- source: salt://roles/paas-docker/wrappers/files/{{ command }}.sh
	- mode: 755
	{% endfor %}

	+{% for command in ['airflow', 'sentry'] %}
	+{{ dirs.bin }}/{{ command }}:
	+ file.managed:
	+ - source: salt://roles/paas-docker/wrappers/files/run-by-realm.sh.jinja
	+ - mode: 755
	+ - template: jinja
	+ - context:
	+ service: {{ command }}
	+{% endfor %}
	+
	{% for command in ['pad-delete'] %}
	{{ dirs.bin }}/{{ command }}:
	file.managed:

File Metadata

Mime Type: text/plain
Expires: Mon, Nov 18, 11:40 (10 h, 18 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 2250613
Default Alt Text: D2943.id7502.diff (12 KB)

D2943.id7502.diffNo OneTemporaryActions

D2943.id7502.diffView Options

File Metadata

Event Timeline

D2943.id7502.diff
No OneTemporary
Actions

D2943.id7502.diff
View Options