Page MenuHomeDevCentral
Files F3751684

D2943.id7493.diff
No OneTemporary
Actions

D2943.id7493.diff
View Options

diff --git a/PORTS b/PORTS
--- a/PORTS
+++ b/PORTS
@@ -41,6 +41,8 @@
43080 Hauk
44080 Hound
# 45080 should be reserved for OpenGrok to compare with Hound
+ 46080 Airflow - HTTP
+ 46555 Airflow - Flower
47080 Jenkins HTTP - Test
50000 Jenkins controller's port for JNLP-based Jenkins agents - CD
52000 Jenkins controller's port for JNLP-based Jenkins agents - Test
diff --git a/_modules/credentials.py b/_modules/credentials.py
--- a/_modules/credentials.py
+++ b/_modules/credentials.py
@@ -117,6 +117,14 @@
return get_password(key, prefix)
+def get_dsn(host, key, prefix=None):
+ if _are_credentials_hidden():
+ return "credential for " + key
+
+ secret = read_secret(key, prefix)
+ return f"{secret['username']}:{secret['password']}@{host}"
+
+
# -------------------------------------------------------------
# Helpers for Sentry credentials
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
diff --git a/pillar/credentials/vault.sls b/pillar/credentials/vault.sls
--- a/pillar/credentials/vault.sls
+++ b/pillar/credentials/vault.sls
@@ -103,6 +103,10 @@
# Format: ops/secrets/nasqueron/service/<...>
#
+ - ops/secrets/nasqueron/airflow/admin_account
+ - ops/secrets/nasqueron/airflow/fernet
+ - ops/secrets/dbserver/cluster-A/users/airflow
+
- ops/secrets/nasqueron/rabbitmq/white-rabbit/erlang-cookie
- ops/secrets/nasqueron/rabbitmq/white-rabbit/root
@@ -163,6 +167,15 @@
paas-docker-dev:
+ #
+ # Credentials used by Nasqueron services
+ # Format: ops/secrets/nasqueron/service/<...>
+ #
+
+ - ops/secrets/nasqueron/airflow/admin_account
+ - ops/secrets/nasqueron/airflow/fernet
+ - ops/secrets/dbserver/cluster-A/users/airflow
+
#
# Credentials used by projects hosted by Nasqueron
# Format: <project name>.<service>.<type>
diff --git a/pillar/paas/docker/dwellers/airflow.sls b/pillar/paas/docker/dwellers/airflow.sls
new file mode 100644
--- /dev/null
+++ b/pillar/paas/docker/dwellers/airflow.sls
@@ -0,0 +1,68 @@
+# -------------------------------------------------------------
+# Salt — Provision Docker engine
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# License: Trivial work, not eligible to copyright
+# Service: Airfllow
+# -------------------------------------------------------------
+
+docker_images:
+ - apache/airflow:2.5.2
+ - redis
+
+airflow_default_container_args: &airflow
+ realm: nasqueron
+ network: airflow
+
+docker_containers:
+ redis:
+ airflow_redis: *airflow
+
+ airflow:
+ airflow_web:
+ <<: *airflow
+ command: web
+ command_port: 8080
+ app_port: 46080
+ healthcheck: ["CMD", "curl", "--fail", "http://localhost:8080/health"]
+
+ airflow_scheduler:
+ <<: *airflow
+ command: scheduler
+ healthcheck: ["CMD", "curl", "--fail", "http://localhost:8974/health"]
+
+ airflow_worker:
+ <<: *airflow
+ command: celery worker
+ healthcheck:
+ - CMD-SHELL
+ - celery --app airflow.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}"
+
+ airflow_triggerer:
+ <<: *airflow
+ command: triggerer
+ healthcheck:
+ - CMD-SHELL
+ - airflow jobs check --job-type TriggererJob --hostname "$${HOSTNAME}"
+
+ airflow_flower:
+ <<: *airflow
+ command: celery flower
+ command_port: 5555
+ app_port: 46555
+ healthcheck: ["CMD", "curl", "--fail", "http://localhost:5555/"]
+
+docker_networks:
+ airflow:
+ subnet: 172.18.4.0/24
+
+airflow_realms:
+ nasqueron:
+ network: airflow
+ services:
+ redis: airflow_redis
+ postgresql: 172.27.27.8 # db-A-001.nasqueron.drake
+ credentials:
+ admin_account: nasqueron/airflow/admin_account
+ fernet_key: nasqueron/airflow/fernet
+ postgresql: dbserver/cluster-A/users/airflow
diff --git a/roles/paas-docker/containers/airflow.sls b/roles/paas-docker/containers/airflow.sls
new file mode 100644
--- /dev/null
+++ b/roles/paas-docker/containers/airflow.sls
@@ -0,0 +1,117 @@
+# -------------------------------------------------------------
+# Salt — Provision Docker engine
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# License: Trivial work, not eligible to copyright
+# -------------------------------------------------------------
+
+{% set has_selinux = salt['grains.get']('selinux:enabled', False) %}
+
+# -------------------------------------------------------------
+# Data directory
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+{% for realm, realm_args in pillar['airflow_realms'].items() %}
+
+/srv/airflow/{{ realm }}:
+ file.directory:
+ - user: 50000
+ - group: 0
+ - makedirs: True
+
+{% for subdir in ["dags", "logs", "plugins"] %}
+/srv/airflow/{{ realm }}/{{ subdir }}:
+ file.directory:
+ - user: 50000
+ - group: 0
+{% endfor %}
+
+/srv/airflow/{{ realm }}/bin/airflow:
+ file.managed:
+ - source: salt://roles/paas-docker/containers/files/airflow/airflow.sh.jinja
+ - makedirs: True
+ - mode: 755
+ - template: jinja
+ - context:
+ realm: {{ realm }}
+ network: {{ realm_args.network }}
+ credentials: {{ realm_args.credentials }}
+ services: {{ realm_args.services }}
+
+{% if has_selinux %}
+selinux_context_{{ realm }}_airflow_data:
+ selinux.fcontext_policy_present:
+ - name: /srv/airflow/{{ realm }}
+ - sel_type: container_file_t
+
+selinux_context_{{ realm }}_airflow_data_applied:
+ selinux.fcontext_policy_applied:
+ - name: /srv/airflow/{{ realm }}
+{% endif %}
+
+# -------------------------------------------------------------
+# Service initialization
+#
+# The first time Airflow is installed for a realm,
+# it needs to run database migrations.
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+airflow_init_{{ realm }}:
+ cmd.run:
+ - name: |
+ airflow {{ realm }} upgrade
+ touch /srv/airflow/{{ realm }}/.initialized
+ - environment:
+ - _AIRFLOW_WWW_USER: {{ realm_args["credentials"]["admin_account"] }}
+ - creates: /srv/airflow/{{ realm }}/.initialized
+
+{% endfor %}
+
+# -------------------------------------------------------------
+# Containers
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+{% for instance, container in pillar['docker_containers']['airflow'].items() %}
+
+{% set realm = container["realm"] %}
+{% set realm_args = pillar["airflow_realms"][realm] %}
+
+{% set postgresql_dsn = salt["credentials.get_dsn"](realm_args["services"]["postgresql"], realm_args["credentials"]["postgresql"]) %}
+
+{{ instance }}:
+ docker_container.running:
+ - detach: True
+ - interactive: True
+ - image: apache/airflow:2.5.2
+ - command: {{ container["command"] }}
+ - binds:
+ - /srv/airflow/{{ realm }}/dags:/opt/airflow/dags
+ - /srv/airflow/{{ realm }}/logs:/opt/airflow/logs
+ - /srv/airflow/{{ realm }}/plugins:/opt/airflow/plugins
+ - environment:
+ - AIRFLOW__CORE__EXECUTOR: CeleryExecutor
+ - AIRFLOW__CORE__FERNET_KEY: {{ salt["credentials.get_password"](realm_args["credentials"]["fernet_key"]) }}
+ - AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "true"
+ - AIRFLOW__CORE__LOAD_EXAMPLES: "false"
+
+ - AIRFLOW__API__AUTH_BACKENDS: airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session
+
+ - AIRFLOW__CELERY__BROKER_URL: redis://:@{{ realm_args["services"]["redis"] }}:6379/0
+ - AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://{{ postgresql_dsn }}/airflow
+
+ - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://{{ postgresql_dsn }}/airflow
+
+ - AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK: "true"
+ {% if "app_port" in container %}
+ - ports:
+ - {{ container['command_port'] }}
+ - port_bindings:
+ - {{ container['app_port'] }}:{{ container['command_port'] }}
+ {% endif %}
+ - healthcheck:
+ Test: {{ container['healthcheck'] }}
+ Interval: 30000000000
+ - networks:
+ - {{ container['network'] }}
+
+{% endfor %}
diff --git a/roles/paas-docker/containers/files/airflow/airflow.sh.jinja b/roles/paas-docker/containers/files/airflow/airflow.sh.jinja
new file mode 100644
--- /dev/null
+++ b/roles/paas-docker/containers/files/airflow/airflow.sh.jinja
@@ -0,0 +1,51 @@
+#!/bin/sh
+
+# -------------------------------------------------------------
+# PaaS Docker
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# License: Trivial work, not eligible to copyright
+# Description: Wrapper for airflow command (local instance)
+# Source file: roles/paas-docker/containers/files/sentry/airflow.sh.jinja
+# Realm: {{ realm }}
+# -------------------------------------------------------------
+#
+# <auto-generated>
+# This file is managed by our rOPS SaltStack repository.
+#
+# Changes to this file may cause incorrect behavior
+# and will be lost if the state is redeployed.
+# </auto-generated>
+
+set -e
+
+COMMAND=$1
+shift
+
+EXTRA_ARGS=""
+
+if [ -n "$_AIRFLOW_WWW_USER" ]; then
+ # Used by initialization to create a default first user
+ user="$(credential $_AIRFLOW_WWW_USER username)"
+ password="$(credential $_AIRFLOW_WWW_USER)"
+
+ EXTRA_ARGS="$EXTRA_ARGS \
+ -e _AIRFLOW_WWW_USER_CREATE=true \
+ -e _AIRFLOW_WWW_USER_USERNAME=$user \
+ -e _AIRFLOW_WWW_USER_PASSWORD=$password"
+fi
+
+if [ "$COMMAND" = "upgrade" ]; then
+ EXTRA_ARGS="$EXTRA_ARGS \
+ -e _AIRFLOW_DB_UPGRADE=true"
+ COMMAND=version
+fi
+
+dsn="$(credential {{ credentials.postgresql }} username):$(credential {{ credentials.postgresql }})@{{ services.postgresql }}"
+docker run -it --rm \
+ --network {{ network }} \
+ -e _PIP_ADDITIONAL_REQUIREMENTS="" \
+ -e CONNECTION_CHECK_MAX_COUNT=0 \
+ -e AIRFLOW__DATABASE__SQL_ALCHEMY_CONN="postgresql+psycopg2://$dsn/airflow" \
+ $EXTRA_ARGS \
+ apache/airflow:2.5.2 bash -c "airflow $COMMAND $@"
diff --git a/roles/paas-docker/wrappers/files/sentry.sh b/roles/paas-docker/wrappers/files/run-by-realm.sh.jinja
rename from roles/paas-docker/wrappers/files/sentry.sh
rename to roles/paas-docker/wrappers/files/run-by-realm.sh.jinja
--- a/roles/paas-docker/wrappers/files/sentry.sh
+++ b/roles/paas-docker/wrappers/files/run-by-realm.sh.jinja
@@ -4,9 +4,11 @@
# PaaS Docker
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Project: Nasqueron
-# Created: 2018-11-11
# License: Trivial work, not eligible to copyright
-# Source file: roles/paas-docker/wrappers/files/sentry.sh
+# Source file: roles/paas-docker/wrappers/files/run-by-realm.sh
+# Description: Redirect to a specific realm wrapper,
+# customized with correct volumes and environment.
+# Command: {{ service }}
# -------------------------------------------------------------
#
# <auto-generated>
@@ -24,16 +26,16 @@
REALM=$1
shift
-if [ ! -d "/srv/sentry/$REALM" ]; then
+if [ ! -d "/srv/{{ service }}/$REALM" ]; then
echo "Realm doesn't exist: $REALM" 1>&2;
exit 2
fi
-DOCKER_RUN_SCRIPT=/srv/sentry/$REALM/bin/sentry
+DOCKER_RUN_SCRIPT=/srv/{{ service }}/$REALM/bin/{{ service }}
if [ ! -f "$DOCKER_RUN_SCRIPT" ]; then
echo "File doesn't exist: $DOCKER_RUN_SCRIPT" 1>&2;
- echo "You can generate it running 'deploy-container sentry' command on the Salt primary server. 1>&2;"
+ echo "You can generate it running 'deploy-container {{ service }}' command on the Salt primary server. 1>&2;"
exit 4
fi
diff --git a/roles/paas-docker/wrappers/init.sls b/roles/paas-docker/wrappers/init.sls
--- a/roles/paas-docker/wrappers/init.sls
+++ b/roles/paas-docker/wrappers/init.sls
@@ -12,13 +12,23 @@
# Wrapper binaries
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-{% for command in ['certbot', 'jenkins', 'phpbb', 'mysql', 'sentry', 'openfire', 'geoipupdate'] %}
+{% for command in ['certbot', 'jenkins', 'phpbb', 'mysql', 'openfire', 'geoipupdate'] %}
{{ dirs.bin }}/{{ command }}:
file.managed:
- source: salt://roles/paas-docker/wrappers/files/{{ command }}.sh
- mode: 755
{% endfor %}
+{% for command in ['airflow', 'sentry'] %}
+{{ dirs.bin }}/{{ command }}:
+ file.managed:
+ - source: salt://roles/paas-docker/wrappers/files/run-by-realm.sh.jinja
+ - mode: 755
+ - template: jinja
+ - context:
+ service: {{ command }}
+{% endfor %}
+
{% for command in ['pad-delete'] %}
{{ dirs.bin }}/{{ command }}:
file.managed:

File Metadata

Mime Type
text/plain
Expires
Mon, Nov 18, 11:39 (10 h, 13 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
2250610
Default Alt Text
D2943.id7493.diff (12 KB)

Event Timeline

Nasqueron DevCentral · If it had been much bigger the moon would have had a core of ice. · Powered by Phabricator