Page MenuHomeDevCentral
Files F3775181

D3273.id8412.diff
No OneTemporary
Actions

D3273.id8412.diff
View Options

diff --git a/pillar/credentials/vault.sls b/pillar/credentials/vault.sls
--- a/pillar/credentials/vault.sls
+++ b/pillar/credentials/vault.sls
@@ -194,6 +194,7 @@
- ops/secrets/nasqueron/airflow/admin_account
- ops/secrets/nasqueron/airflow/fernet
- ops/secrets/nasqueron/airflow/sentry
+ - ops/secrets/nasqueron/airflow/vault
- ops/secrets/dbserver/cluster-A/users/airflow
- ops/secrets/nasqueron/orbeon/oxf.crypto.password
diff --git a/pillar/paas/docker/dwellers/airflow.sls b/pillar/paas/docker/dwellers/airflow.sls
--- a/pillar/paas/docker/dwellers/airflow.sls
+++ b/pillar/paas/docker/dwellers/airflow.sls
@@ -57,6 +57,11 @@
admin_account: nasqueron/airflow/admin_account
fernet_key: nasqueron/airflow/fernet
postgresql: dbserver/cluster-A/users/airflow
+ vault: nasqueron/airflow/vault
+ vault:
+ url: https://172.27.27.7:8200
+ mount_point: apps
+ secrets_path: airflow
sentry:
realm: nasqueron
project_id: 4
diff --git a/roles/paas-docker/containers/airflow.sls b/roles/paas-docker/containers/airflow.sls
--- a/roles/paas-docker/containers/airflow.sls
+++ b/roles/paas-docker/containers/airflow.sls
@@ -7,12 +7,13 @@
{% set has_selinux = salt['grains.get']('selinux:enabled', False) %}
+
+{% for realm, realm_args in pillar['airflow_realms'].items() %}
+
# -------------------------------------------------------------
# Data directory
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-{% for realm, realm_args in pillar['airflow_realms'].items() %}
-
/srv/airflow/{{ realm }}:
file.directory:
- user: 50000
@@ -49,6 +50,29 @@
- name: /srv/airflow/{{ realm }}
{% endif %}
+# -------------------------------------------------------------
+# Airflow configuration for this realm
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+{% set postgresql_dsn = salt["credentials.get_dsn"](realm_args["services"]["postgresql"], realm_args["credentials"]["postgresql"]) %}
+
+/srv/airflow/{{ realm }}/airflow.cfg:
+ file.managed:
+ - source: salt://roles/paas-docker/containers/files/airflow/airflow.cfg.jinja
+ - mode: 400
+ - user: 50000 # As defined in Airflow upstream Docker image
+ - template: jinja
+ - context:
+ realm: {{ realm }}
+ vault: {{ realm_args["vault"] }}
+ services:
+ redis: {{ realm_args["services"]["redis"] }}
+ credentials:
+ fernet_key: {{ salt["credentials.get_password"](realm_args["credentials"]["fernet_key"]) }}
+ db: db+postgresql://{{ postgresql_dsn }}/airflow
+ sentry: {{ salt["credentials.get_sentry_dsn"](realm_args["sentry"]) }}
+ vault: {{ salt["credentials.read_secret"](realm_args["credentials"]["vault"]) }}
+
# -------------------------------------------------------------
# Service initialization
#
@@ -67,6 +91,7 @@
{% endfor %}
+
# -------------------------------------------------------------
# Containers
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
@@ -74,9 +99,6 @@
{% for instance, container in pillar['docker_containers']['airflow'].items() %}
{% set realm = container["realm"] %}
-{% set realm_args = pillar["airflow_realms"][realm] %}
-
-{% set postgresql_dsn = salt["credentials.get_dsn"](realm_args["services"]["postgresql"], realm_args["credentials"]["postgresql"]) %}
{{ instance }}:
docker_container.running:
@@ -88,21 +110,7 @@
- /srv/airflow/{{ realm }}/dags:/opt/airflow/dags
- /srv/airflow/{{ realm }}/logs:/opt/airflow/logs
- /srv/airflow/{{ realm }}/plugins:/opt/airflow/plugins
- - environment:
- - AIRFLOW__CORE__EXECUTOR: CeleryExecutor
- - AIRFLOW__CORE__FERNET_KEY: {{ salt["credentials.get_password"](realm_args["credentials"]["fernet_key"]) }}
- - AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "True"
- - AIRFLOW__CORE__LOAD_EXAMPLES: "False"
-
- - AIRFLOW__API__AUTH_BACKENDS: airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session
-
- - AIRFLOW__CELERY__BROKER_URL: redis://:@{{ realm_args["services"]["redis"] }}:6379/0
- - AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://{{ postgresql_dsn }}/airflow
-
- - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://{{ postgresql_dsn }}/airflow
-
- - AIRFLOW__SENTRY__SENTRY_ON: "True"
- - AIRFLOW__SENTRY__SENTRY_DSN: {{ salt["credentials.get_sentry_dsn"](realm_args["sentry"]) }}
+ - /srv/airflow/{{ realm }}/airflow.cfg:/opt/airflow/airflow.cfg
{% if "app_port" in container %}
- ports:
- {{ container['command_port'] }}
diff --git a/roles/paas-docker/containers/files/airflow/airflow.cfg.jinja b/roles/paas-docker/containers/files/airflow/airflow.cfg.jinja
new file mode 100644
--- /dev/null
+++ b/roles/paas-docker/containers/files/airflow/airflow.cfg.jinja
@@ -0,0 +1,53 @@
+# -------------------------------------------------------------
+# PaaS Docker - Airflow configuration
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# License: Trivial work, not eligible to copyright
+# Source file: roles/paas-docker/containers/files/airflow/airflow.cfg.jinja
+# Realm: {{ realm }}
+# -------------------------------------------------------------
+#
+# <auto-generated>
+# This file is managed by our rOPS SaltStack repository.
+#
+# Changes to this file may cause incorrect behavior
+# and will be lost if the state is redeployed.
+# </auto-generated>
+
+[core]
+executor = CeleryExecutor
+
+dags_are_paused_at_creation = True
+load_examples = False
+
+fernet_key = {{ credentials.fernet_key }}
+
+[database]
+sql_alchemy_conn = {{ credentials.db }}
+
+load_default_connections = False
+
+[secrets]
+backend = airflow.providers.hashicorp.secrets.vault.VaultBackend
+backend_kwargs = {
+ "url": "{{ vault.url }}",
+ "auth_type": "approle",
+ "role_id": "{{ credentials.vault.username }}",
+ "secret_id": "{{ credentials.vault.password }}",
+
+ "mount_point": "{{ vault.mount_point }}",
+ "connections_path": "{{ vault.secrets_path }}/connections",
+ "config_path": "{{ vault.secrets_path }}/config",
+ "variables_path": "{{ vault.secrets_path }}/variables"
+ }
+
+[api]
+auth_backends = airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session
+
+[sentry]
+sentry_on = True
+sentry_dsn = {{ credentials.sentry }}
+
+[celery]
+broker_url = redis://:@{{ services.redis }}:6379/0
+result_backend = {{ credentials.db }}

File Metadata

Mime Type
text/plain
Expires
Mon, Nov 25, 16:56 (21 h, 23 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
2262835
Default Alt Text
D3273.id8412.diff (6 KB)

Event Timeline

Nasqueron DevCentral · If it had been much bigger the moon would have had a core of ice. · Powered by Phabricator