Page MenuHomeDevCentral
Files F11723743

No OneTemporary
Actions

View Options

diff --git a/pillar/credentials/vault.sls b/pillar/credentials/vault.sls
index 5d5e63f..b7c9763 100644
--- a/pillar/credentials/vault.sls
+++ b/pillar/credentials/vault.sls
@@ -1,269 +1,270 @@
# -------------------------------------------------------------
# Salt configuration for Nasqueron servers
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Project: Nasqueron
# License: Trivial work, not eligible to copyright
# -------------------------------------------------------------
# -------------------------------------------------------------
# Vault configuration
#
# :: vault_policies_path: path on vault server where to store policies
#
# :: vault_policies_source: path to fetch policies from
# if starting by salt://, from salt files server
#
# :: vault_mount_paths: translates secrets paths in policies paths
#
# Generally, Vault paths are the same for policies and data access.
#
# For kv secrets engine, version 2, writing and reading versions
# of a kv value are prefixed with the data/ path.
#
# credentials.build_policies_by_node will use this dictionary
# to be able to rewrite secrets paths in data paths.
#
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
vault_policies_path: /srv/policies/vault
vault_policies_source: /srv/policies/vault
vault_mount_paths:
ops/secrets: ops/data/secrets
ops/privacy: ops/data/privacy
apps: apps/data
# -------------------------------------------------------------
# Vault policies to deploy as-is, ie without templating.
#
# Entries of vault_policies must match a .hcl file in
# roles/vault/policies/files folder.
#
# If you need a template, create a new pillar entry instead
# and add the parsing logic either:
# - directly to roles/vault/policies/
#
# - through _modules/credentials.py for policies to apply
# to Salt nodes, like e.g. vault_secrets_by_role
#
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
vault_policies:
- admin
- airflow
- salt-primary
- sentry
- vault_bootstrap
- viperserv
# -------------------------------------------------------------
# Vault policies for Salt
#
# Declare the extra policies each nodes need.
#
# In adition of those extra policies, the vault_secrets_by_role
# will be parsed for the keys.
#
# IMPORTANT: as grains['roles'] can be modified by the node,
# roles are extracted directly from the pillar.
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
vault_extra_policies_by_role:
salt-primary:
- salt-primary
# -------------------------------------------------------------
# Vault secrets by role
#
# Paths of the keys the specified role needs access to.
#
# Avoid * notation as this namespace is shared between Vault
# and the applications. As such, only secrets the Salt nodes
# needs in a state they need to deploy should be listed here.
#
# Use %%node%% as variable for node name.
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
vault_secrets_by_role:
devserver:
- ops/secrets/nasqueron/notifications/notifications-cli/%%node%%
- ops/secrets/nasqueron/deploy/deploy_keys/alken-orin
- ops/secrets/nasqueron/deploy/deploy_keys/by_repo/bitbucket/dereckson/www
- ops/secrets/nasqueron/deploy/deploy_keys/by_repo/bitbucket/ewosp/www
- ops/secrets/nasqueron/deploy/deploy_keys/by_repo/github/wolfplex/api-www
opensearch:
- ops/secrets/nasqueron.opensearch.infra-logs.internal_users.admin
- ops/secrets/nasqueron.opensearch.infra-logs.internal_users.dashboards
paas-docker-prod:
#
# Personal data or personally identifiable information (PII)
# related to Nasqueron Operations SIG members.
#
- ops/privacy/ops-cidr
#
# Credentials used by Nasqueron services
# Format: ops/secrets/nasqueron/service/<...>
#
- ops/secrets/nasqueron/airflow/admin_account
- ops/secrets/nasqueron/airflow/fernet
- ops/secrets/nasqueron/airflow/sentry
- ops/secrets/dbserver/cluster-A/users/airflow
- ops/secrets/nasqueron/etherpad/mysql
- ops/secrets/nasqueron/etherpad/users/dereckson
- ops/secrets/nasqueron/penpot/github
- ops/secrets/nasqueron/penpot/postgresql
- ops/secrets/nasqueron/penpot/secret_key
- ops/secrets/nasqueron/rabbitmq/white-rabbit/erlang-cookie
- ops/secrets/nasqueron/rabbitmq/white-rabbit/root
- ops/secrets/nasqueron/sentry/geoipupdate
#
# Credentials used by Nasqueron services
# Format: ops/secrets/nasqueron.<service>.<type>
#
- ops/secrets/nasqueron.acquisitariat.mysql
- ops/secrets/nasqueron.auth-grove.mysql
- ops/secrets/nasqueron.cachet.app_key
- ops/secrets/nasqueron.cachet.mysql
- ops/secrets/nasqueron.etherpad.api
- ops/secrets/nasqueron.notifications.broker
- ops/secrets/nasqueron.notifications.mailgun
- ops/secrets/nasqueron.notifications.sentry
- ops/secrets/nasqueron.notifications.credentials_github_nasqueron
- ops/secrets/nasqueron.notifications.credentials_github_wolfplex
- ops/secrets/nasqueron.notifications.credentials_github_keruald
- ops/secrets/nasqueron.notifications.credentials_github_trustspace
- ops/secrets/nasqueron.notifications.credentials_github_eglide
- ops/secrets/nasqueron.notifications.credentials_phabricator_nasqueron
- apps/notifications-center/dockerhub/notifications
- apps/notifications-center/dockerhub/auth-grove
- ops/secrets/nasqueron.pixelfed.app_key
- ops/secrets/nasqueron.pixelfed.mailgun
- ops/secrets/nasqueron.pixelfed.mysql
- ops/secrets/nasqueron.sentry.app_key
- ops/secrets/nasqueron.sentry.postgresql
- ops/secrets/nasqueron.sentry.vault
#
# Credentials used by Nasqueron members private services
# Format: <username>.<service>.<type>
#
- ops/secrets/dereckson.phabricator.mysql
#
# Credentials used by projects hosted by Nasqueron
# Format: <project name>.<service>.<type>
#
- ops/secrets/espacewin.phpbb.mysql_root
- ops/secrets/wolfplex.phabricator.mailgun
- ops/secrets/wolfplex.phabricator.mysql
- ops/secrets/zed.phabricator.mysql
- ops/secrets/zed.phabricator.sendgrid
paas-docker-dev:
#
# Credentials used by Nasqueron services
# Format: ops/secrets/nasqueron/service/<...>
#
- ops/secrets/nasqueron/airflow/admin_account
- ops/secrets/nasqueron/airflow/fernet
- ops/secrets/nasqueron/airflow/sentry
+ - ops/secrets/nasqueron/airflow/vault
- ops/secrets/dbserver/cluster-A/users/airflow
- ops/secrets/nasqueron/orbeon/oxf.crypto.password
- ops/secrets/nasqueron/orbeon/users/dereckson
- ops/secrets/dbserver/cluster-A/users/orbeon
- ops/secrets/nasqueron/rabbitmq/orange-rabbit/erlang-cookie
- ops/secrets/nasqueron/rabbitmq/orange-rabbit/root
- ops/secrets/nasqueron/rabbitmq/orange-rabbit/notifications
- ops/secrets/nasqueron.notifications.sentry
#
# Credentials used by projects hosted by Nasqueron
# Format: <project name>.<service>.<type>
#
- ops/secrets/espacewin.bugzilla.mysql
- ops/secrets/espacewin.bugzilla.mysql_root
saas-mediawiki:
- ops/secrets/dbserver/cluster-B/users/saas-mediawiki
- ops/secrets/nasqueron/mediawiki/secret_key
saas-wordpress:
- ops/secrets/dbserver/cluster-B/users/dereckson_blog
- ops/secrets/dereckson/wordpress/secrets
viperserv:
- ops/secrets/nasqueron.viperserv.vault
webserver-alkane-prod:
- ops/secrets/dbserver/cluster-B/users/dereckson_www
- ops/secrets/dbserver/cluster-B/users/zed
- ops/secrets/nasqueron/deploy/deploy_keys/by_repo/github/hypership/content_users
- ops/secrets/zed/hypership/secret_key
#
# Wolfplex credentials
#
- ops/secrets/nasqueron.etherpad.api
webserver-alkane-dev:
- ops/secrets/dbserver/cluster-B/users/dereckson_www51
webserver-legacy:
#
# Wolfplex credentials
#
- ops/secrets/nasqueron.etherpad.api
# -------------------------------------------------------------
# Vault secrets by dbserver cluster
#
# Paths of the keys the specified role needs access to.
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
vault_secrets_by_dbserver_cluster:
# Main PostgreSQL cluster
A:
- ops/secrets/dbserver/cluster-A/users/*
# Main MariaDB cluster - Alkane PaaS, ViperServ
B:
- ops/secrets/dbserver/cluster-B/users/*
diff --git a/pillar/paas/docker/dwellers/airflow.sls b/pillar/paas/docker/dwellers/airflow.sls
index cd25ca6..2aa9210 100644
--- a/pillar/paas/docker/dwellers/airflow.sls
+++ b/pillar/paas/docker/dwellers/airflow.sls
@@ -1,75 +1,80 @@
# -------------------------------------------------------------
# Salt — Provision Docker engine
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Project: Nasqueron
# License: Trivial work, not eligible to copyright
# Service: Airfllow
# -------------------------------------------------------------
docker_images:
- nasqueron/airflow
- redis
airflow_default_container_args: &airflow
realm: nasqueron
network: airflow
docker_containers:
redis:
airflow_redis: *airflow
airflow:
airflow_web:
<<: *airflow
command: webserver
command_port: 8080
app_port: 46080
airflow_scheduler:
<<: *airflow
command: scheduler
airflow_worker:
<<: *airflow
command: celery worker
airflow_triggerer:
<<: *airflow
command: triggerer
airflow_flower:
<<: *airflow
command: celery flower
command_port: 5555
app_port: 46555
docker_networks:
airflow:
subnet: 172.18.4.0/24
airflow_realms:
nasqueron:
network: airflow
services:
redis: airflow_redis
postgresql: 172.27.27.8 # db-A-001.nasqueron.drake
credentials:
admin_account: nasqueron/airflow/admin_account
fernet_key: nasqueron/airflow/fernet
postgresql: dbserver/cluster-A/users/airflow
+ vault: nasqueron/airflow/vault
+ vault:
+ url: https://172.27.27.7:8200
+ mount_point: apps
+ secrets_path: airflow
sentry:
realm: nasqueron
project_id: 4
credential: nasqueron/airflow/sentry
# -------------------------------------------------------------
# Airflow specific monitorng
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
docker_containers_monitoring:
check_by_container_name:
airflow:
check_http_200:
airflow_web: /health
airflow_scheduler: /health
airflow_flower: /
diff --git a/roles/paas-docker/containers/airflow.sls b/roles/paas-docker/containers/airflow.sls
index 20f339e..786616e 100644
--- a/roles/paas-docker/containers/airflow.sls
+++ b/roles/paas-docker/containers/airflow.sls
@@ -1,115 +1,123 @@
# -------------------------------------------------------------
# Salt — Provision Docker engine
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Project: Nasqueron
# License: Trivial work, not eligible to copyright
# -------------------------------------------------------------
{% set has_selinux = salt['grains.get']('selinux:enabled', False) %}
+
+{% for realm, realm_args in pillar['airflow_realms'].items() %}
+
# -------------------------------------------------------------
# Data directory
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-{% for realm, realm_args in pillar['airflow_realms'].items() %}
-
/srv/airflow/{{ realm }}:
file.directory:
- user: 50000
- group: 0
- makedirs: True
{% for subdir in ["dags", "logs", "plugins"] %}
/srv/airflow/{{ realm }}/{{ subdir }}:
file.directory:
- user: 50000
- group: 0
{% endfor %}
/srv/airflow/{{ realm }}/bin/airflow:
file.managed:
- source: salt://roles/paas-docker/containers/files/airflow/airflow.sh.jinja
- makedirs: True
- mode: 755
- template: jinja
- context:
realm: {{ realm }}
network: {{ realm_args.network }}
credentials: {{ realm_args.credentials }}
services: {{ realm_args.services }}
{% if has_selinux %}
selinux_context_{{ realm }}_airflow_data:
selinux.fcontext_policy_present:
- name: /srv/airflow/{{ realm }}
- sel_type: container_file_t
selinux_context_{{ realm }}_airflow_data_applied:
selinux.fcontext_policy_applied:
- name: /srv/airflow/{{ realm }}
{% endif %}
+# -------------------------------------------------------------
+# Airflow configuration for this realm
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+{% set postgresql_dsn = salt["credentials.get_dsn"](realm_args["services"]["postgresql"], realm_args["credentials"]["postgresql"]) %}
+
+/srv/airflow/{{ realm }}/airflow.cfg:
+ file.managed:
+ - source: salt://roles/paas-docker/containers/files/airflow/airflow.cfg.jinja
+ - mode: 400
+ - user: 50000 # As defined in Airflow upstream Docker image
+ - template: jinja
+ - context:
+ realm: {{ realm }}
+ vault: {{ realm_args["vault"] }}
+ services:
+ redis: {{ realm_args["services"]["redis"] }}
+ credentials:
+ fernet_key: {{ salt["credentials.get_password"](realm_args["credentials"]["fernet_key"]) }}
+ db: {{ postgresql_dsn }}/airflow
+ sentry: {{ salt["credentials.get_sentry_dsn"](realm_args["sentry"]) }}
+ vault: {{ salt["credentials.read_secret"](realm_args["credentials"]["vault"]) }}
+
# -------------------------------------------------------------
# Service initialization
#
# The first time Airflow is installed for a realm,
# it needs to run database migrations.
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
airflow_init_{{ realm }}:
cmd.run:
- name: |
airflow {{ realm }} upgrade
touch /srv/airflow/{{ realm }}/.initialized
- environment:
- _AIRFLOW_WWW_USER: {{ realm_args["credentials"]["admin_account"] }}
- creates: /srv/airflow/{{ realm }}/.initialized
{% endfor %}
+
# -------------------------------------------------------------
# Containers
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
{% for instance, container in pillar['docker_containers']['airflow'].items() %}
{% set realm = container["realm"] %}
-{% set realm_args = pillar["airflow_realms"][realm] %}
-
-{% set postgresql_dsn = salt["credentials.get_dsn"](realm_args["services"]["postgresql"], realm_args["credentials"]["postgresql"]) %}
{{ instance }}:
docker_container.running:
- detach: True
- interactive: True
- image: nasqueron/airflow
- command: {{ container["command"] }}
- binds:
- /srv/airflow/{{ realm }}/dags:/opt/airflow/dags
- /srv/airflow/{{ realm }}/logs:/opt/airflow/logs
- /srv/airflow/{{ realm }}/plugins:/opt/airflow/plugins
- - environment:
- - AIRFLOW__CORE__EXECUTOR: CeleryExecutor
- - AIRFLOW__CORE__FERNET_KEY: {{ salt["credentials.get_password"](realm_args["credentials"]["fernet_key"]) }}
- - AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "True"
- - AIRFLOW__CORE__LOAD_EXAMPLES: "False"
-
- - AIRFLOW__API__AUTH_BACKENDS: airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session
-
- - AIRFLOW__CELERY__BROKER_URL: redis://:@{{ realm_args["services"]["redis"] }}:6379/0
- - AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://{{ postgresql_dsn }}/airflow
-
- - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://{{ postgresql_dsn }}/airflow
-
- - AIRFLOW__SENTRY__SENTRY_ON: "True"
- - AIRFLOW__SENTRY__SENTRY_DSN: {{ salt["credentials.get_sentry_dsn"](realm_args["sentry"]) }}
+ - /srv/airflow/{{ realm }}/airflow.cfg:/opt/airflow/airflow.cfg
{% if "app_port" in container %}
- ports:
- {{ container['command_port'] }}
- port_bindings:
- 127.0.0.1:{{ container['app_port'] }}:{{ container['command_port'] }}
{% endif %}
- networks:
- {{ container['network'] }}
{% endfor %}
diff --git a/roles/paas-docker/containers/files/airflow/airflow.cfg.jinja b/roles/paas-docker/containers/files/airflow/airflow.cfg.jinja
new file mode 100644
index 0000000..3e03595
--- /dev/null
+++ b/roles/paas-docker/containers/files/airflow/airflow.cfg.jinja
@@ -0,0 +1,53 @@
+# -------------------------------------------------------------
+# PaaS Docker - Airflow configuration
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# License: Trivial work, not eligible to copyright
+# Source file: roles/paas-docker/containers/files/airflow/airflow.cfg.jinja
+# Realm: {{ realm }}
+# -------------------------------------------------------------
+#
+# <auto-generated>
+# This file is managed by our rOPS SaltStack repository.
+#
+# Changes to this file may cause incorrect behavior
+# and will be lost if the state is redeployed.
+# </auto-generated>
+
+[core]
+executor = CeleryExecutor
+
+dags_are_paused_at_creation = True
+load_examples = False
+
+fernet_key = {{ credentials.fernet_key }}
+
+[database]
+sql_alchemy_conn = postgresql+psycopg2://{{ credentials.db }}
+
+load_default_connections = False
+
+[secrets]
+backend = airflow.providers.hashicorp.secrets.vault.VaultBackend
+backend_kwargs = {
+ "url": "{{ vault.url }}",
+ "auth_type": "approle",
+ "role_id": "{{ credentials.vault.username }}",
+ "secret_id": "{{ credentials.vault.password }}",
+
+ "mount_point": "{{ vault.mount_point }}",
+ "connections_path": "{{ vault.secrets_path }}/connections",
+ "config_path": "{{ vault.secrets_path }}/config",
+ "variables_path": "{{ vault.secrets_path }}/variables"
+ }
+
+[api]
+auth_backends = airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session
+
+[sentry]
+sentry_on = True
+sentry_dsn = {{ credentials.sentry }}
+
+[celery]
+broker_url = redis://:@{{ services.redis }}:6379/0
+result_backend = db+postgresql://{{ credentials.db }}
diff --git a/roles/paas-docker/containers/files/airflow/airflow.sh.jinja b/roles/paas-docker/containers/files/airflow/airflow.sh.jinja
index 9aa8620..933da3e 100644
--- a/roles/paas-docker/containers/files/airflow/airflow.sh.jinja
+++ b/roles/paas-docker/containers/files/airflow/airflow.sh.jinja
@@ -1,60 +1,59 @@
#!/bin/sh
# -------------------------------------------------------------
# PaaS Docker
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Project: Nasqueron
# License: Trivial work, not eligible to copyright
# Description: Wrapper for airflow command (local instance)
# Source file: roles/paas-docker/containers/files/airflow/airflow.sh.jinja
# Realm: {{ realm }}
# -------------------------------------------------------------
#
# <auto-generated>
# This file is managed by our rOPS SaltStack repository.
#
# Changes to this file may cause incorrect behavior
# and will be lost if the state is redeployed.
# </auto-generated>
set -e
COMMAND=$1
shift
EXTRA_ARGS=""
if [ -n "$_AIRFLOW_WWW_USER" ]; then
# Used by initialization to create a default first user
user="$(credential $_AIRFLOW_WWW_USER username)"
password="$(credential $_AIRFLOW_WWW_USER)"
EXTRA_ARGS="$EXTRA_ARGS \
-e _AIRFLOW_WWW_USER_CREATE=true \
-e _AIRFLOW_WWW_USER_USERNAME=$user \
-e _AIRFLOW_WWW_USER_PASSWORD=$password"
fi
if [ "$COMMAND" = "upgrade" ]; then
EXTRA_ARGS="$EXTRA_ARGS \
-e _AIRFLOW_DB_UPGRADE=true"
COMMAND=version
fi
if [ "$COMMAND" = "shell" ]; then
PROGRAM=bash
COMMAND=
COMMAND_ARGS=
else
PROGRAM=airflow
COMMAND_ARGS=$@
fi
-dsn="$(credential {{ credentials.postgresql }} username):$(credential {{ credentials.postgresql }})@{{ services.postgresql }}"
docker run -it --rm \
--network {{ network }} \
+ -v /srv/airflow/{{ realm }}/airflow.cfg:/opt/airflow/airflow.cfg \
-e _PIP_ADDITIONAL_REQUIREMENTS="" \
-e CONNECTION_CHECK_MAX_COUNT=0 \
- -e AIRFLOW__DATABASE__SQL_ALCHEMY_CONN="postgresql+psycopg2://$dsn/airflow" \
$EXTRA_ARGS \
nasqueron/airflow $PROGRAM $COMMAND $COMMAND_ARGS

File Metadata

Mime Type
text/x-diff
Expires
Thu, Sep 18, 11:19 (19 h, 35 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
2989413
Default Alt Text
(20 KB)

Event Timeline

Nasqueron DevCentral · If it had been much bigger the moon would have had a core of ice. · Powered by Phabricator