Page MenuHomeDevCentral
Files F28579050

No OneTemporary
Actions

View Options

diff --git a/pillar/paas/docker/dwellers/airflow.sls b/pillar/paas/docker/dwellers/airflow.sls
index e2c82b1..e0591db 100644
--- a/pillar/paas/docker/dwellers/airflow.sls
+++ b/pillar/paas/docker/dwellers/airflow.sls
@@ -1,68 +1,71 @@
# -------------------------------------------------------------
# Salt — Provision Docker engine
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Project: Nasqueron
# License: Trivial work, not eligible to copyright
# Service: Airfllow
# -------------------------------------------------------------
docker_images:
- apache/airflow:2.5.2
- redis
airflow_default_container_args: &airflow
realm: nasqueron
network: airflow
docker_containers:
redis:
airflow_redis: *airflow
airflow:
airflow_web:
<<: *airflow
command: webserver
command_port: 8080
app_port: 46080
- healthcheck: ["CMD", "curl", "--fail", "http://localhost:8080/health"]
airflow_scheduler:
<<: *airflow
command: scheduler
- healthcheck: ["CMD", "curl", "--fail", "http://localhost:8974/health"]
airflow_worker:
<<: *airflow
command: celery worker
- healthcheck:
- - CMD-SHELL
- - celery --app airflow.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}"
airflow_triggerer:
<<: *airflow
command: triggerer
- healthcheck:
- - CMD-SHELL
- - airflow jobs check --job-type TriggererJob --hostname "$${HOSTNAME}"
airflow_flower:
<<: *airflow
command: celery flower
command_port: 5555
app_port: 46555
- healthcheck: ["CMD", "curl", "--fail", "http://localhost:5555/"]
docker_networks:
airflow:
subnet: 172.18.4.0/24
airflow_realms:
nasqueron:
network: airflow
services:
redis: airflow_redis
postgresql: 172.27.27.8 # db-A-001.nasqueron.drake
credentials:
admin_account: nasqueron/airflow/admin_account
fernet_key: nasqueron/airflow/fernet
postgresql: dbserver/cluster-A/users/airflow
+
+# -------------------------------------------------------------
+# Airflow specific monitorng
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+docker_containers_monitoring:
+ check_by_container_name:
+ airflow:
+ check_http_200:
+ airflow_web: /health
+ airflow_scheduler: /health
+ airflow_flower: /
diff --git a/roles/paas-docker/containers/airflow.sls b/roles/paas-docker/containers/airflow.sls
index 84f5061..3651227 100644
--- a/roles/paas-docker/containers/airflow.sls
+++ b/roles/paas-docker/containers/airflow.sls
@@ -1,117 +1,114 @@
# -------------------------------------------------------------
# Salt — Provision Docker engine
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Project: Nasqueron
# License: Trivial work, not eligible to copyright
# -------------------------------------------------------------
{% set has_selinux = salt['grains.get']('selinux:enabled', False) %}
# -------------------------------------------------------------
# Data directory
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
{% for realm, realm_args in pillar['airflow_realms'].items() %}
/srv/airflow/{{ realm }}:
file.directory:
- user: 50000
- group: 0
- makedirs: True
{% for subdir in ["dags", "logs", "plugins"] %}
/srv/airflow/{{ realm }}/{{ subdir }}:
file.directory:
- user: 50000
- group: 0
{% endfor %}
/srv/airflow/{{ realm }}/bin/airflow:
file.managed:
- source: salt://roles/paas-docker/containers/files/airflow/airflow.sh.jinja
- makedirs: True
- mode: 755
- template: jinja
- context:
realm: {{ realm }}
network: {{ realm_args.network }}
credentials: {{ realm_args.credentials }}
services: {{ realm_args.services }}
{% if has_selinux %}
selinux_context_{{ realm }}_airflow_data:
selinux.fcontext_policy_present:
- name: /srv/airflow/{{ realm }}
- sel_type: container_file_t
selinux_context_{{ realm }}_airflow_data_applied:
selinux.fcontext_policy_applied:
- name: /srv/airflow/{{ realm }}
{% endif %}
# -------------------------------------------------------------
# Service initialization
#
# The first time Airflow is installed for a realm,
# it needs to run database migrations.
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
airflow_init_{{ realm }}:
cmd.run:
- name: |
airflow {{ realm }} upgrade
touch /srv/airflow/{{ realm }}/.initialized
- environment:
- _AIRFLOW_WWW_USER: {{ realm_args["credentials"]["admin_account"] }}
- creates: /srv/airflow/{{ realm }}/.initialized
{% endfor %}
# -------------------------------------------------------------
# Containers
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
{% for instance, container in pillar['docker_containers']['airflow'].items() %}
{% set realm = container["realm"] %}
{% set realm_args = pillar["airflow_realms"][realm] %}
{% set postgresql_dsn = salt["credentials.get_dsn"](realm_args["services"]["postgresql"], realm_args["credentials"]["postgresql"]) %}
{{ instance }}:
docker_container.running:
- detach: True
- interactive: True
- image: apache/airflow:2.5.2
- command: {{ container["command"] }}
- binds:
- /srv/airflow/{{ realm }}/dags:/opt/airflow/dags
- /srv/airflow/{{ realm }}/logs:/opt/airflow/logs
- /srv/airflow/{{ realm }}/plugins:/opt/airflow/plugins
- environment:
- AIRFLOW__CORE__EXECUTOR: CeleryExecutor
- AIRFLOW__CORE__FERNET_KEY: {{ salt["credentials.get_password"](realm_args["credentials"]["fernet_key"]) }}
- AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "true"
- AIRFLOW__CORE__LOAD_EXAMPLES: "false"
- AIRFLOW__API__AUTH_BACKENDS: airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session
- AIRFLOW__CELERY__BROKER_URL: redis://:@{{ realm_args["services"]["redis"] }}:6379/0
- AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://{{ postgresql_dsn }}/airflow
- AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://{{ postgresql_dsn }}/airflow
- AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK: "true"
{% if "app_port" in container %}
- ports:
- {{ container['command_port'] }}
- port_bindings:
- 127.0.0.1:{{ container['app_port'] }}:{{ container['command_port'] }}
{% endif %}
- - healthcheck:
- Test: {{ container['healthcheck'] }}
- Interval: 30000000000
- networks:
- {{ container['network'] }}
{% endfor %}

File Metadata

Mime Type
text/x-diff
Expires
Fri, May 15, 12:03 (1 d, 2 h)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3716716
Default Alt Text
(6 KB)

Event Timeline

Nasqueron DevCentral · If it had been much bigger the moon would have had a core of ice. · Powered by Phabricator