Page Menu
Home
DevCentral
Search
Configure Global Search
Log In
Files
F11724242
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
6 KB
Referenced Files
None
Subscribers
None
View Options
diff --git a/roles/paas-docker/containers/airflow.sls b/roles/paas-docker/containers/airflow.sls
index 786616e..5728b3e 100644
--- a/roles/paas-docker/containers/airflow.sls
+++ b/roles/paas-docker/containers/airflow.sls
@@ -1,123 +1,124 @@
# -------------------------------------------------------------
# Salt — Provision Docker engine
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Project: Nasqueron
# License: Trivial work, not eligible to copyright
# -------------------------------------------------------------
{% set has_selinux = salt['grains.get']('selinux:enabled', False) %}
{% for realm, realm_args in pillar['airflow_realms'].items() %}
# -------------------------------------------------------------
# Data directory
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
/srv/airflow/{{ realm }}:
file.directory:
- user: 50000
- group: 0
- makedirs: True
-{% for subdir in ["dags", "logs", "plugins"] %}
+{% for subdir in ["dags", "logs", "plugins", "workspace"] %}
/srv/airflow/{{ realm }}/{{ subdir }}:
file.directory:
- user: 50000
- group: 0
{% endfor %}
/srv/airflow/{{ realm }}/bin/airflow:
file.managed:
- source: salt://roles/paas-docker/containers/files/airflow/airflow.sh.jinja
- makedirs: True
- mode: 755
- template: jinja
- context:
realm: {{ realm }}
network: {{ realm_args.network }}
credentials: {{ realm_args.credentials }}
services: {{ realm_args.services }}
{% if has_selinux %}
selinux_context_{{ realm }}_airflow_data:
selinux.fcontext_policy_present:
- name: /srv/airflow/{{ realm }}
- sel_type: container_file_t
selinux_context_{{ realm }}_airflow_data_applied:
selinux.fcontext_policy_applied:
- name: /srv/airflow/{{ realm }}
{% endif %}
# -------------------------------------------------------------
# Airflow configuration for this realm
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
{% set postgresql_dsn = salt["credentials.get_dsn"](realm_args["services"]["postgresql"], realm_args["credentials"]["postgresql"]) %}
/srv/airflow/{{ realm }}/airflow.cfg:
file.managed:
- source: salt://roles/paas-docker/containers/files/airflow/airflow.cfg.jinja
- mode: 400
- user: 50000 # As defined in Airflow upstream Docker image
- template: jinja
- context:
realm: {{ realm }}
vault: {{ realm_args["vault"] }}
services:
redis: {{ realm_args["services"]["redis"] }}
credentials:
fernet_key: {{ salt["credentials.get_password"](realm_args["credentials"]["fernet_key"]) }}
db: {{ postgresql_dsn }}/airflow
sentry: {{ salt["credentials.get_sentry_dsn"](realm_args["sentry"]) }}
vault: {{ salt["credentials.read_secret"](realm_args["credentials"]["vault"]) }}
# -------------------------------------------------------------
# Service initialization
#
# The first time Airflow is installed for a realm,
# it needs to run database migrations.
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
airflow_init_{{ realm }}:
cmd.run:
- name: |
airflow {{ realm }} upgrade
touch /srv/airflow/{{ realm }}/.initialized
- environment:
- _AIRFLOW_WWW_USER: {{ realm_args["credentials"]["admin_account"] }}
- creates: /srv/airflow/{{ realm }}/.initialized
{% endfor %}
# -------------------------------------------------------------
# Containers
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
{% for instance, container in pillar['docker_containers']['airflow'].items() %}
{% set realm = container["realm"] %}
{{ instance }}:
docker_container.running:
- detach: True
- interactive: True
- image: nasqueron/airflow
- command: {{ container["command"] }}
- binds:
- /srv/airflow/{{ realm }}/dags:/opt/airflow/dags
- /srv/airflow/{{ realm }}/logs:/opt/airflow/logs
- /srv/airflow/{{ realm }}/plugins:/opt/airflow/plugins
+ - /srv/airflow/{{ realm }}/workspace:/home/airflow/workspace
- /srv/airflow/{{ realm }}/airflow.cfg:/opt/airflow/airflow.cfg
{% if "app_port" in container %}
- ports:
- {{ container['command_port'] }}
- port_bindings:
- 127.0.0.1:{{ container['app_port'] }}:{{ container['command_port'] }}
{% endif %}
- networks:
- {{ container['network'] }}
{% endfor %}
diff --git a/roles/paas-docker/containers/files/airflow/airflow.sh.jinja b/roles/paas-docker/containers/files/airflow/airflow.sh.jinja
index 933da3e..bce68b2 100644
--- a/roles/paas-docker/containers/files/airflow/airflow.sh.jinja
+++ b/roles/paas-docker/containers/files/airflow/airflow.sh.jinja
@@ -1,59 +1,60 @@
#!/bin/sh
# -------------------------------------------------------------
# PaaS Docker
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Project: Nasqueron
# License: Trivial work, not eligible to copyright
# Description: Wrapper for airflow command (local instance)
# Source file: roles/paas-docker/containers/files/airflow/airflow.sh.jinja
# Realm: {{ realm }}
# -------------------------------------------------------------
#
# <auto-generated>
# This file is managed by our rOPS SaltStack repository.
#
# Changes to this file may cause incorrect behavior
# and will be lost if the state is redeployed.
# </auto-generated>
set -e
COMMAND=$1
shift
EXTRA_ARGS=""
if [ -n "$_AIRFLOW_WWW_USER" ]; then
# Used by initialization to create a default first user
user="$(credential $_AIRFLOW_WWW_USER username)"
password="$(credential $_AIRFLOW_WWW_USER)"
EXTRA_ARGS="$EXTRA_ARGS \
-e _AIRFLOW_WWW_USER_CREATE=true \
-e _AIRFLOW_WWW_USER_USERNAME=$user \
-e _AIRFLOW_WWW_USER_PASSWORD=$password"
fi
if [ "$COMMAND" = "upgrade" ]; then
EXTRA_ARGS="$EXTRA_ARGS \
-e _AIRFLOW_DB_UPGRADE=true"
COMMAND=version
fi
if [ "$COMMAND" = "shell" ]; then
PROGRAM=bash
COMMAND=
COMMAND_ARGS=
else
PROGRAM=airflow
COMMAND_ARGS=$@
fi
docker run -it --rm \
--network {{ network }} \
+ -v /srv/airflow/{{ realm }}/workspace:/home/airflow/workspace \
-v /srv/airflow/{{ realm }}/airflow.cfg:/opt/airflow/airflow.cfg \
-e _PIP_ADDITIONAL_REQUIREMENTS="" \
-e CONNECTION_CHECK_MAX_COUNT=0 \
$EXTRA_ARGS \
nasqueron/airflow $PROGRAM $COMMAND $COMMAND_ARGS
File Metadata
Details
Attached
Mime Type
text/x-diff
Expires
Thu, Sep 18, 13:49 (16 h, 15 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
2988401
Default Alt Text
(6 KB)
Attached To
Mode
rOPS Nasqueron Operations
Attached
Detach File
Event Timeline
Log In to Comment