Page MenuHomeDevCentral
Files F3769739

D3281.diff
No OneTemporary
Actions

D3281.diff
View Options

diff --git a/pillar/saas/airflow.sls b/pillar/saas/airflow.sls
new file mode 100644
--- /dev/null
+++ b/pillar/saas/airflow.sls
@@ -0,0 +1,27 @@
+# -------------------------------------------------------------
+# Salt — Airflow
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# License: Trivial work, not eligible to copyright
+# -------------------------------------------------------------
+
+# -------------------------------------------------------------
+# Airflow DAGs composition
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+airflow_pipelines:
+
+ # Realm: nasqueron
+
+ nasqueron:
+
+ datasources:
+ source:
+ # DAGs currently living in test branch - D2754
+ repository: https://devcentral.nasqueron.org/source/datasources.git
+ ref: pipelines
+ directory: _pipelines/dags
+
+ python_dependencies:
+ - requests
+
diff --git a/pillar/top.sls b/pillar/top.sls
--- a/pillar/top.sls
+++ b/pillar/top.sls
@@ -45,6 +45,7 @@
dwellers:
- paas.docker
+ - saas.airflow
- saas.jenkins
eglide:
diff --git a/roles/saas-airflow/dags/init.sls b/roles/saas-airflow/dags/init.sls
new file mode 100644
--- /dev/null
+++ b/roles/saas-airflow/dags/init.sls
@@ -0,0 +1,36 @@
+# -------------------------------------------------------------
+# Salt — Airflow
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# License: Trivial work, not eligible to copyright
+# -------------------------------------------------------------
+
+# -------------------------------------------------------------
+# Airflow DAGs composition
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+{% for realm, dags_to_import in pillar.get("airflow_pipelines", {}).items() %}
+
+/srv/airflow/{{ realm }}/src:
+ file.directory
+
+{% for name, args in dags_to_import.items() %}
+
+{% set rev = args["source"].get("ref", "main") %}
+{% set directory = args["source"].get("directory", "") %}
+
+airflow_dags_{{ realm }}_{{ name }}_repo:
+ git.latest:
+ - name: {{ args["source"]["repository"] }}
+ - target: /srv/airflow/{{ realm }}/src/{{ name }}
+ - rev: {{ rev }}
+
+airflow_dags_{{ realm }}_{{ name }}:
+ file.copy:
+ - name: /srv/airflow/{{ realm }}/dags
+ - source: /srv/airflow/{{ realm }}/src/{{ name }}/{{ directory }}
+ - force: True
+ - user: 50000
+
+{% endfor %}
+{% endfor %}
diff --git a/roles/saas-airflow/init.sls b/roles/saas-airflow/init.sls
new file mode 100644
--- /dev/null
+++ b/roles/saas-airflow/init.sls
@@ -0,0 +1,9 @@
+# -------------------------------------------------------------
+# Salt — Airflow
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# License: Trivial work, not eligible to copyright
+# -------------------------------------------------------------
+
+include:
+ - .dags
diff --git a/top.sls b/top.sls
--- a/top.sls
+++ b/top.sls
@@ -39,6 +39,7 @@
'dwellers':
- roles/paas-docker/docker
- roles/paas-lxc/lxc
+ - roles/saas-airflow
'eglide':
- roles/webserver-core
- roles/shellserver

File Metadata

Mime Type
text/plain
Expires
Sun, Nov 24, 16:44 (54 m, 32 s)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
2257729
Default Alt Text
D3281.diff (3 KB)

Event Timeline

Nasqueron DevCentral · If it had been much bigger the moon would have had a core of ice. · Powered by Phabricator