Page MenuHomeDevCentral
Files F3769248

No OneTemporary
Actions

View Options

diff --git a/pillar/credentials/vault.sls b/pillar/credentials/vault.sls
index 1d76fe8..42bcec7 100644
--- a/pillar/credentials/vault.sls
+++ b/pillar/credentials/vault.sls
@@ -1,208 +1,210 @@
# -------------------------------------------------------------
# Salt configuration for Nasqueron servers
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Project: Nasqueron
# License: Trivial work, not eligible to copyright
# -------------------------------------------------------------
# -------------------------------------------------------------
# Vault configuration
#
# :: vault_policies_path: path on vault server where to store policies
#
# :: vault_policies_source: path to fetch policies from
# if starting by salt://, from salt files server
#
# :: vault_mount_paths: translates secrets paths in policies paths
#
# Generally, Vault paths are the same for policies and data access.
#
# For kv secrets engine, version 2, writing and reading versions
# of a kv value are prefixed with the data/ path.
#
# credentials.build_policies_by_node will use this dictionary
# to be able to rewrite secrets paths in data paths.
#
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
vault_policies_path: /srv/policies/vault
vault_policies_source: salt://roles/vault/policies/files
vault_mount_paths:
ops/secrets: ops/data/secrets
ops/privacy: ops/data/privacy
# -------------------------------------------------------------
# Vault policies to deploy as-is, ie without templating.
#
# Entries of vault_policies must match a .hcl file in
# roles/vault/policies/files folder.
#
# If you need a template, create a new pillar entry instead
# and add the parsing logic either:
# - directly to roles/vault/policies/
#
# - through _modules/credentials.py for policies to apply
# to Salt nodes, like e.g. vault_secrets_by_role
#
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
vault_policies:
- salt-primary
- sentry
- viperserv
# -------------------------------------------------------------
# Vault policies for Salt
#
# Declare the extra policies each nodes need.
#
# In adition of those extra policies, the vault_secrets_by_role
# will be parsed for the keys.
#
# IMPORTANT: as grains['roles'] can be modified by the node,
# roles are extracted directly from the pillar.
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
vault_extra_policies_by_role:
salt-primary:
- salt-primary
# -------------------------------------------------------------
# Vault secrets by role
#
# Paths of the keys the specified role needs access to.
#
# Avoid * notation as this namespace is shared between Vault
# and the applications. As such, only secrets the Salt nodes
# needs in a state they need to deploy should be listed here.
#
# Use %%node%% as variable for node name.
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
vault_secrets_by_role:
devserver:
- ops/secrets/nasqueron/notifications/notifications-cli/%%node%%
opensearch:
- ops/secrets/nasqueron.opensearch.infra-logs.internal_users.admin
- ops/secrets/nasqueron.opensearch.infra-logs.internal_users.dashboards
paas-docker-prod:
#
# Personal data or personally identifiable information (PII)
# related to Nasqueron Operations SIG members.
#
- ops/privacy/ops-cidr
#
# Credentials used by Nasqueron services
# Format: ops/secrets/nasqueron/service/<...>
#
- ops/secrets/nasqueron/airflow/admin_account
- ops/secrets/nasqueron/airflow/fernet
+ - ops/secrets/nasqueron/airflow/sentry
- ops/secrets/dbserver/cluster-A/users/airflow
- ops/secrets/nasqueron/rabbitmq/white-rabbit/erlang-cookie
- ops/secrets/nasqueron/rabbitmq/white-rabbit/root
- ops/secrets/nasqueron/sentry/geoipupdate
#
# Credentials used by Nasqueron services
# Format: ops/secrets/nasqueron.<service>.<type>
#
- ops/secrets/nasqueron.acquisitariat.mysql
- ops/secrets/nasqueron.auth-grove.mysql
- ops/secrets/nasqueron.cachet.app_key
- ops/secrets/nasqueron.cachet.mysql
- ops/secrets/nasqueron.etherpad.api
- ops/secrets/nasqueron.notifications.broker
- ops/secrets/nasqueron.notifications.mailgun
- ops/secrets/nasqueron.notifications.sentry
- ops/secrets/nasqueron.notifications.credentials_github_nasqueron
- ops/secrets/nasqueron.notifications.credentials_github_wolfplex
- ops/secrets/nasqueron.notifications.credentials_github_keruald
- ops/secrets/nasqueron.notifications.credentials_github_trustspace
- ops/secrets/nasqueron.notifications.credentials_github_eglide
- ops/secrets/nasqueron.notifications.credentials_phabricator_nasqueron
- ops/secrets/nasqueron.pixelfed.app_key
- ops/secrets/nasqueron.pixelfed.mailgun
- ops/secrets/nasqueron.pixelfed.mysql
- ops/secrets/nasqueron.sentry.app_key
- ops/secrets/nasqueron.sentry.postgresql
- ops/secrets/nasqueron.sentry.vault
#
# Credentials used by Nasqueron members private services
# Format: <username>.<service>.<type>
#
- ops/secrets/dereckson.phabricator.mysql
#
# Credentials used by projects hosted by Nasqueron
# Format: <project name>.<service>.<type>
#
- ops/secrets/espacewin.phpbb.mysql_root
- ops/secrets/wolfplex.phabricator.mailgun
- ops/secrets/wolfplex.phabricator.mysql
- ops/secrets/zed.phabricator.mysql
- ops/secrets/zed.phabricator.sendgrid
paas-docker-dev:
#
# Credentials used by Nasqueron services
# Format: ops/secrets/nasqueron/service/<...>
#
- ops/secrets/nasqueron/airflow/admin_account
- ops/secrets/nasqueron/airflow/fernet
+ - ops/secrets/nasqueron/airflow/sentry
- ops/secrets/dbserver/cluster-A/users/airflow
#
# Credentials used by projects hosted by Nasqueron
# Format: <project name>.<service>.<type>
#
- ops/secrets/espacewin.bugzilla.mysql
- ops/secrets/espacewin.bugzilla.mysql_root
viperserv:
- ops/secrets/nasqueron.viperserv.vault
webserver-legacy:
#
# Wolfplex credentials
#
- ops/secrets/nasqueron.etherpad.api
# -------------------------------------------------------------
# Vault secrets by dbserver cluster
#
# Paths of the keys the specified role needs access to.
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
vault_secrets_by_dbserver_cluster:
# Main PostgreSQL cluster
A:
- ops/secrets/dbserver/cluster-A/users/*
diff --git a/pillar/paas/docker/dwellers/airflow.sls b/pillar/paas/docker/dwellers/airflow.sls
index e0591db..cd25ca6 100644
--- a/pillar/paas/docker/dwellers/airflow.sls
+++ b/pillar/paas/docker/dwellers/airflow.sls
@@ -1,71 +1,75 @@
# -------------------------------------------------------------
# Salt — Provision Docker engine
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Project: Nasqueron
# License: Trivial work, not eligible to copyright
# Service: Airfllow
# -------------------------------------------------------------
docker_images:
- - apache/airflow:2.5.2
+ - nasqueron/airflow
- redis
airflow_default_container_args: &airflow
realm: nasqueron
network: airflow
docker_containers:
redis:
airflow_redis: *airflow
airflow:
airflow_web:
<<: *airflow
command: webserver
command_port: 8080
app_port: 46080
airflow_scheduler:
<<: *airflow
command: scheduler
airflow_worker:
<<: *airflow
command: celery worker
airflow_triggerer:
<<: *airflow
command: triggerer
airflow_flower:
<<: *airflow
command: celery flower
command_port: 5555
app_port: 46555
docker_networks:
airflow:
subnet: 172.18.4.0/24
airflow_realms:
nasqueron:
network: airflow
services:
redis: airflow_redis
postgresql: 172.27.27.8 # db-A-001.nasqueron.drake
credentials:
admin_account: nasqueron/airflow/admin_account
fernet_key: nasqueron/airflow/fernet
postgresql: dbserver/cluster-A/users/airflow
+ sentry:
+ realm: nasqueron
+ project_id: 4
+ credential: nasqueron/airflow/sentry
# -------------------------------------------------------------
# Airflow specific monitorng
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
docker_containers_monitoring:
check_by_container_name:
airflow:
check_http_200:
airflow_web: /health
airflow_scheduler: /health
airflow_flower: /
diff --git a/pillar/services/monitoring-reporting.sls b/pillar/services/monitoring-reporting.sls
new file mode 100644
index 0000000..4ad863a
--- /dev/null
+++ b/pillar/services/monitoring-reporting.sls
@@ -0,0 +1,16 @@
+# -------------------------------------------------------------
+# Salt — Monitoring and reporting services
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# License: Trivial work, not eligible to copyright
+# Notes: A more comprehensive definition could coexist
+# where the service need to be deployed.
+# -------------------------------------------------------------
+
+# -------------------------------------------------------------
+# Sentry
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+sentry_realms:
+ nasqueron:
+ hostname: sentry.nasqueron.org
diff --git a/pillar/top.sls b/pillar/top.sls
index 589c807..bc66d08 100644
--- a/pillar/top.sls
+++ b/pillar/top.sls
@@ -1,55 +1,56 @@
# -------------------------------------------------------------
# Salt configuration for Nasqueron servers
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Project: Nasqueron
# Created: 2016-04-10
# License: Trivial work, not eligible to copyright
# -------------------------------------------------------------
base:
'*':
- core.users
- core.groups
- core.network
- certificates.certificates
- nodes.nodes
- nodes.forests
- hotfixes.roles
+ - services.monitoring-reporting
- webserver.sites
cloudhugger:
- opensearch.software
- opensearch.clusters
complector:
- credentials.vault
docker-002:
- notifications.config
- paas.docker
- saas.jenkins
- saas.phpbb
db-A-001:
- dbserver.cluster-A
dwellers:
- paas.docker
- saas.jenkins
eglide:
- shellserver.quassel
ysul:
- devserver.repos
- saas.mediawiki
- viperserv.bots
- viperserv.fantoir
- webserver.labs
- webserver.wwwroot51
windriver:
- devserver.ports
- devserver.repos
- webserver.labs
- webserver.wwwroot51
diff --git a/roles/paas-docker/containers/airflow.sls b/roles/paas-docker/containers/airflow.sls
index d81cda0..125cf17 100644
--- a/roles/paas-docker/containers/airflow.sls
+++ b/roles/paas-docker/containers/airflow.sls
@@ -1,112 +1,115 @@
# -------------------------------------------------------------
# Salt — Provision Docker engine
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Project: Nasqueron
# License: Trivial work, not eligible to copyright
# -------------------------------------------------------------
{% set has_selinux = salt['grains.get']('selinux:enabled', False) %}
# -------------------------------------------------------------
# Data directory
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
{% for realm, realm_args in pillar['airflow_realms'].items() %}
/srv/airflow/{{ realm }}:
file.directory:
- user: 50000
- group: 0
- makedirs: True
{% for subdir in ["dags", "logs", "plugins"] %}
/srv/airflow/{{ realm }}/{{ subdir }}:
file.directory:
- user: 50000
- group: 0
{% endfor %}
/srv/airflow/{{ realm }}/bin/airflow:
file.managed:
- source: salt://roles/paas-docker/containers/files/airflow/airflow.sh.jinja
- makedirs: True
- mode: 755
- template: jinja
- context:
realm: {{ realm }}
network: {{ realm_args.network }}
credentials: {{ realm_args.credentials }}
services: {{ realm_args.services }}
{% if has_selinux %}
selinux_context_{{ realm }}_airflow_data:
selinux.fcontext_policy_present:
- name: /srv/airflow/{{ realm }}
- sel_type: container_file_t
selinux_context_{{ realm }}_airflow_data_applied:
selinux.fcontext_policy_applied:
- name: /srv/airflow/{{ realm }}
{% endif %}
# -------------------------------------------------------------
# Service initialization
#
# The first time Airflow is installed for a realm,
# it needs to run database migrations.
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
airflow_init_{{ realm }}:
cmd.run:
- name: |
airflow {{ realm }} upgrade
touch /srv/airflow/{{ realm }}/.initialized
- environment:
- _AIRFLOW_WWW_USER: {{ realm_args["credentials"]["admin_account"] }}
- creates: /srv/airflow/{{ realm }}/.initialized
{% endfor %}
# -------------------------------------------------------------
# Containers
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
{% for instance, container in pillar['docker_containers']['airflow'].items() %}
{% set realm = container["realm"] %}
{% set realm_args = pillar["airflow_realms"][realm] %}
{% set postgresql_dsn = salt["credentials.get_dsn"](realm_args["services"]["postgresql"], realm_args["credentials"]["postgresql"]) %}
{{ instance }}:
docker_container.running:
- detach: True
- interactive: True
- - image: apache/airflow:2.5.2
+ - image: nasqueron/airflow
- command: {{ container["command"] }}
- binds:
- /srv/airflow/{{ realm }}/dags:/opt/airflow/dags
- /srv/airflow/{{ realm }}/logs:/opt/airflow/logs
- /srv/airflow/{{ realm }}/plugins:/opt/airflow/plugins
- environment:
- AIRFLOW__CORE__EXECUTOR: CeleryExecutor
- AIRFLOW__CORE__FERNET_KEY: {{ salt["credentials.get_password"](realm_args["credentials"]["fernet_key"]) }}
- AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "true"
- AIRFLOW__CORE__LOAD_EXAMPLES: "false"
- AIRFLOW__API__AUTH_BACKENDS: airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session
- AIRFLOW__CELERY__BROKER_URL: redis://:@{{ realm_args["services"]["redis"] }}:6379/0
- AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://{{ postgresql_dsn }}/airflow
- AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://{{ postgresql_dsn }}/airflow
+
+ - AIRFLOW__SENTRY__SENTRY_ON: "True"
+ - AIRFLOW__SENTRY__SENTRY_DSN: {{ salt["credentials.get_sentry_dsn"](realm_args["sentry"]) }}
{% if "app_port" in container %}
- ports:
- {{ container['command_port'] }}
- port_bindings:
- 127.0.0.1:{{ container['app_port'] }}:{{ container['command_port'] }}
{% endif %}
- networks:
- {{ container['network'] }}
{% endfor %}

File Metadata

Mime Type
text/x-diff
Expires
Mon, Nov 25, 13:53 (1 d, 6 h)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
2256962
Default Alt Text
(15 KB)

Event Timeline

Nasqueron DevCentral · If it had been much bigger the moon would have had a core of ice. · Powered by Phabricator