No OneTemporary
Actions

Size

21 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/_modules/convert.py b/_modules/convert.py
	index ed55a7e..d301656 100644
	--- a/_modules/convert.py
	+++ b/_modules/convert.py
	@@ -1,36 +1,82 @@
	# -- coding: utf-8 --

	# -------------------------------------------------------------
	# Salt — Convert execution module
	# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
	# Project: Nasqueron
	# Created: 2018-09-08
	# Description: Functions related to data format conversions
	# License: BSD-2-Clause
	# -------------------------------------------------------------


	import json
	+import salt.serializers.yaml
	+
	+
	+# -------------------------------------------------------------
	+# JSON
	+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -


	def to_json_from_pillar_key(key):
	"""
	A function to output a pillar key in JSON.

	CLI Example::

	salt-call --local convert.to_json "Hello world"
	"""
	data = __pillar__.get(key, {})
	return to_json(data)


	def to_json(data):
	"""
	A function to convert data to JSON.

	CLI Example::

	salt-call --local convert.to_json "Hello world"
	"""
	return json.dumps(data, indent=4, sort_keys=True)
	+
	+
	+# -------------------------------------------------------------
	+# YAML
	+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
	+
	+
	+def _to_dictionary(data, root=None):
	+ if root is not None:
	+ return {root: _to_dictionary(data)}
	+
	+ if type(data) is list:
	+ dictionary = {}
	+ for item in data:
	+ dictionary.update(_to_dictionary(item))
	+ return dictionary
	+
	+ if type(data) is tuple and len(data) == 2:
	+ return dict({data})
	+
	+ return dict(data)
	+
	+
	+def to_yaml_dictionary(data, root=None):
	+ """
	+ A function to convert data to YAML dictionary.
	+
	+ CLI Example::
	+
	+ salt * convert.to_yaml_dictionary '[{"a": "bar"}, {"b": "foo"}]'
	+
	+ That example will return:
	+ ```
	+ a: bar
	+ b: foo
	+ ```
	+ """
	+ return salt.serializers.yaml.serialize(
	+ _to_dictionary(data, root), default_flow_style=False
	+ )
	diff --git a/_modules/paas_docker.py b/_modules/paas_docker.py
	index 167c33b..326799e 100644
	--- a/_modules/paas_docker.py
	+++ b/_modules/paas_docker.py
	@@ -1,50 +1,84 @@
	# -- coding: utf-8 --

	# -------------------------------------------------------------
	# Salt — PaaS Docker execution module
	# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
	# Project: Nasqueron
	# Created: 2018-10-07
	# Description: Functions related to data format conversions
	# License: BSD-2-Clause
	# -------------------------------------------------------------


	def get_image(default_image, args):
	"""
	A function to output a pillar key in JSON.

	State Example::

	{% image = salt['paas_docker.get_image']("nasqueron/mysql", container) %}
	"""
	image = default_image

	if "image" in args:
	image = args["image"]

	if "version" in args:
	image += ":" + str(args["version"])

	return image


	def get_subnets():
	"""
	A function to get the Docker subnets list.

	CLI Example:

	salt * paas_docker.get_subnets
	"""
	try:
	networks = __pillar__["docker_networks"][__grains__["id"]]
	except KeyError:
	networks = {}

	# Defined Docker subnet
	subnets = [network["subnet"] for network in networks.values()]
	# Default Docker subnet
	subnets.append("172.17.0.0/16")

	return subnets
	+
	+
	+# -------------------------------------------------------------
	+# Monitoring
	+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
	+
	+
	+def _get_health_check_url(check_type, container, url):
	+ if check_type[-6:] == "_proxy":
	+ return f"https://{container['host']}{url}"
	+
	+ return f"http://localhost:{container['app_port']}{url}"
	+
	+
	+def get_health_checks():
	+ """
	+ A function to get a dictionary with health checks
	+ for known containers to use with our monitoring.
	+
	+ CLI Example:
	+
	+ salt * paas_docker.get_health_checks
	+ """
	+ containers = __pillar__["docker_containers"][__grains__["id"]]
	+ monitoring = __pillar__["docker_containers_monitoring"]
	+
	+ return {
	+ check_type: {
	+ instance: _get_health_check_url(check_type, container, url)
	+ for service, url in monitoring[check_type].items()
	+ for instance, container in containers[service].items()
	+ }
	+ for check_type in monitoring.keys()
	+ }
	diff --git a/pillar/paas/docker.sls b/pillar/paas/docker.sls
	index b1ab2f6..75f64fc 100644
	--- a/pillar/paas/docker.sls
	+++ b/pillar/paas/docker.sls
	@@ -1,512 +1,547 @@
	# -------------------------------------------------------------
	# Salt — Provision Docker engine
	# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
	# Project: Nasqueron
	# Created: 2018-03-10
	# License: Trivial work, not eligible to copyright
	# -------------------------------------------------------------

	docker_aliases:
	- &ipv4_docker001 51.255.124.9
	- &ipv4_docker001_restricted 51.255.124.9

	# -------------------------------------------------------------
	# Images
	#
	# You can append a :tag (by default, latest is used).
	#
	# It's not possible to specify Docker library images only by final name.
	# See https://docs.saltstack.com/en/latest/ref/states/all/salt.states.docker_image.html
	# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

	docker_images:
	'*':
	- certbot/certbot

	dwellers:
	# Core services
	- nasqueron/mysql:5.7

	docker-001:
	# Core services
	- library/postgres
	- library/redis:3.2-alpine
	- library/registry
	- nasqueron/mysql
	- nasqueron/rabbitmq

	# ACME DNS server
	- joohoi/acme-dns

	# Nasqueron services
	- nasqueron/auth-grove

	# Nasqueron API microservices
	- nasqueron/docker-registry-api
	- nasqueron/api-datasources

	# Infrastructure and development services
	- nasqueron/aphlict
	- nasqueron/cachet
	- nasqueron/etherpad:production
	- nasqueron/notifications
	- nasqueron/phabricator
	- ghcr.io/hound-search/hound

	# Continuous deployment jobs
	- jenkins/jenkins
	- nasqueron/jenkins-agent-node
	- nasqueron/jenkins-agent-php
	- nasqueron/jenkins-agent-rust
	- nasqueron/tommy

	# Pixelfed
	- nasqueron/pixelfed

	# Sentry
	- library/sentry
	- tianon/exim4

	# Hauk
	- bilde2910/hauk

	# -------------------------------------------------------------
	# Networks
	#
	# Containers can be grouped by network, instead to use links.
	# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

	docker_networks:
	dwellers:
	bugzilla:
	subnet: 172.21.3.0/24
	docker-001:
	cd:
	subnet: 172.18.1.0/24
	ci:
	subnet: 172.18.2.0/24

	# -------------------------------------------------------------
	# Docker engine configuration
	# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

	docker_daemon:
	docker-001:
	storage-driver: devicemapper
	storage-opts:
	- "dm.thinpooldev=/dev/mapper/wharf-thinpool"
	- "dm.use_deferred_removal=true"
	- "dm.use_deferred_deletion=true"

	docker_devicemapper:
	docker-001:
	thinpool: wharf-thinpool

	# -------------------------------------------------------------
	# Containers
	#
	# The docker_containers entry allow to declare
	# containers by image by servers
	#
	# The hierarchy is so as following.
	#
	# docker_containers:
	# server with the Docker engine:
	# service codename:
	# instance name:
	# container properties
	#
	# The service codename must match a state file in
	# the roles/paas-docker/containers/ directory.
	#
	# The container will be run with the specified instance name.
	#
	# nginx
	#
	# The container properties can also describe the information
	# needed to configure nginx with the host and app_port key.
	#
	# In such case, a matching vhost file should be declared as
	# roles/paas-docker/nginx/files/vhosts/<service codename>.sls
	# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

	docker_containers:

	#
	# Dwellers is the engine for Mastodon and CI intelligent bus services
	#
	dwellers:

	#
	# Core services
	#

	mysql:
	bugzilla_db:
	network: bugzilla
	version: 5.7

	#
	# Bugzilla
	#

	bugzilla:
	ew_bugzilla:
	host: bugzilla.espace-win.org
	app_port: 33080
	network: bugzilla
	mysql:
	host: bugzilla_db
	db: EspaceWin_Bugs
	credential: espacewin.bugzilla.mysql

	#
	# Mastodon
	#

	# Mastodon is currently deployed manually through docker-compose
	# and not yet integrated to the platform. This declaration is
	# currently only used for extra utilities deployment.

	mastodon_sidekiq:
	mastodon_sidekiq_1:
	realm: nasqueron

	#
	# Current production engine
	#
	docker-001:

	#
	# Core services
	#

	mysql:
	acquisitariat: {}
	phpbb_db: {}

	postgresql:
	sentry_db:
	credential: nasqueron.sentry.postgresql

	rabbitmq:
	white-rabbit:
	ip: *ipv4_docker001_restricted
	host: white-rabbit.nasqueron.org
	app_port: 15672

	redis:
	sentry_redis: {}
	pixelfed_redis: {}

	registry:
	registry:
	host: registry.nasqueron.org
	app_port: 5000
	allowed_ips:
	# Localhost
	- 127.0.0.1

	# Dwellers
	- 51.255.124.11
	- 2001:470:1f13:ce7:ca5:cade:fab:1e

	# docker-001
	- 51.255.124.9
	- 2001:470:1f13:365::50f7:ba11

	#
	# Let's Encrypt
	#

	acme_dns:
	acme:
	ip: *ipv4_docker001
	app_port: 41080
	host: acme.nasqueron.org
	nsadmin: ops.nasqueron.org

	#
	# CI and CD
	#

	jenkins:
	jenkins_cd:
	realm: cd
	host: cd.nasqueron.org
	app_port: 38080
	jnlp_port: 50000
	jenkins_ci:
	realm: ci
	host: ci.nasqueron.org
	app_port: 42080
	jnlp_port: 55000

	jenkins_agent:
	# Agents for CD

	apsile: &php_for_cd
	image: php
	realm: cd

	elapsi: *php_for_cd

	rust_brown:
	image: rust
	realm: cd

	yarabokin:
	image: node
	realm: cd

	# Agents for CI

	zateki: &php_for_ci
	image: php
	realm: ci

	zenerre: *php_for_ci

	tommy:
	tommy_ci:
	app_port: 24080
	host: builds.nasqueron.org
	aliases:
	- build.nasqueron.org
	jenkins_url: https://ci.nasqueron.org

	tommy_cd:
	# No host definition, as this dashboard is mounted on infra.nasqueron.org
	app_port: 24180
	jenkins_url: https://cd.nasqueron.org

	# Infrastructure and development services

	hound:
	hound:
	app_port: 44080
	host: code.nasqueron.org
	github_account: nasqueron

	notifications:
	notifications:
	host: notifications.nasqueron.org
	app_port: 37080
	broker_link: white-rabbit
	credentials:
	broker: nasqueron.notifications.broker
	mailgun: nasqueron.notifications.mailgun
	sentry:
	realm: nasqueron
	project_id: 2
	credential: nasqueron.notifications.sentry

	phabricator:
	# Nasqueron instance
	devcentral:
	app_port: 31080
	host: devcentral.nasqueron.org
	aliases:
	- phabricator.nasqueron.org
	blogs:
	servers:
	host: servers.nasqueron.org
	aliases:
	- server.nasqueron.org
	- serveur.nasqueron.org
	- serveurs.nasqueron.org
	mailer: mailgun
	credentials:
	mysql: zed.phabricator.mysql
	static_host: devcentral.nasqueron-user-content.org
	title: Nasqueron DevCentral
	mysql_link: acquisitariat
	skip_container: True

	# Private instance for Dereckson
	river_sector:
	app_port: 23080
	host: river-sector.dereckson.be
	static_host: river-sector.nasqueron-user-content.org
	mailer: _
	credentials:
	mysql: dereckson.phabricator.mysql
	storage:
	namespace: river_sector
	title: River Sector
	mysql_link: acquisitariat

	# Wolfplex instance
	wolfplex_phab:
	app_port: 35080
	host: phabricator.wolfplex.org
	aliases:
	- phabricator.wolfplex.be
	static_host: wolfplex.phabricator.nasqueron-user-content.org
	mailer: mailgun
	credentials:
	mailgun: wolfplex.phabricator.mailgun
	mysql: wolfplex.phabricator.mysql
	storage:
	namespace: wolfphab
	title: Wolfplex Phabricator
	mysql_link: acquisitariat

	# Zed instance
	zed_code:
	app_port: 36080
	host: code.zed.dereckson.be
	static_host: zed.phabricator.nasqueron-user-content.org
	mailer: sendgrid
	credentials:
	mysql: zed.phabricator.mysql
	sendgrid: zed.phabricator.sendgrid
	storage:
	namespace: zedphab
	title: Zed
	mysql_link: acquisitariat

	aphlict:
	aphlict:
	ports:
	client: 22280
	admin: 22281

	cachet:
	cachet:
	app_port: 39080
	host: status.nasqueron.org
	credential: nasqueron.cachet.mysql
	app_key: nasqueron.cachet.app_key
	mysql_link: acquisitariat

	etherpad:
	pad:
	app_port: 34080
	host: pad.nasqueron.org
	aliases:
	- pad.wolfplex.org
	- pad.wolfplex.be
	credential: nasqueron.etherpad.api
	mysql_link: acquisitariat

	auth-grove:
	login:
	app_port: 25080
	host: login.nasqueron.org
	credential: nasqueron.auth-grove.mysql
	mysql_link: acquisitariat

	# API microservices

	docker-registry-api:
	api-docker-registry:
	app_port: 20080
	api_entry_point: /docker/registry
	registry_instance: registry

	api-datasources:
	api-datasources:
	app_port: 19080
	api_entry_point: /datasources

	# phpBB SaaS
	# The SaaS uses a MySQL instance, declared in the MySQL section.

	# Openfire
	openfire:
	openfire:
	ip: *ipv4_docker001
	app_port: 9090
	host: xmpp.nasqueron.org

	# Other subservices for XMPP
	# listening to their own subdomain
	aliases:
	- conference.nasqueron.org

	# Pixelfed
	pixelfed:
	pixelfed:
	app_port: 30080
	host: photos.nasqueron.org
	aliases:
	- photo.nasqueron.org
	links:
	mysql: acquisitariat
	redis: pixelfed_redis
	credentials:
	app_key: nasqueron.pixelfed.app_key
	mailgun: nasqueron.pixelfed.mailgun
	mysql: nasqueron.pixelfed.mysql
	app:
	title: Nasqueron Photos
	max_album_length: 16

	# Hauk
	hauk:
	hauk:
	app_port: 43080
	host: geo.nasqueron.org
	api_entry_point: /hauk

	# Sentry
	# The Sentry instance uses a Redis and a PostgreSQL instance,
	# declared above.
	exim:
	sentry_smtp:
	mailname: mx.sentry.nasqueron.org

	sentry:
	sentry_web_1:
	app_port: 26080
	host: sentry.nasqueron.org

	# As an instance is divided between a web, a cron and a worker
	# containers, we need an identified to share a data volume.
	realm: nasqueron

	sentry_worker:
	sentry_worker_1:
	realm: nasqueron

	sentry_cron:
	sentry_cron:
	realm: nasqueron

	+# -------------------------------------------------------------
	+# Monitoring
	+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
	+
	+docker_containers_monitoring:
	+
	+ # Go to URL, check it's an HTTP 200 response
	+ check_http_200:
	+ acme_dns: /health
	+ cachet: /api/v1/ping
	+ hound: /healthz
	+
	+ # Test a regular URL for services without health check
	+ api-datasources: /datasources
	+ etherpad: /stats
	+ hauk: /
	+ jenkins: /login
	+ registry: /
	+
	+ # Go to URL, check it's an HTTP 200 response code + "ALIVE" as content
	+ check_http_200_alive:
	+ auth-grove: /status
	+ docker-registry-api: /status
	+ notifications: /status
	+ tommy: /status
	+
	+ # Same than check_http_200, but we need to query the proxy
	+ check_http_200_proxy:
	+ openfire: /login.jsp
	+ pixelfed: /api/nodeinfo/2.0.json
	+
	+ # Same than check_http_200_alive, but we need to query the proxy
	+ check_http_200_alive_proxy:
	+ phabricator: /status
	+
	# -------------------------------------------------------------
	# Ports listened by known applications
	# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

	rabbitmq_ports:
	- 4369 # epmd, Erlang peer discovery service used by RabbitMQ and CLI tools
	- 5671 # AMQP with TLS (AMQPS)
	- 5672 # AMQP
	- 15672 # Management UI, HTTP API, rabbitmqadmin (management plugin port)
	- 25672 # Erlang distribution server port - Federation, rabbitmqctl

	# Not implemented ports, as we don't use those protocols:
	#
	# - 1883 # MQTT
	# - 8883 # MQTT with TLS
	# - 15674 # STOMP over a WebSocket connection (rabbitmq_web_stomp plugin port)
	# - 15675 # MQTT over a WebSocket connection (rabbitmq_web_mqtt plugin port)
	# - 15692 # Prometheus metrics (rabbitmq_prometheus plugin port)
	# - 61613 # STOMP
	# - 61614 # STOMP with TLS

	xmpp_ports:
	- 3478
	- 5222 # Client to server
	- 5223 # Client to server (Encrypted (legacy-mode) connections)
	- 5229 # Flash Cross Domain
	- 5262 # Connections managers
	- 5269 # Server to server
	- 5270 # Server to server (Encrypted (legacy-mode) connections)
	- 5275 # External components
	- 5276 # External components (Encrypted (legacy-mode) connections)
	- 7070 # HTTP binding
	- 7443 # HTTP binding with TLS
	- 7777 # File transfer proxy
	- 9090 # Web administration server
	- 9091 # Web administration server with TLS
	diff --git a/roles/paas-docker/init.sls b/roles/paas-docker/init.sls
	index d2ff1c0..396fb3e 100644
	--- a/roles/paas-docker/init.sls
	+++ b/roles/paas-docker/init.sls
	@@ -1,27 +1,28 @@
	# -------------------------------------------------------------
	# Salt — Provision Docker engine
	# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
	# Project: Nasqueron
	# Created: 2018-09-13
	# License: Trivial work, not eligible to copyright
	# -------------------------------------------------------------

	{% from "map.jinja" import dirs with context %}

	include:
	- .kernel
	- .salt
	- .docker
	- .zemke-rhyne
	{% if salt['file.file_exists'](dirs['bin'] + '/zr') %}
	- .containers
	{% endif %}
	- .systemd-unit
	- .wwwroot-502
	- .wwwroot-content
	- .nginx
	+ - .monitoring
	- .letsencrypt
	- .wrappers
	{% if salt['node.has']('flags:install_docker_devel_tools') %}
	- .devel
	{% endif %}
	diff --git a/roles/paas-docker/monitoring/files/checks.yml.jinja b/roles/paas-docker/monitoring/files/checks.yml.jinja
	new file mode 100644
	index 0000000..c3ee3c7
	--- /dev/null
	+++ b/roles/paas-docker/monitoring/files/checks.yml.jinja
	@@ -0,0 +1,19 @@
	+# -------------------------------------------------------------
	+# Configuration for Docker PaaS monitoring
	+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
	+# Project: Nasqueron
	+# Source file: roles/paas-docker/monitoring/files/checks.yml.jinja
	+# -------------------------------------------------------------
	+#
	+# <auto-generated>
	+# This file is managed by our rOPS SaltStack repository.
	+#
	+# Changes to this file may cause incorrect behavior
	+# and will be lost if the state is redeployed.
	+# </auto-generated>
	+
	+# -------------------------------------------------------------
	+# Checks configuration
	+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
	+
	+{{ salt["convert.to_yaml_dictionary"](checks, root="checks") }}
	diff --git a/roles/paas-docker/monitoring/init.sls b/roles/paas-docker/monitoring/init.sls
	new file mode 100644
	index 0000000..d62831a
	--- /dev/null
	+++ b/roles/paas-docker/monitoring/init.sls
	@@ -0,0 +1,29 @@
	+# -------------------------------------------------------------
	+# Salt — Provision Docker engine
	+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
	+# Project: Nasqueron
	+# License: Trivial work, not eligible to copyright
	+# -------------------------------------------------------------
	+
	+{% from "map.jinja" import dirs with context %}
	+
	+# -------------------------------------------------------------
	+# Platform checks
	+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
	+
	+platform-checks:
	+ pip.installed
	+
	+# -------------------------------------------------------------
	+# Health check configuration
	+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
	+
	+{{ dirs.etc }}/monitoring/checks.yml:
	+ file.managed:
	+ - source: salt://roles/paas-docker/monitoring/files/checks.yml.jinja
	+ - makedirs: True
	+ - mode: 0644
	+ - template: jinja
	+ - context:
	+ checks:
	+ - {{ salt['paas_docker.get_health_checks']() }}

File Metadata

Mime Type: text/x-diff
Expires: Mon, Nov 25, 01:14 (17 h, 35 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 2259310
Default Alt Text: (21 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions