Page MenuHomeDevCentral

No OneTemporary

diff --git a/_modules/paas_docker.py b/_modules/paas_docker.py
index 73241ef..afe4c6a 100644
--- a/_modules/paas_docker.py
+++ b/_modules/paas_docker.py
@@ -1,125 +1,134 @@
# -*- coding: utf-8 -*-
# -------------------------------------------------------------
# Salt — PaaS Docker execution module
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Project: Nasqueron
# Created: 2018-10-07
# Description: Functions related to data format conversions
# License: BSD-2-Clause
# -------------------------------------------------------------
def get_image(default_image, args):
"""
A function to output a pillar key in JSON.
State Example::
{% image = salt['paas_docker.get_image']("nasqueron/mysql", container) %}
"""
image = default_image
if "image" in args:
image = args["image"]
if "version" in args:
image += ":" + str(args["version"])
return image
def list_images():
"""
A function to get the list of images used on a Docker engine.
Example:
salt docker-002 paas_docker.list_images
"""
images = __pillar__.get("docker_images", [])
# Workaround for a merge issue for lists:
# Salt Tower concatenates them, a set will dedup them.
return set(images)
def get_subnets():
"""
A function to get the Docker subnets list.
CLI Example:
salt * paas_docker.get_subnets
"""
try:
networks = __pillar__["docker_networks"]
except KeyError:
networks = {}
# Defined Docker subnet
subnets = [network["subnet"] for network in networks.values()]
# Default Docker subnet
subnets.append("172.17.0.0/16")
return subnets
def _get_containers():
try:
return __pillar__["docker_containers"]
except KeyError:
return {}
def list_containers():
"""
A function to list all the containers provisionned on a Docker engine.
This function uses the pillar docker_containers as authoritative source,
so it documents the expected configuration, not the actual containers
running. That allows to compare both states.
CLI Example:
salt * paas_docker.list_containers
"""
return [
key
for service, service_containers in _get_containers().items()
for key in service_containers.keys()
]
# -------------------------------------------------------------
# Monitoring
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
def _get_health_check_url(check_type, container, url):
if check_type[-6:] == "_proxy":
return f"https://{container['host']}{url}"
return f"http://localhost:{container['app_port']}{url}"
def get_health_checks():
"""
A function to get a dictionary with health checks
for known containers to use with our monitoring.
CLI Example:
salt * paas_docker.get_health_checks
"""
containers = _get_containers()
monitoring = __pillar__["docker_containers_monitoring"]
return {
check_type: {
instance: _get_health_check_url(check_type, container, url)
for service, url in monitoring[check_type].items()
for instance, container in containers.get(service, {}).items()
}
for check_type in monitoring.keys()
}
+
+
+# -------------------------------------------------------------
+# Format
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+
+def format_env_list(values, separator=",", assign_op="-"):
+ return separator.join([f"{k}{assign_op}{v}" for k, v in values.items()])
diff --git a/_tests/modules/test_paas_docker.py b/_tests/modules/test_paas_docker.py
index 0323384..b96387c 100755
--- a/_tests/modules/test_paas_docker.py
+++ b/_tests/modules/test_paas_docker.py
@@ -1,64 +1,72 @@
#!/usr/bin/env python3
from importlib.machinery import SourceFileLoader
import unittest
salt_test_case = SourceFileLoader("salt_test_case", "salt_test_case.py").load_module()
docker = SourceFileLoader("docker", "../_modules/paas_docker.py").load_module()
class Testinstance(unittest.TestCase, salt_test_case.SaltTestCase):
def setUp(self):
self.initialize_mocks()
self.instance = docker
self.mock_pillar("data/paas_docker.yaml")
self.mock_grains()
self.grains["id"] = "egladil"
def test_get_image(self):
container = {"image": "foo", "version": "42"}
self.assertEqual("foo:42", docker.get_image("not_foo", container))
def test_list_images(self):
expected = {"foo", "bar", "quux"}
self.assertEqual(expected, docker.list_images())
def test_get_image_without_version(self):
container = {
"image": "foo",
}
self.assertEqual("foo", docker.get_image("not_foo", container))
def test_get_image_without_image(self):
container = {"version": "42"}
self.assertEqual("not_foo:42", docker.get_image("not_foo", container))
def test_get_image_without_anything(self):
self.assertEqual("not_foo", docker.get_image("not_foo", {}))
def test_get_image_with_numeric_version(self):
container = {"image": "foo", "version": 2.5}
self.assertEqual("foo:2.5", docker.get_image("not_foo", container))
def test_get_subnets(self):
expected = ["172.18.1.0/24", "172.18.2.0/24", "172.17.0.0/16"]
self.assertEqual(expected, docker.get_subnets())
def test_get_subnets_when_none_are_defined(self):
# Only the default Docker one
expected = ["172.17.0.0/16"]
self.grains["id"] = "voidserver"
self.assertEqual(expected, docker.get_subnets())
+ def test_format_env_list(self):
+ expression = {"foo": "bar", "quux": 42}
+
+ expected = "foo: bar; quux: 42"
+ self.assertEqual(
+ expected, docker.format_env_list(expression, assign_op=": ", separator="; ")
+ )
+
if __name__ == "__main__":
unittest.main()
diff --git a/pillar/paas/docker/docker-002/sentry.sls b/pillar/paas/docker/docker-002/sentry.sls
index bfff949..ebec57c 100644
--- a/pillar/paas/docker/docker-002/sentry.sls
+++ b/pillar/paas/docker/docker-002/sentry.sls
@@ -1,65 +1,98 @@
# -------------------------------------------------------------
# Salt — Provision Docker engine
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Project: Nasqueron
# License: Trivial work, not eligible to copyright
# Service: Sentry
# -------------------------------------------------------------
docker_networks:
sentry:
subnet: 172.18.3.0/24
docker_images:
- library/postgres
- library/redis:3.2-alpine
- library/sentry
- tianon/exim4
docker_containers:
#
# Core services used by Sentry
#
exim:
sentry_smtp:
mailname: mx.sentry.nasqueron.org
network: sentry
memcached:
sentry_memcached:
version: 1.6.9-alpine
network: sentry
redis:
sentry_redis:
network: sentry
postgresql:
sentry_db:
credential: nasqueron.sentry.postgresql
+ #
+ # Kafka instance
+ #
+
+ zookeeper:
+ sentry_zookeeper:
+ version: 5.5.0
+ network: sentry
+
+ kafka:
+ sentry_kafka:
+ version: 5.5.0
+ zookeeper: sentry_zookeeper
+ network: sentry
+ topics:
+ - ingest-attachments
+ - ingest-transactions
+ - ingest-events
+ - ingest-replay-recordings
+
#
# Services maintained by Sentry
#
sentry:
sentry_web_1:
app_port: 26080
host: sentry.nasqueron.org
# As an instance is divided between a web, a cron and a worker
# containers, we need an identified to share a data volume.
realm: nasqueron
network: sentry
sentry_worker:
sentry_worker_1:
realm: nasqueron
network: sentry
sentry_cron:
sentry_cron:
realm: nasqueron
network: sentry
+
+# -------------------------------------------------------------
+# Services configuration
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+kakfa_loggers:
+ kafka.cluster: WARN
+ kafka.controller: WARN
+ kafka.coordinator: WARN
+ kafka.log: WARN
+ kafka.server: WARN
+ kafka.zookeeper: WARN
+ state.change.logger: WARN
diff --git a/roles/paas-docker/containers/kafka.sls b/roles/paas-docker/containers/kafka.sls
new file mode 100644
index 0000000..de404e4
--- /dev/null
+++ b/roles/paas-docker/containers/kafka.sls
@@ -0,0 +1,98 @@
+# -------------------------------------------------------------
+# Salt — Provision Docker engine
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# License: Trivial work, not eligible to copyright
+# -------------------------------------------------------------
+
+{% set has_selinux = salt['grains.get']('selinux:enabled', False) %}
+
+{% for instance, container in pillar['docker_containers']['kafka'].items() %}
+{% set image = salt['paas_docker.get_image']("confluentinc/cp-kafka", container) %}
+
+# -------------------------------------------------------------
+# Data directory
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+/srv/kafka/{{ instance }}:
+ file.directory:
+ - makedirs: True
+
+{% for subdir in ['data', 'log'] %}
+# There are several releases of the cp-kafka instance,
+# Some using "appuser", some "cp-kafka" and some "root".
+/srv/kafka/{{ instance }}/{{ subdir }}:
+ file.directory:
+ - user: {{ container["uid"] | default(0) }}
+ - group: {{ container["did"] | default(0) }}
+{% endfor %}
+
+{% if has_selinux %}
+selinux_context_{{ instance }}_kafka_data:
+ selinux.fcontext_policy_present:
+ - name: /srv/kafka/{{ instance }}
+ - sel_type: container_file_t
+
+selinux_context_{{ instance }}_kafka_data_applied:
+ selinux.fcontext_policy_applied:
+ - name: /srv/kafka/{{ instance }}
+{% endif %}
+
+# -------------------------------------------------------------
+# Container
+#
+# Environment is configured per Sentry recommandation
+# Source: https://github.com/getsentry/self-hosted/blob/master/docker-compose.yml
+#
+# The secrets volume it to be shared with Zookeeper
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+{% set loggers = salt["paas_docker.format_env_list"](pillar["kakfa_loggers"]) %}
+
+{{ instance }}:
+ docker_container.running:
+ - detach: True
+ - interactive: True
+ - image: {{ image }}
+ - binds:
+ - /srv/kafka/{{ instance }}/data:/var/lib/kafka/data
+ - /srv/kafka/{{ instance }}/log:/var/lib/kafka/log
+ - /srv/zookeeper/{{ container["zookeeper"] }}/secrets:/etc/kafka/secrets
+ - environment:
+ KAFKA_ZOOKEEPER_CONNECT: "{{ container["zookeeper"] }}:2181"
+ KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://{{ instance }}:9092"
+ KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "1"
+ KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS: "1"
+ KAFKA_LOG_RETENTION_HOURS: "24"
+ KAFKA_MESSAGE_MAX_BYTES: "50000000" # 50MB or bust
+ KAFKA_MAX_REQUEST_SIZE: "50000000" # 50MB on requests apparently too
+ CONFLUENT_SUPPORT_METRICS_ENABLE: "false"
+ KAFKA_LOG4J_LOGGERS: {{ loggers }}
+ KAFKA_LOG4J_ROOT_LOGLEVEL: "WARN"
+ KAFKA_TOOLS_LOG4J_LOGLEVEL: "WARN"
+ - healthcheck:
+ Test: nc -z localhost 9092
+ Interval: 30000000000
+ - networks:
+ - {{ container['network'] }}
+
+# -------------------------------------------------------------
+# Kafka topics
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+{% if "topics" in container %}
+{{ instance }}_topics:
+ cmd.run:
+ - name: |
+ sleep 20
+ {% for topic in container["topics"] %}
+ docker exec {{ instance }} kafka-topics --create --topic {{ topic }} --bootstrap-server localhost:9092
+ {% endfor %}
+ touch /srv/kafka/{{ instance }}/.topics_initialized
+ - require:
+ - docker_container: {{ instance }}
+ - creates: /srv/kafka/{{ instance }}/.topics_initialized
+{% endif %}
+
+
+{% endfor %}
diff --git a/roles/paas-docker/containers/zookeeper.sls b/roles/paas-docker/containers/zookeeper.sls
new file mode 100644
index 0000000..e4bef4a
--- /dev/null
+++ b/roles/paas-docker/containers/zookeeper.sls
@@ -0,0 +1,73 @@
+# -------------------------------------------------------------
+# Salt — Provision Docker engine
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# License: Trivial work, not eligible to copyright
+# -------------------------------------------------------------
+
+{% set has_selinux = salt['grains.get']('selinux:enabled', False) %}
+
+{% for instance, container in pillar['docker_containers']['zookeeper'].items() %}
+{% set image = salt['paas_docker.get_image']("confluentinc/cp-zookeeper", container) %}
+
+# -------------------------------------------------------------
+# Data directory
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+/srv/zookeeper/{{ instance }}:
+ file.directory:
+ - makedirs: True
+
+{% for subdir in ['data', 'log', 'secrets'] %}
+# There are several releases of the cp-zookeeper instance,
+# Some using "appuser", some "cp-kafka" and some "root".
+/srv/zookeeper/{{ instance }}/{{ subdir }}:
+ file.directory:
+ - user: {{ container["uid"] | default(0) }}
+ - group: {{ container["did"] | default(0) }}
+{% endfor %}
+
+{% if has_selinux %}
+selinux_context_{{ instance }}_zookeeper_data:
+ selinux.fcontext_policy_present:
+ - name: /srv/zookeeper/{{ instance }}
+ - sel_type: container_file_t
+
+selinux_context_{{ instance }}_zookeeper_data_applied:
+ selinux.fcontext_policy_applied:
+ - name: /srv/zookeeper/{{ instance }}
+{% endif %}
+
+# -------------------------------------------------------------
+# Container
+#
+# Environment is configured per Sentry recommandation
+# Source: https://github.com/getsentry/self-hosted/blob/master/docker-compose.yml
+#
+# The secrets volume it to be shared with Zookeeper applications
+# like kafka.
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+{{ instance }}:
+ docker_container.running:
+ - detach: True
+ - interactive: True
+ - image: {{ image }}
+ - binds:
+ - /srv/zookeeper/{{ instance }}/data:/var/lib/zookeper/data
+ - /srv/zookeeper/{{ instance }}/log:/var/lib/zookeeper/log
+ - /srv/zookeeper/{{ instance }}/secrets:/etc/zookeeper/secrets
+ - environment:
+ ZOOKEEPER_CLIENT_PORT: "2181"
+ CONFLUENT_SUPPORT_METRICS_ENABLE: "false"
+ ZOOKEEPER_LOG4J_ROOT_LOGLEVEL: "WARN"
+ ZOOKEEPER_TOOLS_LOG4J_LOGLEVEL: "WARN"
+ KAFKA_OPTS: "-Dzookeeper.4lw.commands.whitelist=ruok"
+ - healthcheck:
+ Test: echo "ruok" | nc -w 2 -q 2 localhost 2181 | grep imok
+ Interval: 30000000000
+{% if 'network' in container %}
+ - networks:
+ - {{ container['network'] }}
+{% endif %}
+{% endfor %}

File Metadata

Mime Type
text/x-diff
Expires
Sat, Nov 7, 12:22 (1 w, 5 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
4250084
Default Alt Text
(14 KB)

Event Timeline