Page MenuHomeDevCentral
Files F3770382

D2871.diff
No OneTemporary
Actions

D2871.diff
View Options

diff --git a/_modules/paas_docker.py b/_modules/paas_docker.py
--- a/_modules/paas_docker.py
+++ b/_modules/paas_docker.py
@@ -123,3 +123,12 @@
}
for check_type in monitoring.keys()
}
+
+
+# -------------------------------------------------------------
+# Format
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+
+def format_env_list(values, separator=",", assign_op="-"):
+ return separator.join([f"{k}{assign_op}{v}" for k, v in values.items()])
diff --git a/_tests/modules/test_paas_docker.py b/_tests/modules/test_paas_docker.py
--- a/_tests/modules/test_paas_docker.py
+++ b/_tests/modules/test_paas_docker.py
@@ -59,6 +59,14 @@
self.grains["id"] = "voidserver"
self.assertEqual(expected, docker.get_subnets())
+ def test_format_env_list(self):
+ expression = {"foo": "bar", "quux": 42}
+
+ expected = "foo: bar; quux: 42"
+ self.assertEqual(
+ expected, docker.format_env_list(expression, assign_op=": ", separator="; ")
+ )
+
if __name__ == "__main__":
unittest.main()
diff --git a/pillar/paas/docker/docker-002/sentry.sls b/pillar/paas/docker/docker-002/sentry.sls
--- a/pillar/paas/docker/docker-002/sentry.sls
+++ b/pillar/paas/docker/docker-002/sentry.sls
@@ -40,6 +40,26 @@
sentry_db:
credential: nasqueron.sentry.postgresql
+ #
+ # Kafka instance
+ #
+
+ zookeeper:
+ sentry_zookeeper:
+ version: 5.5.0
+ network: sentry
+
+ kafka:
+ sentry_kafka:
+ version: 5.5.0
+ zookeeper: sentry_zookeeper
+ network: sentry
+ topics:
+ - ingest-attachments
+ - ingest-transactions
+ - ingest-events
+ - ingest-replay-recordings
+
#
# Services maintained by Sentry
#
@@ -63,3 +83,16 @@
sentry_cron:
realm: nasqueron
network: sentry
+
+# -------------------------------------------------------------
+# Services configuration
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+kakfa_loggers:
+ kafka.cluster: WARN
+ kafka.controller: WARN
+ kafka.coordinator: WARN
+ kafka.log: WARN
+ kafka.server: WARN
+ kafka.zookeeper: WARN
+ state.change.logger: WARN
diff --git a/roles/paas-docker/containers/kafka.sls b/roles/paas-docker/containers/kafka.sls
new file mode 100644
--- /dev/null
+++ b/roles/paas-docker/containers/kafka.sls
@@ -0,0 +1,98 @@
+# -------------------------------------------------------------
+# Salt — Provision Docker engine
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# License: Trivial work, not eligible to copyright
+# -------------------------------------------------------------
+
+{% set has_selinux = salt['grains.get']('selinux:enabled', False) %}
+
+{% for instance, container in pillar['docker_containers']['kafka'].items() %}
+{% set image = salt['paas_docker.get_image']("confluentinc/cp-kafka", container) %}
+
+# -------------------------------------------------------------
+# Data directory
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+/srv/kafka/{{ instance }}:
+ file.directory:
+ - makedirs: True
+
+{% for subdir in ['data', 'log'] %}
+# There are several releases of the cp-kafka instance,
+# Some using "appuser", some "cp-kafka" and some "root".
+/srv/kafka/{{ instance }}/{{ subdir }}:
+ file.directory:
+ - user: {{ container["uid"] | default(0) }}
+ - group: {{ container["did"] | default(0) }}
+{% endfor %}
+
+{% if has_selinux %}
+selinux_context_{{ instance }}_kafka_data:
+ selinux.fcontext_policy_present:
+ - name: /srv/kafka/{{ instance }}
+ - sel_type: container_file_t
+
+selinux_context_{{ instance }}_kafka_data_applied:
+ selinux.fcontext_policy_applied:
+ - name: /srv/kafka/{{ instance }}
+{% endif %}
+
+# -------------------------------------------------------------
+# Container
+#
+# Environment is configured per Sentry recommandation
+# Source: https://github.com/getsentry/self-hosted/blob/master/docker-compose.yml
+#
+# The secrets volume it to be shared with Zookeeper
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+{% set loggers = salt["paas_docker.format_env_list"](pillar["kakfa_loggers"]) %}
+
+{{ instance }}:
+ docker_container.running:
+ - detach: True
+ - interactive: True
+ - image: {{ image }}
+ - binds:
+ - /srv/kafka/{{ instance }}/data:/var/lib/kafka/data
+ - /srv/kafka/{{ instance }}/log:/var/lib/kafka/log
+ - /srv/zookeeper/{{ container["zookeeper"] }}/secrets:/etc/kafka/secrets
+ - environment:
+ KAFKA_ZOOKEEPER_CONNECT: "{{ container["zookeeper"] }}:2181"
+ KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://{{ instance }}:9092"
+ KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "1"
+ KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS: "1"
+ KAFKA_LOG_RETENTION_HOURS: "24"
+ KAFKA_MESSAGE_MAX_BYTES: "50000000" # 50MB or bust
+ KAFKA_MAX_REQUEST_SIZE: "50000000" # 50MB on requests apparently too
+ CONFLUENT_SUPPORT_METRICS_ENABLE: "false"
+ KAFKA_LOG4J_LOGGERS: {{ loggers }}
+ KAFKA_LOG4J_ROOT_LOGLEVEL: "WARN"
+ KAFKA_TOOLS_LOG4J_LOGLEVEL: "WARN"
+ - healthcheck:
+ Test: nc -z localhost 9092
+ Interval: 30000000000
+ - networks:
+ - {{ container['network'] }}
+
+# -------------------------------------------------------------
+# Kafka topics
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+{% if "topics" in container %}
+{{ instance }}_topics:
+ cmd.run:
+ - name: |
+ sleep 20
+ {% for topic in container["topics"] %}
+ docker exec {{ instance }} kafka-topics --create --topic {{ topic }} --bootstrap-server localhost:9092
+ {% endfor %}
+ touch /srv/kafka/{{ instance }}/.topics_initialized
+ - require:
+ - docker_container: {{ instance }}
+ - creates: /srv/kafka/{{ instance }}/.topics_initialized
+{% endif %}
+
+
+{% endfor %}
diff --git a/roles/paas-docker/containers/zookeeper.sls b/roles/paas-docker/containers/zookeeper.sls
new file mode 100644
--- /dev/null
+++ b/roles/paas-docker/containers/zookeeper.sls
@@ -0,0 +1,73 @@
+# -------------------------------------------------------------
+# Salt — Provision Docker engine
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# License: Trivial work, not eligible to copyright
+# -------------------------------------------------------------
+
+{% set has_selinux = salt['grains.get']('selinux:enabled', False) %}
+
+{% for instance, container in pillar['docker_containers']['zookeeper'].items() %}
+{% set image = salt['paas_docker.get_image']("confluentinc/cp-zookeeper", container) %}
+
+# -------------------------------------------------------------
+# Data directory
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+/srv/zookeeper/{{ instance }}:
+ file.directory:
+ - makedirs: True
+
+{% for subdir in ['data', 'log', 'secrets'] %}
+# There are several releases of the cp-zookeeper instance,
+# Some using "appuser", some "cp-kafka" and some "root".
+/srv/zookeeper/{{ instance }}/{{ subdir }}:
+ file.directory:
+ - user: {{ container["uid"] | default(0) }}
+ - group: {{ container["did"] | default(0) }}
+{% endfor %}
+
+{% if has_selinux %}
+selinux_context_{{ instance }}_zookeeper_data:
+ selinux.fcontext_policy_present:
+ - name: /srv/zookeeper/{{ instance }}
+ - sel_type: container_file_t
+
+selinux_context_{{ instance }}_zookeeper_data_applied:
+ selinux.fcontext_policy_applied:
+ - name: /srv/zookeeper/{{ instance }}
+{% endif %}
+
+# -------------------------------------------------------------
+# Container
+#
+# Environment is configured per Sentry recommandation
+# Source: https://github.com/getsentry/self-hosted/blob/master/docker-compose.yml
+#
+# The secrets volume it to be shared with Zookeeper applications
+# like kafka.
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+{{ instance }}:
+ docker_container.running:
+ - detach: True
+ - interactive: True
+ - image: {{ image }}
+ - binds:
+ - /srv/zookeeper/{{ instance }}/data:/var/lib/zookeper/data
+ - /srv/zookeeper/{{ instance }}/log:/var/lib/zookeeper/log
+ - /srv/zookeeper/{{ instance }}/secrets:/etc/zookeeper/secrets
+ - environment:
+ ZOOKEEPER_CLIENT_PORT: "2181"
+ CONFLUENT_SUPPORT_METRICS_ENABLE: "false"
+ ZOOKEEPER_LOG4J_ROOT_LOGLEVEL: "WARN"
+ ZOOKEEPER_TOOLS_LOG4J_LOGLEVEL: "WARN"
+ KAFKA_OPTS: "-Dzookeeper.4lw.commands.whitelist=ruok"
+ - healthcheck:
+ Test: echo "ruok" | nc -w 2 -q 2 localhost 2181 | grep imok
+ Interval: 30000000000
+{% if 'network' in container %}
+ - networks:
+ - {{ container['network'] }}
+{% endif %}
+{% endfor %}

File Metadata

Mime Type
text/plain
Expires
Sun, Nov 24, 20:07 (19 h, 42 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
2257982
Default Alt Text
D2871.diff (8 KB)

Event Timeline

Nasqueron DevCentral · If it had been much bigger the moon would have had a core of ice. · Powered by Phabricator