Page Menu
Home
DevCentral
Search
Configure Global Search
Log In
Files
F3824661
D2871.id7318.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
8 KB
Referenced Files
None
Subscribers
None
D2871.id7318.diff
View Options
diff --git a/_modules/paas_docker.py b/_modules/paas_docker.py
--- a/_modules/paas_docker.py
+++ b/_modules/paas_docker.py
@@ -123,3 +123,12 @@
}
for check_type in monitoring.keys()
}
+
+
+# -------------------------------------------------------------
+# Format
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+
+def format_env_list(values, separator=",", assign_op="-"):
+ return separator.join([f"{k}{assign_op}{v}" for k, v in values.items()])
diff --git a/_tests/modules/test_paas_docker.py b/_tests/modules/test_paas_docker.py
--- a/_tests/modules/test_paas_docker.py
+++ b/_tests/modules/test_paas_docker.py
@@ -59,6 +59,14 @@
self.grains["id"] = "voidserver"
self.assertEqual(expected, docker.get_subnets())
+ def test_format_env_list(self):
+ expression = {"foo": "bar", "quux": 42}
+
+ expected = "foo: bar; quux: 42"
+ self.assertEqual(
+ expected, docker.format_env_list(expression, assign_op=": ", separator="; ")
+ )
+
if __name__ == "__main__":
unittest.main()
diff --git a/pillar/paas/docker/docker-002/sentry.sls b/pillar/paas/docker/docker-002/sentry.sls
--- a/pillar/paas/docker/docker-002/sentry.sls
+++ b/pillar/paas/docker/docker-002/sentry.sls
@@ -40,6 +40,26 @@
sentry_db:
credential: nasqueron.sentry.postgresql
+ #
+ # Kafka instance
+ #
+
+ zookeeper:
+ sentry_zookeeper:
+ version: 5.5.0
+ network: sentry
+
+ kafka:
+ sentry_kafka:
+ version: 5.5.0
+ zookeeper: sentry_zookeeper
+ network: sentry
+ topics:
+ - ingest-attachments
+ - ingest-transactions
+ - ingest-events
+ - ingest-replay-recordings
+
#
# Services maintained by Sentry
#
@@ -63,3 +83,16 @@
sentry_cron:
realm: nasqueron
network: sentry
+
+# -------------------------------------------------------------
+# Services configuration
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+kakfa_loggers:
+ kafka.cluster: WARN
+ kafka.controller: WARN
+ kafka.coordinator: WARN
+ kafka.log: WARN
+ kafka.server: WARN
+ kafka.zookeeper: WARN
+ state.change.logger: WARN
diff --git a/roles/paas-docker/containers/kafka.sls b/roles/paas-docker/containers/kafka.sls
new file mode 100644
--- /dev/null
+++ b/roles/paas-docker/containers/kafka.sls
@@ -0,0 +1,98 @@
+# -------------------------------------------------------------
+# Salt — Provision Docker engine
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# License: Trivial work, not eligible to copyright
+# -------------------------------------------------------------
+
+{% set has_selinux = salt['grains.get']('selinux:enabled', False) %}
+
+{% for instance, container in pillar['docker_containers']['kafka'].items() %}
+{% set image = salt['paas_docker.get_image']("confluentinc/cp-kafka", container) %}
+
+# -------------------------------------------------------------
+# Data directory
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+/srv/kafka/{{ instance }}:
+ file.directory:
+ - makedirs: True
+
+{% for subdir in ['data', 'log'] %}
+# There are several releases of the cp-kafka instance,
+# Some using "appuser", some "cp-kafka" and some "root".
+/srv/kafka/{{ instance }}/{{ subdir }}:
+ file.directory:
+ - user: {{ container["uid"] | default(0) }}
+ - group: {{ container["did"] | default(0) }}
+{% endfor %}
+
+{% if has_selinux %}
+selinux_context_{{ instance }}_kafka_data:
+ selinux.fcontext_policy_present:
+ - name: /srv/kafka/{{ instance }}
+ - sel_type: container_file_t
+
+selinux_context_{{ instance }}_kafka_data_applied:
+ selinux.fcontext_policy_applied:
+ - name: /srv/kafka/{{ instance }}
+{% endif %}
+
+# -------------------------------------------------------------
+# Container
+#
+# Environment is configured per Sentry recommandation
+# Source: https://github.com/getsentry/self-hosted/blob/master/docker-compose.yml
+#
+# The secrets volume it to be shared with Zookeeper
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+{% set loggers = salt["paas_docker.format_env_list"](pillar["kakfa_loggers"]) %}
+
+{{ instance }}:
+ docker_container.running:
+ - detach: True
+ - interactive: True
+ - image: {{ image }}
+ - binds:
+ - /srv/kafka/{{ instance }}/data:/var/lib/kafka/data
+ - /srv/kafka/{{ instance }}/log:/var/lib/kafka/log
+ - /srv/zookeeper/{{ container["zookeeper"] }}/secrets:/etc/kafka/secrets
+ - environment:
+ KAFKA_ZOOKEEPER_CONNECT: "{{ container["zookeeper"] }}:2181"
+ KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://{{ instance }}:9092"
+ KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "1"
+ KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS: "1"
+ KAFKA_LOG_RETENTION_HOURS: "24"
+ KAFKA_MESSAGE_MAX_BYTES: "50000000" # 50MB or bust
+ KAFKA_MAX_REQUEST_SIZE: "50000000" # 50MB on requests apparently too
+ CONFLUENT_SUPPORT_METRICS_ENABLE: "false"
+ KAFKA_LOG4J_LOGGERS: {{ loggers }}
+ KAFKA_LOG4J_ROOT_LOGLEVEL: "WARN"
+ KAFKA_TOOLS_LOG4J_LOGLEVEL: "WARN"
+ - healthcheck:
+ Test: nc -z localhost 9092
+ Interval: 30000000000
+ - networks:
+ - {{ container['network'] }}
+
+# -------------------------------------------------------------
+# Kafka topics
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+{% if "topics" in container %}
+{{ instance }}_topics:
+ cmd.run:
+ - name: |
+ sleep 20
+ {% for topic in container["topics"] %}
+ docker exec {{ instance }} kafka-topics --create --topic {{ topic }} --bootstrap-server localhost:9092
+ {% endfor %}
+ touch /srv/kafka/{{ instance }}/.topics_initialized
+ - require:
+ - docker_container: {{ instance }}
+ - creates: /srv/kafka/{{ instance }}/.topics_initialized
+{% endif %}
+
+
+{% endfor %}
diff --git a/roles/paas-docker/containers/zookeeper.sls b/roles/paas-docker/containers/zookeeper.sls
new file mode 100644
--- /dev/null
+++ b/roles/paas-docker/containers/zookeeper.sls
@@ -0,0 +1,73 @@
+# -------------------------------------------------------------
+# Salt — Provision Docker engine
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# License: Trivial work, not eligible to copyright
+# -------------------------------------------------------------
+
+{% set has_selinux = salt['grains.get']('selinux:enabled', False) %}
+
+{% for instance, container in pillar['docker_containers']['zookeeper'].items() %}
+{% set image = salt['paas_docker.get_image']("confluentinc/cp-zookeeper", container) %}
+
+# -------------------------------------------------------------
+# Data directory
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+/srv/zookeeper/{{ instance }}:
+ file.directory:
+ - makedirs: True
+
+{% for subdir in ['data', 'log', 'secrets'] %}
+# There are several releases of the cp-zookeeper instance,
+# Some using "appuser", some "cp-kafka" and some "root".
+/srv/zookeeper/{{ instance }}/{{ subdir }}:
+ file.directory:
+ - user: {{ container["uid"] | default(0) }}
+ - group: {{ container["did"] | default(0) }}
+{% endfor %}
+
+{% if has_selinux %}
+selinux_context_{{ instance }}_zookeeper_data:
+ selinux.fcontext_policy_present:
+ - name: /srv/zookeeper/{{ instance }}
+ - sel_type: container_file_t
+
+selinux_context_{{ instance }}_zookeeper_data_applied:
+ selinux.fcontext_policy_applied:
+ - name: /srv/zookeeper/{{ instance }}
+{% endif %}
+
+# -------------------------------------------------------------
+# Container
+#
+# Environment is configured per Sentry recommandation
+# Source: https://github.com/getsentry/self-hosted/blob/master/docker-compose.yml
+#
+# The secrets volume it to be shared with Zookeeper applications
+# like kafka.
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+{{ instance }}:
+ docker_container.running:
+ - detach: True
+ - interactive: True
+ - image: {{ image }}
+ - binds:
+ - /srv/zookeeper/{{ instance }}/data:/var/lib/zookeper/data
+ - /srv/zookeeper/{{ instance }}/log:/var/lib/zookeeper/log
+ - /srv/zookeeper/{{ instance }}/secrets:/etc/zookeeper/secrets
+ - environment:
+ ZOOKEEPER_CLIENT_PORT: "2181"
+ CONFLUENT_SUPPORT_METRICS_ENABLE: "false"
+ ZOOKEEPER_LOG4J_ROOT_LOGLEVEL: "WARN"
+ ZOOKEEPER_TOOLS_LOG4J_LOGLEVEL: "WARN"
+ KAFKA_OPTS: "-Dzookeeper.4lw.commands.whitelist=ruok"
+ - healthcheck:
+ Test: echo "ruok" | nc -w 2 -q 2 localhost 2181 | grep imok
+ Interval: 30000000000
+{% if 'network' in container %}
+ - networks:
+ - {{ container['network'] }}
+{% endif %}
+{% endfor %}
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Wed, Dec 4, 14:05 (4 h, 47 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
2276405
Default Alt Text
D2871.id7318.diff (8 KB)
Attached To
Mode
D2871: Provide Kafka and ZooKeeper instances for Sentry
Attached
Detach File
Event Timeline
Log In to Comment