Page MenuHomeDevCentral
Files F7700918

D2455.id6178.diff
No OneTemporary
Actions

D2455.id6178.diff
View Options

diff --git a/pillar/credentials/zr.sls b/pillar/credentials/zr.sls
--- a/pillar/credentials/zr.sls
+++ b/pillar/credentials/zr.sls
@@ -46,6 +46,7 @@
internal_users:
admin: 163
dashboards: 164
+ beat_docker: 165
# photos.nasqueron.org
pixelfed:
diff --git a/pillar/opensearch/clusters.sls b/pillar/opensearch/clusters.sls
--- a/pillar/opensearch/clusters.sls
+++ b/pillar/opensearch/clusters.sls
@@ -16,3 +16,6 @@
users:
admin: nasqueron.opensearch.infra-logs.internal_users.admin
dashboards: nasqueron.opensearch.infra-logs.internal_users.dashboards
+ beat_docker: nasqueron.opensearch.infra-logs.internal_users.beat_docker
+ ingest_clients_users:
+ - beat_docker
diff --git a/pillar/paas/docker.sls b/pillar/paas/docker.sls
--- a/pillar/paas/docker.sls
+++ b/pillar/paas/docker.sls
@@ -145,6 +145,9 @@
network: bugzilla
version: 5.7
+ filebeat:
+ filebeat_docker: *filebeat_docker
+
#
# Bugzilla
#
@@ -225,6 +228,19 @@
host: acme.nasqueron.org
nsadmin: ops.nasqueron.org
+ #
+ # Logs collection
+ # Docker logs -> filebeat -> OpenSearch
+ #
+
+ filebeat:
+ filebeat_docker: &filebeat_docker
+ opensearch:
+ server: cloudhugger.nasqueron.org
+ index: "docker-%{+yyyy.MM.dd}"
+ credentials:
+ opensearch: nasqueron.opensearch.infra-logs.internal_users.beat_docker
+
#
# CI and CD
#
diff --git a/roles/opensearch/opensearch/files/internal_users.yml.jinja b/roles/opensearch/opensearch/files/internal_users.yml.jinja
--- a/roles/opensearch/opensearch/files/internal_users.yml.jinja
+++ b/roles/opensearch/opensearch/files/internal_users.yml.jinja
@@ -39,3 +39,18 @@
hash: {{ salt['opensearch.hash_password'](users['dashboards']['password']) }}
reserved: true
description: "Dashboards to OpenSearch machine user"
+
+# -------------------------------------------------------------
+# OpenSearch clients to ingest pipelines
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+{% for user in ingest_clients_users %}
+
+{{ users[user]['username'] }}:
+ hash: {{ salt['opensearch.hash_password'](users[user]['password']) }}
+ reserved: true
+ backend_roles:
+ - "ingest_client"
+ description: "Ingest client machine user"
+
+{% endfor %}
diff --git a/roles/opensearch/opensearch/security.sls b/roles/opensearch/opensearch/security.sls
--- a/roles/opensearch/opensearch/security.sls
+++ b/roles/opensearch/opensearch/security.sls
@@ -18,6 +18,7 @@
- group: opensearch
- template: jinja
- context:
+ ingest_clients_users: {{ config['ingest_clients_users'] }}
users:
{% for user, credential in config['users'].items() %}
{{ user }}:
diff --git a/roles/paas-docker/containers/filebeat.sls b/roles/paas-docker/containers/filebeat.sls
new file mode 100755
--- /dev/null
+++ b/roles/paas-docker/containers/filebeat.sls
@@ -0,0 +1,65 @@
+# -------------------------------------------------------------
+# Salt — Provision Docker engine
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# License: Trivial work, not eligible to copyright
+# -------------------------------------------------------------
+
+{% set has_selinux = salt['grains.get']('selinux:enabled', False) %}
+{% set containers = pillar['docker_containers'][grains['id']] %}
+
+{% for instance, container in containers['filebeat'].items() %}
+
+# -------------------------------------------------------------
+# Storage directory
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+/srv/filebeat/{{ instance }}:
+ file.directory:
+ - user: 9001
+ - makedirs: True
+
+{% if has_selinux %}
+selinux_context_{{ instance }}_data:
+ selinux.fcontext_policy_present:
+ - name: /srv/filebeat/{{ instance }}
+ - sel_type: container_file_t
+
+selinux_context_{{ instance }}_data_applied:
+ selinux.fcontext_policy_applied:
+ - name: /srv/filebeat/{{ instance }}
+{% endif %}
+
+# -------------------------------------------------------------
+# Configuration
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+/srv/filebeat/{{ instance }}/filebeat.yml:
+ file.managed:
+ - source: salt://roles/paas-docker/containers/files/filebeat/filebeat.yml.jinja
+ - mode: 600
+ - template: jinja
+ - context:
+ elastic: {{ container['opensearch'] }}
+ elastic_username: {{ salt['zr.get_username'](container['credentials']['opensearch']) }}
+ elastic_password: {{ salt['zr.get_password'](container['credentials']['opensearch']) }}
+
+# -------------------------------------------------------------
+# Container
+#
+# To be compatible with OpenSearch, currently it's recommended
+# to use filebeat 7.10.2.
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+{{ instance }}:
+ docker_container.running:
+ - detach: True
+ - interactive: True
+ - image: docker.elastic.co/beats/filebeat:7.10.2
+ - user: root
+ - binds:
+ - /srv/filebeat/{{ instance }}/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
+ - /var/lib/docker:/var/lib/docker:ro
+ - /var/run/docker.sock:/var/run/docker.sock
+
+{% endfor %}
diff --git a/roles/paas-docker/containers/files/filebeat/filebeat.yml.jinja b/roles/paas-docker/containers/files/filebeat/filebeat.yml.jinja
new file mode 100755
--- /dev/null
+++ b/roles/paas-docker/containers/files/filebeat/filebeat.yml.jinja
@@ -0,0 +1,40 @@
+# -------------------------------------------------------------
+# OpenSearch
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# License: Trivial work, not eligible to copyright
+# Source file: roles/paas-docker/containers/files/filebeat/filebeat.yml.jinja
+# -------------------------------------------------------------
+#
+# <auto-generated>
+# This file is managed by our rOPS SaltStack repository.
+#
+# Changes to this file may cause incorrect behavior
+# and will be lost if the state is redeployed.
+# </auto-generated>
+
+filebeat.inputs:
+- type: container
+ paths:
+ - /var/lib/docker/containers/*/*.log
+
+processors:
+- add_docker_metadata:
+ host: unix:///var/run/docker.sock
+
+- decode_json_fields:
+ fields:
+ - message
+ target: json
+ overwrite_keys: True
+
+output.elasticsearch:
+ hosts:
+ - {{ elastic['server'] }}
+ username: {{ elastic_username }}
+ password: {{ elastic_password }}
+ indices:
+ - index: {{ elastic['index'] }}
+
+logging.json: True
+logging.metrics.enabled: False

File Metadata

Mime Type
text/plain
Expires
Fri, May 2, 07:58 (16 h, 43 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
2623844
Default Alt Text
D2455.id6178.diff (6 KB)

Event Timeline

Nasqueron DevCentral · If it had been much bigger the moon would have had a core of ice. · Powered by Phabricator