Page MenuHomeDevCentral
Files F3762180

D2455.id6192.diff
No OneTemporary
Actions

D2455.id6192.diff
View Options

diff --git a/pillar/credentials/zr.sls b/pillar/credentials/zr.sls
--- a/pillar/credentials/zr.sls
+++ b/pillar/credentials/zr.sls
@@ -46,6 +46,7 @@
internal_users:
admin: 163
dashboards: 164
+ beat_docker: 165
# photos.nasqueron.org
pixelfed:
diff --git a/pillar/opensearch/clusters.sls b/pillar/opensearch/clusters.sls
--- a/pillar/opensearch/clusters.sls
+++ b/pillar/opensearch/clusters.sls
@@ -16,3 +16,6 @@
users:
admin: nasqueron.opensearch.infra-logs.internal_users.admin
dashboards: nasqueron.opensearch.infra-logs.internal_users.dashboards
+ beat_docker: nasqueron.opensearch.infra-logs.internal_users.beat_docker
+ ingest_clients_users:
+ - beat_docker
diff --git a/pillar/paas/docker.sls b/pillar/paas/docker.sls
--- a/pillar/paas/docker.sls
+++ b/pillar/paas/docker.sls
@@ -145,6 +145,37 @@
network: bugzilla
version: 5.7
+ filebeat:
+ filebeat_docker: &filebeat_docker
+ opensearch:
+ server: https://cloudhugger.nasqueron.org:9200
+ index: "docker-%{+yyyy.MM.dd}"
+ cacert: |
+ -----BEGIN CERTIFICATE-----
+ MIIDrjCCApagAwIBAgIBATANBgkqhkiG9w0BAQsFADBoMRMwEQYKCZImiZPyLGQB
+ GRYDb3JnMRkwFwYKCZImiZPyLGQBGRYJbmFzcXVlcm9uMQswCQYDVQQLDAJDQTEp
+ MCcGA1UEAwwgcm9vdC5jYS1pbmZyYS1sb2dzLm5hc3F1ZXJvbi5vcmcwHhcNMjIw
+ MTA4MjEzNTUyWhcNMjQwMTA4MjEzNTUyWjBoMRMwEQYKCZImiZPyLGQBGRYDb3Jn
+ MRkwFwYKCZImiZPyLGQBGRYJbmFzcXVlcm9uMQswCQYDVQQLDAJDQTEpMCcGA1UE
+ Awwgcm9vdC5jYS1pbmZyYS1sb2dzLm5hc3F1ZXJvbi5vcmcwggEiMA0GCSqGSIb3
+ DQEBAQUAA4IBDwAwggEKAoIBAQDEbntmlQMPdhuPbt1n7/IDp1frzf8SWPaZ5z7B
+ t97hdaderXWHS9xMttpzVmIJMXx/yhmpaDVK4UnafgeoHxV34mnzi3NfdtG+iviO
+ sP8mNuoDdl1eKSNA1J32Y1qztx93vkmuPTsH+Yrn726lEc88mda/W9QW9odYvTP9
+ 3RGIGwpmEnohq4abN2IMqdkOgr68sEy1MfvrU/4bVpAsMdJZrMJWMAR5XS11f0M6
+ CyI46vvLQuSny4+F5Ed/KOD2HotgxBpL0fiJCHEANusgfCTShKDkmfg5WtA7mxsD
+ Ylp3ipfojr8PWvUqqAZxN96rbBSf6LKtMsw+CpufHpcHy2SdAgMBAAGjYzBhMA8G
+ A1UdEwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAU5bhKpZ9/Xlubxda+AlWk4TWRogcw
+ HQYDVR0OBBYEFOW4SqWff15bm8XWvgJVpOE1kaIHMA4GA1UdDwEB/wQEAwIBhjAN
+ BgkqhkiG9w0BAQsFAAOCAQEAYm9F/HgKBbceEY2j3NHbZWVbu9YZvJK1/83HyNc/
+ yQy31VlauKazRJ7gXfmhLss1IeLB5ihe4bCCsiDz5g+kdrmQ0sOsHBQ58vzHXufZ
+ tn1pM3dpWn5cnEaf/Jfc+Pf1qSBiLlUgU651EP3e3QKZr2rD3yuf1/Ru+n6dWPWt
+ pqh1cP7Ev+UDx5LPMyy4wTV16IfbHATs9j2ROywNaz4g7Y+3FJ46stOmILa2qf+l
+ 2phfjv89sqQHbXf9a5ugS5q0lKXcWo/e3uVFIjhzrD4wuuNNiSiicqGslPRVOtDu
+ hl3DFJNWt9JztddhqmGORDsKyzyEvZ/E8hbhv18xSv2hjQ==
+ -----END CERTIFICATE-----
+ credentials:
+ opensearch: nasqueron.opensearch.infra-logs.internal_users.beat_docker
+
#
# Bugzilla
#
@@ -225,6 +256,14 @@
host: acme.nasqueron.org
nsadmin: ops.nasqueron.org
+ #
+ # Logs collection
+ # Docker logs -> filebeat -> OpenSearch
+ #
+
+ filebeat:
+ filebeat_docker: *filebeat_docker
+
#
# CI and CD
#
diff --git a/roles/opensearch/opensearch/files/internal_users.yml.jinja b/roles/opensearch/opensearch/files/internal_users.yml.jinja
--- a/roles/opensearch/opensearch/files/internal_users.yml.jinja
+++ b/roles/opensearch/opensearch/files/internal_users.yml.jinja
@@ -39,3 +39,18 @@
hash: {{ salt['opensearch.hash_password'](users['dashboards']['password']) }}
reserved: true
description: "Dashboards to OpenSearch machine user"
+
+# -------------------------------------------------------------
+# OpenSearch clients to ingest pipelines
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+{% for user in ingest_clients_users %}
+
+{{ users[user]['username'] }}:
+ hash: {{ salt['opensearch.hash_password'](users[user]['password']) }}
+ reserved: true
+ opendistro_security_roles:
+ - "ingest_client"
+ description: "Ingest client machine user"
+
+{% endfor %}
diff --git a/roles/opensearch/opensearch/files/roles.yml b/roles/opensearch/opensearch/files/roles.yml
new file mode 100644
--- /dev/null
+++ b/roles/opensearch/opensearch/files/roles.yml
@@ -0,0 +1,228 @@
+# -------------------------------------------------------------
+# OpenSearch
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# Source file: roles/opensearch/opensearch/files/roles.yml
+# -------------------------------------------------------------
+#
+# <auto-generated>
+# This file is managed by our rOPS SaltStack repository.
+#
+# Changes to this file may cause incorrect behavior
+# and will be lost if the state is redeployed.
+# </auto-generated>
+
+_meta:
+ type: "roles"
+ config_version: 2
+
+# -------------------------------------------------------------
+# Roles defined by OpenSearch default distribution
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+# Restrict users so they can only view visualization and dashboard on OpenSearchDashboards
+kibana_read_only:
+ reserved: true
+
+# The security REST API access role is used to assign specific users access to change the security settings through the REST API.
+security_rest_api_access:
+ reserved: true
+
+# Allows users to view monitors, destinations and alerts
+alerting_read_access:
+ reserved: true
+ cluster_permissions:
+ - 'cluster:admin/opendistro/alerting/alerts/get'
+ - 'cluster:admin/opendistro/alerting/destination/get'
+ - 'cluster:admin/opendistro/alerting/monitor/get'
+ - 'cluster:admin/opendistro/alerting/monitor/search'
+
+# Allows users to view and acknowledge alerts
+alerting_ack_alerts:
+ reserved: true
+ cluster_permissions:
+ - 'cluster:admin/opendistro/alerting/alerts/*'
+
+# Allows users to use all alerting functionality
+alerting_full_access:
+ reserved: true
+ cluster_permissions:
+ - 'cluster_monitor'
+ - 'cluster:admin/opendistro/alerting/*'
+ index_permissions:
+ - index_patterns:
+ - '*'
+ allowed_actions:
+ - 'indices_monitor'
+ - 'indices:admin/aliases/get'
+ - 'indices:admin/mappings/get'
+
+# Allow users to read Anomaly Detection detectors and results
+anomaly_read_access:
+ reserved: true
+ cluster_permissions:
+ - 'cluster:admin/opendistro/ad/detector/info'
+ - 'cluster:admin/opendistro/ad/detector/search'
+ - 'cluster:admin/opendistro/ad/detectors/get'
+ - 'cluster:admin/opendistro/ad/result/search'
+ - 'cluster:admin/opendistro/ad/tasks/search'
+ - 'cluster:admin/opendistro/ad/detector/validate'
+ - 'cluster:admin/opendistro/ad/result/topAnomalies'
+
+# Allows users to use all Anomaly Detection functionality
+anomaly_full_access:
+ reserved: true
+ cluster_permissions:
+ - 'cluster_monitor'
+ - 'cluster:admin/opendistro/ad/*'
+ index_permissions:
+ - index_patterns:
+ - '*'
+ allowed_actions:
+ - 'indices_monitor'
+ - 'indices:admin/aliases/get'
+ - 'indices:admin/mappings/get'
+
+# Allows users to read Notebooks
+notebooks_read_access:
+ reserved: true
+ cluster_permissions:
+ - 'cluster:admin/opendistro/notebooks/list'
+ - 'cluster:admin/opendistro/notebooks/get'
+
+# Allows users to all Notebooks functionality
+notebooks_full_access:
+ reserved: true
+ cluster_permissions:
+ - 'cluster:admin/opendistro/notebooks/create'
+ - 'cluster:admin/opendistro/notebooks/update'
+ - 'cluster:admin/opendistro/notebooks/delete'
+ - 'cluster:admin/opendistro/notebooks/get'
+ - 'cluster:admin/opendistro/notebooks/list'
+
+# Allows users to read observability objects
+observability_read_access:
+ reserved: true
+ cluster_permissions:
+ - 'cluster:admin/opensearch/observability/get'
+
+# Allows users to all Observability functionality
+observability_full_access:
+ reserved: true
+ cluster_permissions:
+ - 'cluster:admin/opensearch/observability/create'
+ - 'cluster:admin/opensearch/observability/update'
+ - 'cluster:admin/opensearch/observability/delete'
+ - 'cluster:admin/opensearch/observability/get'
+
+# Allows users to read and download Reports
+reports_instances_read_access:
+ reserved: true
+ cluster_permissions:
+ - 'cluster:admin/opendistro/reports/instance/list'
+ - 'cluster:admin/opendistro/reports/instance/get'
+ - 'cluster:admin/opendistro/reports/menu/download'
+
+# Allows users to read and download Reports and Report-definitions
+reports_read_access:
+ reserved: true
+ cluster_permissions:
+ - 'cluster:admin/opendistro/reports/definition/get'
+ - 'cluster:admin/opendistro/reports/definition/list'
+ - 'cluster:admin/opendistro/reports/instance/list'
+ - 'cluster:admin/opendistro/reports/instance/get'
+ - 'cluster:admin/opendistro/reports/menu/download'
+
+# Allows users to all Reports functionality
+reports_full_access:
+ reserved: true
+ cluster_permissions:
+ - 'cluster:admin/opendistro/reports/definition/create'
+ - 'cluster:admin/opendistro/reports/definition/update'
+ - 'cluster:admin/opendistro/reports/definition/on_demand'
+ - 'cluster:admin/opendistro/reports/definition/delete'
+ - 'cluster:admin/opendistro/reports/definition/get'
+ - 'cluster:admin/opendistro/reports/definition/list'
+ - 'cluster:admin/opendistro/reports/instance/list'
+ - 'cluster:admin/opendistro/reports/instance/get'
+ - 'cluster:admin/opendistro/reports/menu/download'
+
+# Allows users to use all asynchronous-search functionality
+asynchronous_search_full_access:
+ reserved: true
+ cluster_permissions:
+ - 'cluster:admin/opendistro/asynchronous_search/*'
+ index_permissions:
+ - index_patterns:
+ - '*'
+ allowed_actions:
+ - 'indices:data/read/search*'
+
+# Allows users to read stored asynchronous-search results
+asynchronous_search_read_access:
+ reserved: true
+ cluster_permissions:
+ - 'cluster:admin/opendistro/asynchronous_search/get'
+
+# Allows user to use all index_management actions - ism policies, rollups, transforms
+index_management_full_access:
+ reserved: true
+ cluster_permissions:
+ - "cluster:admin/opendistro/ism/*"
+ - "cluster:admin/opendistro/rollup/*"
+ - "cluster:admin/opendistro/transform/*"
+ index_permissions:
+ - index_patterns:
+ - '*'
+ allowed_actions:
+ - 'indices:admin/opensearch/ism/*'
+
+# Allows users to use all cross cluster replication functionality at leader cluster
+cross_cluster_replication_leader_full_access:
+ reserved: true
+ index_permissions:
+ - index_patterns:
+ - '*'
+ allowed_actions:
+ - "indices:admin/plugins/replication/index/setup/validate"
+ - "indices:data/read/plugins/replication/changes"
+ - "indices:data/read/plugins/replication/file_chunk"
+
+# Allows users to use all cross cluster replication functionality at follower cluster
+cross_cluster_replication_follower_full_access:
+ reserved: true
+ cluster_permissions:
+ - "cluster:admin/plugins/replication/autofollow/update"
+ index_permissions:
+ - index_patterns:
+ - '*'
+ allowed_actions:
+ - "indices:admin/plugins/replication/index/setup/validate"
+ - "indices:data/write/plugins/replication/changes"
+ - "indices:admin/plugins/replication/index/start"
+ - "indices:admin/plugins/replication/index/pause"
+ - "indices:admin/plugins/replication/index/resume"
+ - "indices:admin/plugins/replication/index/stop"
+ - "indices:admin/plugins/replication/index/update"
+ - "indices:admin/plugins/replication/index/status_check"
+
+# -------------------------------------------------------------
+# Custom roles
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+ingest_client:
+ reserved: True
+ description: Access for ingestion client to all indices
+ cluster_permissions:
+ - "cluster:monitor/*"
+ - "cluster_composite_ops"
+ - "indices:admin/template/get"
+ - "indices:admin/template/put"
+ - "cluster:admin/ingest/pipeline/put"
+ - "cluster:admin/ingest/pipeline/get"
+ index_permissions:
+ - index_patterns:
+ - '*'
+ allowed_actions:
+ - "crud"
+ - "create_index"
diff --git a/roles/opensearch/opensearch/files/security_initialize.sh b/roles/opensearch/opensearch/files/security_initialize.sh
--- a/roles/opensearch/opensearch/files/security_initialize.sh
+++ b/roles/opensearch/opensearch/files/security_initialize.sh
@@ -12,7 +12,7 @@
-cacert $ROOT/config/root-ca.pem \
-cert $ROOT/config/admin.pem \
-key $ROOT/config/admin.key \
- -f $ROOT/plugins/opensearch-security/securityconfig/internal_users.yml \
+ -cd $ROOT/plugins/opensearch-security/securityconfig \
-nhnv -icl \
-h "$OPENSEARCH_HOSTNAME"
diff --git a/roles/opensearch/opensearch/security.sls b/roles/opensearch/opensearch/security.sls
--- a/roles/opensearch/opensearch/security.sls
+++ b/roles/opensearch/opensearch/security.sls
@@ -19,6 +19,7 @@
- mode: 0600
- template: jinja
- context:
+ ingest_clients_users: {{ config['ingest_clients_users'] }}
users:
{% for user, credential in config['users'].items() %}
{{ user }}:
@@ -26,6 +27,16 @@
password: {{ salt['zr.get_password'](credential) }}
{% endfor %}
+/opt/opensearch/plugins/opensearch-security/securityconfig/roles.yml:
+ file.managed:
+ - source: salt://roles/opensearch/opensearch/files/roles.yml
+ - user: opensearch
+ - group: opensearch
+ - mode: 0600
+
+/opt/opensearch/plugins/opensearch-security/securityconfig/opensearch.yml.example:
+ file.absent
+
opensearch_security_initialize:
cmd.script:
- source: salt://roles/opensearch/opensearch/files/security_initialize.sh
diff --git a/roles/paas-docker/containers/filebeat.sls b/roles/paas-docker/containers/filebeat.sls
new file mode 100644
--- /dev/null
+++ b/roles/paas-docker/containers/filebeat.sls
@@ -0,0 +1,82 @@
+# -------------------------------------------------------------
+# Salt — Provision Docker engine
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# License: Trivial work, not eligible to copyright
+# -------------------------------------------------------------
+
+{% set has_selinux = salt['grains.get']('selinux:enabled', False) %}
+{% set containers = pillar['docker_containers'][grains['id']] %}
+
+{% for instance, container in containers['filebeat'].items() %}
+
+# -------------------------------------------------------------
+# Storage directory
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+/srv/filebeat/{{ instance }}:
+ file.directory:
+ - user: 9001
+ - makedirs: True
+
+{% if has_selinux %}
+selinux_context_{{ instance }}_data:
+ selinux.fcontext_policy_present:
+ - name: /srv/filebeat/{{ instance }}
+ - sel_type: container_file_t
+
+selinux_context_{{ instance }}_data_applied:
+ selinux.fcontext_policy_applied:
+ - name: /srv/filebeat/{{ instance }}
+{% endif %}
+
+# -------------------------------------------------------------
+# Configuration
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+{% set provide_cacert = 'cacert' in container['opensearch'] %}
+
+/srv/filebeat/{{ instance }}/filebeat.yml:
+ file.managed:
+ - source: salt://roles/paas-docker/containers/files/filebeat/filebeat.yml.jinja
+ - mode: 0600
+ - template: jinja
+ - context:
+ host: {{ grains['id'] }}
+ elastic: {{ container['opensearch'] }}
+ elastic_username: {{ salt['zr.get_username'](container['credentials']['opensearch']) }}
+ elastic_password: {{ salt['zr.get_password'](container['credentials']['opensearch']) }}
+ provide_cacert: {{ provide_cacert }}
+
+{% if provide_cacert %}
+/srv/filebeat/{{ instance }}/cacert.pem:
+ file.managed:
+ - contents: |
+{% filter indent(width=8) %}
+{{ container['opensearch']['cacert'] }}
+{% endfilter %}
+ - mode: 0644
+{% endif %}
+
+# -------------------------------------------------------------
+# Container
+#
+# To be compatible with OpenSearch, currently it's recommended
+# to use filebeat 7.10.2.
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+{{ instance }}:
+ docker_container.running:
+ - detach: True
+ - interactive: True
+ - image: docker.elastic.co/beats/filebeat-oss:7.10.2
+ - user: root
+ - binds:
+ - /srv/filebeat/{{ instance }}/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
+{% if provide_cacert %}
+ - /srv/filebeat/{{ instance }}/cacert.pem:/usr/share/filebeat/cacert.pem:ro
+{% endif %}
+ - /var/lib/docker:/var/lib/docker:ro
+ - /var/run/docker.sock:/var/run/docker.sock
+
+{% endfor %}
diff --git a/roles/paas-docker/containers/files/filebeat/filebeat.yml.jinja b/roles/paas-docker/containers/files/filebeat/filebeat.yml.jinja
new file mode 100644
--- /dev/null
+++ b/roles/paas-docker/containers/files/filebeat/filebeat.yml.jinja
@@ -0,0 +1,51 @@
+# -------------------------------------------------------------
+# OpenSearch
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# License: Trivial work, not eligible to copyright
+# Source file: roles/paas-docker/containers/files/filebeat/filebeat.yml.jinja
+# -------------------------------------------------------------
+#
+# <auto-generated>
+# This file is managed by our rOPS SaltStack repository.
+#
+# Changes to this file may cause incorrect behavior
+# and will be lost if the state is redeployed.
+# </auto-generated>
+
+filebeat.inputs:
+- type: container
+ paths:
+ - /var/lib/docker/containers/*/*.log
+
+processors:
+- add_fields:
+ target: ""
+ fields:
+ host: {{ host }}
+ forest: {{ forest }}
+
+- add_docker_metadata:
+ host: unix:///var/run/docker.sock
+
+- decode_json_fields:
+ fields:
+ - message
+ target: json
+ overwrite_keys: True
+
+output.elasticsearch:
+ hosts:
+ - {{ elastic['server'] }}
+{% if provide_cacert %}
+ ssl:
+ certificate_authorities:
+ - /usr/share/filebeat/cacert.pem
+{% endif %}
+ username: {{ elastic_username }}
+ password: {{ elastic_password }}
+ indices:
+ - index: {{ elastic['index'] }}
+
+logging.json: True
+logging.metrics.enabled: False

File Metadata

Mime Type
text/plain
Expires
Fri, Nov 22, 07:55 (3 h, 8 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
2255882
Default Alt Text
D2455.id6192.diff (18 KB)

Event Timeline

Nasqueron DevCentral · If it had been much bigger the moon would have had a core of ice. · Powered by Phabricator