diff --git a/pillar/credentials/zr.sls b/pillar/credentials/zr.sls --- a/pillar/credentials/zr.sls +++ b/pillar/credentials/zr.sls @@ -46,6 +46,7 @@ internal_users: admin: 163 dashboards: 164 + beat_docker: 165 # photos.nasqueron.org pixelfed: diff --git a/pillar/opensearch/clusters.sls b/pillar/opensearch/clusters.sls --- a/pillar/opensearch/clusters.sls +++ b/pillar/opensearch/clusters.sls @@ -11,9 +11,19 @@ infra_logs: cluster_name: infra-logs cluster_type: single-node + nodes: - cloudhugger + + heap_size: 26G + users: admin: nasqueron.opensearch.infra-logs.internal_users.admin dashboards: nasqueron.opensearch.infra-logs.internal_users.dashboards - heap_size: 26G + beat_docker: nasqueron.opensearch.infra-logs.internal_users.beat_docker + ingest_clients_users: + - beat_docker + + index_management: + index_policies: + - prune_old_indices diff --git a/pillar/paas/docker.sls b/pillar/paas/docker.sls --- a/pillar/paas/docker.sls +++ b/pillar/paas/docker.sls @@ -145,6 +145,38 @@ network: bugzilla version: 5.7 + filebeat: + filebeat_docker: &filebeat_docker + retention: 60d + opensearch: + server: https://cloudhugger.nasqueron.org:9200 + index: "docker-%{+yyyy.MM.dd}" + cacert: | + -----BEGIN CERTIFICATE----- + MIIDrjCCApagAwIBAgIBATANBgkqhkiG9w0BAQsFADBoMRMwEQYKCZImiZPyLGQB + GRYDb3JnMRkwFwYKCZImiZPyLGQBGRYJbmFzcXVlcm9uMQswCQYDVQQLDAJDQTEp + MCcGA1UEAwwgcm9vdC5jYS1pbmZyYS1sb2dzLm5hc3F1ZXJvbi5vcmcwHhcNMjIw + MTA4MjEzNTUyWhcNMjQwMTA4MjEzNTUyWjBoMRMwEQYKCZImiZPyLGQBGRYDb3Jn + MRkwFwYKCZImiZPyLGQBGRYJbmFzcXVlcm9uMQswCQYDVQQLDAJDQTEpMCcGA1UE + Awwgcm9vdC5jYS1pbmZyYS1sb2dzLm5hc3F1ZXJvbi5vcmcwggEiMA0GCSqGSIb3 + DQEBAQUAA4IBDwAwggEKAoIBAQDEbntmlQMPdhuPbt1n7/IDp1frzf8SWPaZ5z7B + t97hdaderXWHS9xMttpzVmIJMXx/yhmpaDVK4UnafgeoHxV34mnzi3NfdtG+iviO + sP8mNuoDdl1eKSNA1J32Y1qztx93vkmuPTsH+Yrn726lEc88mda/W9QW9odYvTP9 + 3RGIGwpmEnohq4abN2IMqdkOgr68sEy1MfvrU/4bVpAsMdJZrMJWMAR5XS11f0M6 + CyI46vvLQuSny4+F5Ed/KOD2HotgxBpL0fiJCHEANusgfCTShKDkmfg5WtA7mxsD + Ylp3ipfojr8PWvUqqAZxN96rbBSf6LKtMsw+CpufHpcHy2SdAgMBAAGjYzBhMA8G + A1UdEwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAU5bhKpZ9/Xlubxda+AlWk4TWRogcw + HQYDVR0OBBYEFOW4SqWff15bm8XWvgJVpOE1kaIHMA4GA1UdDwEB/wQEAwIBhjAN + BgkqhkiG9w0BAQsFAAOCAQEAYm9F/HgKBbceEY2j3NHbZWVbu9YZvJK1/83HyNc/ + yQy31VlauKazRJ7gXfmhLss1IeLB5ihe4bCCsiDz5g+kdrmQ0sOsHBQ58vzHXufZ + tn1pM3dpWn5cnEaf/Jfc+Pf1qSBiLlUgU651EP3e3QKZr2rD3yuf1/Ru+n6dWPWt + pqh1cP7Ev+UDx5LPMyy4wTV16IfbHATs9j2ROywNaz4g7Y+3FJ46stOmILa2qf+l + 2phfjv89sqQHbXf9a5ugS5q0lKXcWo/e3uVFIjhzrD4wuuNNiSiicqGslPRVOtDu + hl3DFJNWt9JztddhqmGORDsKyzyEvZ/E8hbhv18xSv2hjQ== + -----END CERTIFICATE----- + credentials: + opensearch: nasqueron.opensearch.infra-logs.internal_users.beat_docker + # # Bugzilla # @@ -225,6 +257,14 @@ host: acme.nasqueron.org nsadmin: ops.nasqueron.org + # + # Logs collection + # Docker logs -> filebeat -> OpenSearch + # + + filebeat: + filebeat_docker: *filebeat_docker + # # CI and CD # diff --git a/roles/opensearch/opensearch/files/index_policies/prune_old_indices.json b/roles/opensearch/opensearch/files/index_policies/prune_old_indices.json new file mode 100755 --- /dev/null +++ b/roles/opensearch/opensearch/files/index_policies/prune_old_indices.json @@ -0,0 +1,58 @@ +{ + "policy": { + "policy_id": "prune_old_indices", + "description": "Prune indices after 60 days.", + "schema_version": 12, + "error_notification": { + "destination": { + "custom_webhook": { + "path": null, + "header_params": {}, + "password": null, + "scheme": null, + "port": -1, + "query_params": {}, + "host": null, + "url": "https://notifications.nasqueron.org/gate/notification", + "username": null + } + }, + "message_template": { + "source": "{ \"service\": \"OpenSearch\", \"project\": \"Nasqueron\", \"group\": \"ops\", \"type\": \"index-management-error\", \"text\": \"[infra-logs] The index {{ctx.index}} failed during policy execution.\"}", + "lang": "mustache" + } + }, + "default_state": "Hot", + "states": [ + { + "name": "Hot", + "actions": [], + "transitions": [ + { + "state_name": "Deleted", + "conditions": { + "min_index_age": "60d" + } + } + ] + }, + { + "name": "Deleted", + "actions": [ + { + "delete": {} + } + ], + "transitions": [] + } + ], + "ism_template": [ + { + "index_patterns": [ + "docker-*" + ], + "priority": 1 + } + ] + } +} diff --git a/roles/opensearch/opensearch/files/internal_users.yml.jinja b/roles/opensearch/opensearch/files/internal_users.yml.jinja --- a/roles/opensearch/opensearch/files/internal_users.yml.jinja +++ b/roles/opensearch/opensearch/files/internal_users.yml.jinja @@ -39,3 +39,18 @@ hash: {{ salt['opensearch.hash_password'](users['dashboards']['password']) }} reserved: true description: "Dashboards to OpenSearch machine user" + +# ------------------------------------------------------------- +# OpenSearch clients to ingest pipelines +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +{% for user in ingest_clients_users %} + +{{ users[user]['username'] }}: + hash: {{ salt['opensearch.hash_password'](users[user]['password']) }} + reserved: true + opendistro_security_roles: + - "ingest_client" + description: "Ingest client machine user" + +{% endfor %} diff --git a/roles/opensearch/opensearch/files/roles.yml b/roles/opensearch/opensearch/files/roles.yml new file mode 100644 --- /dev/null +++ b/roles/opensearch/opensearch/files/roles.yml @@ -0,0 +1,228 @@ +# ------------------------------------------------------------- +# OpenSearch +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Project: Nasqueron +# Source file: roles/opensearch/opensearch/files/roles.yml +# ------------------------------------------------------------- +# +# +# This file is managed by our rOPS SaltStack repository. +# +# Changes to this file may cause incorrect behavior +# and will be lost if the state is redeployed. +# + +_meta: + type: "roles" + config_version: 2 + +# ------------------------------------------------------------- +# Roles defined by OpenSearch default distribution +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +# Restrict users so they can only view visualization and dashboard on OpenSearchDashboards +kibana_read_only: + reserved: true + +# The security REST API access role is used to assign specific users access to change the security settings through the REST API. +security_rest_api_access: + reserved: true + +# Allows users to view monitors, destinations and alerts +alerting_read_access: + reserved: true + cluster_permissions: + - 'cluster:admin/opendistro/alerting/alerts/get' + - 'cluster:admin/opendistro/alerting/destination/get' + - 'cluster:admin/opendistro/alerting/monitor/get' + - 'cluster:admin/opendistro/alerting/monitor/search' + +# Allows users to view and acknowledge alerts +alerting_ack_alerts: + reserved: true + cluster_permissions: + - 'cluster:admin/opendistro/alerting/alerts/*' + +# Allows users to use all alerting functionality +alerting_full_access: + reserved: true + cluster_permissions: + - 'cluster_monitor' + - 'cluster:admin/opendistro/alerting/*' + index_permissions: + - index_patterns: + - '*' + allowed_actions: + - 'indices_monitor' + - 'indices:admin/aliases/get' + - 'indices:admin/mappings/get' + +# Allow users to read Anomaly Detection detectors and results +anomaly_read_access: + reserved: true + cluster_permissions: + - 'cluster:admin/opendistro/ad/detector/info' + - 'cluster:admin/opendistro/ad/detector/search' + - 'cluster:admin/opendistro/ad/detectors/get' + - 'cluster:admin/opendistro/ad/result/search' + - 'cluster:admin/opendistro/ad/tasks/search' + - 'cluster:admin/opendistro/ad/detector/validate' + - 'cluster:admin/opendistro/ad/result/topAnomalies' + +# Allows users to use all Anomaly Detection functionality +anomaly_full_access: + reserved: true + cluster_permissions: + - 'cluster_monitor' + - 'cluster:admin/opendistro/ad/*' + index_permissions: + - index_patterns: + - '*' + allowed_actions: + - 'indices_monitor' + - 'indices:admin/aliases/get' + - 'indices:admin/mappings/get' + +# Allows users to read Notebooks +notebooks_read_access: + reserved: true + cluster_permissions: + - 'cluster:admin/opendistro/notebooks/list' + - 'cluster:admin/opendistro/notebooks/get' + +# Allows users to all Notebooks functionality +notebooks_full_access: + reserved: true + cluster_permissions: + - 'cluster:admin/opendistro/notebooks/create' + - 'cluster:admin/opendistro/notebooks/update' + - 'cluster:admin/opendistro/notebooks/delete' + - 'cluster:admin/opendistro/notebooks/get' + - 'cluster:admin/opendistro/notebooks/list' + +# Allows users to read observability objects +observability_read_access: + reserved: true + cluster_permissions: + - 'cluster:admin/opensearch/observability/get' + +# Allows users to all Observability functionality +observability_full_access: + reserved: true + cluster_permissions: + - 'cluster:admin/opensearch/observability/create' + - 'cluster:admin/opensearch/observability/update' + - 'cluster:admin/opensearch/observability/delete' + - 'cluster:admin/opensearch/observability/get' + +# Allows users to read and download Reports +reports_instances_read_access: + reserved: true + cluster_permissions: + - 'cluster:admin/opendistro/reports/instance/list' + - 'cluster:admin/opendistro/reports/instance/get' + - 'cluster:admin/opendistro/reports/menu/download' + +# Allows users to read and download Reports and Report-definitions +reports_read_access: + reserved: true + cluster_permissions: + - 'cluster:admin/opendistro/reports/definition/get' + - 'cluster:admin/opendistro/reports/definition/list' + - 'cluster:admin/opendistro/reports/instance/list' + - 'cluster:admin/opendistro/reports/instance/get' + - 'cluster:admin/opendistro/reports/menu/download' + +# Allows users to all Reports functionality +reports_full_access: + reserved: true + cluster_permissions: + - 'cluster:admin/opendistro/reports/definition/create' + - 'cluster:admin/opendistro/reports/definition/update' + - 'cluster:admin/opendistro/reports/definition/on_demand' + - 'cluster:admin/opendistro/reports/definition/delete' + - 'cluster:admin/opendistro/reports/definition/get' + - 'cluster:admin/opendistro/reports/definition/list' + - 'cluster:admin/opendistro/reports/instance/list' + - 'cluster:admin/opendistro/reports/instance/get' + - 'cluster:admin/opendistro/reports/menu/download' + +# Allows users to use all asynchronous-search functionality +asynchronous_search_full_access: + reserved: true + cluster_permissions: + - 'cluster:admin/opendistro/asynchronous_search/*' + index_permissions: + - index_patterns: + - '*' + allowed_actions: + - 'indices:data/read/search*' + +# Allows users to read stored asynchronous-search results +asynchronous_search_read_access: + reserved: true + cluster_permissions: + - 'cluster:admin/opendistro/asynchronous_search/get' + +# Allows user to use all index_management actions - ism policies, rollups, transforms +index_management_full_access: + reserved: true + cluster_permissions: + - "cluster:admin/opendistro/ism/*" + - "cluster:admin/opendistro/rollup/*" + - "cluster:admin/opendistro/transform/*" + index_permissions: + - index_patterns: + - '*' + allowed_actions: + - 'indices:admin/opensearch/ism/*' + +# Allows users to use all cross cluster replication functionality at leader cluster +cross_cluster_replication_leader_full_access: + reserved: true + index_permissions: + - index_patterns: + - '*' + allowed_actions: + - "indices:admin/plugins/replication/index/setup/validate" + - "indices:data/read/plugins/replication/changes" + - "indices:data/read/plugins/replication/file_chunk" + +# Allows users to use all cross cluster replication functionality at follower cluster +cross_cluster_replication_follower_full_access: + reserved: true + cluster_permissions: + - "cluster:admin/plugins/replication/autofollow/update" + index_permissions: + - index_patterns: + - '*' + allowed_actions: + - "indices:admin/plugins/replication/index/setup/validate" + - "indices:data/write/plugins/replication/changes" + - "indices:admin/plugins/replication/index/start" + - "indices:admin/plugins/replication/index/pause" + - "indices:admin/plugins/replication/index/resume" + - "indices:admin/plugins/replication/index/stop" + - "indices:admin/plugins/replication/index/update" + - "indices:admin/plugins/replication/index/status_check" + +# ------------------------------------------------------------- +# Custom roles +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +ingest_client: + reserved: True + description: Access for ingestion client to all indices + cluster_permissions: + - "cluster:monitor/*" + - "cluster_composite_ops" + - "indices:admin/template/get" + - "indices:admin/template/put" + - "cluster:admin/ingest/pipeline/put" + - "cluster:admin/ingest/pipeline/get" + index_permissions: + - index_patterns: + - '*' + allowed_actions: + - "crud" + - "create_index" diff --git a/roles/opensearch/opensearch/files/security_initialize.sh b/roles/opensearch/opensearch/files/security_initialize.sh --- a/roles/opensearch/opensearch/files/security_initialize.sh +++ b/roles/opensearch/opensearch/files/security_initialize.sh @@ -12,7 +12,7 @@ -cacert $ROOT/config/root-ca.pem \ -cert $ROOT/config/admin.pem \ -key $ROOT/config/admin.key \ - -f $ROOT/plugins/opensearch-security/securityconfig/internal_users.yml \ + -cd $ROOT/plugins/opensearch-security/securityconfig \ -nhnv -icl \ -h "$OPENSEARCH_HOSTNAME" diff --git a/roles/opensearch/opensearch/indices.sls b/roles/opensearch/opensearch/indices.sls new file mode 100755 --- /dev/null +++ b/roles/opensearch/opensearch/indices.sls @@ -0,0 +1,27 @@ +# ------------------------------------------------------------- +# Salt — Provision OpenSearch +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Project: Nasqueron +# License: Trivial work, not eligible to copyright +# ------------------------------------------------------------- + +{% set config = salt['opensearch.get_config']() %} +{% set root = "/opt/opensearch/config/index_policies" %} + +{% for policy in config['index_management']['index_policies'] %} + +{{ root }}/{{ policy }}.json: + file.managed: + - source: salt://roles/opensearch/files/index_policies/{{ policy }}.json + - user: opensearch + - group: opensearch + - mode: 0644 + +apply_index_policy_{{ policy }}: + cmd.run: + - name: | + es-query _plugins/_ism/policies/{{ policy }} -XPUT -d @{{ root }}/{{ policy }}.json + touch {{ root }}/.policy-applied-{{ policy }} + - creates: {{ root }}/.policy-applied-{{ policy }} + +{% endfor %} diff --git a/roles/opensearch/opensearch/init.sls b/roles/opensearch/opensearch/init.sls --- a/roles/opensearch/opensearch/init.sls +++ b/roles/opensearch/opensearch/init.sls @@ -12,3 +12,4 @@ - .service - .security - .wrapper + - .indices diff --git a/roles/opensearch/opensearch/security.sls b/roles/opensearch/opensearch/security.sls --- a/roles/opensearch/opensearch/security.sls +++ b/roles/opensearch/opensearch/security.sls @@ -19,6 +19,7 @@ - mode: 0600 - template: jinja - context: + ingest_clients_users: {{ config['ingest_clients_users'] }} users: {% for user, credential in config['users'].items() %} {{ user }}: @@ -26,6 +27,16 @@ password: {{ salt['zr.get_password'](credential) }} {% endfor %} +/opt/opensearch/plugins/opensearch-security/securityconfig/roles.yml: + file.managed: + - source: salt://roles/opensearch/opensearch/files/roles.yml + - user: opensearch + - group: opensearch + - mode: 0600 + +/opt/opensearch/plugins/opensearch-security/securityconfig/opensearch.yml.example: + file.absent + opensearch_security_initialize: cmd.script: - source: salt://roles/opensearch/opensearch/files/security_initialize.sh diff --git a/roles/paas-docker/containers/filebeat.sls b/roles/paas-docker/containers/filebeat.sls new file mode 100644 --- /dev/null +++ b/roles/paas-docker/containers/filebeat.sls @@ -0,0 +1,86 @@ +# ------------------------------------------------------------- +# Salt — Provision Docker engine +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Project: Nasqueron +# License: Trivial work, not eligible to copyright +# ------------------------------------------------------------- + +{% set has_selinux = salt['grains.get']('selinux:enabled', False) %} +{% set containers = pillar['docker_containers'][grains['id']] %} + +{% for instance, container in containers['filebeat'].items() %} + +# ------------------------------------------------------------- +# Storage directory +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +/srv/filebeat/{{ instance }}: + file.directory: + - user: 9001 + - makedirs: True + +{% if has_selinux %} +selinux_context_{{ instance }}_data: + selinux.fcontext_policy_present: + - name: /srv/filebeat/{{ instance }} + - sel_type: container_file_t + +selinux_context_{{ instance }}_data_applied: + selinux.fcontext_policy_applied: + - name: /srv/filebeat/{{ instance }} +{% endif %} + +# ------------------------------------------------------------- +# Configuration +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +{% set with_cacert = 'cacert' in container['opensearch'] %} + +/srv/filebeat/{{ instance }}/filebeat.yml: + file.managed: + - source: salt://roles/paas-docker/containers/files/filebeat/filebeat.yml.jinja + - mode: 0600 + - template: jinja + - context: + host: {{ grains['id'] }} + forest: {{ salt['forest.get']() }} + + retention: {{ container['retention'] }} + + elastic: {{ container['opensearch'] }} + elastic_username: {{ salt['zr.get_username'](container['credentials']['opensearch']) }} + elastic_password: {{ salt['zr.get_password'](container['credentials']['opensearch']) }} + with_cacert: {{ with_cacert }} + +{% if with_cacert %} +/srv/filebeat/{{ instance }}/cacert.pem: + file.managed: + - contents: | +{% filter indent(width=8) %} +{{ container['opensearch']['cacert'] }} +{% endfilter %} + - mode: 0644 +{% endif %} + +# ------------------------------------------------------------- +# Container +# +# To be compatible with OpenSearch, currently it's recommended +# to use filebeat 7.10.2. +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +{{ instance }}: + docker_container.running: + - detach: True + - interactive: True + - image: docker.elastic.co/beats/filebeat-oss:7.10.2 + - user: root + - binds: + - /srv/filebeat/{{ instance }}/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro +{% if with_cacert %} + - /srv/filebeat/{{ instance }}/cacert.pem:/usr/share/filebeat/cacert.pem:ro +{% endif %} + - /var/lib/docker:/var/lib/docker:ro + - /var/run/docker.sock:/var/run/docker.sock + +{% endfor %} diff --git a/roles/paas-docker/containers/files/filebeat/filebeat.yml.jinja b/roles/paas-docker/containers/files/filebeat/filebeat.yml.jinja new file mode 100644 --- /dev/null +++ b/roles/paas-docker/containers/files/filebeat/filebeat.yml.jinja @@ -0,0 +1,52 @@ +# ------------------------------------------------------------- +# OpenSearch +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Project: Nasqueron +# License: Trivial work, not eligible to copyright +# Source file: roles/paas-docker/containers/files/filebeat/filebeat.yml.jinja +# ------------------------------------------------------------- +# +# +# This file is managed by our rOPS SaltStack repository. +# +# Changes to this file may cause incorrect behavior +# and will be lost if the state is redeployed. +# + +filebeat.inputs: +- type: container + paths: + - /var/lib/docker/containers/*/*.log + ignore_older: {{ retention }} + +processors: +- add_fields: + target: "" + fields: + host: {{ host }} + forest: {{ forest }} + +- add_docker_metadata: + host: unix:///var/run/docker.sock + +- decode_json_fields: + fields: + - message + target: json + overwrite_keys: True + +output.elasticsearch: + hosts: + - {{ elastic['server'] }} +{% if provide_cacert %} + ssl: + certificate_authorities: + - /usr/share/filebeat/cacert.pem +{% endif %} + username: {{ elastic_username }} + password: {{ elastic_password }} + indices: + - index: {{ elastic['index'] }} + +logging.json: True +logging.metrics.enabled: False