Page Menu
Home
DevCentral
Search
Configure Global Search
Log In
Files
F3929517
D2455.id6181.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
15 KB
Referenced Files
None
Subscribers
None
D2455.id6181.diff
View Options
diff --git a/pillar/credentials/zr.sls b/pillar/credentials/zr.sls
--- a/pillar/credentials/zr.sls
+++ b/pillar/credentials/zr.sls
@@ -46,6 +46,7 @@
internal_users:
admin: 163
dashboards: 164
+ beat_docker: 165
# photos.nasqueron.org
pixelfed:
diff --git a/pillar/opensearch/clusters.sls b/pillar/opensearch/clusters.sls
--- a/pillar/opensearch/clusters.sls
+++ b/pillar/opensearch/clusters.sls
@@ -16,3 +16,6 @@
users:
admin: nasqueron.opensearch.infra-logs.internal_users.admin
dashboards: nasqueron.opensearch.infra-logs.internal_users.dashboards
+ beat_docker: nasqueron.opensearch.infra-logs.internal_users.beat_docker
+ ingest_clients_users:
+ - beat_docker
diff --git a/pillar/paas/docker.sls b/pillar/paas/docker.sls
--- a/pillar/paas/docker.sls
+++ b/pillar/paas/docker.sls
@@ -145,6 +145,9 @@
network: bugzilla
version: 5.7
+ filebeat:
+ filebeat_docker: *filebeat_docker
+
#
# Bugzilla
#
@@ -225,6 +228,19 @@
host: acme.nasqueron.org
nsadmin: ops.nasqueron.org
+ #
+ # Logs collection
+ # Docker logs -> filebeat -> OpenSearch
+ #
+
+ filebeat:
+ filebeat_docker: &filebeat_docker
+ opensearch:
+ server: cloudhugger.nasqueron.org
+ index: "docker-%{+yyyy.MM.dd}"
+ credentials:
+ opensearch: nasqueron.opensearch.infra-logs.internal_users.beat_docker
+
#
# CI and CD
#
diff --git a/roles/opensearch/opensearch/files/internal_users.yml.jinja b/roles/opensearch/opensearch/files/internal_users.yml.jinja
--- a/roles/opensearch/opensearch/files/internal_users.yml.jinja
+++ b/roles/opensearch/opensearch/files/internal_users.yml.jinja
@@ -39,3 +39,18 @@
hash: {{ salt['opensearch.hash_password'](users['dashboards']['password']) }}
reserved: true
description: "Dashboards to OpenSearch machine user"
+
+# -------------------------------------------------------------
+# OpenSearch clients to ingest pipelines
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+{% for user in ingest_clients_users %}
+
+{{ users[user]['username'] }}:
+ hash: {{ salt['opensearch.hash_password'](users[user]['password']) }}
+ reserved: true
+ backend_roles:
+ - "ingest_client"
+ description: "Ingest client machine user"
+
+{% endfor %}
diff --git a/roles/opensearch/opensearch/files/roles.yml b/roles/opensearch/opensearch/files/roles.yml
new file mode 100644
--- /dev/null
+++ b/roles/opensearch/opensearch/files/roles.yml
@@ -0,0 +1,228 @@
+# -------------------------------------------------------------
+# OpenSearch
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# Source file: roles/opensearch/opensearch/files/roles.yml
+# -------------------------------------------------------------
+#
+# <auto-generated>
+# This file is managed by our rOPS SaltStack repository.
+#
+# Changes to this file may cause incorrect behavior
+# and will be lost if the state is redeployed.
+# </auto-generated>
+
+_meta:
+ type: "roles"
+ config_version: 2
+
+# -------------------------------------------------------------
+# Roles defined by OpenSearch default distribution
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+# Restrict users so they can only view visualization and dashboard on OpenSearchDashboards
+kibana_read_only:
+ reserved: true
+
+# The security REST API access role is used to assign specific users access to change the security settings through the REST API.
+security_rest_api_access:
+ reserved: true
+
+# Allows users to view monitors, destinations and alerts
+alerting_read_access:
+ reserved: true
+ cluster_permissions:
+ - 'cluster:admin/opendistro/alerting/alerts/get'
+ - 'cluster:admin/opendistro/alerting/destination/get'
+ - 'cluster:admin/opendistro/alerting/monitor/get'
+ - 'cluster:admin/opendistro/alerting/monitor/search'
+
+# Allows users to view and acknowledge alerts
+alerting_ack_alerts:
+ reserved: true
+ cluster_permissions:
+ - 'cluster:admin/opendistro/alerting/alerts/*'
+
+# Allows users to use all alerting functionality
+alerting_full_access:
+ reserved: true
+ cluster_permissions:
+ - 'cluster_monitor'
+ - 'cluster:admin/opendistro/alerting/*'
+ index_permissions:
+ - index_patterns:
+ - '*'
+ allowed_actions:
+ - 'indices_monitor'
+ - 'indices:admin/aliases/get'
+ - 'indices:admin/mappings/get'
+
+# Allow users to read Anomaly Detection detectors and results
+anomaly_read_access:
+ reserved: true
+ cluster_permissions:
+ - 'cluster:admin/opendistro/ad/detector/info'
+ - 'cluster:admin/opendistro/ad/detector/search'
+ - 'cluster:admin/opendistro/ad/detectors/get'
+ - 'cluster:admin/opendistro/ad/result/search'
+ - 'cluster:admin/opendistro/ad/tasks/search'
+ - 'cluster:admin/opendistro/ad/detector/validate'
+ - 'cluster:admin/opendistro/ad/result/topAnomalies'
+
+# Allows users to use all Anomaly Detection functionality
+anomaly_full_access:
+ reserved: true
+ cluster_permissions:
+ - 'cluster_monitor'
+ - 'cluster:admin/opendistro/ad/*'
+ index_permissions:
+ - index_patterns:
+ - '*'
+ allowed_actions:
+ - 'indices_monitor'
+ - 'indices:admin/aliases/get'
+ - 'indices:admin/mappings/get'
+
+# Allows users to read Notebooks
+notebooks_read_access:
+ reserved: true
+ cluster_permissions:
+ - 'cluster:admin/opendistro/notebooks/list'
+ - 'cluster:admin/opendistro/notebooks/get'
+
+# Allows users to all Notebooks functionality
+notebooks_full_access:
+ reserved: true
+ cluster_permissions:
+ - 'cluster:admin/opendistro/notebooks/create'
+ - 'cluster:admin/opendistro/notebooks/update'
+ - 'cluster:admin/opendistro/notebooks/delete'
+ - 'cluster:admin/opendistro/notebooks/get'
+ - 'cluster:admin/opendistro/notebooks/list'
+
+# Allows users to read observability objects
+observability_read_access:
+ reserved: true
+ cluster_permissions:
+ - 'cluster:admin/opensearch/observability/get'
+
+# Allows users to all Observability functionality
+observability_full_access:
+ reserved: true
+ cluster_permissions:
+ - 'cluster:admin/opensearch/observability/create'
+ - 'cluster:admin/opensearch/observability/update'
+ - 'cluster:admin/opensearch/observability/delete'
+ - 'cluster:admin/opensearch/observability/get'
+
+# Allows users to read and download Reports
+reports_instances_read_access:
+ reserved: true
+ cluster_permissions:
+ - 'cluster:admin/opendistro/reports/instance/list'
+ - 'cluster:admin/opendistro/reports/instance/get'
+ - 'cluster:admin/opendistro/reports/menu/download'
+
+# Allows users to read and download Reports and Report-definitions
+reports_read_access:
+ reserved: true
+ cluster_permissions:
+ - 'cluster:admin/opendistro/reports/definition/get'
+ - 'cluster:admin/opendistro/reports/definition/list'
+ - 'cluster:admin/opendistro/reports/instance/list'
+ - 'cluster:admin/opendistro/reports/instance/get'
+ - 'cluster:admin/opendistro/reports/menu/download'
+
+# Allows users to all Reports functionality
+reports_full_access:
+ reserved: true
+ cluster_permissions:
+ - 'cluster:admin/opendistro/reports/definition/create'
+ - 'cluster:admin/opendistro/reports/definition/update'
+ - 'cluster:admin/opendistro/reports/definition/on_demand'
+ - 'cluster:admin/opendistro/reports/definition/delete'
+ - 'cluster:admin/opendistro/reports/definition/get'
+ - 'cluster:admin/opendistro/reports/definition/list'
+ - 'cluster:admin/opendistro/reports/instance/list'
+ - 'cluster:admin/opendistro/reports/instance/get'
+ - 'cluster:admin/opendistro/reports/menu/download'
+
+# Allows users to use all asynchronous-search functionality
+asynchronous_search_full_access:
+ reserved: true
+ cluster_permissions:
+ - 'cluster:admin/opendistro/asynchronous_search/*'
+ index_permissions:
+ - index_patterns:
+ - '*'
+ allowed_actions:
+ - 'indices:data/read/search*'
+
+# Allows users to read stored asynchronous-search results
+asynchronous_search_read_access:
+ reserved: true
+ cluster_permissions:
+ - 'cluster:admin/opendistro/asynchronous_search/get'
+
+# Allows user to use all index_management actions - ism policies, rollups, transforms
+index_management_full_access:
+ reserved: true
+ cluster_permissions:
+ - "cluster:admin/opendistro/ism/*"
+ - "cluster:admin/opendistro/rollup/*"
+ - "cluster:admin/opendistro/transform/*"
+ index_permissions:
+ - index_patterns:
+ - '*'
+ allowed_actions:
+ - 'indices:admin/opensearch/ism/*'
+
+# Allows users to use all cross cluster replication functionality at leader cluster
+cross_cluster_replication_leader_full_access:
+ reserved: true
+ index_permissions:
+ - index_patterns:
+ - '*'
+ allowed_actions:
+ - "indices:admin/plugins/replication/index/setup/validate"
+ - "indices:data/read/plugins/replication/changes"
+ - "indices:data/read/plugins/replication/file_chunk"
+
+# Allows users to use all cross cluster replication functionality at follower cluster
+cross_cluster_replication_follower_full_access:
+ reserved: true
+ cluster_permissions:
+ - "cluster:admin/plugins/replication/autofollow/update"
+ index_permissions:
+ - index_patterns:
+ - '*'
+ allowed_actions:
+ - "indices:admin/plugins/replication/index/setup/validate"
+ - "indices:data/write/plugins/replication/changes"
+ - "indices:admin/plugins/replication/index/start"
+ - "indices:admin/plugins/replication/index/pause"
+ - "indices:admin/plugins/replication/index/resume"
+ - "indices:admin/plugins/replication/index/stop"
+ - "indices:admin/plugins/replication/index/update"
+ - "indices:admin/plugins/replication/index/status_check"
+
+# -------------------------------------------------------------
+# Custom roles
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+ingest_client:
+ reserved: True
+ description: Access for ingestion client to all indices
+ cluster_permissions:
+ - "cluster_monitor",
+ - "cluster_composite_ops",
+ - "indices:admin/template/get",
+ - "indices:admin/template/put",
+ - "cluster:admin/ingest/pipeline/put",
+ - "cluster:admin/ingest/pipeline/get"
+ index_permissions:
+ - index_patterns:
+ - '*'
+ allowed_actions:
+ - "crud"
+ - "create_index"
diff --git a/roles/opensearch/opensearch/security.sls b/roles/opensearch/opensearch/security.sls
--- a/roles/opensearch/opensearch/security.sls
+++ b/roles/opensearch/opensearch/security.sls
@@ -19,6 +19,7 @@
- mode: 0600
- template: jinja
- context:
+ ingest_clients_users: {{ config['ingest_clients_users'] }}
users:
{% for user, credential in config['users'].items() %}
{{ user }}:
@@ -26,6 +27,13 @@
password: {{ salt['zr.get_password'](credential) }}
{% endfor %}
+/opt/opensearch/plugins/opensearch-security/securityconfig/roles.yml:
+ file.managed:
+ - source: salt://roles/opensearch/opensearch/files/roles.yml
+ - user: opensearch
+ - group: opensearch
+ - mode: 0600
+
opensearch_security_initialize:
cmd.script:
- source: salt://roles/opensearch/opensearch/files/security_initialize.sh
diff --git a/roles/paas-docker/containers/filebeat.sls b/roles/paas-docker/containers/filebeat.sls
new file mode 100755
--- /dev/null
+++ b/roles/paas-docker/containers/filebeat.sls
@@ -0,0 +1,65 @@
+# -------------------------------------------------------------
+# Salt — Provision Docker engine
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# License: Trivial work, not eligible to copyright
+# -------------------------------------------------------------
+
+{% set has_selinux = salt['grains.get']('selinux:enabled', False) %}
+{% set containers = pillar['docker_containers'][grains['id']] %}
+
+{% for instance, container in containers['filebeat'].items() %}
+
+# -------------------------------------------------------------
+# Storage directory
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+/srv/filebeat/{{ instance }}:
+ file.directory:
+ - user: 9001
+ - makedirs: True
+
+{% if has_selinux %}
+selinux_context_{{ instance }}_data:
+ selinux.fcontext_policy_present:
+ - name: /srv/filebeat/{{ instance }}
+ - sel_type: container_file_t
+
+selinux_context_{{ instance }}_data_applied:
+ selinux.fcontext_policy_applied:
+ - name: /srv/filebeat/{{ instance }}
+{% endif %}
+
+# -------------------------------------------------------------
+# Configuration
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+/srv/filebeat/{{ instance }}/filebeat.yml:
+ file.managed:
+ - source: salt://roles/paas-docker/containers/files/filebeat/filebeat.yml.jinja
+ - mode: 0600
+ - template: jinja
+ - context:
+ elastic: {{ container['opensearch'] }}
+ elastic_username: {{ salt['zr.get_username'](container['credentials']['opensearch']) }}
+ elastic_password: {{ salt['zr.get_password'](container['credentials']['opensearch']) }}
+
+# -------------------------------------------------------------
+# Container
+#
+# To be compatible with OpenSearch, currently it's recommended
+# to use filebeat 7.10.2.
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+{{ instance }}:
+ docker_container.running:
+ - detach: True
+ - interactive: True
+ - image: docker.elastic.co/beats/filebeat:7.10.2
+ - user: root
+ - binds:
+ - /srv/filebeat/{{ instance }}/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
+ - /var/lib/docker:/var/lib/docker:ro
+ - /var/run/docker.sock:/var/run/docker.sock
+
+{% endfor %}
diff --git a/roles/paas-docker/containers/files/filebeat/filebeat.yml.jinja b/roles/paas-docker/containers/files/filebeat/filebeat.yml.jinja
new file mode 100755
--- /dev/null
+++ b/roles/paas-docker/containers/files/filebeat/filebeat.yml.jinja
@@ -0,0 +1,40 @@
+# -------------------------------------------------------------
+# OpenSearch
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# License: Trivial work, not eligible to copyright
+# Source file: roles/paas-docker/containers/files/filebeat/filebeat.yml.jinja
+# -------------------------------------------------------------
+#
+# <auto-generated>
+# This file is managed by our rOPS SaltStack repository.
+#
+# Changes to this file may cause incorrect behavior
+# and will be lost if the state is redeployed.
+# </auto-generated>
+
+filebeat.inputs:
+- type: container
+ paths:
+ - /var/lib/docker/containers/*/*.log
+
+processors:
+- add_docker_metadata:
+ host: unix:///var/run/docker.sock
+
+- decode_json_fields:
+ fields:
+ - message
+ target: json
+ overwrite_keys: True
+
+output.elasticsearch:
+ hosts:
+ - {{ elastic['server'] }}
+ username: {{ elastic_username }}
+ password: {{ elastic_password }}
+ indices:
+ - index: {{ elastic['index'] }}
+
+logging.json: True
+logging.metrics.enabled: False
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Mon, Dec 23, 09:38 (16 h, 48 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
2312208
Default Alt Text
D2455.id6181.diff (15 KB)
Attached To
Mode
D2455: Collect logs from Docker
Attached
Detach File
Event Timeline
Log In to Comment