diff --git a/.arclint b/.arclint --- a/.arclint +++ b/.arclint @@ -28,6 +28,9 @@ "F821": "advice" }, "flake8.builtins": [ + "KAFKA_CLUSTERS", + "SENTRY_FEATURES", + "SENTRY_OPTIONS", "__context__", "__executors__", "__ext_pillar__", diff --git a/_modules/credentials.py b/_modules/credentials.py --- a/_modules/credentials.py +++ b/_modules/credentials.py @@ -117,6 +117,11 @@ return get_password(key, prefix) +# ------------------------------------------------------------- +# Helpers for Sentry credentials +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + def get_sentry_dsn(args): if _are_credentials_hidden(): return "credential for " + args["credential"] diff --git a/pillar/credentials/vault.sls b/pillar/credentials/vault.sls --- a/pillar/credentials/vault.sls +++ b/pillar/credentials/vault.sls @@ -49,6 +49,7 @@ vault_policies: - salt-primary + - sentry - viperserv # ------------------------------------------------------------- @@ -138,6 +139,7 @@ - ops/secrets/nasqueron.sentry.app_key - ops/secrets/nasqueron.sentry.postgresql + - ops/secrets/nasqueron.sentry.vault # # Credentials used by Nasqueron members private services diff --git a/pillar/paas/docker/docker-002/sentry.sls b/pillar/paas/docker/docker-002/sentry.sls --- a/pillar/paas/docker/docker-002/sentry.sls +++ b/pillar/paas/docker/docker-002/sentry.sls @@ -155,22 +155,54 @@ # sentry: - sentry_web_1: + sentry_web: app_port: 26080 host: sentry.nasqueron.org - - # As an instance is divided between a web, a cron and a worker - # containers, we need an identified to share a data volume. + command: run web realm: nasqueron network: sentry - sentry_worker: - sentry_worker_1: + sentry_worker: + command: run worker realm: nasqueron network: sentry - sentry_cron: sentry_cron: + command: run cron + realm: nasqueron + network: sentry + + sentry_ingest_consumer: + command: run ingest-consumer --all-consumer-types + realm: nasqueron + network: sentry + + sentry_ingest_replay_recordings: + command: run ingest-replay-recordings + realm: nasqueron + network: sentry + + sentry_post_process_forwarder_errors: + command: run post-process-forwarder --entity errors + realm: nasqueron + network: sentry + + sentry_post_process_forwarder_transations: + command: run post-process-forwarder --entity transactions + --commit-log-topic=snuba-transactions-commit-log + --synchronize-commit-group transactions_group + realm: nasqueron + network: sentry + + sentry_consumer_events: + command: run query-subscription-consumer --commit-batch-size 1 + --topic events-subscription-results + realm: nasqueron + network: sentry + + sentry_consumer_transactions: + command: run query-subscription-consumer --commit-batch-size 1 + --topic transactions-subscription-results realm: nasqueron network: sentry @@ -186,3 +218,23 @@ kafka.server: WARN kafka.zookeeper: WARN state.change.logger: WARN + +sentry_realms: + nasqueron: + network: sentry + services: + kafka: sentry_kafka + memcached: sentry_memcached + postgresql: sentry_db + redis: sentry_redis + smtp: sentry_smtp + snuba: sentry_snuba_api + symbolicator: sentry_symbolicator + web: sentry_web + credentials: + secret_key: nasqueron.sentry.app_key + postgresql: nasqueron.sentry.postgresql + vault: nasqueron.sentry.vault + + hostname: sentry.nasqueron.org + email_from: noreply@sentry.nasqueron.org diff --git a/pillar/saas/sentry.sls b/pillar/saas/sentry.sls deleted file mode 100644 --- a/pillar/saas/sentry.sls +++ /dev/null @@ -1,21 +0,0 @@ -# ------------------------------------------------------------- -# Salt — Sentry instances -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# Project: Nasqueron -# Created: 2018-11-10 -# License: Trivial work, not eligible to copyright -# ------------------------------------------------------------- - -# ------------------------------------------------------------- -# Sentry realms -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -sentry_realms: - nasqueron: - links: - postgresql: sentry_db - redis: sentry_redis - smtp: sentry_smtp - credential: nasqueron.sentry.app_key - email_from: no-reply@sentry.nasqueron.org - host: sentry.nasqueron.org diff --git a/pillar/top.sls b/pillar/top.sls --- a/pillar/top.sls +++ b/pillar/top.sls @@ -29,7 +29,6 @@ - paas.docker - saas.jenkins - saas.phpbb - - saas.sentry db-A-001: - dbserver.cluster-A diff --git a/roles/paas-docker/containers/files/sentry/etc/config.yml b/roles/paas-docker/containers/files/sentry/etc/config.yml new file mode 100644 --- /dev/null +++ b/roles/paas-docker/containers/files/sentry/etc/config.yml @@ -0,0 +1,30 @@ +# ------------------------------------------------------------- +# Sentry configuration +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Project: Nasqueron +# Realm: {{ realm }} +# License: Trivial work, not eligible to copyright +# Source file: roles/paas-docker/containers/files/sentry/etc/config.yml +# ------------------------------------------------------------- +# +# <auto-generated> +# This file is managed by our rOPS SaltStack repository. +# +# Changes to this file may cause incorrect behavior +# and will be lost if the state is redeployed. +# </auto-generated> + +mail.host: "{{ args.services.smtp }}" + +filestore.backend: filesystem +filestore.options: + location: /data/files +dsym.cache-path: /data/dsym-cache +releasefile.cache-path: /data/releasefile-cache + +system.internal-url-prefix: 'http://{{ args.services.web }}:9000' +symbolicator.enabled: true +symbolicator.options: + url: "http://{{ args.services.symbolicator }}:3021" + +transaction-events.force-disable-internal-project: true diff --git a/roles/paas-docker/containers/files/sentry/etc/sentry.conf.py b/roles/paas-docker/containers/files/sentry/etc/sentry.conf.py new file mode 100644 --- /dev/null +++ b/roles/paas-docker/containers/files/sentry/etc/sentry.conf.py @@ -0,0 +1,289 @@ +# ------------------------------------------------------------- +# Sentry configuration +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Project: Nasqueron +# Realm: {{ realm }} +# License: Trivial work, not eligible to copyright +# Source file: roles/paas-docker/containers/files/sentry/etc/sentry.conf.py +# ------------------------------------------------------------- +# +# <auto-generated> +# This file is managed by our rOPS SaltStack repository. +# +# Changes to this file may cause incorrect behavior +# and will be lost if the state is redeployed. +# </auto-generated> + + +import hvac + +from sentry.conf.server import * # NOQA + + +# ------------------------------------------------------------- +# Helper methods +# +# The get_internal_network has been adapted from pynetlinux: +# https://github.com/rlisagor/pynetlinux/blob/e3f16978855c6649685f0c43d4c3fcf768427ae5/pynetlinux/ifconfig.py#L197-L223 +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + +def get_internal_network(): + import ctypes + import fcntl + import math + import socket + import struct + + iface = b"eth0" + sockfd = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + ifreq = struct.pack(b"16sH14s", iface, socket.AF_INET, b"\x00" * 14) + + try: + ip = struct.unpack( + b"!I", struct.unpack(b"16sH2x4s8x", fcntl.ioctl(sockfd, 0x8915, ifreq))[2] + )[0] + netmask = socket.ntohl( + struct.unpack(b"16sH2xI8x", fcntl.ioctl(sockfd, 0x891B, ifreq))[2] + ) + except IOError: + return () + base = socket.inet_ntoa(struct.pack(b"!I", ip & netmask)) + netmask_bits = 32 - int(round(math.log(ctypes.c_uint32(~netmask).value + 1, 2), 1)) + return "{0:s}/{1:d}".format(base, netmask_bits) + + +def read_secret(key): + secret = vault_client.secrets.kv.read_secret_version( + mount_point="ops", path="secrets/" + key + ) + return secret["data"]["data"] + + +# ------------------------------------------------------------- +# Authenticate to Vault +# +# Credentials for PostgreSQL or the secret key are stored in Vault. +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + +vault_client = hvac.Client( + url="{{ vault.addr }}", verify="/etc/sentry/certificates/nasqueron-vault-ca.crt" +) + +vault_client.auth.approle.login( + role_id="{{ vault.approle.roleID }}", + secret_id="{{ vault.approle.secretID }}", +) + + +# ------------------------------------------------------------- +# Sentry configuration +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + +INTERNAL_SYSTEM_IPS = (get_internal_network(),) + +secret = read_secret("{{ args.credentials.postgresql }}") +DATABASES = { + "default": { + "ENGINE": "sentry.db.postgres", + "NAME": "postgres", + "USER": secret["username"], + "PASSWORD": secret["password"], + "HOST": "{{ args.services.postgresql }}", + "PORT": "", + } +} + +SENTRY_USE_BIG_INTS = True + + +# ------------------------------------------------------------- +# General +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + +SENTRY_SINGLE_ORGANIZATION = False + +SENTRY_OPTIONS["system.event-retention-days"] = int( + env("SENTRY_EVENT_RETENTION_DAYS", "90") +) + +secret_key = read_secret("{{ args.credentials.secret_key }}") +SENTRY_OPTIONS["system.secret-key"] = secret_key["password"] + +GEOIP_PATH_MMDB = "/usr/local/share/geoip/GeoLite2-City.mmdb" + + +# ------------------------------------------------------------- +# Redis +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + +SENTRY_OPTIONS["redis.clusters"] = { + "default": { + "hosts": { + 0: { + "host": "{{ args.services.redis }}", + "password": "", + "port": "6379", + "db": "0", + } + } + } +} + +SENTRY_BUFFER = "sentry.buffer.redis.RedisBuffer" +SENTRY_QUOTAS = "sentry.quotas.redis.RedisQuota" +SENTRY_RATELIMITER = "sentry.ratelimits.redis.RedisRateLimiter" +SENTRY_DIGESTS = "sentry.digests.backends.redis.RedisBackend" + + +# ------------------------------------------------------------- +# Queue - Celery +# +# Reference: https://develop.sentry.dev/services/queue/ +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + +BROKER_URL = "redis://:{password}@{host}:{port}/{db}".format( + **SENTRY_OPTIONS["redis.clusters"]["default"]["hosts"][0] +) + + +# ------------------------------------------------------------- +# Caching +# +# :: Redis +# :: Memcached +# :: Kafka +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + +SENTRY_CACHE = "sentry.cache.redis.RedisCache" + +CACHES = { + "default": { + "BACKEND": "django.core.cache.backends.memcached.MemcachedCache", + "LOCATION": ["{{ args.services.memcached }}:11211"], + "TIMEOUT": 3600, + } +} + +DEFAULT_KAFKA_OPTIONS = { + "bootstrap.servers": "{{ args.services.kafka }}:9092", + "message.max.bytes": 50000000, + "socket.timeout.ms": 1000, +} +KAFKA_CLUSTERS["default"] = DEFAULT_KAFKA_OPTIONS + +SENTRY_EVENTSTREAM = "sentry.eventstream.kafka.KafkaEventStream" +SENTRY_EVENTSTREAM_OPTIONS = {"producer_configuration": DEFAULT_KAFKA_OPTIONS} + + +# ------------------------------------------------------------- +# TSDB / Snuba +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + +SENTRY_TSDB = "sentry.tsdb.redissnuba.RedisSnubaTSDB" +SENTRY_SNUBA = "http://{{ args.services.snuba }}:1218" + +SENTRY_SEARCH = "sentry.search.snuba.EventsDatasetSnubaSearchBackend" +SENTRY_SEARCH_OPTIONS = {} +SENTRY_TAGSTORE_OPTIONS = {} + + +# ------------------------------------------------------------- +# Web server +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + +SENTRY_WEB_HOST = "0.0.0.0" +SENTRY_WEB_PORT = 9000 +SENTRY_WEB_OPTIONS = { + "http": "%s:%s" % (SENTRY_WEB_HOST, SENTRY_WEB_PORT), + "protocol": "uwsgi", + # This is needed in order to prevent https://github.com/getsentry/sentry/blob/c6f9660e37fcd9c1bbda8ff4af1dcfd0442f5155/src/sentry/services/http.py#L70 + "uwsgi-socket": None, + "so-keepalive": True, + # Keep this between 15s-75s as that's what Relay supports + "http-keepalive": 15, + "http-chunked-input": True, + # the number of web workers + "workers": 3, + "threads": 4, + "memory-report": False, + # Some stuff so uwsgi will cycle workers sensibly + "max-requests": 100000, + "max-requests-delta": 500, + "max-worker-lifetime": 86400, + # Duplicate options from sentry default just so we don't get + # bit by sentry changing a default value that we depend on. + "thunder-lock": True, + "log-x-forwarded-for": False, + "buffer-size": 32768, + "limit-post": 209715200, + "disable-logging": True, + "reload-on-rss": 600, + "ignore-sigpipe": True, + "ignore-write-errors": True, + "disable-write-exception": True, +} + + +# ------------------------------------------------------------- +# TLS +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + +SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") +USE_X_FORWARDED_HOST = True +SESSION_COOKIE_SECURE = True +CSRF_COOKIE_SECURE = True +SOCIAL_AUTH_REDIRECT_IS_HTTPS = True + + +# ------------------------------------------------------------- +# Mail +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + +SENTRY_OPTIONS["mail.list-namespace"] = "{{ args.hostname }}" +SENTRY_OPTIONS["mail.from"] = "{{ args.email_from }}" + + +# ------------------------------------------------------------- +# Features +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + +SENTRY_FEATURES["projects:sample-events"] = False +SENTRY_FEATURES.update( + { + feature: True + for feature in ( + "organizations:discover", + "organizations:events", + "organizations:global-views", + "organizations:incidents", + "organizations:integrations-issue-basic", + "organizations:integrations-issue-sync", + "organizations:invite-members", + "organizations:metric-alert-builder-aggregate", + "organizations:sso-basic", + "organizations:sso-rippling", + "organizations:sso-saml2", + "organizations:performance-view", + "organizations:advanced-search", + "organizations:session-replay", + "projects:custom-inbound-filters", + "projects:data-forwarding", + "projects:discard-groups", + "projects:plugins", + "projects:rate-limits", + "projects:servicehooks", + ) + } +) diff --git a/roles/paas-docker/containers/files/sentry/sentry.sh.jinja b/roles/paas-docker/containers/files/sentry/sentry.sh.jinja --- a/roles/paas-docker/containers/files/sentry/sentry.sh.jinja +++ b/roles/paas-docker/containers/files/sentry/sentry.sh.jinja @@ -19,10 +19,10 @@ set -e -SECRET_KEY=$(credential {{ credential_key }}) - docker run -it --rm \ - -e SENTRY_SECRET_KEY=$SECRET_KEY \ - --link {{ links.postgresql }}:postgres \ - --link {{ links.redis }}:redis \ - sentry "$@" + --network {{ network }} \ + -v /srv/sentry/{{ realm }}/etc:/etc/sentry \ + -v /srv/sentry/{{ realm }}/data:/data \ + -v /srv/geoip:/usr/local/share/geoip:ro \ + -e PYTHONUSERBASE=/data/custom-packages \ + nasqueron/sentry "$@" diff --git a/roles/paas-docker/containers/sentry.sls b/roles/paas-docker/containers/sentry.sls --- a/roles/paas-docker/containers/sentry.sls +++ b/roles/paas-docker/containers/sentry.sls @@ -12,7 +12,7 @@ # Data directory # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -{% for realm, args in pillar['sentry_realms'].items() %} +{% for realm, realm_args in pillar['sentry_realms'].items() %} /srv/sentry/{{ realm }}: file.directory: @@ -20,15 +20,46 @@ - group: 999 - makedirs: True +/srv/sentry/{{ realm }}/data: + file.directory: + - user: 999 + - group: 999 + +/srv/sentry/{{ realm }}/data/files: + file.directory: + - user: 999 + - group: 999 + +/srv/sentry/{{ realm }}/etc: + file.recurse: + - source: salt://roles/paas-docker/containers/files/sentry/etc + - user: 999 + - group: 999 + - dir_mode: 700 + - file_mode: 400 + - template: jinja + - context: + realm: {{ realm }} + args: {{ realm_args }} + vault: + approle: {{ salt["credentials.read_secret"](realm_args["credentials"]["vault"]) }} + addr: https://172.27.27.7:8200 + +sentry_{{ realm }}_vault_certificate: + file.managed: + - name: /srv/sentry/{{ realm }}/etc/certificates/nasqueron-vault-ca.crt + - source: salt://roles/core/certificates/files/nasqueron-vault-ca.crt + - mode: 644 + - makedirs: True + /srv/sentry/{{ realm }}/bin/sentry: file.managed: - source: salt://roles/paas-docker/containers/files/sentry/sentry.sh.jinja - - template: jinja - mode: 755 - - makedirs: True + - template: jinja - context: - links: {{ args['links'] }} - credential_key: {{ args['credential'] }} + realm: {{ realm }} + network: {{ realm_args["network"] }} {% if has_selinux %} selinux_context_{{ realm }}_sentry_data: @@ -49,49 +80,26 @@ {% for instance, container in pillar['docker_containers']['sentry'].items() %} -{% set args = pillar['sentry_realms'][container['realm']] %} - {{ instance }}: docker_container.running: - detach: True - interactive: True - - image: library/sentry - - binds: &binds /srv/sentry/{{ container['realm'] }}:/var/lib/sentry/files - - links: &links - - {{ args['links']['postgresql'] }}:postgres - - {{ args['links']['redis'] }}:redis - - {{ args['links']['smtp'] }}:smtp - - environment: &env - - SENTRY_SECRET_KEY: {{ salt['credentials.get_token'](args['credential']) }} - - SENTRY_FILESTORE_DIR: - - SENTRY_USE_SSL: 1 - - SENTRY_SERVER_EMAIL: {{ args['email_from'] }} - - SENTRY_FILESTORE_DIR: /var/lib/sentry/files + - image: nasqueron/sentry + - command: {{ container["command"] }} + - binds: + - /srv/sentry/{{ container["realm"] }}/etc:/etc/sentry + - /srv/sentry/{{ container["realm"] }}/data:/data + - /srv/geoip:/usr/local/share/geoip:ro + - environment: + - PYTHONUSERBASE: /data/custom-packages + - SENTRY_EVENT_RETENTION_DAYS: 90 + {% if "app_port" in container %} - ports: - - 80 + - 9000 - port_bindings: - {{ container['app_port'] }}:9000 + {% endif %} + - networks: + - {{ container['network'] }} {% endfor %} - -# ------------------------------------------------------------- -# Services containers -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -{% for service in ['worker', 'cron'] %} -{% for instance, container in pillar['docker_containers']['sentry_' + service].items() %} - -{% set args = pillar['sentry_realms'][container['realm']] %} - -{{ instance }}: - docker_container.running: - - detach: True - - interactive: True - - image: library/sentry - - binds: *binds - - links: *links - - environment: *env - - command: run {{ service }} - -{% endfor %} -{% endfor %} diff --git a/roles/paas-docker/containers/sentry_cron.sls b/roles/paas-docker/containers/sentry_cron.sls deleted file mode 100644 --- a/roles/paas-docker/containers/sentry_cron.sls +++ /dev/null @@ -1,10 +0,0 @@ -# ------------------------------------------------------------- -# Salt — Provision Docker engine -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# Project: Nasqueron -# Created: 2020-02-14 -# License: Trivial work, not eligible to copyright -# ------------------------------------------------------------- - -# This section is intentionally left blank. -# Containers are provided by sentry.sls diff --git a/roles/paas-docker/containers/sentry_worker.sls b/roles/paas-docker/containers/sentry_worker.sls deleted file mode 100644 --- a/roles/paas-docker/containers/sentry_worker.sls +++ /dev/null @@ -1,10 +0,0 @@ -# ------------------------------------------------------------- -# Salt — Provision Docker engine -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# Project: Nasqueron -# Created: 2020-02-14 -# License: Trivial work, not eligible to copyright -# ------------------------------------------------------------- - -# This section is intentionally left blank. -# Containers are provided by sentry.sls diff --git a/roles/paas-docker/containers/files/sentry/sentry.sh.jinja b/roles/vault/policies/files/sentry.hcl copy from roles/paas-docker/containers/files/sentry/sentry.sh.jinja copy to roles/vault/policies/files/sentry.hcl --- a/roles/paas-docker/containers/files/sentry/sentry.sh.jinja +++ b/roles/vault/policies/files/sentry.hcl @@ -1,13 +1,9 @@ -#!/bin/sh - # ------------------------------------------------------------- -# PaaS Docker +# Vault configuration - Policy for Sentry # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # Project: Nasqueron -# Created: 2018-11-10 # License: Trivial work, not eligible to copyright -# Description: Wrapper for sentry command (local instance) -# Source file: roles/paas-docker/containers/files/sentry/sentry.sh.jinja +# Source file: roles/vault/vault/files/sentry.hcl # ------------------------------------------------------------- # # <auto-generated> @@ -17,12 +13,10 @@ # and will be lost if the state is redeployed. # </auto-generated> -set -e - -SECRET_KEY=$(credential {{ credential_key }}) +path "ops/data/secrets/nasqueron.sentry.app_key" { + capabilities = [ "read" ] +} -docker run -it --rm \ - -e SENTRY_SECRET_KEY=$SECRET_KEY \ - --link {{ links.postgresql }}:postgres \ - --link {{ links.redis }}:redis \ - sentry "$@" +path "ops/data/secrets/nasqueron.sentry.postgresql" { + capabilities = [ "read" ] +}