Page MenuHomeDevCentral
Files F51390666

D4218.diff
No OneTemporary
Actions

D4218.diff
View Options

diff --git a/pillar/credentials/vault.sls b/pillar/credentials/vault.sls
--- a/pillar/credentials/vault.sls
+++ b/pillar/credentials/vault.sls
@@ -103,6 +103,8 @@
vault_secrets_by_role:
devserver:
+ - ops/secrets/backups/ovh/s3/darak-%%node%%
+
- ops/secrets/dbserver/windriver-mariadb/users/*
- ops/secrets/dbserver/windriver-pgsql/users/*
@@ -268,6 +270,9 @@
- ops/secrets/dereckson/wordpress/secrets
+ salt-primary:
+ - ops/secrets/backups/ovh/s3/vakor-%%node%%
+
viperserv:
- ops/secrets/nasqueron/viperserv/vault
@@ -315,7 +320,7 @@
- ops/secrets/dbserver/cluster-B/users/*
vault_secrets_ubiquity:
+ - ops/secrets/backups/ovh/s3/amaris-%%node%%
# IPsec tunnels
-
- ops/secrets/network/ipsec/key
diff --git a/requirements.txt b/requirements.txt
--- a/requirements.txt
+++ b/requirements.txt
@@ -16,3 +16,7 @@
jsondiff==2.0.0
requests>=2.32.3,<3.0
unittest-data-provider>=1.0.1,<2.0
+
+# Utilities
+hvac>=2.4.0,<3.0
+pyhcl>=0.4.5,<0.5
diff --git a/roles/core/backups/config.sls b/roles/core/backups/config.sls
new file mode 100644
--- /dev/null
+++ b/roles/core/backups/config.sls
@@ -0,0 +1,20 @@
+# -------------------------------------------------------------
+# Salt — Backups
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# License: Trivial work, not eligible to copyright
+# -------------------------------------------------------------
+
+{% from "map.jinja" import dirs, services with context %}
+
+{{ dirs.etc }}/restic.conf:
+ file.managed:
+ - source: salt://roles/core/backups/files/restic.conf.jinja
+ - template: jinja
+ - user: root
+ - group: ops
+ - mode: 600
+ - context:
+ restic_repository: "{{ salt["pillar.get"]("backups:repository", "") }}"
+ restic_wrapper: {{ dirs.bin }}/restic-vault-wrapper
+ restic_credential: nasqueron/backups/{{ grains["id"] }}
diff --git a/roles/core/backups/files/periodic/999.restic.sh b/roles/core/backups/files/periodic/999.restic.sh
new file mode 100644
--- /dev/null
+++ b/roles/core/backups/files/periodic/999.restic.sh
@@ -0,0 +1,18 @@
+#!/bin/sh
+
+# -------------------------------------------------------------
+# Salt — Backups (Restic) :: daily periodic
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# License: Trivial work, not eligible to copyright
+# Source file: roles/core/backups/files/periodic/999.restic.sh
+# -------------------------------------------------------------
+#
+# <auto-generated>
+# This file is managed by our rOPS SaltStack repository.
+#
+# Changes to this file may cause incorrect behavior
+# and will be lost if the state is redeployed.
+# </auto-generated>
+
+/usr/local/bin/restic-s3 backup /etc /var/log /home
diff --git a/roles/core/backups/files/rc/restic.conf b/roles/core/backups/files/rc/restic.conf
new file mode 100644
--- /dev/null
+++ b/roles/core/backups/files/rc/restic.conf
@@ -0,0 +1,16 @@
+# -------------------------------------------------------------
+# Salt — Backups (Restic) :: rc configuration
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# License: Trivial work, not eligible to copyright
+# Source file: roles/core/backups/files/rc/restic.conf
+# -------------------------------------------------------------
+#
+# <auto-generated>
+# This file is managed by our rOPS SaltStack repository.
+#
+# Changes to this file may cause incorrect behavior
+# and will be lost if the state is redeployed.
+# </auto-generated>
+
+restic_enable="YES"
diff --git a/roles/core/backups/files/rc/restic.rc b/roles/core/backups/files/rc/restic.rc
new file mode 100644
--- /dev/null
+++ b/roles/core/backups/files/rc/restic.rc
@@ -0,0 +1,29 @@
+#!/bin/sh
+
+# -------------------------------------------------------------
+# Salt — Backups (Restic) :: rc script
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# License: Trivial work, not eligible to copyright
+# Source file: roles/core/backups/files/rc/restic.rc
+# -------------------------------------------------------------
+#
+# <auto-generated>
+# This file is managed by our rOPS SaltStack repository.
+#
+# Changes to this file may cause incorrect behavior
+# and will be lost if the state is redeployed.
+# </auto-generated>
+
+# PROVIDE: restic
+# REQUIRE: LOGIN
+# KEYWORD: shutdown
+
+. /etc/rc.subr
+
+name="restic"
+rcvar="restic_enable"
+command="/usr/local/bin/restic-s3"
+command_args="backup /etc /var/log /home"
+load_rc_config $name
+run_rc_command "$1"
diff --git a/roles/core/backups/files/restic-s3.sh.jinja b/roles/core/backups/files/restic-s3.sh.jinja
new file mode 100644
--- /dev/null
+++ b/roles/core/backups/files/restic-s3.sh.jinja
@@ -0,0 +1,28 @@
+#!/bin/sh
+
+# -------------------------------------------------------------
+# Salt — Backups (Restic) :: S3 launcher
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# License: Trivial work, not eligible to copyright
+# Source file: roles/core/backups/files/restic-s3.sh.jinja
+# -------------------------------------------------------------
+#
+# <auto-generated>
+# This file is managed by our rOPS SaltStack repository.
+#
+# Changes to this file may cause incorrect behavior
+# and will be lost if the state is redeployed.
+# </auto-generated>
+
+set -eu
+
+. "{{ restic_conf }}"
+
+SECRET_JSON=$(salt-call credentials.read_secret "{{ s3_credential }}" --out=json)
+
+AWS_ACCESS_KEY_ID=$(echo "$SECRET_JSON" | jq -r '.local.username')
+AWS_SECRET_ACCESS_KEY=$(echo "$SECRET_JSON" | jq -r '.local.password')
+export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY
+
+exec "{{ restic_bin }}" "$@"
diff --git a/roles/core/backups/files/restic-vault-wrapper.sh b/roles/core/backups/files/restic-vault-wrapper.sh
new file mode 100644
--- /dev/null
+++ b/roles/core/backups/files/restic-vault-wrapper.sh
@@ -0,0 +1,25 @@
+#!/bin/sh
+
+# -------------------------------------------------------------
+# Salt — Backups (Restic) :: Vault Wrapper
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# License: Trivial work, not eligible to copyright
+# Source file: roles/core/backups/files/restic-vault-wrapper.sh
+# -------------------------------------------------------------
+#
+# <auto-generated>
+# This file is managed by our rOPS SaltStack repository.
+#
+# Changes to this file may cause incorrect behavior
+# and will be lost if the state is redeployed.
+# </auto-generated>
+
+set -eu
+
+if [ "$#" -ne 1 ]; then
+ echo "Usage: $0 <credential key>" 1>&2
+ exit 1
+fi
+
+salt-call credentials.get_password "$1" --out=json | jq -r .local
diff --git a/roles/core/backups/files/restic.conf.jinja b/roles/core/backups/files/restic.conf.jinja
new file mode 100644
--- /dev/null
+++ b/roles/core/backups/files/restic.conf.jinja
@@ -0,0 +1,17 @@
+# -------------------------------------------------------------
+# Salt — Backups (Restic) :: Configuration
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# License: Trivial work, not eligible to copyright
+# Source file: roles/core/backups/files/restic.conf.jinja
+# -------------------------------------------------------------
+#
+# <auto-generated>
+# This file is managed by our rOPS SaltStack repository.
+#
+# Changes to this file may cause incorrect behavior
+# and will be lost if the state is redeployed.
+# </auto-generated>
+
+export RESTIC_REPOSITORY="{{ restic_repository }}"
+export RESTIC_PASSWORD_COMMAND="{{ restic_wrapper }} {{ restic_credential }}"
diff --git a/roles/core/backups/files/systemd/restic.service b/roles/core/backups/files/systemd/restic.service
new file mode 100644
--- /dev/null
+++ b/roles/core/backups/files/systemd/restic.service
@@ -0,0 +1,23 @@
+# -------------------------------------------------------------
+# Salt — Backups (Restic) :: systemd service
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# License: Trivial work, not eligible to copyright
+# Source file: roles/core/backups/files/systemd/restic.service
+# -------------------------------------------------------------
+#
+# <auto-generated>
+# This file is managed by our rOPS SaltStack repository.
+#
+# Changes to this file may cause incorrect behavior
+# and will be lost if the state is redeployed.
+# </auto-generated>
+
+[Unit]
+Description=Restic Backup
+After=network.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/restic-s3 backup /etc /var/log /home
+User=root
diff --git a/roles/core/backups/files/systemd/restic.timer b/roles/core/backups/files/systemd/restic.timer
new file mode 100644
--- /dev/null
+++ b/roles/core/backups/files/systemd/restic.timer
@@ -0,0 +1,24 @@
+# -------------------------------------------------------------
+# Salt — Backups (Restic) :: systemd timer
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# License: Trivial work, not eligible to copyright
+# Source file: roles/core/backups/files/systemd/restic.timer
+# -------------------------------------------------------------
+#
+# <auto-generated>
+# This file is managed by our rOPS SaltStack repository.
+#
+# Changes to this file may cause incorrect behavior
+# and will be lost if the state is redeployed.
+# </auto-generated>
+
+[Unit]
+Description=Run Restic Backup Daily
+
+[Timer]
+OnCalendar=daily
+Persistent=true
+
+[Install]
+WantedBy=timers.target
diff --git a/roles/core/backups/init.sls b/roles/core/backups/init.sls
new file mode 100644
--- /dev/null
+++ b/roles/core/backups/init.sls
@@ -0,0 +1,11 @@
+# -------------------------------------------------------------
+# Salt — Backups
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# License: Trivial work, not eligible to copyright
+# -------------------------------------------------------------
+
+include:
+ - .software
+ - .config
+ - .service
diff --git a/roles/core/backups/service.sls b/roles/core/backups/service.sls
new file mode 100644
--- /dev/null
+++ b/roles/core/backups/service.sls
@@ -0,0 +1,46 @@
+# -------------------------------------------------------------
+# Salt — Backups
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# License: Trivial work, not eligible to copyright
+# -------------------------------------------------------------
+
+{% from "map.jinja" import dirs, services with context %}
+
+# -------------------------------------------------------------
+# Linux / systemd
+# - - - -- - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+{% if services["manager"] == "systemd" %}
+/etc/systemd/system/restic.service:
+ file.managed:
+ - source: salt://roles/core/backups/files/systemd/restic.service
+
+/etc/systemd/system/restic.timer:
+ file.managed:
+ - source: salt://roles/core/backups/files/systemd/restic.timer
+
+restic.timer:
+ service.running:
+ - enable: True
+{% endif %}
+
+# -------------------------------------------------------------
+# FreeBSD / rc
+# - - - -- - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+{% if services["manager"] == "rc" %}
+{{ dirs.etc }}/rc.d/restic:
+ file.managed:
+ - source: salt://roles/core/backups/files/rc/restic.rc
+ - mode: 755
+
+/etc/rc.conf.d/restic:
+ file.managed:
+ - source: salt://roles/core/backups/files/rc/restic.conf
+
+/etc/periodic/daily/999.restic:
+ file.managed:
+ - source: salt://roles/core/backups/files/periodic/999.restic.sh
+ - mode: 755
+{% endif %}
diff --git a/roles/core/backups/software.sls b/roles/core/backups/software.sls
new file mode 100644
--- /dev/null
+++ b/roles/core/backups/software.sls
@@ -0,0 +1,33 @@
+# -------------------------------------------------------------
+# Salt — Backups
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# License: Trivial work, not eligible to copyright
+# -------------------------------------------------------------
+
+{% from "map.jinja" import dirs with context %}
+
+restic_software:
+ pkg.installed:
+ - pkgs:
+ - jq
+ - restic
+
+{{ dirs.bin }}/restic-vault-wrapper:
+ file.managed:
+ - source: salt://roles/core/backups/files/restic-vault-wrapper.sh
+ - user: root
+ - group: ops
+ - mode: 755
+
+{{ dirs.bin }}/restic-s3:
+ file.managed:
+ - source: salt://roles/core/backups/files/restic-s3.sh.jinja
+ - template: jinja
+ - user: root
+ - group: ops
+ - mode: 755
+ - context:
+ restic_conf: {{ dirs.etc }}/restic.conf
+ restic_bin: {{ dirs.bin }}/restic
+ s3_credential: nasqueron/backups/s3
diff --git a/utils/vault/rotate-backups-encryption-keys.py b/utils/vault/rotate-backups-encryption-keys.py
new file mode 100755
--- /dev/null
+++ b/utils/vault/rotate-backups-encryption-keys.py
@@ -0,0 +1,261 @@
+#!/usr/bin/env python3
+
+# -------------------------------------------------------------
+# Vault :: Rotate restic backup encryption keys
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# Description: Generate a new restic password for each
+# (container, prefix) pair and PATCH it to
+# Vault without touching S3 access keys.
+# Dependencies: hvac, pyhcl
+# License: BSD-2-Clause
+# -------------------------------------------------------------
+
+
+import os
+import secrets
+import sys
+from pathlib import Path
+
+import hcl
+import hvac
+from hvac.exceptions import InvalidPath
+
+# -------------------------------------------------------------
+# Default paths
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+
+DEFAULT_LOCALS_PATH = REPO_ROOT / "terraform" / "ovh-ops-backups" / "locals.tf"
+
+VAULT_CA_CERTIFICATE = (
+ REPO_ROOT / "roles" / "core" / "certificates" / "files" / "nasqueron-vault-ca.crt"
+)
+
+# -------------------------------------------------------------
+# Vault kv2
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+VAULT_MOUNT_POINT = "ops"
+SECRET_PATH_PREFIX = "secrets/backups/ovh/s3/"
+
+# -------------------------------------------------------------
+# Backup map
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+
+def load_hcl(path):
+ with open(path) as fd:
+ return hcl.load(fd)
+
+
+def extract_backup_containers(config):
+ return (config.get("locals") or {}).get("backup_containers") or {}
+
+
+def load_backup_containers(source_path=None):
+ path = Path(source_path) if source_path else DEFAULT_LOCALS_PATH
+
+ if not path.is_file():
+ print(f"Backup map not found: {path}", file=sys.stderr)
+ sys.exit(1)
+
+ containers = extract_backup_containers(load_hcl(path))
+ if any(spec.get("clients") for spec in containers.values()):
+ return containers
+
+ print(
+ f"Could not find locals.backup_containers with clients in {path}",
+ file=sys.stderr,
+ )
+ sys.exit(1)
+
+
+# -------------------------------------------------------------
+# Password generation
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+
+def generate_password():
+ return secrets.token_urlsafe(32)
+
+
+def passwords_by_prefix(clients):
+ """
+ One restic password per prefix.
+
+ Clients that share a prefix in the same container therefore
+ receive the same encryption key. Prefixes are not shared
+ across containers.
+ """
+ passwords = {}
+
+ for spec in clients.values():
+ for prefix in spec["prefixes"]:
+ if prefix not in passwords:
+ passwords[prefix] = generate_password()
+
+ return passwords
+
+
+def plan_rotations(backup_containers):
+ plan = []
+
+ for container, spec in backup_containers.items():
+ clients = spec.get("clients") or {}
+ if not clients:
+ print(f"Skipping {container}: no clients declared", file=sys.stderr)
+ continue
+
+ passwords = passwords_by_prefix(clients)
+
+ for client, client_spec in clients.items():
+ prefixes = client_spec["prefixes"]
+ plan.append(
+ {
+ "container": container,
+ "client": client,
+ "prefixes": prefixes,
+ "password": passwords[prefixes[0]],
+ "path": f"{SECRET_PATH_PREFIX}{container}-{client}",
+ }
+ )
+
+ return plan
+
+
+# -------------------------------------------------------------
+# Vault
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+
+def read_vault_token():
+ if "VAULT_TOKEN" in os.environ:
+ return True, os.environ["VAULT_TOKEN"]
+
+ if "HOME" in os.environ:
+ token_path = os.path.join(os.environ["HOME"], ".vault-token")
+ if os.path.isfile(token_path):
+ with open(token_path) as f:
+ return True, f.read().strip()
+
+ return False, None
+
+
+def connect_to_vault(url, token):
+ return hvac.Client(url=url, token=token, verify=str(VAULT_CA_CERTIFICATE))
+
+
+def patch_password(client, path, password):
+ client.secrets.kv.v2.patch(
+ mount_point=VAULT_MOUNT_POINT,
+ path=path,
+ secret={"password": password},
+ )
+
+
+def logical_path(path):
+ return f"{VAULT_MOUNT_POINT}/{path}"
+
+
+def format_target(item):
+ prefixes = ", ".join(item["prefixes"])
+ return f"{logical_path(item['path'])} (prefix: {prefixes})"
+
+
+# -------------------------------------------------------------
+# Application entry-point
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+
+def print_usage():
+ print(
+ f"Usage: {sys.argv[0]} [--dry-run] [path-to-locals.tf]",
+ file=sys.stderr,
+ )
+
+
+def parse_args(argv):
+ dry_run = False
+ source_path = None
+
+ for arg in argv:
+ if arg in ("-h", "--help"):
+ print_usage()
+ sys.exit(0)
+ if arg == "--dry-run":
+ dry_run = True
+ continue
+ if arg.startswith("-"):
+ print(f"Unknown option: {arg}", file=sys.stderr)
+ print_usage()
+ sys.exit(1)
+ source_path = arg
+
+ return dry_run, source_path
+
+
+def apply_rotations(plan, dry_run):
+ if dry_run:
+ for item in plan:
+ print(f"Would update {format_target(item)}")
+ return 0
+
+ if "VAULT_ADDR" not in os.environ:
+ print(
+ "Set VAULT_ADDR environment variable to point to your current Vault installation.",
+ file=sys.stderr,
+ )
+ print(
+ "For example, `export VAULT_ADDR=https://172.27.27.7:8200`",
+ file=sys.stderr,
+ )
+ return 2
+
+ success, token = read_vault_token()
+ if not success:
+ print(
+ "Set VAULT_TOKEN environment variable to your Vault token to authenticate the request.",
+ file=sys.stderr,
+ )
+ print(
+ "Alternatively, you can also store your token in ~/.vault-token.",
+ file=sys.stderr,
+ )
+ return 2
+
+ client = connect_to_vault(os.environ["VAULT_ADDR"], token)
+ errors = 0
+
+ for item in plan:
+ try:
+ patch_password(client, item["path"], item["password"])
+ except InvalidPath:
+ print(
+ f"Secret does not exist: {logical_path(item['path'])}",
+ file=sys.stderr,
+ )
+ errors += 1
+ continue
+
+ print(f"Updated {format_target(item)}")
+
+ return 1 if errors else 0
+
+
+def run(dry_run, source_path):
+ plan = plan_rotations(load_backup_containers(source_path))
+
+ if not plan:
+ print("No backup clients found to update.", file=sys.stderr)
+ return 1
+
+ return apply_rotations(plan, dry_run)
+
+
+if __name__ == "__main__":
+ arg_dry_run, arg_source_path = parse_args(sys.argv[1:])
+ exit_code = run(arg_dry_run, arg_source_path)
+
+ sys.exit(exit_code)

File Metadata

Mime Type
text/plain
Expires
Sat, Sep 26, 06:43 (21 h, 5 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
4122273
Default Alt Text
D4218.diff (20 KB)

Event Timeline

Nasqueron DevCentral · If it had been much bigger the moon would have had a core of ice. · Powered by Phabricator