diff --git a/_modules/credentials.py b/_modules/credentials.py index 556630e..b47818b 100644 --- a/_modules/credentials.py +++ b/_modules/credentials.py @@ -1,237 +1,237 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------- # Salt — Credentials # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # Project: Nasqueron # Description: Credentials-related execution module methods # License: BSD-2-Clause # ------------------------------------------------------------- import os from salt.utils.files import fopen VAULT_PREFIX = "ops/secrets/" # ------------------------------------------------------------- # Configuration # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - def _are_credentials_hidden(): return "CONFIG_PUBLISHER" in os.environ or "state.show_sls" in os.environ.get( "SUDO_COMMAND", "" ) # ------------------------------------------------------------- # HOF utilities # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - def _filter_discard_empty_string_values(haystack): if type(haystack) is dict: return {k: v for k, v in haystack.items() if v != ""} if type(haystack) is list: return [v for v in haystack if v != ""] raise ValueError("Argument isn't a list or a dict: " + str(type(haystack))) def _join_document_fragments(fragments): filtered = _filter_discard_empty_string_values(fragments) return "\n\n".join(filtered) # ------------------------------------------------------------- # Fetch credentials from Vault # # Methods signatures are compatible with Zemke-Rhyne module. # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - def _get_default_secret_path(): return VAULT_PREFIX -def _read_secret(key, prefix=None): +def read_secret(key, prefix=None): + if _are_credentials_hidden(): + return "credential for " + key + if prefix is None: prefix = _get_default_secret_path() return __salt__["vault.read_secret"](f"{prefix}/{key}") def get_password(key, prefix=None): """ A function to fetch credential on Vault CLI Example: salt docker-001 credentials.get_password nasqueron.foo.bar :param key: The key in ops/secrets namespace :param prefix: the prefix path for that key, by default "ops/secrets/" :return: The username """ - if _are_credentials_hidden(): - return "credential for " + key - - return _read_secret(key, prefix)["password"] + return read_secret(key, prefix)["password"] def get_username(key, prefix=None): """ A function to fetch the username associated to a credential through Vault CLI Example: salt docker-001 credentials.get_username nasqueron.foo.bar :param key: The key in ops/secrets namespace :param prefix: the prefix path for that key, by default "ops/secrets/" :return: The secret value """ - return _read_secret(key, prefix)["username"] + return read_secret(key, prefix)["username"] def get_token(key, prefix=None): """ A function to fetch credential through Vault CLI Example: salt docker-001 credentials.get_token nasqueron.foo.bar :param key: The key in ops/secrets namespace :param prefix: the prefix path for that key, by default "ops/secrets/" :return: The secret value For Vault, this is actually an alias of the get_password method. """ return get_password(key, prefix) def get_sentry_dsn(args): if _are_credentials_hidden(): return "credential for " + args["credential"] host = __pillar__["sentry_realms"][args["realm"]]["host"] - credential = _read_secret(args["credential"]) + credential = read_secret(args["credential"]) return ( f"https://{credential['username']}:{credential['password']}" f"@{host}/{args['project_id']}" ) # ------------------------------------------------------------- # Build Vault policies # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - class VaultSaltRolePolicy: def __init__(self, role): self.role = role def build_policy(self): return _join_document_fragments( [ self.build_read_secrets_policy(), self.import_extra_policies(), ] ) # # Secrets from pillar entry vault_secrets_by_role # def build_read_secrets_policy(self): vault_paths = __pillar__["vault_secrets_by_role"].get(self.role, []) return _join_document_fragments( [self.get_read_rule(vault_path) for vault_path in vault_paths] ) def get_read_rule(self, vault_path): resolved_vault_path = self.resolve_vault_path(vault_path) return f"""path \"{resolved_vault_path}\" {{ capabilities = [ \"read\" ] }}""" @staticmethod def resolve_vault_path(vault_path): for pillar_path, mount_path in __pillar__.get("vault_mount_paths", {}).items(): if vault_path.startswith(pillar_path): start_position = len(pillar_path) return mount_path + vault_path[start_position:] return vault_path # # Import policies from pillar entry vault_extra_policies_by_role # def import_extra_policies(self): extra_policies = __pillar__["vault_extra_policies_by_role"].get(self.role, []) return _join_document_fragments( [self.import_policy(policy) for policy in extra_policies] ) @staticmethod def import_policy(policy): policy_file = f"{__pillar__['vault_policies_source']}/{policy}.hcl" if policy_file.startswith("salt://"): policy_file = __salt__["cp.cache_file"](policy_file) with fopen(policy_file) as fd: return fd.read() def _compile_roles_policies(): return { role: VaultSaltRolePolicy(role).build_policy() for role in _get_relevant_roles() } def _get_relevant_roles(): return { role for pillar_entry in [ "vault_extra_policies_by_role", "vault_secrets_by_role", ] for role in __pillar__[pillar_entry].keys() } def _build_node_policy(node, roles_policies): rules = [ roles_policies[role] for role in __salt__["node.get"]("roles", node) if role in roles_policies ] policy = _join_document_fragments(rules) if not policy: policy = "# This policy is intentionally left blank." return policy def build_policies_by_node(): roles_policies = _compile_roles_policies() policies = { node: _build_node_policy(node, roles_policies) for node in __pillar__["nodes"].keys() } return policies diff --git a/pillar/credentials/vault.sls b/pillar/credentials/vault.sls index dff398c..5d439a9 100644 --- a/pillar/credentials/vault.sls +++ b/pillar/credentials/vault.sls @@ -1,137 +1,141 @@ # ------------------------------------------------------------- # Salt configuration for Nasqueron servers # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # Project: Nasqueron # License: Trivial work, not eligible to copyright # ------------------------------------------------------------- # ------------------------------------------------------------- # Vault configuration # # :: vault_policies_path: path on vault server where to store policies # # :: vault_policies_source: path to fetch policies from # if starting by salt://, from salt files server # # :: vault_mount_paths: translates secrets paths in policies paths # # Generally, Vault paths are the same for policies and data access. # # For kv secrets engine, version 2, writing and reading versions # of a kv value are prefixed with the data/ path. # # credentials.build_policies_by_node will use this dictionary # to be able to rewrite secrets paths in data paths. # # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - vault_policies_path: /srv/policies/vault vault_policies_source: salt://roles/vault/policies/files vault_mount_paths: ops/secrets: ops/data/secrets ops/privacy: ops/data/privacy # ------------------------------------------------------------- # Vault policies to deploy as-is, ie without templating. # # Entries of vault_policies must match a .hcl file in # roles/vault/policies/files folder. # # If you need a template, create a new pillar entry instead # and add the parsing logic either: # - directly to roles/vault/policies/ # # - through _modules/credentials.py for policies to apply # to Salt nodes, like e.g. vault_secrets_by_role # # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - vault_policies: - salt-primary + - viperserv # ------------------------------------------------------------- # Vault policies for Salt # # Declare the extra policies each nodes need. # # In adition of those extra policies, the vault_secrets_by_role # will be parsed for the keys. # # IMPORTANT: as grains['roles'] can be modified by the node, # roles are extracted directly from the pillar. # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - vault_extra_policies_by_role: salt-primary: - salt-primary # ------------------------------------------------------------- # Vault secrets by role # # Paths of the keys the specified role needs access to. # # Avoid * notation as this namespace is shared between Vault # and the applications. As such, only secrets the Salt nodes # needs in a state they need to deploy should be listed here. # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - vault_secrets_by_role: opensearch: - ops/secrets/nasqueron.opensearch.infra-logs.internal_users.admin - ops/secrets/nasqueron.opensearch.infra-logs.internal_users.dashboards paas-docker-prod: # # Personal data or personally identifiable information (PII) # related to Nasqueron Operations SIG members. # - ops/privacy/ops-cidr # # Credentials used by Nasqueron services # Format: ops/secrets/nasqueron.. # - ops/secrets/nasqueron.acquisitariat.mysql - ops/secrets/nasqueron.auth-grove.mysql - ops/secrets/nasqueron.cachet.app_key - ops/secrets/nasqueron.cachet.mysql - ops/secrets/nasqueron.etherpad.api - ops/secrets/nasqueron.notifications.broker - ops/secrets/nasqueron.notifications.mailgun - ops/secrets/nasqueron.notifications.sentry - ops/secrets/nasqueron.pixelfed.app_key - ops/secrets/nasqueron.pixelfed.mailgun - ops/secrets/nasqueron.pixelfed.mysql - ops/secrets/nasqueron.sentry.app_key - ops/secrets/nasqueron.sentry.postgresql # # Credentials used by Nasqueron members private services # Format: .. # - ops/secrets/dereckson.phabricator.mysql # # Credentials used by projects hosted by Nasqueron # Format: .. # - ops/secrets/espacewin.bugzilla.mysql - ops/secrets/wolfplex.phabricator.mailgun - ops/secrets/wolfplex.phabricator.mysql - ops/secrets/zed.phabricator.mysql - ops/secrets/zed.phabricator.sendgrid + + viperserv: + - ops/secrets/nasqueron.viperserv.vault diff --git a/pillar/nodes/nodes.sls b/pillar/nodes/nodes.sls index ede2240..a618031 100644 --- a/pillar/nodes/nodes.sls +++ b/pillar/nodes/nodes.sls @@ -1,228 +1,229 @@ # ------------------------------------------------------------- # Salt — Nodes # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # Project: Nasqueron # Created: 2017-10-20 # License: Trivial work, not eligible to copyright # ------------------------------------------------------------- nodes_aliases: netmasks: intranought: &intranought_netmask 255.255.255.240 nodes: ## ## Forest: Nasqueron ## Semantic field: https://devcentral.nasqueron.org/P27 ## cloudhugger: forest: nasqueron-infra hostname: cloudhugger.nasqueron.org roles: - opensearch network: ipv6_native: True ipv6_tunnel: False canonical_public_ipv4: 188.165.200.229 interfaces: eno1: device: eno1 ipv4: address: 188.165.200.229 gateway: 188.165.200.254 ipv6: address: fe80::ec4:7aff:fe6a:36e8 prefix: 64 gateway: fe80::ee30:91ff:fee0:df80 complector: forest: nasqueron-infra hostname: complector.nasqueron.org roles: - vault - salt-primary zfs: pool: zroot network: ipv6_tunnel: False interfaces: intranought: device: vmx0 ipv4: address: 172.27.27.7 netmask: *intranought_netmask gateway: 172.27.27.1 dwellers: forest: nasqueron-dev-docker hostname: dwellers.nasqueron.org roles: - paas-lxc - paas-docker - paas-docker-dev - mastodon flags: install_docker_devel_tools: True network: ipv6_tunnel: True canonical_public_ipv4: 51.255.124.11 interfaces: public: device: ens192 uuid: 6e05ebea-f2fd-4ca1-a21f-78a778664d8c ipv4: address: 51.255.124.11 netmask: 255.255.255.252 gateway: 91.121.86.254 intranought: device: ens224 uuid: 8e8ca793-b2eb-46d8-9266-125aba6d06c4 ipv4: address: 172.27.27.4 netmask: *intranought_netmask gateway: 172.27.27.1 docker-001: forest: nasqueron-infra hostname: docker-001.nasqueron.org roles: - paas-docker - paas-docker-prod network: ipv6_tunnel: False canonical_public_ipv4: 51.255.124.9 interfaces: public: device: ens192 uuid: ef7370c5-5060-4d89-82bb-dbeabf4a35f6 ipv4: address: 51.255.124.9 netmask: 255.255.255.252 gateway: 91.121.86.254 intranought: device: ens224 uuid: 3fd0b9f8-ecc3-400d-bc61-3ba21d0b6337 ipv4: address: 172.27.27.6 netmask: *intranought_netmask gateway: 172.27.27.1 router-001: forest: nasqueron-infra hostname: router-001.nasqueron.org roles: - router network: ipv6_tunnel: False canonical_public_ipv4: 51.255.124.8 interfaces: public: device: vmx0 ipv4: address: 51.255.124.8 netmask: 255.255.255.252 gateway: 91.121.86.254 flags: - ipv4_ovh_failover intranought: device: vmx1 ipv4: address: 172.27.27.1 netmask: *intranought_netmask ysul: forest: nasqueron-dev hostname: ysul.nasqueron.org roles: - devserver - dbserver-mysql + - viperserv - webserver-legacy zfs: pool: arcology network: ipv6_tunnel: True ipv6_gateway: 2001:470:1f12:9e1::1 canonical_public_ipv4: 212.83.187.132 interfaces: igb0: device: igb0 ipv4: address: 163.172.49.16 netmask: 255.255.255.0 gateway: 163.172.49.1 aliases: - 212.83.187.132 windriver: forest: nasqueron-dev hostname: windriver.nasqueron.org roles: - devserver - dbserver-mysql - webserver-legacy zfs: pool: arcology network: ipv6_native: True ipv6_tunnel: False canonical_public_ipv4: 51.159.18.59 interfaces: igb0: device: igb0 ipv4: address: 51.159.18.59 netmask: 255.255.255.0 gateway: 51.159.18.1 ipv6: address: 2001:0bc8:6005:0005:aa1e:84ff:fef3:5d9c gateway: fe80::a293:51ff:feb7:5073 prefix: 128 ## ## Forest: Eglide ## Semantic field: ? (P27 used for "Eglide" too) ## ## This forest is intended to separate credentials ## between Eglide and Nasqueron servers. ## eglide: forest: eglide hostname: eglide.org roles: - shellserver network: ipv6_tunnel: True canonical_public_ipv4: 51.159.150.221 interfaces: ens2: device: ens2 ipv4: address: 51.159.150.221 gateway: "" flags: # This interface is configured by cloud-init - skip_interface_configuration fixes: rsyslog_xconsole: True diff --git a/roles/vault/policies/files/viperserv.hcl b/roles/vault/policies/files/viperserv.hcl new file mode 100644 index 0000000..40465fc --- /dev/null +++ b/roles/vault/policies/files/viperserv.hcl @@ -0,0 +1,18 @@ +# ------------------------------------------------------------- +# Vault configuration - Policy for ViperServ eggdrops +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Project: Nasqueron +# License: Trivial work, not eligible to copyright +# Source file: roles/vault/vault/files/viperserv.hcl +# ------------------------------------------------------------- +# +# +# This file is managed by our rOPS SaltStack repository. +# +# Changes to this file may cause incorrect behavior +# and will be lost if the state is redeployed. +# + +path "apps/data/viperserv/*" { + capabilities = [ "read" ] +} diff --git a/roles/viperserv/eggdrop/config.sls b/roles/viperserv/eggdrop/config.sls index 85a4fc0..c0609a8 100644 --- a/roles/viperserv/eggdrop/config.sls +++ b/roles/viperserv/eggdrop/config.sls @@ -1,96 +1,103 @@ # ------------------------------------------------------------- # Salt — Deploy eggdrop park # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # Project: Nasqueron # Created: 2017-11-14 # License: Trivial work, not eligible to copyright # ------------------------------------------------------------- # ------------------------------------------------------------- # Directory for configuration # # Each bot gets a directory to store userlist, chanlist, motd, # and specific configuration file. # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - {% for botname, bot in pillar['viperserv_bots'].items() %} /srv/viperserv/{{ botname }}: file.directory: - user: {{ bot['runas'] | default('viperserv') }} - group: nasqueron-irc - dir_mode: 770 {% endfor %} # ------------------------------------------------------------- # Logs # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - {% for botname, bot in pillar['viperserv_bots'].items() %} /srv/viperserv/logs/{{ botname }}: file.directory: - user: {{ bot['runas'] | default('viperserv') }} - group: nasqueron-irc /srv/viperserv/logs/{{ botname }}.log: file.managed: - user: {{ bot['runas'] | default('viperserv') }} - group: nasqueron-irc - mode: 660 - replace: False {% endfor %} # ------------------------------------------------------------- # Configuration files # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - /srv/viperserv/core.conf: file.managed: - source: salt://roles/viperserv/eggdrop/files/eggdrop-core.conf - user: viperserv - group: nasqueron-irc /srv/viperserv/.credentials: file.managed: - source: salt://roles/viperserv/eggdrop/files/dot.credentials - user: viperserv - group: nasqueron-irc - - replace: False - - mode: 660 + - mode: 400 + - template: jinja + - context: + db: + host: localhost + database: Nasqueron + vault: + approle: {{ salt['credentials.read_secret']('nasqueron.viperserv.vault') }} + addr: https://172.27.27.7:8200 {% for botname, bot in pillar['viperserv_bots'].items() %} /srv/viperserv/{{ botname }}/eggdrop.conf: file.managed: - source: salt://roles/viperserv/eggdrop/files/eggdrop-bot.conf - user: {{ bot['runas'] | default('viperserv') }} - group: nasqueron-irc - mode: 755 - template: jinja - context: botname: {{ botname }} realname: {{ bot['realname'] | default(botname) }} scripts: {{ bot['scripts'] }} modules: {{ bot['modules'] | default([]) }} runas: {{ bot['runas'] | default('viperserv') }} nickserv: {{ bot['nickserv'] | default(False) }} /srv/viperserv/{{ botname }}/motd: file.managed: - source: salt://roles/viperserv/eggdrop/files/motd/{{ botname }} - user: {{ bot['runas'] | default('viperserv') }} - group: nasqueron-irc /srv/viperserv/{{ botname }}/banner: file.managed: - source: salt://roles/viperserv/eggdrop/files/banner - user: {{ bot['runas'] | default('viperserv') }} - group: nasqueron-irc - template: jinja - context: bot: {{ botname }} server: {{ grains['id'] }} {% endfor %} diff --git a/roles/viperserv/eggdrop/files/dot.credentials b/roles/viperserv/eggdrop/files/dot.credentials index 75a114d..b4619b8 100644 --- a/roles/viperserv/eggdrop/files/dot.credentials +++ b/roles/viperserv/eggdrop/files/dot.credentials @@ -1,21 +1,31 @@ # ------------------------------------------------------------- # Eggdrop configuration file # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# License: Trivial work, not eligible to copyright +# Source file: roles/viperserv/eggdrop/files/dot.credentials +# ------------------------------------------------------------- +# +# +# This file is managed by our rOPS SaltStack repository. +# +# Changes to this file may cause incorrect behavior +# and will be lost if the state is redeployed. +# # # _ ___ _____ # | | / (_)___ ___ _____/ ___/___ ______ __ # | | / / / __ \/ _ \/ ___/\__ \/ _ \/ ___/ | / / # | |/ / / /_/ / __/ / ___/ / __/ / | |/ / # |___/_/ .___/\___/_/ /____/\___/_/ |___/ # /_/ # # [ 1993 technology for 2017 hackers ] # # ------------------------------------------------------------- -set sql(host) localhost -set sql(user) someuser -set sql(pass) somepass -set sql(database) Nasqueron +set sql(host) {{ db.host }} +set sql(database) {{ db.database }} -die "Please configure MySQL credentials in /srv/viperserv/.credentials" +set vault(roleID) {{ vault.approle.roleID }} +set vault(secretID) {{ vault.approle.secretID }} +set vault(host) {{ vault.addr }} diff --git a/roles/viperserv/eggdrop/files/eggdrop-bot.conf b/roles/viperserv/eggdrop/files/eggdrop-bot.conf index 425c0c4..9e214af 100755 --- a/roles/viperserv/eggdrop/files/eggdrop-bot.conf +++ b/roles/viperserv/eggdrop/files/eggdrop-bot.conf @@ -1,98 +1,111 @@ #!/usr/bin/env eggdrop # ------------------------------------------------------------- # Eggdrop configuration file # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # License: Trivial work, not eligible to copyright # Source file: roles/viperserv/eggdrop/files/eggdrop-bot.conf # Pillar file: pillar/viperserv/bots.sls # ------------------------------------------------------------- # # # This file is managed by our rOPS SaltStack repository. # # Changes to this file may cause incorrect behavior # and will be lost if the state is redeployed. # # # _ ___ _____ # | | / (_)___ ___ _____/ ___/___ ______ __ # | | / / / __ \/ _ \/ ___/\__ \/ _ \/ ___/ | / / # | |/ / / /_/ / __/ / ___/ / __/ / | |/ / # |___/_/ .___/\___/_/ /____/\___/_/ |___/ # /_/ # # [ 1993 technology for 2017 hackers ] # # ------------------------------------------------------------- # ------------------------------------------------------------- # Settings for {{ botname }} # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - set runas {{ runas }} set username {{ botname }} set nick $username set altnick {{ botname }}` set realname "{{ realname }}" set vhost6 viperserv.nasqueron.org set listen-addr 2001:470:1f13:9e1:0:c0ff:ee:7 set prefer-ipv6 1 set network libera set net-type 5 # Main eggdrop settings common to all ViperServ bots source core.conf +# Credentials for Vault (work in progress) and MySQL (deprecated) +source .credentials + +# ------------------------------------------------------------- +# Vault +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +source scripts/vendor/vault.tcl +source scripts/Vault.tcl + # ------------------------------------------------------------- # MySQL # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - load lib/fbsql.so -source .credentials -sql connect $sql(host) $sql(user) $sql(pass) -sql2 connect $sql(host) $sql(user) $sql(pass) +set sql_credentials [dict get [vault_get mysql] data] + +sql connect $sql(host) [dict get $sql_credentials username] [dict get $sql_credentials password] +sql2 connect $sql(host) [dict get $sql_credentials username] [dict get $sql_credentials password] sql selectdb $sql(database) sql2 selectdb $sql(database) +unset sql_credentials + # ------------------------------------------------------------- # Base settings with scripts dependencies # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # Dependencies source scripts/Core.tcl # Settings {%- if nickserv %} -set nickserv_password [registry get nickserv.$username.password] +set nickserv_password [vault_get nickserv/$username password] set servers " irc.libera.chat:+6697:$username:$nickserv_password " {% else %} set servers { irc.libera.chat:+6697 } {%- endif %} # ------------------------------------------------------------- # Eggdrop modules # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - {% for module in modules %} loadmodule {{ module }}{% endfor %} # ------------------------------------------------------------- # Scripts # # These scripts are provided by the rVIPER distribution. # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - source scripts/IRC.tcl source scripts/Tech.tcl {% for script in scripts %} source scripts/{{ script }}{% endfor %} source scripts/vendor/action.fix.tcl