diff --git a/Makefile b/Makefile new file mode 100644 --- /dev/null +++ b/Makefile @@ -0,0 +1,6 @@ +all: generate-webcontent-index + +generate-webcontent-index: + tmpfile=`mktemp /tmp/make-rOPS-generate-webcontent-index.XXXXXX` ; \ + utils/generate-webcontent-index.py > "$$tmpfile" ;\ + mv "$$tmpfile" roles/webserver-content/init.sls diff --git a/_modules/node.py b/_modules/node.py --- a/_modules/node.py +++ b/_modules/node.py @@ -142,6 +142,10 @@ return filtered_list +def has_web_content(content, nodename=None): + return content in filter_by_role('web_content_sls', nodename) + + def get_wwwroot(nodename=None): ''' A function to determine the wwwroot folder to use. diff --git a/_tests/data/forests.yaml b/_tests/data/forests.yaml --- a/_tests/data/forests.yaml +++ b/_tests/data/forests.yaml @@ -39,3 +39,9 @@ amdir: {} amroth: {} galadriel: {} + +web_content_sls: + treecity: + - .ll/carasgaladhon + border: + - .arda/onodlo diff --git a/_tests/modules/test_node.py b/_tests/modules/test_node.py --- a/_tests/modules/test_node.py +++ b/_tests/modules/test_node.py @@ -23,6 +23,14 @@ self.assertEqual("wwwroot/entwash.node/www", node.get_wwwroot('entwash')) + def test_has_web_content(self): + self.assertTrue(node.has_web_content('.ll/carasgaladhon')) + self.assertFalse(node.has_web_content('.arda/onodlo')) + + self.assertTrue(node.has_web_content('.arda/onodlo', 'entwash')) + + self.assertFalse(node.has_web_content('notexisting')) + def test_filter_by_role(self): node_key = self.grains['id'] diff --git a/pillar/top.sls b/pillar/top.sls --- a/pillar/top.sls +++ b/pillar/top.sls @@ -14,8 +14,8 @@ - certificates.certificates - nodes.nodes - nodes.forests + - webserver.sites ysul: - paas-jails.jails - - webserver-legacy.sites - viperserv.bots - viperserv.fantoir diff --git a/pillar/webserver-legacy/sites.sls b/pillar/webserver/sites.sls rename from pillar/webserver-legacy/sites.sls rename to pillar/webserver/sites.sls --- a/pillar/webserver-legacy/sites.sls +++ b/pillar/webserver/sites.sls @@ -57,6 +57,20 @@ autochmod: True php-fpm: prod +# ------------------------------------------------------------- +# States +# +# Sites with states documenting how to build them +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +web_content_sls: + shellserver: + - .com/paysannerebelle + - .org/eglide + webserver-legacy: + - .be/dereckson + - .org/nasqueron/docs + # ------------------------------------------------------------- # Tweaks # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/roles/shellserver/web-hosting/init.sls b/roles/shellserver/web-hosting/init.sls --- a/roles/shellserver/web-hosting/init.sls +++ b/roles/shellserver/web-hosting/init.sls @@ -7,6 +7,7 @@ # ------------------------------------------------------------- {% from "map.jinja" import dirs with context %} +{% set wwwgroup = "www-data" %} # ------------------------------------------------------------- # Nginx configuration files @@ -32,7 +33,19 @@ /var/log/www: file.directory: - user: root - - group: www-data + - group: {{ wwwgroup }} + - dir_mode: 750 + +/var/log/www/eglide.org: + file.directory: + - user: root + - group: {{ wwwgroup }} + - dir_mode: 750 + +/var/log/www/paysannerebelle.com: + file.directory: + - user: hlp + - group: {{ wwwgroup }} - dir_mode: 750 # ------------------------------------------------------------- diff --git a/roles/webserver-content/README.md b/roles/webserver-content/README.md new file mode 100644 --- /dev/null +++ b/roles/webserver-content/README.md @@ -0,0 +1,65 @@ +# Webserver content + +## Goal of this role + +This role provisions the `/var/wwwroot` folder with the website content, +when there is a custom logic to prepare it, like a specific Git repository +to clone, or a build process to follow. + +This roles does NOT describe web server configuration, +which is done in other `webserver-` roles. + +## Structure + +This role doesn't follow the role/unit folder hierarchy. + +Instead, it follows a tld/domain/subdomain.sls logic. + +For example, the folder for the `*.acme.tld` sites will be `tld/acme`. +This structure goal is to play nice with the Salt include syntax, as dots +are a directory spearator. + +The bipbip.acme.tld site will be described in `tld/acme/bipbip.sls` file. + +## Add a new domain + + 1. Create a new folder hierarchy for the domain + 2. Include a `init.sls` file for your subdomains + 3. Declare the new domain in pillar/webserver/sites.sls + 4. Regenerate the role index with utils/generate-webcontent-index.py (or make) + utils/generate-webcontent-index.py > roles/webserver-content/init.sls + +For example the tld/acme/init.sls file could be: +``` +include: + - .www + - .acme +``` + +Alphabetical order is followed, but www is generally first. + +In the pillar file, website are assigned to a role. + +If you wish to deploy all the sites on one role, you can directly include +the folder, and your init.sls will do the rest. + +If not, two strategies exist: you can use node.filter_by_role in your +init.sls too or perhaps more simply you can document in init.sls this +roles can't be deployed directly, and make references to sls files in +the pillar (without final .sls extension). + +For example to deploy bipbip.acme.tld (`tld/acme/bipbip.sls`) on servers +with the shellserver role: + +``` +shellserver: + - .tld/acme/bibpip +``` + +## Prune old files + +If you need to prune a former website, you can add +the directory to the /hotfixes/old-directories.sls state. + +There is no need to revert your commit when the +directories or files are deleted. diff --git a/roles/webserver-legacy/be/dereckson/assets.sls b/roles/webserver-content/be/dereckson/assets.sls rename from roles/webserver-legacy/be/dereckson/assets.sls rename to roles/webserver-content/be/dereckson/assets.sls --- a/roles/webserver-legacy/be/dereckson/assets.sls +++ b/roles/webserver-content/be/dereckson/assets.sls @@ -5,6 +5,8 @@ # License: Trivial work, not eligible to copyright # ------------------------------------------------------------- +{% if salt['node.has_web_content'](".org/nasqueron/assets") %} + # ------------------------------------------------------------- # Deploy /opt/staging/wwwroot/d.be/assets to assets.d.be # @@ -23,3 +25,5 @@ - file_mode: 644 - user: dereckson.be - group: web + +{% endif %} diff --git a/roles/webserver-legacy/be/dereckson/init.sls b/roles/webserver-content/be/dereckson/init.sls rename from roles/webserver-legacy/be/dereckson/init.sls rename to roles/webserver-content/be/dereckson/init.sls --- a/roles/webserver-legacy/be/dereckson/init.sls +++ b/roles/webserver-content/be/dereckson/init.sls @@ -7,4 +7,3 @@ include: - .assets - - .www diff --git a/roles/webserver-legacy/be/dereckson/init.sls b/roles/webserver-content/com/paysannerebelle/init.sls rename from roles/webserver-legacy/be/dereckson/init.sls rename to roles/webserver-content/com/paysannerebelle/init.sls --- a/roles/webserver-legacy/be/dereckson/init.sls +++ b/roles/webserver-content/com/paysannerebelle/init.sls @@ -1,10 +1,9 @@ # ------------------------------------------------------------- -# Salt — Provision *.dereckson.be sites +# Salt — Provision *.paysannerebelle.com sites # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# Project: DcK Area +# Project: Collectif des paysannes et paysans rebelles # License: Trivial work, not eligible to copyright # ------------------------------------------------------------- include: - - .assets - - .www + - .robot diff --git a/roles/webserver-content/com/paysannerebelle/robot.sls b/roles/webserver-content/com/paysannerebelle/robot.sls new file mode 100644 --- /dev/null +++ b/roles/webserver-content/com/paysannerebelle/robot.sls @@ -0,0 +1,24 @@ +# ------------------------------------------------------------- +# Salt — Provision robot.paysannerebelle.com website +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Project: Collectif des paysannes et paysans rebelles +# Created: 2017-04-16 +# License: Trivial work, not eligible to copyright +# ------------------------------------------------------------- + +{% if salt['node.has_web_content'](".com/paysannerebelle") %} + +{% set wwwgroup = "www-data" %} + +# ------------------------------------------------------------- +# Site directory +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +/var/wwwroot/paysannerebelle.com/robot: + file.directory: + - user: hlp + - group: {{ wwwgroup }} + - dir_mode: 711 + - makedirs: True + +{% endif %} diff --git a/roles/webserver-legacy/org/nasqueron/init.sls b/roles/webserver-content/init.sls rename from roles/webserver-legacy/org/nasqueron/init.sls rename to roles/webserver-content/init.sls --- a/roles/webserver-legacy/org/nasqueron/init.sls +++ b/roles/webserver-content/init.sls @@ -1,9 +1,13 @@ # ------------------------------------------------------------- -# Salt — Provision *.nasqueron.org sites +# Salt — Webserver content # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# Project: Nasqueron +# Project: Eglide +# Created: 2017-11-23 # License: Trivial work, not eligible to copyright # ------------------------------------------------------------- include: - - .docs + - .be/dereckson + - .com/paysannerebelle + - .org/eglide + - .org/nasqueron/docs diff --git a/roles/webserver-legacy/org/nasqueron/init.sls b/roles/webserver-content/org/eglide/init.sls copy from roles/webserver-legacy/org/nasqueron/init.sls copy to roles/webserver-content/org/eglide/init.sls --- a/roles/webserver-legacy/org/nasqueron/init.sls +++ b/roles/webserver-content/org/eglide/init.sls @@ -1,9 +1,9 @@ # ------------------------------------------------------------- -# Salt — Provision *.nasqueron.org sites +# Salt — Provision *.eglide.org sites # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# Project: Nasqueron +# Project: Eglide # License: Trivial work, not eligible to copyright # ------------------------------------------------------------- include: - - .docs + - .www diff --git a/roles/shellserver/eglide-website/init.sls b/roles/webserver-content/org/eglide/www.sls rename from roles/shellserver/eglide-website/init.sls rename to roles/webserver-content/org/eglide/www.sls --- a/roles/shellserver/eglide-website/init.sls +++ b/roles/webserver-content/org/eglide/www.sls @@ -6,6 +6,8 @@ # License: Trivial work, not eligible to copyright # ------------------------------------------------------------- +{% if salt['node.has_web_content'](".org/eglide") %} + # ------------------------------------------------------------- # Deploy /opt/staging/wwwroot/eglide.org/www to www.eglide.org # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -33,25 +35,4 @@ - dir_mode: 711 - file_mode: 644 -/var/wwwroot/paysannerebelle.com/robot/: - file.directory: - - user: hlp - - group: {{ wwwgroup }} - - dir_mode: 711 - - makedirs: True - -# ------------------------------------------------------------- -# Nginx logs -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -/var/log/www/eglide.org: - file.directory: - - user: root - - group: {{ wwwgroup }} - - dir_mode: 750 - -/var/log/www/paysannerebelle.com: - file.directory: - - user: hlp - - group: {{ wwwgroup }} - - dir_mode: 750 +{% endif %} diff --git a/roles/webserver-legacy/org/nasqueron/docs.sls b/roles/webserver-content/org/nasqueron/docs.sls rename from roles/webserver-legacy/org/nasqueron/docs.sls rename to roles/webserver-content/org/nasqueron/docs.sls --- a/roles/webserver-legacy/org/nasqueron/docs.sls +++ b/roles/webserver-content/org/nasqueron/docs.sls @@ -5,6 +5,8 @@ # License: Trivial work, not eligible to copyright # ------------------------------------------------------------- +{% if salt['node.has_web_content'](".org/nasqueron/docs") %} + {% from "map.jinja" import packages with context %} # ------------------------------------------------------------- @@ -44,3 +46,5 @@ sphinx: pkg.installed: - name: {{ packages.sphinx }} + +{% endif %} diff --git a/roles/webserver-legacy/org/nasqueron/files/build-docs-salt-wrapper.sh b/roles/webserver-content/org/nasqueron/files/build-docs-salt-wrapper.sh rename from roles/webserver-legacy/org/nasqueron/files/build-docs-salt-wrapper.sh rename to roles/webserver-content/org/nasqueron/files/build-docs-salt-wrapper.sh diff --git a/roles/webserver-legacy/org/nasqueron/init.sls b/roles/webserver-content/org/nasqueron/init.sls rename from roles/webserver-legacy/org/nasqueron/init.sls rename to roles/webserver-content/org/nasqueron/init.sls --- a/roles/webserver-legacy/org/nasqueron/init.sls +++ b/roles/webserver-content/org/nasqueron/init.sls @@ -5,5 +5,7 @@ # License: Trivial work, not eligible to copyright # ------------------------------------------------------------- -include: - - .docs +# This section is intentionally left blank. + +# As Nasqueron sites are distributed among several servers, +# per domain files should be directly included instead. diff --git a/top.sls b/top.sls --- a/top.sls +++ b/top.sls @@ -17,6 +17,7 @@ - roles/core/sshd - roles/core/sysctl - roles/core/users + - roles/webserver-content 'local': - roles/saltmaster 'ysul': @@ -34,7 +35,6 @@ 'eglide': - roles/webserver-core - roles/shellserver/userland-software - - roles/shellserver/eglide-website - roles/shellserver/vhosts - roles/shellserver/web-hosting - roles/shellserver/database diff --git a/utils/generate-webcontent-index.py b/utils/generate-webcontent-index.py new file mode 100755 --- /dev/null +++ b/utils/generate-webcontent-index.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 + +# ------------------------------------------------------------- +# rOPS — regenerate roles/webserver-content/init.sls +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Project: Nasqueron +# Created: 2017-11-24 +# Description: Read the web_content_sls pillar entry +# and regenerate the webserver-content include. +# License: BSD-2-Clause +# ------------------------------------------------------------- + + +import yaml + + +# ------------------------------------------------------------- +# Table of contents +# ------------------------------------------------------------- +# +# :: Configuration +# :: Update code +# :: Run task +# +# ------------------------------------------------------------- + +# ------------------------------------------------------------- +# Configuration +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + +pillar_file = "pillar/webserver/sites.sls" +file_to_update = "roles/webserver-content/init.sls" + + +# ------------------------------------------------------------- +# Update code +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + +def do_update(pillar_file, file_to_update): + print_header(file_to_update) + print("\ninclude:") + for site in get_sites(pillar_file): + print(" - {}".format(site)) + + +def get_pillar_entry(pillar_file, key): + with open(pillar_file) as fd: + pillar = yaml.load(fd.read()) + return pillar[key] + + +def get_sites(pillar_file): + sites = get_pillar_entry(pillar_file, 'web_content_sls') + return sorted([site for sublist in + [sites[role] for role in sites] + for site in sublist]) + + +def print_header(file_to_update): + with open(file_to_update) as fd: + for line in fd: + if not line.startswith("#"): + break + print(line, end="") + + +# ------------------------------------------------------------- +# Run task +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + +if __name__ == "__main__": + do_update(pillar_file, file_to_update)