diff --git a/pillar/webserver/sites.sls b/pillar/webserver/sites.sls index 7e5de72..b2fa380 100644 --- a/pillar/webserver/sites.sls +++ b/pillar/webserver/sites.sls @@ -1,186 +1,190 @@ # ------------------------------------------------------------- # Salt — Sites to provision on the legacy web server # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # Project: Nasqueron # License: Trivial work, not eligible to copyright # ------------------------------------------------------------- # ------------------------------------------------------------- # Domains we deploy # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - web_domains: # # Directly managed by Nasqueron # nasqueron: - nasqueron.org # # Nasqueron members # nasqueron_members: - dereckson.be # # Projects ICT is managed by Nasqueron # espacewin: - espace-win.org wolfplex: - wolfplex.org # ------------------------------------------------------------- # Static sites # # Sites to deploy from the staging repository # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - web_static_sites: nasqueron.org: - www - assets - docker - ftp - packages - trustspace wolfplex.org: - www # ------------------------------------------------------------- # PHP sites # # Username must be unique and use max 31 characters. # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - php_custom_builds: php56: mode: release version: 5.6.38 hash: d65b231bbdd63be4439ef5ced965cfd63e62983429dbd4dfcfb49981593ebc03 php_fpm_instances: # PHP 7.2, generally installed as package/port prod: command: /usr/local/sbin/php-fpm + # PHP 5.6, installed through php-builder unit + legacy: + command: /opt/php/php56/sbin/php-fpm + web_php_sites: # Nasqueron members mediawiki.dereckson.be: domain: dereckson.be subdomain: mediawiki user: web-be-dereckson-mw php-fpm: prod www.dereckson.be: domain: dereckson.be subdomain: www user: web-be-dereckson-www source: wwwroot/dereckson.be/www target: /var/wwwroot/dereckson.be/www php-fpm: prod www51.dereckson.be: domain: dereckson.be subdomain: www51 user: web-be-dereckson-www51 php-fpm: prod # Directly managed by Nasqueron api.nasqueron.org: domain: nasqueron.org subdomain: api user: web-org-nasqueron-api-serverslog php-fpm: prod env: SERVERS_LOG_FILE: /srv/api/data/servers-log-all.json wikis.nasqueron.org: domain: nasqueron.org subdomain: wikis user: mediawiki php-fpm: prod skipCreateAccount: True env: MEDIAWIKI_ENTRY_POINT: /srv/mediawiki/index.php DB_HOST: localhost DB_USER: mediawiki-saas # Espace Win www.espace-win.org: domain: espace-win.org subdomain: www user: web-org-espacewin-www source: wwwroot/espace-win.org/www target: /var/wwwroot/espace-win.org/www - php-fpm: prod + php-fpm: legacy www51.espace-win.org: domain: espace-win.org subdomain: www51 user: web-org-espacewin-www51 php-fpm: prod # Wolfplex Hackerspace www.wolfplex.org: domain: wolfplex.org subdomain: www user: web-org-wolfplex-www php-fpm: prod env: DATASTORE: /var/dataroot/wolfplex CREDENTIAL_PATH_DATASOURCES_SECURITYDATA: /var/dataroot/wolfplex/secrets.json # ------------------------------------------------------------- # States # # Sites with states documenting how to build them # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - web_content_sls: # # Eglide # shellserver: # Third party sites hosted to Eglide - .com/paysannerebelle # Directly managed by Eglide project - .org/eglide # # Nasqueron servers # mastodon: - .org/nasqueron/social webserver-legacy: # Nasqueron members - .be/dereckson # Projects hosted - .space/hypership # Directly managed by Nasqueron - .org/nasqueron/api - .org/nasqueron/daeghrefn - .org/nasqueron/docs - .org/nasqueron/infra - .org/nasqueron/labs - .org/nasqueron/rain # Wolfplex Hackerspace - .org/wolfplex/api - .org/wolfplex/www # ------------------------------------------------------------- # Tweaks # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - web_autochmod: - /var/wwwroot/dereckson.be/www diff --git a/roles/webserver-legacy/php-sites/cleanup.sls b/roles/webserver-legacy/php-sites/cleanup.sls new file mode 100644 index 0000000..25f9ec2 --- /dev/null +++ b/roles/webserver-legacy/php-sites/cleanup.sls @@ -0,0 +1,40 @@ +#!py + +# ------------------------------------------------------------- +# Salt — Provision PHP websites — php-fpm pools +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Project: Nasqueron +# License: Trivial work, not eligible to copyright +# Description: When a site is declared to be served by +# an instance, pools from other instances +# should be deleted if they exist. +# +# That allows to move pools among instances. +# ------------------------------------------------------------- + +def get_etc_dir(): + if __grains__['os'] == 'FreeBSD': + return "/usr/local/etc" + + return "/etc" + + +def files_to_delete_if_they_exist(): + files = [] + etc_dir = get_etc_dir() + for instance in __pillar__['php_fpm_instances']: + files.extend([etc_dir + "/php-fpm.d/" + instance + "-pools/" + site['user'] + ".conf" + for _, site in __pillar__['web_php_sites'].items() + if site['php-fpm'] != instance]) + + return files + + +def run(): + config = {} + + # Task: delete php-fpm stale files + for file in files_to_delete_if_they_exist(): + config[file] = {"file.absent": []} + + return config diff --git a/roles/webserver-legacy/php-sites/files/rc/php-fpm b/roles/webserver-legacy/php-sites/files/rc/php-fpm index bf9de6c..58a0ed5 100644 --- a/roles/webserver-legacy/php-sites/files/rc/php-fpm +++ b/roles/webserver-legacy/php-sites/files/rc/php-fpm @@ -1,125 +1,126 @@ #!/bin/sh # PROVIDE: php-fpm # REQUIRE: LOGIN # KEYWORD: shutdown # Add the following lines to /etc/rc.conf to enable php-fpm: # php_fpm_(instance_)?enable (bool): Set to "NO" by default. # Set it to "YES" to enable php-fpm. # php_fpm_(instance_)?umask (str): Custom PID file path and name. # Set it to to define umask before process start # php_fpm_(instance_)?command (str): Command to run # Default to /usr/local/sbin/php-fpm (port binary) # php_fpm_instances (str): Set to "" by default. # If defined, list of instances to enable . /etc/rc.subr name="php_fpm" rcvar=php_fpm_enable start_precmd="php_fpm_prestart" restart_precmd="php_fpm_checkconfig" reload_precmd="php_fpm_checkconfig" configtest_cmd="php_fpm_checkconfig" load_rc_config "$name" : ${php_fpm_enable="NO"} : ${php_fpm_umask=""} : ${php_fpm_command="/usr/local/sbin/php-fpm"} extra_commands="reload configtest logrotate" sig_stop="QUIT" sig_reload="USR2" logrotate_cmd="php_fpm_logrotate" # Instances logic has been forked from the MySQL port rc service code. if [ -n "$2" ]; then instance="$2" load_rc_config ${name}_${instance} case "$php_fpm_instances" in "$2 "*|*" $2 "*|*" $2"|"$2") eval php_fpm_umask="\${php_fpm_${instance}_umask:-\"${php_fpm_umask}\"}" eval php_fpm_command="\${php_fpm_${instance}_command:-\"${php_fpm_command}\"}" php_fpm_pidfile="/var/run/php-fpm-${instance}.pid" php_fpm_conf="/usr/local/etc/php-fpm.d/${instance}.conf" php_fpm_name=${instance} ;; *) err 1 "$2 not found in php_fpm_instances" ;; esac else if [ -n "${php_fpm_instances}" -a -n "$1" ]; then for instance in ${php_fpm_instances}; do eval _enable="\${php_fpm_${instance}_enable}" case "${_enable:-${php_fpm_enable}}" in [Nn][Oo]|[Ff][Aa][Ll][Ss][Ee]|[Oo][Ff][Ff]|0) continue ;; [Yy][Ee][Ss]|[Tt][Rr][Uu][Ee]|[Oo][Nn]|1) ;; *) if [ -z "$_enable" ]; then _var=php_fpm_enable else _var=php_fpm_${instance}_enable fi warn "Bad value" \ "'${_enable:-${php_fpm_enable}}'" \ "for ${_var}. " \ "Instance ${instance} skipped." continue ;; esac echo "===> php-fpm instance: ${instance}" if /usr/local/etc/rc.d/php-fpm $1 ${instance}; then success="${instance} ${success}" else failed="${instance} (${retcode}) ${failed}" fi done exit 0 else php_fpm_pidfile=/var/run/php-fpm.pid php_fpm_conf=/usr/local/etc/php-fpm.conf fi fi command=${php_fpm_command} +command_args="--fpm-config ${php_fpm_conf}" pidfile="${php_fpm_pidfile}" required_files="${php_fpm_conf}" php_fpm_logrotate() { if [ -z "$rc_pid" ]; then _run_rc_notrunning return 1 fi echo "Rotating logs $name." kill -USR1 $rc_pid } php_fpm_checkconfig() { echo "Performing sanity check on php-fpm configuration:" eval ${command} -t --fpm-config "${php_fpm_conf}" } php_fpm_prestart() { php_fpm_checkconfig checkconfig=$? if [ $checkconfig -ne 0 ]; then return $checkconfig fi if [ ! -z "$php_fpm_umask" ]; then echo "Setting umask to: ${php_fpm_umask}" umask $php_fpm_umask fi } run_rc_command "$1" diff --git a/roles/webserver-legacy/php-sites/init.sls b/roles/webserver-legacy/php-sites/init.sls index bc899f7..d483c42 100644 --- a/roles/webserver-legacy/php-sites/init.sls +++ b/roles/webserver-legacy/php-sites/init.sls @@ -1,12 +1,13 @@ # ------------------------------------------------------------- # Salt — Provision PHP websites # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # Project: Nasqueron # License: Trivial work, not eligible to copyright # ------------------------------------------------------------- include: - .account - .files - .php - .php-fpm + - .cleanup diff --git a/roles/webserver-legacy/php-sites/php-fpm.sls b/roles/webserver-legacy/php-sites/php-fpm.sls index 2865c1f..11c4b85 100644 --- a/roles/webserver-legacy/php-sites/php-fpm.sls +++ b/roles/webserver-legacy/php-sites/php-fpm.sls @@ -1,93 +1,93 @@ # ------------------------------------------------------------- # Salt — Provision PHP websites — php-fpm pools # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # Project: Nasqueron # License: Trivial work, not eligible to copyright # ------------------------------------------------------------- {% from "map.jinja" import dirs with context %} # ------------------------------------------------------------- # Configuration : instances # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - {% for instance, config in pillar['php_fpm_instances'].items() %} php-fpm_config_{{ instance }}: file.managed: - name: {{ dirs.etc }}/php-fpm.d/{{ instance }}.conf - source: salt://roles/webserver-legacy/php-sites/files/php-fpm.conf - template: jinja - context: instance: {{ instance }} {{ dirs.etc }}/php-fpm.d/{{ instance }}-pools: file.directory {% endfor %} # ------------------------------------------------------------- # Configuration : pools # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - {% for fqdn, site in pillar['web_php_sites'].items() %} php-fpm_pool_{{ site['user'] }}: file.managed: - - name: {{ dirs.etc }}/php-fpm.d/prod-pools/{{ site['user'] }}.conf + - name: {{ dirs.etc }}/php-fpm.d/{{ site['php-fpm'] }}-pools/{{ site['user'] }}.conf - source: salt://roles/webserver-legacy/php-sites/files/php-fpm-pool.conf - template: jinja - context: fqdn: {{ fqdn }} domain: {{ site['domain'] }} subdomain: {{ site['subdomain'] }} user: {{ site['user' ]}} display_errors: {{ site['display_errors']|default('off') }} slow_delay: {{ site['slow_delay']|default('5s') }} env : {{ site['env']|default({}) }} /var/log/www/{{ site['domain' ]}}/{{ site['subdomain' ]}}-php.log: file.managed: - replace: False - user: {{ site['user'] }} - group: web - chmod: 600 {% endfor %} # ------------------------------------------------------------- # Service # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - {% if grains['os'] == 'FreeBSD' %} {% set instances = " ".join(pillar['php_fpm_instances'].keys()) %} # roles/webserver-legacy/php-sites/files/rc/php-fpm /usr/local/etc/rc.d/php-fpm: file.managed: - source: salt://roles/webserver-legacy/php-sites/files/rc/php-fpm - mode: 755 /etc/rc.conf.d/php_fpm: file.directory /etc/rc.conf.d/php_fpm/instances: file.managed: - source: salt://roles/webserver-legacy/php-sites/files/rc/instances - template: jinja - context: instances: {{ instances }} {% for instance, config in pillar['php_fpm_instances'].items() %} /etc/rc.conf.d/php_fpm/{{ instance }}: file.managed: - source: salt://roles/webserver-legacy/php-sites/files/rc/per_instance - template: jinja - context: instance: {{ instance }} command: {{ config['command'] | default('') }} {% endfor %} {% endif %} diff --git a/roles/webserver-legacy/php-sites/php.sls b/roles/webserver-legacy/php-sites/php.sls index e8bc379..26f1ae6 100644 --- a/roles/webserver-legacy/php-sites/php.sls +++ b/roles/webserver-legacy/php-sites/php.sls @@ -1,37 +1,43 @@ # ------------------------------------------------------------- # Salt — Provision PHP websites — php-fpm pools # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # Project: Nasqueron # License: Trivial work, not eligible to copyright # ------------------------------------------------------------- {% from "map.jinja" import dirs with context %} # ------------------------------------------------------------- # PHP global configuration # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - {{ dirs.etc }}/php.ini: file.managed: - source: salt://roles/webserver-legacy/php-sites/files/php.ini +{% for build in pillar['php_custom_builds'] %} +/opt/php/{{ build }}/lib/php.ini: + file.managed: + - source: salt://roles/webserver-legacy/php-sites/files/php.ini +{% endfor %} + # ------------------------------------------------------------- # Sessions directories # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - /var/tmp/php: file.directory: - mode: 1770 - group: web /var/tmp/php/sessions: file.directory: - mode: 1770 - group: web {% for fqdn, site in pillar['web_php_sites'].items() %} /var/tmp/php/sessions/{{ fqdn }}: file.directory: - mode: 0700 - user: {{ site['user']}} {% endfor %} diff --git a/utils/dump-py-state.py b/utils/dump-py-state.py index 2782082..079a30b 100755 --- a/utils/dump-py-state.py +++ b/utils/dump-py-state.py @@ -1,84 +1,99 @@ #!/usr/bin/env python3 # ------------------------------------------------------------- # rOPS — compile a #!py .sls file and dump result in YAML # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # Project: Nasqueron # Created: 2018-10-17 # Description: Read the web_content_sls pillar entry # and regenerate the webserver-content include. # License: BSD-2-Clause # ------------------------------------------------------------- import os +import subprocess import sys import yaml # ------------------------------------------------------------- # Pillar helper # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - def get_pillar_files(pillar_directory): pillar_files = [] for dir_path, dir_names, file_names in os.walk(pillar_directory): files = [os.path.join(dir_path, file_name) for file_name in file_names if file_name.endswith(".sls")] pillar_files.extend(files) return pillar_files def load_pillar(pillar_directory): pillar = {} for pillar_file in get_pillar_files(pillar_directory): data = yaml.load(open(pillar_file, "r")) pillar.update(data) return pillar +# ------------------------------------------------------------- +# Grains helper +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + +def system(args): + result = subprocess.run(args, stdout=subprocess.PIPE) + return result.stdout.decode('utf-8').strip() + + # ------------------------------------------------------------- # Source code helper # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - def run_shim(): return "\n\nif __name__ == '__main__':\n\tprint(yaml.dump(run(), default_flow_style=False))" def assemble_source_code(filename): with open(filename, 'r') as fd: source_code = fd.read() return source_code + run_shim() # ------------------------------------------------------------- # Run task # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - if __name__ == "__main__": argc = len(sys.argv) if argc < 2: print("Usage: dump-py-state.py ", file=sys.stderr) exit(1) sls_file = sys.argv[1] try: source_code = assemble_source_code(sls_file) except OSError as ex: print(ex, file=sys.stderr) exit(ex.errno) __pillar__ = load_pillar("pillar") + __grains__ = { + 'os': system(["uname", "-o"]) + } + exec(source_code)