diff --git a/PORTS b/PORTS index 8cb4b00..6514d5d 100644 --- a/PORTS +++ b/PORTS @@ -1,52 +1,53 @@ reserved-for-legacy-docker-migration-medium-priority 3000 Mastodon public HTTP 4000 Mastodon streaming HTTP 15674 RabbitMQ 41080 Nasqueron Tools HTTP reserved-for-legacy-docker-migration-low-priority 4440 Rundeck HTTP 21080 Drupal CRM HTTP 22080 Zammad HTTP 27080 Grafana HTTP 28080 phragile HTTP 29080 etcd HTTP 32080 Discourse HTTP 40080 RocketChat HTTP paas-docker 5000 Docker registry HTTP 9090 Openfire HTTP + 16080 Orbeon HTTP 17080 Penpot - back-end 17300 Penpot - exporter 19080 Nasqueron API - Datasources 20080 Nasqueron API - Docker registry API 22220 Phabricator Aphlict (client) 22221 Phabricator Aphlict (admin) 23080 Phabricator HTTP - River Sector 24080 Tommy HTTP - CI 24180 Tommy HTTP - CD 25080 Auth Grove HTTP 26080 Sentry HTTP 26300 Sentry - Relay 30080 Pixelfed HTTP 31080 Phabricator HTTP - DevCentral 33080 Bugzilla HTTP - Espace Win 34080 Etherpad 35080 Phabricator HTTP - Wolfplex 36080 Phabricator HTTP - Zed 37080 Notifications center HTTP 38080 Jenkins HTTP - CD 39080 Cachet HTTP 41080 ACME DNS server HTTP 42080 Jenkins HTTP - CI 43080 Hauk 44080 Hound # 45080 should be reserved for OpenGrok to compare with Hound 46080 Airflow - HTTP 46555 Airflow - Flower 47080 Jenkins HTTP - Test 48080 Vault - Notifications - Integration 50000 Jenkins controller's port for JNLP-based Jenkins agents - CD 52000 Jenkins controller's port for JNLP-based Jenkins agents - Test 55000 Jenkins controller's port for JNKP-based Jenkins agents - CI diff --git a/_modules/tomcat.py b/_modules/tomcat.py new file mode 100644 index 0000000..e06b064 --- /dev/null +++ b/_modules/tomcat.py @@ -0,0 +1,16 @@ +# ------------------------------------------------------------- +# Salt — Tomcat execution module +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Project: Nasqueron +# License: BSD-2-Clause +# ------------------------------------------------------------- + +# ------------------------------------------------------------- +# Tomcat users and roles +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + +def extract_roles_from_users(users): + return set( + role for _, args in users.items() if "roles" in args for role in args["roles"] + ) diff --git a/pillar/credentials/vault.sls b/pillar/credentials/vault.sls index 5649bc7..5a8a613 100644 --- a/pillar/credentials/vault.sls +++ b/pillar/credentials/vault.sls @@ -1,259 +1,263 @@ # ------------------------------------------------------------- # Salt configuration for Nasqueron servers # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # Project: Nasqueron # License: Trivial work, not eligible to copyright # ------------------------------------------------------------- # ------------------------------------------------------------- # Vault configuration # # :: vault_policies_path: path on vault server where to store policies # # :: vault_policies_source: path to fetch policies from # if starting by salt://, from salt files server # # :: vault_mount_paths: translates secrets paths in policies paths # # Generally, Vault paths are the same for policies and data access. # # For kv secrets engine, version 2, writing and reading versions # of a kv value are prefixed with the data/ path. # # credentials.build_policies_by_node will use this dictionary # to be able to rewrite secrets paths in data paths. # # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - vault_policies_path: /srv/policies/vault vault_policies_source: /srv/policies/vault vault_mount_paths: ops/secrets: ops/data/secrets ops/privacy: ops/data/privacy apps: apps/data # ------------------------------------------------------------- # Vault policies to deploy as-is, ie without templating. # # Entries of vault_policies must match a .hcl file in # roles/vault/policies/files folder. # # If you need a template, create a new pillar entry instead # and add the parsing logic either: # - directly to roles/vault/policies/ # # - through _modules/credentials.py for policies to apply # to Salt nodes, like e.g. vault_secrets_by_role # # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - vault_policies: - admin - salt-primary - sentry - vault_bootstrap - viperserv # ------------------------------------------------------------- # Vault policies for Salt # # Declare the extra policies each nodes need. # # In adition of those extra policies, the vault_secrets_by_role # will be parsed for the keys. # # IMPORTANT: as grains['roles'] can be modified by the node, # roles are extracted directly from the pillar. # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - vault_extra_policies_by_role: salt-primary: - salt-primary # ------------------------------------------------------------- # Vault secrets by role # # Paths of the keys the specified role needs access to. # # Avoid * notation as this namespace is shared between Vault # and the applications. As such, only secrets the Salt nodes # needs in a state they need to deploy should be listed here. # # Use %%node%% as variable for node name. # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - vault_secrets_by_role: devserver: - ops/secrets/nasqueron/notifications/notifications-cli/%%node%% - ops/secrets/nasqueron/deploy/deploy_keys/alken-orin - ops/secrets/nasqueron/deploy/deploy_keys/by_repo/bitbucket/dereckson/www - ops/secrets/nasqueron/deploy/deploy_keys/by_repo/bitbucket/ewosp/www - ops/secrets/nasqueron/deploy/deploy_keys/by_repo/github/wolfplex/api-www opensearch: - ops/secrets/nasqueron.opensearch.infra-logs.internal_users.admin - ops/secrets/nasqueron.opensearch.infra-logs.internal_users.dashboards paas-docker-prod: # # Personal data or personally identifiable information (PII) # related to Nasqueron Operations SIG members. # - ops/privacy/ops-cidr # # Credentials used by Nasqueron services # Format: ops/secrets/nasqueron/service/<...> # - ops/secrets/nasqueron/airflow/admin_account - ops/secrets/nasqueron/airflow/fernet - ops/secrets/nasqueron/airflow/sentry - ops/secrets/dbserver/cluster-A/users/airflow - ops/secrets/nasqueron/etherpad/mysql - ops/secrets/nasqueron/etherpad/users/dereckson - ops/secrets/nasqueron/penpot/github - ops/secrets/nasqueron/penpot/postgresql - ops/secrets/nasqueron/penpot/secret_key - ops/secrets/nasqueron/rabbitmq/white-rabbit/erlang-cookie - ops/secrets/nasqueron/rabbitmq/white-rabbit/root - ops/secrets/nasqueron/sentry/geoipupdate # # Credentials used by Nasqueron services # Format: ops/secrets/nasqueron.. # - ops/secrets/nasqueron.acquisitariat.mysql - ops/secrets/nasqueron.auth-grove.mysql - ops/secrets/nasqueron.cachet.app_key - ops/secrets/nasqueron.cachet.mysql - ops/secrets/nasqueron.etherpad.api - ops/secrets/nasqueron.notifications.broker - ops/secrets/nasqueron.notifications.mailgun - ops/secrets/nasqueron.notifications.sentry - ops/secrets/nasqueron.notifications.credentials_github_nasqueron - ops/secrets/nasqueron.notifications.credentials_github_wolfplex - ops/secrets/nasqueron.notifications.credentials_github_keruald - ops/secrets/nasqueron.notifications.credentials_github_trustspace - ops/secrets/nasqueron.notifications.credentials_github_eglide - ops/secrets/nasqueron.notifications.credentials_phabricator_nasqueron - apps/notifications-center/dockerhub/notifications - apps/notifications-center/dockerhub/auth-grove - ops/secrets/nasqueron.pixelfed.app_key - ops/secrets/nasqueron.pixelfed.mailgun - ops/secrets/nasqueron.pixelfed.mysql - ops/secrets/nasqueron.sentry.app_key - ops/secrets/nasqueron.sentry.postgresql - ops/secrets/nasqueron.sentry.vault # # Credentials used by Nasqueron members private services # Format: .. # - ops/secrets/dereckson.phabricator.mysql # # Credentials used by projects hosted by Nasqueron # Format: .. # - ops/secrets/espacewin.phpbb.mysql_root - ops/secrets/wolfplex.phabricator.mailgun - ops/secrets/wolfplex.phabricator.mysql - ops/secrets/zed.phabricator.mysql - ops/secrets/zed.phabricator.sendgrid paas-docker-dev: # # Credentials used by Nasqueron services # Format: ops/secrets/nasqueron/service/<...> # - ops/secrets/nasqueron/airflow/admin_account - ops/secrets/nasqueron/airflow/fernet - ops/secrets/nasqueron/airflow/sentry - ops/secrets/dbserver/cluster-A/users/airflow + - ops/secrets/nasqueron/orbeon/oxf.crypto.password + - ops/secrets/nasqueron/orbeon/users/dereckson + - ops/secrets/dbserver/cluster-A/users/orbeon + - ops/secrets/nasqueron/rabbitmq/orange-rabbit/erlang-cookie - ops/secrets/nasqueron/rabbitmq/orange-rabbit/root - ops/secrets/nasqueron/rabbitmq/orange-rabbit/notifications - ops/secrets/nasqueron.notifications.sentry # # Credentials used by projects hosted by Nasqueron # Format: .. # - ops/secrets/espacewin.bugzilla.mysql - ops/secrets/espacewin.bugzilla.mysql_root saas-mediawiki: - ops/secrets/dbserver/cluster-B/users/saas-mediawiki - ops/secrets/nasqueron/mediawiki/secret_key saas-wordpress: - ops/secrets/dbserver/cluster-B/users/dereckson_blog - ops/secrets/dereckson/wordpress/secrets viperserv: - ops/secrets/nasqueron.viperserv.vault webserver-alkane: - ops/secrets/dbserver/cluster-B/users/dereckson_www - ops/secrets/dbserver/cluster-B/users/zed - ops/secrets/zed/hypership/secret_key # # Wolfplex credentials # - ops/secrets/nasqueron.etherpad.api webserver-legacy: # # Wolfplex credentials # - ops/secrets/nasqueron.etherpad.api # ------------------------------------------------------------- # Vault secrets by dbserver cluster # # Paths of the keys the specified role needs access to. # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - vault_secrets_by_dbserver_cluster: # Main PostgreSQL cluster A: - ops/secrets/dbserver/cluster-A/users/* # Main MariaDB cluster - Alkane PaaS, ViperServ B: - ops/secrets/dbserver/cluster-B/users/* diff --git a/pillar/paas/docker/dwellers/orbeon.sls b/pillar/paas/docker/dwellers/orbeon.sls new file mode 100644 index 0000000..e6b94fa --- /dev/null +++ b/pillar/paas/docker/dwellers/orbeon.sls @@ -0,0 +1,41 @@ +# ------------------------------------------------------------- +# Salt — Provision Docker engine +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Project: Nasqueron +# License: Trivial work, not eligible to copyright +# Service: Orbeon +# ------------------------------------------------------------- + +docker_networks: + orbeon: + subnet: 172.18.5.0/24 + +docker_images: + - nasqueron/orbeon + - tianon/exim4 + +docker_containers: + exim: + orbeon_smtp: + mailname: forms.nasqueron.org + network: orbeon + + orbeon: + nasqueron_forms: + host: forms.nasqueron.org + app_port: 16080 + network: orbeon + db: + service: db-A + database: forms + credential: dbserver/cluster-A/users/orbeon + secret_key: nasqueron/orbeon/oxf.crypto.password + tomcat: + users: + dereckson: nasqueron/orbeon/users/dereckson + smtp: orbeon_smtp + + # Published forms are categorized by apps. + # List of forapps so nginx can proxy // + apps: + - nasqueron-join diff --git a/roles/paas-docker/containers/files/_tomcat/tomcat-users.xml b/roles/paas-docker/containers/files/_tomcat/tomcat-users.xml new file mode 100644 index 0000000..e6f0f87 --- /dev/null +++ b/roles/paas-docker/containers/files/_tomcat/tomcat-users.xml @@ -0,0 +1,40 @@ + + + + + {% if roles is defined %} + {% for role, role_args in roles.items() %} + + {% endfor %} + {% else %} + {% for role in salt["tomcat.extract_roles_from_users"](users) %} + + {% endfor %} + {% endif %} + + {% for username, user_args in users.items() %} + + {% endfor %} + diff --git a/roles/paas-docker/containers/files/orbeon/nasqueron_forms/form-builder-permissions.xml b/roles/paas-docker/containers/files/orbeon/nasqueron_forms/form-builder-permissions.xml new file mode 100644 index 0000000..d6d5a25 --- /dev/null +++ b/roles/paas-docker/containers/files/orbeon/nasqueron_forms/form-builder-permissions.xml @@ -0,0 +1,21 @@ + + + + + + diff --git a/roles/paas-docker/containers/files/orbeon/nasqueron_forms/properties-local.xml b/roles/paas-docker/containers/files/orbeon/nasqueron_forms/properties-local.xml new file mode 100644 index 0000000..4dad89f --- /dev/null +++ b/roles/paas-docker/containers/files/orbeon/nasqueron_forms/properties-local.xml @@ -0,0 +1,105 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + https://assets.nasqueron.org/css/forms/nasqueron-forms.css + + + + https://assets.nasqueron.org/logos/logo-white-32px.png + + + + + + + save-progress send + + + + + + + + diff --git a/roles/paas-docker/containers/files/orbeon/nasqueron_forms/web.xml b/roles/paas-docker/containers/files/orbeon/nasqueron_forms/web.xml new file mode 100644 index 0000000..dbee43a --- /dev/null +++ b/roles/paas-docker/containers/files/orbeon/nasqueron_forms/web.xml @@ -0,0 +1,409 @@ + + + + + + Nasqueron Forms + + Nasqueron Forms is an Orbeon Forms, a XForms web forms solutions + to create and fill complex forms. + + + + + oxf.resources.factory + org.orbeon.oxf.resources.PriorityResourceManagerFactory + + + + oxf.resources.priority.2 + org.orbeon.oxf.resources.WebAppResourceManagerFactory + + + oxf.resources.priority.2.oxf.resources.webapp.rootdir + /WEB-INF/resources + + + + oxf.resources.priority.6 + org.orbeon.oxf.resources.ClassLoaderResourceManagerFactory + + + + oxf.run-mode + prod + + + + oxf.properties + oxf:/config/properties-${oxf.run-mode}.xml + + + + oxf.initialize-logging + true + + + + + + + + + + + + orbeon-exist-filter + org.orbeon.oxf.servlet.TokenSecurityFilter + + + orbeon-exist-filter + /exist/* + REQUEST + FORWARD + + + + orbeon-limiter-filter + org.orbeon.oxf.servlet.LimiterFilter + + + include + (/fr/.*)|(/xforms-server) + + + + exclude + (?!/([^/]+)/service/).+\.(gif|css|pdf|json|js|coffee|map|png|jpg|xsd|htc|ico|swf|html|htm|txt) + + + + + min-threads + 1 + + + num-threads + x1 + + + max-threads + x1 + + + + orbeon-limiter-filter + /* + REQUEST + + + + orbeon-form-runner-auth-servlet-filter + org.orbeon.oxf.servlet.FormRunnerAuthFilter + + + + orbeon-form-runner-auth-servlet-filter + /* + REQUEST + FORWARD + + + + orbeon-xforms-filter + org.orbeon.oxf.servlet.OrbeonXFormsFilter + + + + + + orbeon-xforms-filter + /xforms-jsp/* + + REQUEST + FORWARD + + + + org.orbeon.oxf.webapp.OrbeonServletContextListener + + + + org.orbeon.oxf.xforms.ReplicationServletContextListener + + + + org.orbeon.oxf.xforms.XFormsServletContextListener + + + + org.orbeon.oxf.webapp.OrbeonSessionListener + + + + net.sf.ehcache.constructs.web.ShutdownListener + + + + orbeon-main-servlet + org.orbeon.oxf.servlet.OrbeonServlet + + + oxf.main-processor.name + {http://www.orbeon.com/oxf/processors}pipeline + + + oxf.main-processor.input.config + oxf:/config/prologue-servlet.xpl + + + + oxf.error-processor.name + {http://www.orbeon.com/oxf/processors}page-flow + + + oxf.error-processor.input.controller + oxf:/config/error-page-flow.xml + + + + oxf.http.accept-methods + get,post,head,put,delete,lock,unlock + + + + + + + + + orbeon-renderer-servlet + org.orbeon.oxf.servlet.OrbeonServlet + + + oxf.main-processor.name + {http://www.orbeon.com/oxf/processors}page-flow + + + oxf.main-processor.input.controller + oxf:/ops/xforms/xforms-renderer-page-flow.xml + + + + oxf.error-processor.name + {http://www.orbeon.com/oxf/processors}pipeline + + + oxf.error-processor.input.config + oxf:/config/error.xpl + + + + + + + exist-rest-servlet + org.exist.http.servlets.EXistServlet + + basedir + WEB-INF/ + + + configuration + exist-conf.xml + + + start + true + + + + orbeon-main-servlet + / + + + orbeon-renderer-servlet + /xforms-renderer + + + exist-rest-servlet + /exist/rest/* + + + + + + + + + + + + + Form Runner + /fr/auth + + + orbeon-user + + + + + + + Form Runner services and public pages and resources + /fr/service/* + /fr/style/* + /fr/not-found + /fr/error + /fr/login + /fr/login-error + + + + + + + Form Builder + /fr/orbeon/builder/* + + + orbeon-admin + + + + + + + Form Administration + /fr/admin + + + orbeon-admin + + + + + + + Form landing pages + /fr/ + /fr/forms + + + orbeon-admin + + + + + + orbeon-user + + + + + FORM + + /fr/login + /fr/login-error + + + + + 60 + + diff --git a/roles/paas-docker/containers/files/orbeon/server.xml b/roles/paas-docker/containers/files/orbeon/server.xml new file mode 100644 index 0000000..3b0db71 --- /dev/null +++ b/roles/paas-docker/containers/files/orbeon/server.xml @@ -0,0 +1,40 @@ + + + + + + diff --git a/roles/paas-docker/containers/orbeon.sls b/roles/paas-docker/containers/orbeon.sls new file mode 100644 index 0000000..4af4886 --- /dev/null +++ b/roles/paas-docker/containers/orbeon.sls @@ -0,0 +1,103 @@ +# ------------------------------------------------------------- +# Salt — Provision Docker engine +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Project: Nasqueron +# License: Trivial work, not eligible to copyright +# ------------------------------------------------------------- + +{% set has_selinux = salt['grains.get']('selinux:enabled', False) %} + +{% for instance, container in pillar['docker_containers']['orbeon'].items() %} + +# ------------------------------------------------------------- +# Storage directory +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +/srv/orbeon/{{ instance }}: + file.directory: + - makedirs: True + +{% if has_selinux %} +selinux_context_{{ instance }}_data: + selinux.fcontext_policy_present: + - name: /srv/orbeon/{{ instance }} + - sel_type: container_file_t + +selinux_context_{{ instance }}_data_applied: + selinux.fcontext_policy_applied: + - name: /srv/orbeon/{{ instance }} +{% endif %} + +# ------------------------------------------------------------- +# Configuration files +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +/srv/orbeon/{{ instance }}/conf/tomcat-users.xml: + file.managed: + - source: salt://roles/paas-docker/containers/files/_tomcat/tomcat-users.xml + - mode: 400 + - makedirs: True + - template: jinja + - show_changes: False + - context: + users: + {% for user, credential in container["tomcat"]["users"].items() %} + {{ user }}: + password: {{ salt["credentials.get_password"](credential) | yaml_dquote }} + roles: + - orbeon-admin + {% endfor %} + +/srv/orbeon/{{ instance }}/conf/properties-local.xml: + file.managed: + - source: salt://roles/paas-docker/containers/files/orbeon/{{ instance }}/properties-local.xml + - mode: 400 + - template: jinja + - show_changes: False + - context: + secret_key: {{ salt["credentials.get_password"](container["secret_key"]) | yaml_dquote }} + host: {{ container["host"] }} + smtp: {{ container["smtp"] }} + +/srv/orbeon/{{ instance }}/conf/orbeon.xml: + file.managed: + - source: salt://roles/paas-docker/containers/files/orbeon/server.xml + - mode: 400 + - template: jinja + - show_changes: False + - context: + db: + host: {{ pillar["nasqueron_services"][container["db"]["service"]] }} + database: {{ container["db"]["database"] }} + user: {{ salt["credentials.get_username"](container["db"]["credential"]) }} + pass: {{ salt["credentials.get_password"](container["db"]["credential"]) | yaml_dquote }} + +{% for config_file in ["web.xml", "form-builder-permissions.xml"] %} +/srv/orbeon/{{ instance }}/conf/{{ config_file }}: + file.managed: + - source: salt://roles/paas-docker/containers/files/orbeon/{{ instance }}/{{ config_file }} +{% endfor %} + +# ------------------------------------------------------------- +# Container +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +{{ instance }}: + docker_container.running: + - detach: True + - interactive: True + - image: nasqueron/orbeon + - binds: + - /srv/orbeon/{{ instance }}/conf/tomcat-users.xml:/usr/local/tomcat/conf/tomcat-users.xml + - /srv/orbeon/{{ instance }}/conf/orbeon.xml:/usr/local/tomcat/conf/Catalina/localhost/orbeon.xml + - /srv/orbeon/{{ instance }}/conf/web.xml:/usr/local/tomcat/webapps/orbeon/WEB-INF/web.xml + - /srv/orbeon/{{ instance }}/conf/form-builder-permissions.xml:/usr/local/tomcat/webapps/orbeon/WEB-INF/resources/config/form-builder-permissions.xml + - /srv/orbeon/{{ instance }}/conf/properties-local.xml:/usr/local/tomcat/webapps/orbeon/WEB-INF/resources/config/properties-local.xml + - ports: + - 8080 + - port_bindings: + - {{ container['app_port'] }}:8080 + - networks: + - {{ container['network'] }} + +{% endfor %} diff --git a/roles/paas-docker/nginx/files/vhosts/orbeon.conf b/roles/paas-docker/nginx/files/vhosts/orbeon.conf new file mode 100644 index 0000000..9f76cb4 --- /dev/null +++ b/roles/paas-docker/nginx/files/vhosts/orbeon.conf @@ -0,0 +1,53 @@ +# ------------------------------------------------------------- +# Configuration for Docker PaaS front-end nginx +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Source file: roles/paas-docker/nginx/files/vhosts/orbeon.conf +# ------------------------------------------------------------- +# +# +# This file is managed by our rOPS SaltStack repository. +# +# Changes to this file may cause incorrect behavior +# and will be lost if the state is redeployed. +# + +server { + listen 80; + listen [::]:80; + server_name {{ fqdn }}; + + include includes/letsencrypt; + + return 301 https://$host$request_uri; +} + +server { + server_name {{ fqdn }}; + + include includes/tls; + ssl_certificate /srv/letsencrypt/etc/live/{{ fqdn }}/fullchain.pem; + ssl_certificate_key /srv/letsencrypt/etc/live/{{ fqdn }}/privkey.pem; + + include includes/letsencrypt; + + {% for app in args["apps"] %} + location /{{ app }}/ { + proxy_pass http://localhost:16080/orbeon/fr/{{ app }}/; + proxy_redirect http://localhost:16080/orbeon/fr/{{ app }}/ /{{ app }}/; + + include includes/proxy_params; + } + {% endfor %} + + location /orbeon { + proxy_pass http://localhost:{{ app_port }}; + proxy_redirect off; + + include includes/proxy_params; + } + + root /var/wwwroot-502/$server_name; + + error_page 502 /502.html; + location /502.html {} +}