diff --git a/_tests/Makefile b/_tests/Makefile --- a/_tests/Makefile +++ b/_tests/Makefile @@ -1,3 +1,4 @@ test: python -m unittest discover modules python -m unittest discover pillar + python -m unittest discover scripts/python diff --git a/_tests/data/T1492-ps-x-sample.txt b/_tests/data/T1492-ps-x-sample.txt new file mode 100644 --- /dev/null +++ b/_tests/data/T1492-ps-x-sample.txt @@ -0,0 +1,10 @@ + PID TTY STAT TIME COMMAND + 1 ? Ss 0:00 /sbin/tini -- bundle exec sidekiq + 7 ? Sl 7:33 sidekiq 5.2.2 mastodon [12 of 12 busy] + 1552 pts/0 Ss 0:00 bash +11562 ? Sl 1:11 ffmpeg -i /tmp/8d777f385d3dfec8815d20f7496026dc201812 +11693 ? Rl 1:11 ffmpeg -i /tmp/8d777f385d3dfec8815d20f7496026dc201812 +11895 ? Sl 1:05 ffmpeg -i /tmp/8d777f385d3dfec8815d20f7496026dc201812 +12033 ? Sl 0:28 ffmpeg -i /tmp/8d777f385d3dfec8815d20f7496026dc201812 +12114 ? Sl 0:11 ffmpeg -i /tmp/8d777f385d3dfec8815d20f7496026dc201812 +12528 pts/0 R+ 0:00 ps x diff --git a/_tests/scripts/python/test_clear_video_queue.py b/_tests/scripts/python/test_clear_video_queue.py new file mode 100644 --- /dev/null +++ b/_tests/scripts/python/test_clear_video_queue.py @@ -0,0 +1,16 @@ +from importlib.machinery import SourceFileLoader +import unittest + + +path = "roles/paas-docker/containers/files/mastodon/clear-video-queue.py" +script = SourceFileLoader('script', "../" + path).load_module() + + +class Testinstance(unittest.TestCase): + + def test_extract_pids(self): + with open("data/T1492-ps-x-sample.txt", "r") as fd: + ps_data = [line.strip() for line in fd] + + expected_pids = [11562, 11693, 11895] + self.assertEqual(expected_pids, script.extract_pids(ps_data)) diff --git a/pillar/paas/docker.sls b/pillar/paas/docker.sls --- a/pillar/paas/docker.sls +++ b/pillar/paas/docker.sls @@ -170,6 +170,18 @@ db: EspaceWin_Bugs credential: espacewin.bugzilla.mysql + # + # Mastodon + # + + # Mastodon is currently deployed manually through docker-compose + # and not yet integrated to the platform. This declaration is + # currently only used for extra utilities deployment. + + mastodon_sidekiq: + mastodon_sidekiq_1: + realm: nasqueron + # # Equatower is the current production engine # diff --git a/roles/paas-docker/containers/files/mastodon/clear-video-queue.py b/roles/paas-docker/containers/files/mastodon/clear-video-queue.py new file mode 100755 --- /dev/null +++ b/roles/paas-docker/containers/files/mastodon/clear-video-queue.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python2 + +# ------------------------------------------------------------- +# Mastodon - clear stuck video tasks from quueue +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Project: Nasqueron +# Created: 2018-12-08 +# License: Trivial work, not eligible to copyright +# Source file: roles/paas-docker/containers/files/mastodon/clear-video-queue.py +# ------------------------------------------------------------- +# +# +# This file is managed by our rOPS SaltStack repository. +# +# Changes to this file may cause incorrect behavior +# and will be lost if the state is redeployed. +# + + +from __future__ import print_function +import subprocess + + +PS_COLUMN_PID = 0 +PS_COLUMN_TIME = 3 +PS_COLUMN_COMMAND = 4 + +SUSPECT_COMMANDS = [ + "ffmpeg" +] + +MAX_TIME = 30 + + +def parse_time(time): + time_parts = [int(token) for token in time.split(":")] + return time_parts[0] * 60 + time_parts[1] + + +def process_time_is_up(time): + return parse_time(time) > MAX_TIME + + +def process_is_suspect(command): + for suspect_command in SUSPECT_COMMANDS: + if command.startswith(suspect_command): + return True + + return False + + +def extract_pid(ps_output_line): + if not ps_output_line[0].isdigit(): + return None + + tokens = ps_output_line.split(None, 4) + extracted_pid = int(tokens[PS_COLUMN_PID]) + time = tokens[PS_COLUMN_TIME] + command = tokens[PS_COLUMN_COMMAND] + + if process_time_is_up(time) and process_is_suspect(command): + return extracted_pid + + return None + + +def extract_pids(output): + extracted_pids = [extract_pid(line) for line in output] + + return [extracted_pid + for extracted_pid in extracted_pids + if extracted_pid is not None] + + +def get_kill_command(pids_to_kill): + command = ["kill", "-9 "] + command.extend([str(pid_to_kill) for pid_to_kill in pids_to_kill]) + + return command + + +if __name__ == "__main__": + ps_output = subprocess.check_output(["ps", "x"]) + ps_data = [line.strip() for line in ps_output.strip().split("\n")] + + pids = extract_pids(ps_data) + kill_command = get_kill_command(pids) + + subprocess.call(kill_command) diff --git a/roles/paas-docker/containers/mastodon.sls b/roles/paas-docker/containers/mastodon.sls new file mode 100644 --- /dev/null +++ b/roles/paas-docker/containers/mastodon.sls @@ -0,0 +1,36 @@ +# ------------------------------------------------------------- +# Salt — Provision Docker engine +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Project: Nasqueron +# Created: 2018-12-08 +# License: Trivial work, not eligible to copyright +# ------------------------------------------------------------- + +{% set containers = pillar['docker_containers'][grains['id']] %} + +# ------------------------------------------------------------- +# Extra utilities +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +/srv/mastodon/extra_utilities: + file.directory: + - makedirs: True + +/srv/mastodon/extra_utilities/clear-video-queue: + file.managed: + - source: salt://roles/paas-docker/containers/files/mastodon/clear-video-queue.py + - mode: 755 + +# ------------------------------------------------------------- +# Provision extra utilities +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +{% for instance in containers['mastodon_sidekiq'] %} + +provision_clear_video_queue_{{ instance }}: + cmd.run: + - name: docker cp /srv/mastodon/extra_utilities/clear-video-queue {{ instance }}:/usr/bin/clear-video-queue + - require: + - file: /srv/mastodon/extra_utilities/clear-video-queue + +{% endfor %}