diff options
Diffstat (limited to '')
| -rw-r--r-- | common-playbook.yml | 23 | ||||
| -rw-r--r-- | group_vars/all.yml | 11 | ||||
| -rw-r--r-- | inventory.yml | 14 | ||||
| -rw-r--r-- | roles/caption/tasks/main.yml | 4 | ||||
| -rwxr-xr-x | roles/caption/templates/process-captions.py | 84 | ||||
| -rw-r--r-- | roles/media/defaults/main.yml | 1 | ||||
| -rw-r--r-- | roles/media/tasks/main.yml | 34 | ||||
| -rw-r--r-- | roles/media/templates/nginx-include | 20 | ||||
| -rw-r--r-- | roles/prerec/templates/reencode.sh | 5 | ||||
| -rw-r--r-- | roles/stream/defaults/main.yml | 4 | ||||
| -rw-r--r-- | roles/stream/tasks/main.yml | 48 | ||||
| -rwxr-xr-x | roles/stream/templates/icecast-emacsconf.init.d | 8 | ||||
| -rw-r--r-- | roles/stream/templates/icecast.xml | 30 | ||||
| -rwxr-xr-x | roles/stream/templates/lowres.init.d | 76 | ||||
| -rwxr-xr-x | roles/stream/templates/lowres.sh | 6 | ||||
| -rwxr-xr-x | roles/stream/templates/on-connect | 12 | ||||
| -rwxr-xr-x | roles/stream/templates/on-disconnect | 29 | ||||
| -rw-r--r-- | roles/wiki-publish/defaults/main.yml | 13 | ||||
| -rw-r--r-- | roles/wiki-publish/tasks/emacs.yml | 38 | ||||
| -rw-r--r-- | roles/wiki-publish/tasks/main.yml | 53 |
20 files changed, 390 insertions, 123 deletions
diff --git a/common-playbook.yml b/common-playbook.yml index 388d84f..2adfdff 100644 --- a/common-playbook.yml +++ b/common-playbook.yml @@ -4,12 +4,12 @@ roles: - wiki - name: Set up wiki for publishing - hosts: front + hosts: res tags: wiki-publish roles: - wiki-publish - name: Set up wiki for interactive editing - hosts: front + hosts: res tags: wiki-edit roles: - wiki-edit @@ -28,14 +28,31 @@ tags: upload roles: - upload - - name: Set up icecast server for streaming hosts: stream tags: stream roles: - stream +- name: Test the icecast server + hosts: res + tags: test, never + tasks: + - include_role: + name: stream + tasks_from: test + when: icecast_test is defined and icecast_test=='file' - name: Set up captioning tools hosts: res tags: caption roles: - caption +- name: Set up media + hosts: media + tags: media + roles: + - media +- name: Set up OBS + hosts: obs + tags: obs + roles: + - obs diff --git a/group_vars/all.yml b/group_vars/all.yml index 8f86f46..f7bbbc4 100644 --- a/group_vars/all.yml +++ b/group_vars/all.yml @@ -1,7 +1,18 @@ docker: false +res_x: 1280 +res_y: 720 +emacsconf_year: 2022 +emacsconf_name: EmacsConf +emacsconf_id: emacsconf emacsconf_user: orga +emacsconf_group: orga emacsconf_tracks: - name: General id: gen + watch: https://emacsconf.org/{{ emacsconf_year }}/watch/gen/ + watch_lowres: https://live.emacsconf.org/{{ emacsconf_year }}/watch/gen-480p/ - name: Development id: dev + watch: https://emacsconf.org/{{ emacsconf_year }}/watch/dev/ + watch_lowres: https://live.emacsconf.org/{{ emacsconf_year }}/watch/dev-480p/ +icecast_emacsconf_user: emacsconf diff --git a/inventory.yml b/inventory.yml index 997e01f..f91434d 100644 --- a/inventory.yml +++ b/inventory.yml @@ -5,6 +5,7 @@ prod: ansible_python_interpreter: /usr/bin/python3 ansible_become: true emacsconf_group: org + cpus: 12 front: ansible_host: front0.emacsconf.org remote_user: orga @@ -23,6 +24,13 @@ prod: ansible_ssh_user: orga ansible_python_interpreter: /usr/bin/python3 ansible_become: true + media: + ansible_host: media.emacsconf.org + remote_user: orga + ansible_ssh_user: orga + ansible_python_interpreter: /usr/bin/python3 + ansible_become: true + host_name: media.emacsconf.org upload: ansible_host: media.emacsconf.org remote_user: orga @@ -35,6 +43,12 @@ prod: ansible_ssh_user: orga ansible_python_interpreter: /usr/bin/python3 ansible_become: true + obs: + ansible_host: obs + remote_user: sacha + ansible_ssh_user: sacha + ansible_python_interpreter: /usr/bin/python3 + ansible_become: true all: hosts: localhost: diff --git a/roles/caption/tasks/main.yml b/roles/caption/tasks/main.yml index 7fe1570..353b83b 100644 --- a/roles/caption/tasks/main.yml +++ b/roles/caption/tasks/main.yml @@ -31,7 +31,7 @@ path: "{{ emacsconf_caption_dir }}" state: directory - name: Copy the script for processing the files - tags: process-captions, wip + tags: process-captions template: src: process-captions.py dest: "{{ emacsconf_caption_dir }}/process-captions.py" @@ -58,7 +58,6 @@ group: root mode: 0755 - name: Change the group for all the files - tags: wip file: dest: "{{ emacsconf_caption_dir }}" group: "{{ emacsconf_group }}" @@ -66,7 +65,6 @@ recurse: true - name: Restart caption monitoring service become: true - tags: wip service: name: captions enabled: true diff --git a/roles/caption/templates/process-captions.py b/roles/caption/templates/process-captions.py index 72e9ad2..66f39dd 100755 --- a/roles/caption/templates/process-captions.py +++ b/roles/caption/templates/process-captions.py @@ -32,16 +32,12 @@ import datetime import sys import webvtt import xml.etree.ElementTree as ET -from lhotse import RecordingSet, Recording, AudioSource, SupervisionSegment, SupervisionSet, create_cut_set_eager, align_with_torchaudio, CutSet, annotate_with_whisper -from tqdm import tqdm -import whisper -import re -import os +from lhotse import RecordinRecording, AudioSource, SupervisionSegment, SupervisionSet, create_cut_set_e import json import torch -THREADS = 12 -VIDEO_REGEXP = '\.(webm|mov)$' +THREADS = {{ cpus }} +VIDEO_REGEXP = '\.(webm|mov|mp4)$' AUDIO_REGEXP = '\.(ogg|opus)$' ALWAYS = False TRIM_AUDIO = False @@ -49,6 +45,8 @@ MODEL = os.environ.get('MODEL', 'large') # Set to tiny for testing WORK_DIR = "{{ emacsconf_caption_dir }}" JSON_FILE = os.path.join(WORK_DIR, 'talks.json') +# ---------------------------------------------------------------- + def get_slug_from_filename(filename): m = re.search('emacsconf-[0-9]+-([a-z]+)--', filename) if m: @@ -155,41 +153,23 @@ def generate_srv2(work): captions.append(SupervisionSegment(id=rec_id + '-sup' + '%05d' % i, channel=recs[0].channel_ids[0], recording_id=rec_id, start=to_sec(caption.start), duration=to_sec(caption.end) - to_sec(caption.start), text=caption.text, language='English')) sups = SupervisionSet.from_segments(captions) main = CutSet.from_manifests(recordings=recs, supervisions=sups) - work['cuts'] = main.trim_to_supervisions(keep_overlapping=False,keep_all_channels=True) + work['cuts'] = main.trim_to_supervisions(keep_all_channels=True) cuts_aligned = align_with_torchaudio(work['cuts']) root = ET.Element("timedtext") doc = ET.SubElement(root, "window") for line, aligned in enumerate(cuts_aligned): - # Numbers are weird - words = re.split(' ', captions[line].text) - tokenized_words = [re.sub('[^\'A-Z0-9]', '', w.upper()) for w in words] - if len(aligned.supervisions) == 0: - print(captions[line], aligned) - continue - aligned_words = list(aligned.supervisions[0].alignment['word']) - aligned_index = 0 - aligned_len = len(aligned_words) - word_index = 0 - word_len = len(words) - while word_index < word_len and aligned_index < aligned_len: - # log("Choosing %s %s" % (words[word_index], aligned_words[aligned_index].symbol)) - ET.SubElement(doc, 'text', - t=str(float(aligned_words[aligned_index].start)*1000), - d=str(float(aligned_words[aligned_index].duration)*1000), - w="1", - append="1").text = words[word_index] - if tokenized_words[word_index] != aligned_words[aligned_index].symbol and word_index < word_len - 1: - # Scan ahead for a word that maches the next word, but don't go too far - cur_aligned = aligned_index - while aligned_index < aligned_len and aligned_index < cur_aligned + 5 and aligned_words[aligned_index].symbol != tokenized_words[word_index + 1]: - log("Sliding to match %s %d %s" % (tokenized_words[word_index + 1], aligned_index, aligned_words[aligned_index].symbol)) - aligned_index = aligned_index + 1 - if not aligned_words[aligned_index].symbol == tokenized_words[word_index + 1]: - log("Resetting, couldn't find") - aligned_index = cur_aligned + 1 - else: - aligned_index = aligned_index + 1 - word_index = word_index + 1 + if len(aligned.supervisions) > 0: + aligned_words = aligned.supervisions[0].alignment['word'] + for w, word in enumerate(aligned_words): + el = ET.SubElement(doc, 'text', + t=str(float(word.start)*1000), + d=str(float(word.duration)*1000), + w="1", + append="1") + el.text = word.symbol + el.tail = "\n" + else: + print("No supervisions", aligned) tree = ET.ElementTree(root) work['srv2'] = work['base'] + '.srv2' with open(work['srv2'], "w") as f: @@ -218,18 +198,20 @@ directory = sys.argv[1] if len(sys.argv) > 1 else WORK_DIR needs_work = get_files_to_work_on(directory) if len(needs_work) > 0: - if THREADS > 0: - torch.set_num_threads(THREADS) - for work in needs_work: - log("Started processing %s" % work['base']) - if work['audio']: - if ALWAYS or not 'vtt' in work: - work = generate_captions(work) - if ALWAYS or not 'srv2' in work: - work = generate_srv2(work) - # print("Aligning words", audio_file, datetime.datetime.now()) - # word_cuts = align_words(cuts) - # convert_cuts_to_word_timing(audio_file, word_cuts) - log("Done %s" % str(work['base'])) + while len(needs_work) > 0: + if THREADS > 0: + torch.set_num_threads(THREADS) + for work in needs_work: + log("Started processing %s" % work['base']) + if work['audio']: + if ALWAYS or not 'vtt' in work: + work = generate_captions(work) + if ALWAYS or not 'srv2' in work: + work = generate_srv2(work) + # print("Aligning words", audio_file, datetime.datetime.now()) + # word_cuts = align_words(cuts) + # convert_cuts_to_word_timing(audio_file, word_cuts) + log("Done %s" % str(work['base'])) + needs_work = get_files_to_work_on(directory) else: log("No work needed.") diff --git a/roles/media/defaults/main.yml b/roles/media/defaults/main.yml new file mode 100644 index 0000000..0d8046d --- /dev/null +++ b/roles/media/defaults/main.yml @@ -0,0 +1 @@ +media_protect_root: false diff --git a/roles/media/tasks/main.yml b/roles/media/tasks/main.yml new file mode 100644 index 0000000..6eb3ab6 --- /dev/null +++ b/roles/media/tasks/main.yml @@ -0,0 +1,34 @@ +- name: Install package for setting htpasswd + package: + name: python3-passlib +- name: Ensure web path exists + file: + path: /var/www/{{ host_name }}/{{ emacsconf_year }}/backstage + state: directory +- name: Change ownership and permissions + file: + path: /var/www/{{ host_name }}/{{ emacsconf_year }} + owner: "{{ emacsconf_user }}" + group: "{{ emacsconf_group }}" + mode: "u=rwX,g=rwX,o=rX" + recurse: true +- name: Create htpasswd entry + htpasswd: + create: yes + name: "{{ emacsconf_backstage_user }}" + password: "{{ emacsconf_backstage_password }}" + path: /etc/nginx/sites-available/{{ host_name }}-{{ emacsconf_year }}-htpasswd +- name: Create Nginx include + template: + src: nginx-include + dest: /etc/nginx/sites-available/{{ host_name }}-{{ emacsconf_year }}-include +- name: Include it in the main configuration + lineinfile: + path: /etc/nginx/sites-available/{{ host_name }} + regexp: "{{ host_name }}-{{ emacsconf_year }}-include" + line: " include sites-available/{{ host_name }}-{{ emacsconf_year }}-include;" + insertafter: root +- name: Reload configuration + service: + name: nginx + state: reloaded diff --git a/roles/media/templates/nginx-include b/roles/media/templates/nginx-include new file mode 100644 index 0000000..3bfb58e --- /dev/null +++ b/roles/media/templates/nginx-include @@ -0,0 +1,20 @@ + location /{{ emacsconf_year }}/backstage { + auth_basic "Restricted"; + auth_basic_user_file /etc/nginx/sites-available/{{ host_name }}-{{ emacsconf_year }}-htpasswd; + autoindex on; + } + {% if media_protect_root %} + location /{{ emacsconf_year }} { + auth_basic "Restricted"; + auth_basic_user_file /etc/nginx/sites-available/{{ host_name }}-{{ emacsconf_year }}-htpasswd; + autoindex on; + } + {% endif %} + location /{{ emacsconf_year }}/emacsconf.ics { + auth_basic off; + } + {% for track in emacsconf_tracks %} + location /{{ emacsconf_year }}/emacsconf-{{ track.id }}.ics { + auth_basic off; + } + {% endfor %}
\ No newline at end of file diff --git a/roles/prerec/templates/reencode.sh b/roles/prerec/templates/reencode.sh new file mode 100644 index 0000000..d0c98c3 --- /dev/null +++ b/roles/prerec/templates/reencode.sh @@ -0,0 +1,5 @@ +Q=32 +CPU=8 +TIME_LIMIT='' +ffmpeg -y -i "$1" $TIME_LIMIT -c:v libvpx-vp9 -b:v 0 -crf $Q -an -row-mt 1 -tile-columns 2 -tile-rows 2 -cpu-used $CPU -g 240 -pass 1 -f webm -threads $CPU /dev/null && + ffmpeg -y -i "$1" $TIME_LIMIT -c:v libvpx-vp9 -b:v 0 -crf $Q -c:a libopus -row-mt 1 -tile-columns 2 -tile-rows 2 -cpu-used $CPU -pass 2 -g 240 -threads $CPU "$2" diff --git a/roles/stream/defaults/main.yml b/roles/stream/defaults/main.yml index cd8984f..5ce637e 100644 --- a/roles/stream/defaults/main.yml +++ b/roles/stream/defaults/main.yml @@ -1,2 +1,6 @@ icecast_admin_email: emacsconf-org@gnu.org icecast_port: 8001 +icecast_lowres_scale: "854:480" +icecast_user: icecast2 +icecast_group: icecast +icecast_restream_dir: /etc/icecast2/restream diff --git a/roles/stream/tasks/main.yml b/roles/stream/tasks/main.yml index 7cabb6b..76febe0 100644 --- a/roles/stream/tasks/main.yml +++ b/roles/stream/tasks/main.yml @@ -8,6 +8,33 @@ template: src: icecast.xml dest: /etc/icecast2/icecast-emacsconf.xml +- name: Create restream dir + file: + path: "{{ icecast_restream_dir }}" + owner: "{{ icecast_user }}" + state: directory +- name: Debug + debug: + var: item.id + loop: "{{ emacsconf_tracks }}" +- name: Copy the lowres restreaming shell script + template: + src: lowres.sh + dest: /usr/local/bin/{{ emacsconf_id }}-lowres-{{ item.id }}.sh + mode: 0755 + loop: "{{ emacsconf_tracks }}" +- name: Copy lowres on-connect + template: + src: on-connect + dest: /usr/local/bin/{{ emacsconf_id }}-lowres-{{ item.id }}-on-connect + mode: 0755 + loop: "{{ emacsconf_tracks }}" +- name: Copy lowres on-disconnect + template: + src: on-disconnect + dest: /usr/local/bin/{{ emacsconf_id }}-lowres-{{ item.id }}-on-disconnect + mode: 0755 + loop: "{{ emacsconf_tracks }}" - name: Set up init file become: true template: @@ -22,36 +49,21 @@ mode: 0644 - name: Enable icecast become: true - sysvinit: + service: name: emacsconf - enabled: yes state: started + enabled: yes - name: Check if icecast is listening wait_for: port: "{{ icecast_port }}" delay: 5 - timeout: 10 + timeout: 20 msg: "Timeout waiting for {{ icecast_port }} to respond" register: port_check ignore_errors: yes - name: Try to restart icecast if not started service: name=emacsconf state=started enabled=yes when: port_check.failed == true -- name: Test by streaming from a file - tags: test-stream-file, never - block: - - debug: - msg: ffmpeg -loglevel 32 -re -i {{ icecast_test }} -loop 1 -f webm -r 30 -ac 2 -cluster_size_limit 2M -cluster_time_limit 5100 -content_type video/webm -c:v libvpx -b:v 1M -crf 30 -g 125 -threads 4 -deadline realtime icecast://emacsconf:{{ icecast_emacsconf_password }}@live0.emacsconf.org:{{ icecast_port }}/emacsconf/{{ icecast_test_track|d("gen") }}.webm - - local_action: shell ffmpeg -re -loglevel 32 -i {{ icecast_test }} -loop 1 -f webm -r 30 -ac 2 -cluster_size_limit 2M -cluster_time_limit 5100 -content_type video/webm -c:v libvpx -b:v 1M -crf 30 -g 125 -threads 4 -deadline realtime icecast://emacsconf:{{ icecast_emacsconf_password }}@live0.emacsconf.org:{{ icecast_port }}/emacsconf/{{ icecast_test_track|d("gen") }}.webm - when: icecast_test is defined -- name: Test by displaying the stream in mpv - tags: test-stream-mpv, never - local_action: shell mpv http://live0.emacsconf.org:{{ icecast_port }}/emacsconf/{{ icecast_test_track|d("gen") }}.webm - when: test is defined and test == 'mpv' -- name: Test by streaming a test pattern - tags: test-stream-pattern - local_action: shell ffmpeg -loglevel 32 -f lavfi -i testsrc=duration=10:size=1280x720:rate=30 -f webm icecast://emacsconf:{{ icecast_emacsconf_password }}@live0.emacsconf.org:{{ icecast_port }}/emacsconf/{{ icecast_test_track|d("gen") }}.webm - when: test is defined and test == 'pattern' # - name: Set up restream script # template: # src: restream-yt.sh diff --git a/roles/stream/templates/icecast-emacsconf.init.d b/roles/stream/templates/icecast-emacsconf.init.d index b3ad491..3d880c1 100755 --- a/roles/stream/templates/icecast-emacsconf.init.d +++ b/roles/stream/templates/icecast-emacsconf.init.d @@ -30,14 +30,10 @@ test -x $DAEMON || exit 0 # Defaults CONFIGFILE="/etc/icecast2/icecast-emacsconf.xml" -CONFIGDEFAULTFILE="/etc/default/icecast2" -USERID=icecast2 -GROUPID=icecast +USERID={{ icecast_user }} +GROUPID={{ icecast_group }} PIDFILE=/var/run/icecast-emacsconf -# Reads config file (will override defaults above) -[ -r "$CONFIGDEFAULTFILE" ] && . $CONFIGDEFAULTFILE - set -e case "$1" in diff --git a/roles/stream/templates/icecast.xml b/roles/stream/templates/icecast.xml index d2bb89c..e09e6a0 100644 --- a/roles/stream/templates/icecast.xml +++ b/roles/stream/templates/icecast.xml @@ -190,33 +190,35 @@ {% for track in emacsconf_tracks %} <mount type="normal"> <mount-name>/emacsconf/{{ track.id }}.webm</mount-name> - <username>emacsconf</username> + <username>{{ icecast_emacsconf_user }}</username> <password>{{ icecast_emacsconf_password }}</password> - <dump-file>/data/emacsconf-{{ emacsconf_year }}-{{ track.id }}_%Y-%m-%d_%H-%M-%S.webm</dump-file> - <stream-name>EmacsConf {{ emacsconf_year }} - {{ track.name }} track</stream-name> - <stream-description>The livestream for the {{ track.name }} track of EmacsConf {{ emacsconf_year }}</stream-description> - <stream-url>https://emacsconf.org/{{ emacsconf_year }}/watch/{{ track.id }}/</stream-url> + <dump-file>/data/{{ emacsconf_id }}-{{ emacsconf_year }}-{{ track.id }}_%Y-%m-%d_%H-%M-%S.webm</dump-file> + <stream-name>{{ emacsconf_name }} {{ emacsconf_year }} - {{ track.name }} track</stream-name> + <stream-description>The livestream for the {{ track.name }} track of {{ emacsconf_name }} {{ emacsconf_year }}</stream-description> + <stream-url>{{ track.watch }}</stream-url> <type>video/webm</type> <public>1</public> + <on-connect>/usr/local/bin/{{ emacsconf_id }}-lowres-{{ track.id }}-on-connect</on-connect> + <on-disconnect>/usr/local/bin/{{ emacsconf_id }}-lowres-{{ track.id }}-on-disconnect</on-disconnect> </mount> <mount type="normal"> - <mount-name>/emacsconf/{{ track.id }}-host.webm</mount-name> - <username>emacsconf</username> + <mount-name>/{{ emacsconf_id }}/{{ track.id }}-host.webm</mount-name> + <username>{{ icecast_emacsconf_user }}</username> <password>{{ icecast_emacsconf_password }}</password> <stream-name>EmacsConf {{ emacsconf_year }} - {{ track.name }} track</stream-name> - <stream-description>The host stream for the {{ track.name }} track of EmacsConf {{ emacsconf_year }}</stream-description> - <stream-url>https://emacsconf.org/{{ emacsconf_year }}/watch/{{ track.id }}/</stream-url> + <stream-description>The host stream for the {{ track.name }} track of {{ emacsconf_name }} {{ emacsconf_year }}</stream-description> + <stream-url>{{ track.watch }}</stream-url> <type>video/webm</type> <public>1</public> <hidden>1</hidden> </mount> <mount type="normal"> - <mount-name>/emacsconf/{{ track.id }}-480p.webm</mount-name> - <username>emacsconf</username> + <mount-name>/{{ emacsconf_id }}/{{ track.id }}-480p.webm</mount-name> + <username>{{ icecast_emacsconf_user }}</username> <password>{{ icecast_emacsconf_password }}</password> - <stream-name>EmacsConf {{ emacsconf_year }} - {{ track.name }} track (480p)</stream-name> - <stream-description>The 480p livestream for the {{ track.name }} track of EmacsConf {{ emacsconf_year }}</stream-description> - <stream-url>https://emacsconf.org/{{ emacsconf_year }}/watch/{{ track.id }}/</stream-url> + <stream-name>{{ emacsconf_name }} {{ emacsconf_year }} - {{ track.name }} track (480p)</stream-name> + <stream-description>The low-res livestream for the {{ track.name }} track of {{ emacsconf_name }} {{ emacsconf_year }}</stream-description> + <stream-url>{{ track.watch_lowres }}</stream-url> <type>video/webm</type> <public>1</public> </mount> diff --git a/roles/stream/templates/lowres.init.d b/roles/stream/templates/lowres.init.d new file mode 100755 index 0000000..41da95c --- /dev/null +++ b/roles/stream/templates/lowres.init.d @@ -0,0 +1,76 @@ +#!/bin/sh +# {{ ansible_managed }} + +### BEGIN INIT INFO +# Provides: {{ emacsconf_id }}-lowres-{{ item.id }} +# Required-Start: $local_fs $remote_fs $network $syslog +# Required-Stop: $local_fs $remote_fs $network $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: 480p restreaming for {{ item.name }} +# Description: 480p restreaming for {{ item.name }} using start-stop-daemon +### END INIT INFO + +PATH="/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin" +USER="{{ icecast_user }}" +GROUP="{{ icecast_group }}" +DESC="Low-res restream for {{ item.name }}" +NAME="{{ emacsconf_id }}-lowres-{{ item.id }}" + +set -e + +. /lib/lsb/init-functions + +start() { + echo "Starting $DESC... " + start-stop-daemon --start --chuid "$USER:$GROUP" --background --make-pidfile --pidfile /var/run/$NAME.pid --exec "/usr/local/bin/{{ emacsconf_id }}-lowres-{{ item.id }}.sh http://localhost:{{ icecast_port }}/{{ item.id }}.webm http:// > /var/log/$NAME.log" || true + echo "done" +} + +#We need this function to ensure the whole process tree will be killed +killtree() { + local _pid=$1 + local _sig=${2-TERM} + for _child in $(ps -o pid --no-headers --ppid ${_pid}); do + killtree ${_child} ${_sig} + done + kill -${_sig} ${_pid} +} + +stop() { + echo "Stopping $DESC... " + if test -f /var/run/$NAME.pid; then + while test -d /proc/$(cat /var/run/$NAME.pid); do + killtree $(cat /var/run/$NAME.pid) 15 + sleep 0.5 + done + rm /var/run/$NAME.pid + fi + echo "done" +} + +status() { + status_of_proc -p /var/run/$NAME.pid "" "$NAME" && exit 0 || exit $? +} + +case "$1" in + start) + start + ;; + stop) + stop + ;; + restart) + stop + start + ;; + status) + status + ;; + *) + echo "Usage: $NAME {start|stop|restart|status}" >&2 + exit 1 + ;; +esac + +exit 0 diff --git a/roles/stream/templates/lowres.sh b/roles/stream/templates/lowres.sh new file mode 100755 index 0000000..1f74215 --- /dev/null +++ b/roles/stream/templates/lowres.sh @@ -0,0 +1,6 @@ +#!/bin/bash +# {{ ansible_managed }} +sleep 10 +for i in 1 2 3 4 5; do + ffmpeg -loglevel 24 -f webm -reconnect_at_eof 1 -reconnect_streamed 1 -re -i "http://localhost:{{ icecast_port }}/{{ emacsconf_id }}/{{ item.id }}.webm" -vf scale="{{ icecast_lowres_scale }}" -f webm -c:a copy -b:v 500k -maxrate 1M -bufsize 1M -content_type video/webm -c:v libvpx "icecast://{{ icecast_emacsconf_user }}:{{ icecast_emacsconf_password }}@localhost:{{ icecast_port }}/{{ emacsconf_id }}/{{ item.id }}-480p.webm" >> {{ icecast_restream_dir }}/{{ emacsconf_id }}-lowres-{{ item.id }}.log || sleep 5 +done diff --git a/roles/stream/templates/on-connect b/roles/stream/templates/on-connect new file mode 100755 index 0000000..0f613bf --- /dev/null +++ b/roles/stream/templates/on-connect @@ -0,0 +1,12 @@ +#!/bin/bash +set -e +PATH="/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin" +PIDFILE="{{ icecast_restream_dir }}/{{ emacsconf_id }}-lowres-{{ item.id }}.pid" +echo $(date) " on connect {{ item.id }} $*" >> {{ icecast_restream_dir }}/restream.log + +. /lib/lsb/init-functions + +start-stop-daemon --start --quiet --background \ + --make-pidfile --pidfile $PIDFILE \ + --oknodo \ + --exec /usr/local/bin/{{ emacsconf_id }}-lowres-{{ item.id }}.sh > /dev/null diff --git a/roles/stream/templates/on-disconnect b/roles/stream/templates/on-disconnect new file mode 100755 index 0000000..5847356 --- /dev/null +++ b/roles/stream/templates/on-disconnect @@ -0,0 +1,29 @@ +#!/bin/bash +set -e +PATH="/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin" +PIDFILE="{{ icecast_restream_dir }}/{{ emacsconf_id }}-lowres-{{ item.id }}.pid" +echo $(date) " on disconnect {{ item.id }} $*" >> {{ icecast_restream_dir }}/restream.log + +. /lib/lsb/init-functions + +#We need this function to ensure the whole process tree will be killed +killtree() { + local _pid=$1 + local _sig=${2-TERM} + for _child in $(ps -o pid --no-headers --ppid ${_pid}); do + killtree ${_child} ${_sig} + done + kill -${_sig} ${_pid} +} + +stop() { + echo "Stopping $DESC... " + if test -f $PIDFILE; then + while test -d /proc/$(cat $PIDFILE); do + killtree $(cat $PIDFILE) 15 + sleep 0.5 + done + rm $PIDFILE + fi + echo "done" +} diff --git a/roles/wiki-publish/defaults/main.yml b/roles/wiki-publish/defaults/main.yml index 863003f..4de7445 100644 --- a/roles/wiki-publish/defaults/main.yml +++ b/roles/wiki-publish/defaults/main.yml @@ -1,8 +1,9 @@ -emacsconf_year: 2022 emacsconf_org_file: "{{ emacsconf_private_dir }}/conf.org" emacsconf_publishing_phase: schedule -emacsconf_el_dir: ~/emacsconf-el -emacsconf_edit_wiki_dir: ~/emacsconf-wiki -emacsconf_private_dir: ~/emacsconf-2022-private -emacs_config_dir: ~/.emacs.d -emacs_package: emacs-snapshot-nox +emacsconf_el_dir: ~{{ emacsconf_user }}/emacsconf-el +emacsconf_edit_wiki_dir: ~{{ emacsconf_user }}/emacsconf-wiki +emacsconf_private_dir: ~{{ emacsconf_user }}/emacsconf-2022-private +emacs_config_dir: ~{{ emacsconf_user }}/.emacs.d +emacs_version: 28.2 +emacs_build_parent: /usr/src/emacs +emacs_build_dir: "{{ emacs_build_parent }}/emacs-{{ emacs_version }}" diff --git a/roles/wiki-publish/tasks/emacs.yml b/roles/wiki-publish/tasks/emacs.yml new file mode 100644 index 0000000..8d75979 --- /dev/null +++ b/roles/wiki-publish/tasks/emacs.yml @@ -0,0 +1,38 @@ +- name: Install dependencies + apt: + name: + - build-essential + - libncurses5-dev + - git + - gnutls-bin + - pkg-config + - libjansson-dev + - libgnutls28-dev + - automake + - texinfo + state: present +- name: Create directory + file: + path: "{{ emacs_build_parent }}" + state: directory +- name: Download Emacs + unarchive: + src: https://ftp.gnu.org/gnu/emacs/emacs-{{emacs_version}}.tar.gz + remote_src: yes + dest: "{{ emacs_build_parent }}" + creates: "{{ emacs_build_dir }}" +- name: Configure + command: ./configure --with-x-toolkit=no + args: + chdir: "{{ emacs_build_dir }}" + creates: '{{ emacs_build_dir }}/Makefile' +- name: Build + command: make -j{{ cpus }} + args: + chdir: "{{ emacs_build_dir }}" + creates: '{{ emacs_build_dir }}/src/emacs' +- name: Install + command: make install + args: + chdir: "{{ emacs_build_dir }}" + creates: '/usr/local/bin/emacs' diff --git a/roles/wiki-publish/tasks/main.yml b/roles/wiki-publish/tasks/main.yml index 960aa01..28566a5 100644 --- a/roles/wiki-publish/tasks/main.yml +++ b/roles/wiki-publish/tasks/main.yml @@ -1,45 +1,44 @@ --- - name: Set up SSH directory ansible.builtin.file: - path: "/home/{{ emacsconf_publish_user }}/.ssh" + path: "/home/{{ emacsconf_user }}/.ssh" state: directory mode: '0700' - name: Install SSH key for EmacsConf wiki ansible.builtin.get_url: url: https://emacsconf.org/id_rsa_anon_git_emacsconf - dest: "/home/{{ emacsconf_publish_user }}/.ssh/id_rsa_anon_git_emacsconf" + dest: "/home/{{ emacsconf_user }}/.ssh/id_rsa_anon_git_emacsconf" mode: '0600' - owner: "{{ emacsconf_publish_user }}" -- name: Set up packages - become: yes - when: emacs_package and emacs_package != "emacs" - block: - # This repository is currently not working - # - name: Add snapshot repository - # ansible.builtin.apt_repository: - # repo: deb http://emacs.ganneff.de/ buster main - - name: Remove old package - ansible.builtin.apt: - name: emacs - state: absent - - name: Install Emacs snapshot - ansible.builtin.apt: - name: "{{ emacs_package }}" - state: present +- name: Install the SSH key for orga + template: + src: id_rsa + dest: "/home/{{ emacsconf_user }}/.ssh/id_rsa" + mode: '0600' +- name: Change ownership of SSH directory + file: + path: "/home/{{ emacsconf_user }}/.ssh" + recurse: true + owner: "{{ emacsconf_user }}" + +- name: Set up Emacs + include: emacs.yml +- name: Ensure configuration directory exists + file: + path: "{{ emacs_config_dir }}" + state: directory - name: Set up Emacs configuration for non-interactive publishing - become: true - become_user: "{{ emacsconf_publish_user }}" template: src: emacsconf-config.el dest: "{{ emacs_config_dir }}/emacsconf-config.el" + - name: Check if Emacs base configuration already exists become: true - become_user: "{{ emacsconf_publish_user }}" lineinfile: dest: "{{ emacs_config_dir }}/init.el" state: present regexp: "emacsconf-config" line: "(load-file \"emacsconf-config.el\")" + create: yes - name: Set up or update repositories become: false tags: publish @@ -71,6 +70,16 @@ - private.failed - not 'Local modifications exist in repository' in private.msg - not 'Failed to checkout branch' in private.msg +- name: Change ownership + file: + path: "{{ item }}" + owner: "{{ emacsconf_user }}" + recurse: true + with_items: + - "{{ emacsconf_config_dir }}" + - "{{ emacsconf_el_dir }}" + - "{{ emacsconf_private_dir }}" + - "{{ emacsconf_edit_wiki_dir }}" - name: Publish tags: publish block: |