media, wiki-publish, caption updates

* common-playbook.yml: Start moving publishing setup to res
* group_vars/all.yml: New common variables.
* inventory.yml (prod): Number of threads, new hosts
* roles/caption/templates/process-captions.py: Add mp4, try to figure
out why script was failing, simplify
* roles/media/tasks/main.yml: New role for setting up
media.emacsconf.org for this year
* roles/prerec/templates/reencode.sh: Keep a copy of zaeph's script
* roles/stream/defaults/main.yml: Add more variables
* roles/stream: Restreaming lowres
* roles/wiki-publish/tasks/emacs.yml: Build Emacs from source

17 files changed, 345 insertions, 120 deletions

diff --git a/roles/caption/tasks/main.yml b/roles/caption/tasks/main.yml
index 7fe1570..353b83b 100644
--- a/roles/caption/tasks/main.yml
+++ b/roles/caption/tasks/main.yml
@@ -31,7 +31,7 @@
     path: "{{ emacsconf_caption_dir }}"
     state: directory
 - name: Copy the script for processing the files
-  tags: process-captions, wip
+  tags: process-captions
   template:
     src: process-captions.py
     dest: "{{ emacsconf_caption_dir }}/process-captions.py"
@@ -58,7 +58,6 @@
     group: root
     mode: 0755
 - name: Change the group for all the files
-  tags: wip
   file:
     dest: "{{ emacsconf_caption_dir }}"
     group: "{{ emacsconf_group }}"
@@ -66,7 +65,6 @@
     recurse: true
 - name: Restart caption monitoring service
   become: true
-  tags: wip
   service:
     name: captions
     enabled: true
diff --git a/roles/caption/templates/process-captions.py b/roles/caption/templates/process-captions.py
index 72e9ad2..66f39dd 100755
--- a/roles/caption/templates/process-captions.py
+++ b/roles/caption/templates/process-captions.py
@@ -32,16 +32,12 @@ import datetime
 import sys
 import webvtt
 import xml.etree.ElementTree as ET
-from lhotse import RecordingSet, Recording, AudioSource, SupervisionSegment, SupervisionSet, create_cut_set_eager, align_with_torchaudio, CutSet, annotate_with_whisper
-from tqdm import tqdm
-import whisper
-import re
-import os
+from lhotse import RecordinRecording, AudioSource, SupervisionSegment, SupervisionSet, create_cut_set_e
 import json
 import torch
 
-THREADS = 12
-VIDEO_REGEXP = '\.(webm|mov)$'
+THREADS = {{ cpus }}
+VIDEO_REGEXP = '\.(webm|mov|mp4)$'
 AUDIO_REGEXP = '\.(ogg|opus)$'
 ALWAYS = False
 TRIM_AUDIO = False
@@ -49,6 +45,8 @@ MODEL = os.environ.get('MODEL', 'large')  # Set to tiny for testing
 WORK_DIR = "{{ emacsconf_caption_dir }}"
 JSON_FILE = os.path.join(WORK_DIR, 'talks.json')
 
+# ----------------------------------------------------------------
+
 def get_slug_from_filename(filename):
     m = re.search('emacsconf-[0-9]+-([a-z]+)--', filename)
     if m:
@@ -155,41 +153,23 @@ def generate_srv2(work):
         captions.append(SupervisionSegment(id=rec_id + '-sup' + '%05d' % i, channel=recs[0].channel_ids[0], recording_id=rec_id, start=to_sec(caption.start), duration=to_sec(caption.end) - to_sec(caption.start), text=caption.text, language='English'))
     sups = SupervisionSet.from_segments(captions)
     main = CutSet.from_manifests(recordings=recs, supervisions=sups)
-    work['cuts'] = main.trim_to_supervisions(keep_overlapping=False,keep_all_channels=True)
+    work['cuts'] = main.trim_to_supervisions(keep_all_channels=True)
     cuts_aligned = align_with_torchaudio(work['cuts'])
     root = ET.Element("timedtext")
     doc = ET.SubElement(root, "window")
     for line, aligned in enumerate(cuts_aligned):
-        # Numbers are weird 
-        words = re.split(' ', captions[line].text)
-        tokenized_words = [re.sub('[^\'A-Z0-9]', '', w.upper()) for w in words]
-        if len(aligned.supervisions) == 0:
-            print(captions[line], aligned)
-            continue
-        aligned_words = list(aligned.supervisions[0].alignment['word'])
-        aligned_index = 0
-        aligned_len = len(aligned_words)
-        word_index = 0
-        word_len = len(words)
-        while word_index < word_len and aligned_index < aligned_len:
-            # log("Choosing %s %s" % (words[word_index], aligned_words[aligned_index].symbol))
-            ET.SubElement(doc, 'text',
-                              t=str(float(aligned_words[aligned_index].start)*1000),
-                              d=str(float(aligned_words[aligned_index].duration)*1000),
-                              w="1",
-                              append="1").text = words[word_index]
-            if tokenized_words[word_index] != aligned_words[aligned_index].symbol and word_index < word_len - 1:
-                # Scan ahead for a word that maches the next word, but don't go too far
-                cur_aligned = aligned_index
-                while aligned_index < aligned_len and aligned_index < cur_aligned + 5 and aligned_words[aligned_index].symbol != tokenized_words[word_index + 1]:
-                    log("Sliding to match %s %d %s" % (tokenized_words[word_index + 1], aligned_index, aligned_words[aligned_index].symbol))
-                    aligned_index = aligned_index + 1
-                if not aligned_words[aligned_index].symbol == tokenized_words[word_index + 1]:
-                    log("Resetting, couldn't find")
-                    aligned_index = cur_aligned + 1
-            else:
-                aligned_index = aligned_index + 1
-            word_index = word_index + 1
+        if len(aligned.supervisions) > 0:
+            aligned_words = aligned.supervisions[0].alignment['word']
+            for w, word in enumerate(aligned_words):
+                el = ET.SubElement(doc, 'text',
+                                  t=str(float(word.start)*1000),
+                                  d=str(float(word.duration)*1000),
+                                  w="1",
+                                  append="1")
+                el.text = word.symbol
+                el.tail = "\n"
+        else:
+            print("No supervisions", aligned)
     tree = ET.ElementTree(root)
     work['srv2'] = work['base'] + '.srv2'
     with open(work['srv2'], "w") as f:
@@ -218,18 +198,20 @@ directory = sys.argv[1] if len(sys.argv) > 1 else WORK_DIR
 
 needs_work = get_files_to_work_on(directory)
 if len(needs_work) > 0:
-    if THREADS > 0:
-        torch.set_num_threads(THREADS)
-    for work in needs_work:
-        log("Started processing %s" % work['base'])
-        if work['audio']:
-            if ALWAYS or not 'vtt' in work:
-                work = generate_captions(work)
-            if ALWAYS or not 'srv2' in work:
-                work = generate_srv2(work)
-    #     print("Aligning words", audio_file, datetime.datetime.now())
-    #     word_cuts = align_words(cuts)
-    #     convert_cuts_to_word_timing(audio_file, word_cuts)
-        log("Done %s" % str(work['base']))
+    while len(needs_work) > 0:
+        if THREADS > 0:
+            torch.set_num_threads(THREADS)
+            for work in needs_work:
+                log("Started processing %s" % work['base'])
+                if work['audio']:
+                    if ALWAYS or not 'vtt' in work:
+                        work = generate_captions(work)
+                    if ALWAYS or not 'srv2' in work:
+                        work = generate_srv2(work)
+                        #     print("Aligning words", audio_file, datetime.datetime.now())
+                        #     word_cuts = align_words(cuts)
+                        #     convert_cuts_to_word_timing(audio_file, word_cuts)
+                    log("Done %s" % str(work['base']))
+            needs_work = get_files_to_work_on(directory)
 else:
     log("No work needed.")
diff --git a/roles/media/defaults/main.yml b/roles/media/defaults/main.yml
new file mode 100644
index 0000000..0d8046d
--- /dev/null
+++ b/roles/media/defaults/main.yml
@@ -0,0 +1 @@
+media_protect_root: false
diff --git a/roles/media/tasks/main.yml b/roles/media/tasks/main.yml
new file mode 100644
index 0000000..6eb3ab6
--- /dev/null
+++ b/roles/media/tasks/main.yml
@@ -0,0 +1,34 @@
+- name: Install package for setting htpasswd
+  package:
+    name: python3-passlib
+- name: Ensure web path exists
+  file:
+    path: /var/www/{{ host_name }}/{{ emacsconf_year }}/backstage
+    state: directory
+- name: Change ownership and permissions
+  file:
+    path: /var/www/{{ host_name }}/{{ emacsconf_year }}
+    owner: "{{ emacsconf_user }}"
+    group: "{{ emacsconf_group }}"
+    mode: "u=rwX,g=rwX,o=rX"
+    recurse: true
+- name: Create htpasswd entry
+  htpasswd:
+    create: yes
+    name: "{{ emacsconf_backstage_user }}"
+    password: "{{ emacsconf_backstage_password }}"
+    path: /etc/nginx/sites-available/{{ host_name }}-{{ emacsconf_year }}-htpasswd
+- name: Create Nginx include
+  template:
+    src: nginx-include
+    dest: /etc/nginx/sites-available/{{ host_name }}-{{ emacsconf_year }}-include
+- name: Include it in the main configuration
+  lineinfile:
+    path: /etc/nginx/sites-available/{{ host_name }}
+    regexp: "{{ host_name }}-{{ emacsconf_year }}-include"
+    line: "    include sites-available/{{ host_name }}-{{ emacsconf_year }}-include;"
+    insertafter: root
+- name: Reload configuration
+  service:
+    name: nginx
+    state: reloaded
diff --git a/roles/media/templates/nginx-include b/roles/media/templates/nginx-include
new file mode 100644
index 0000000..3bfb58e
--- /dev/null
+++ b/roles/media/templates/nginx-include
@@ -0,0 +1,20 @@
+        location /{{ emacsconf_year }}/backstage {
+                auth_basic "Restricted";
+                auth_basic_user_file /etc/nginx/sites-available/{{ host_name }}-{{ emacsconf_year }}-htpasswd;
+                autoindex on;
+        }
+        {% if media_protect_root %}
+        location /{{ emacsconf_year }} {
+                auth_basic "Restricted";
+                auth_basic_user_file /etc/nginx/sites-available/{{ host_name }}-{{ emacsconf_year }}-htpasswd;
+                autoindex on;
+        }
+        {% endif %}
+        location /{{ emacsconf_year }}/emacsconf.ics {
+           auth_basic off;
+        }
+        {% for track in emacsconf_tracks %}
+        location /{{ emacsconf_year }}/emacsconf-{{ track.id }}.ics {
+           auth_basic off;
+        }
+        {% endfor %}
\ No newline at end of file
diff --git a/roles/prerec/templates/reencode.sh b/roles/prerec/templates/reencode.sh
new file mode 100644
index 0000000..d0c98c3
--- /dev/null
+++ b/roles/prerec/templates/reencode.sh
@@ -0,0 +1,5 @@
+Q=32
+CPU=8
+TIME_LIMIT=''
+ffmpeg -y -i "$1" $TIME_LIMIT -c:v libvpx-vp9 -b:v 0 -crf $Q -an -row-mt 1 -tile-columns 2 -tile-rows 2 -cpu-used $CPU -g 240 -pass 1 -f webm -threads $CPU /dev/null &&
+    ffmpeg -y -i "$1" $TIME_LIMIT -c:v libvpx-vp9 -b:v 0 -crf $Q -c:a libopus -row-mt 1 -tile-columns 2 -tile-rows 2 -cpu-used $CPU -pass 2 -g 240 -threads $CPU "$2"
diff --git a/roles/stream/defaults/main.yml b/roles/stream/defaults/main.yml
index cd8984f..5ce637e 100644
--- a/roles/stream/defaults/main.yml
+++ b/roles/stream/defaults/main.yml
@@ -1,2 +1,6 @@
 icecast_admin_email: emacsconf-org@gnu.org
 icecast_port: 8001
+icecast_lowres_scale: "854:480"
+icecast_user: icecast2
+icecast_group: icecast
+icecast_restream_dir: /etc/icecast2/restream
diff --git a/roles/stream/tasks/main.yml b/roles/stream/tasks/main.yml
index 7cabb6b..76febe0 100644
--- a/roles/stream/tasks/main.yml
+++ b/roles/stream/tasks/main.yml
@@ -8,6 +8,33 @@
   template:
     src: icecast.xml
     dest: /etc/icecast2/icecast-emacsconf.xml
+- name: Create restream dir
+  file:
+    path: "{{ icecast_restream_dir }}"
+    owner: "{{ icecast_user }}"
+    state: directory
+- name: Debug
+  debug:
+    var: item.id
+  loop: "{{ emacsconf_tracks }}"
+- name: Copy the lowres restreaming shell script
+  template:
+    src: lowres.sh
+    dest: /usr/local/bin/{{ emacsconf_id }}-lowres-{{ item.id }}.sh
+    mode: 0755
+  loop: "{{ emacsconf_tracks }}"
+- name: Copy lowres on-connect
+  template:
+    src: on-connect
+    dest: /usr/local/bin/{{ emacsconf_id }}-lowres-{{ item.id }}-on-connect
+    mode: 0755
+  loop: "{{ emacsconf_tracks }}"
+- name: Copy lowres on-disconnect
+  template:
+    src: on-disconnect
+    dest: /usr/local/bin/{{ emacsconf_id }}-lowres-{{ item.id }}-on-disconnect
+    mode: 0755
+  loop: "{{ emacsconf_tracks }}"
 - name: Set up init file
   become: true
   template:
@@ -22,36 +49,21 @@
     mode: 0644
 - name: Enable icecast
   become: true
-  sysvinit:
+  service:
     name: emacsconf
-    enabled: yes
     state: started
+    enabled: yes
 - name: Check if icecast is listening
   wait_for:
     port: "{{ icecast_port }}"
     delay: 5
-    timeout: 10
+    timeout: 20
     msg: "Timeout waiting for {{ icecast_port }} to respond"
   register: port_check
   ignore_errors: yes
 - name: Try to restart icecast if not started
   service: name=emacsconf state=started enabled=yes
   when: port_check.failed == true
-- name: Test by streaming from a file
-  tags: test-stream-file, never
-  block:
-  - debug:
-      msg: ffmpeg -loglevel 32 -re -i {{ icecast_test }} -loop 1 -f webm -r 30 -ac 2 -cluster_size_limit 2M -cluster_time_limit 5100 -content_type video/webm -c:v libvpx -b:v 1M -crf 30 -g 125 -threads 4 -deadline realtime  icecast://emacsconf:{{ icecast_emacsconf_password }}@live0.emacsconf.org:{{ icecast_port }}/emacsconf/{{ icecast_test_track|d("gen") }}.webm
-  - local_action: shell ffmpeg -re -loglevel 32 -i {{ icecast_test }} -loop 1 -f webm -r 30 -ac 2 -cluster_size_limit 2M -cluster_time_limit 5100 -content_type video/webm -c:v libvpx -b:v 1M -crf 30 -g 125 -threads 4 -deadline realtime  icecast://emacsconf:{{ icecast_emacsconf_password }}@live0.emacsconf.org:{{ icecast_port }}/emacsconf/{{ icecast_test_track|d("gen") }}.webm
-  when: icecast_test is defined
-- name: Test by displaying the stream in mpv
-  tags: test-stream-mpv, never
-  local_action: shell mpv http://live0.emacsconf.org:{{ icecast_port }}/emacsconf/{{ icecast_test_track|d("gen") }}.webm
-  when: test is defined and test == 'mpv'
-- name: Test by streaming a test pattern
-  tags: test-stream-pattern
-  local_action: shell ffmpeg -loglevel 32 -f lavfi -i testsrc=duration=10:size=1280x720:rate=30 -f webm icecast://emacsconf:{{ icecast_emacsconf_password }}@live0.emacsconf.org:{{ icecast_port }}/emacsconf/{{ icecast_test_track|d("gen") }}.webm
-  when: test is defined and test == 'pattern'
 # - name: Set up restream script
 #   template:
 #     src: restream-yt.sh
diff --git a/roles/stream/templates/icecast-emacsconf.init.d b/roles/stream/templates/icecast-emacsconf.init.d
index b3ad491..3d880c1 100755
--- a/roles/stream/templates/icecast-emacsconf.init.d
+++ b/roles/stream/templates/icecast-emacsconf.init.d
@@ -30,14 +30,10 @@ test -x $DAEMON || exit 0
 
 # Defaults
 CONFIGFILE="/etc/icecast2/icecast-emacsconf.xml"
-CONFIGDEFAULTFILE="/etc/default/icecast2"
-USERID=icecast2
-GROUPID=icecast
+USERID={{ icecast_user }}
+GROUPID={{ icecast_group }}
 PIDFILE=/var/run/icecast-emacsconf
 
-# Reads config file (will override defaults above)
-[ -r "$CONFIGDEFAULTFILE" ] && . $CONFIGDEFAULTFILE
-
 set -e
 
 case "$1" in
diff --git a/roles/stream/templates/icecast.xml b/roles/stream/templates/icecast.xml
index d2bb89c..e09e6a0 100644
--- a/roles/stream/templates/icecast.xml
+++ b/roles/stream/templates/icecast.xml
@@ -190,33 +190,35 @@
     {% for track in emacsconf_tracks %}
     <mount type="normal">
         <mount-name>/emacsconf/{{ track.id }}.webm</mount-name>
-        <username>emacsconf</username>
+        <username>{{ icecast_emacsconf_user }}</username>
         <password>{{ icecast_emacsconf_password }}</password>
-	      <dump-file>/data/emacsconf-{{ emacsconf_year }}-{{ track.id }}_%Y-%m-%d_%H-%M-%S.webm</dump-file>
-        <stream-name>EmacsConf {{ emacsconf_year }} - {{ track.name }} track</stream-name>
-        <stream-description>The livestream for the {{ track.name }} track of EmacsConf {{ emacsconf_year }}</stream-description>
-	      <stream-url>https://emacsconf.org/{{ emacsconf_year }}/watch/{{ track.id }}/</stream-url>
+	      <dump-file>/data/{{ emacsconf_id }}-{{ emacsconf_year }}-{{ track.id }}_%Y-%m-%d_%H-%M-%S.webm</dump-file>
+        <stream-name>{{ emacsconf_name }} {{ emacsconf_year }} - {{ track.name }} track</stream-name>
+        <stream-description>The livestream for the {{ track.name }} track of {{ emacsconf_name }} {{ emacsconf_year }}</stream-description>
+	      <stream-url>{{ track.watch }}</stream-url>
         <type>video/webm</type>
         <public>1</public>
+        <on-connect>/usr/local/bin/{{ emacsconf_id }}-lowres-{{ track.id }}-on-connect</on-connect>
+        <on-disconnect>/usr/local/bin/{{ emacsconf_id }}-lowres-{{ track.id }}-on-disconnect</on-disconnect>
     </mount>
     <mount type="normal">
-        <mount-name>/emacsconf/{{ track.id }}-host.webm</mount-name>
-        <username>emacsconf</username>
+        <mount-name>/{{ emacsconf_id }}/{{ track.id }}-host.webm</mount-name>
+        <username>{{ icecast_emacsconf_user }}</username>
         <password>{{ icecast_emacsconf_password }}</password>
         <stream-name>EmacsConf {{ emacsconf_year }} - {{ track.name }} track</stream-name>
-        <stream-description>The host stream for the {{ track.name }} track of EmacsConf {{ emacsconf_year }}</stream-description>
-	      <stream-url>https://emacsconf.org/{{ emacsconf_year }}/watch/{{ track.id }}/</stream-url>
+        <stream-description>The host stream for the {{ track.name }} track of {{ emacsconf_name }} {{ emacsconf_year }}</stream-description>
+	      <stream-url>{{ track.watch }}</stream-url>
         <type>video/webm</type>
         <public>1</public>
         <hidden>1</hidden>
     </mount>
     <mount type="normal">
-        <mount-name>/emacsconf/{{ track.id }}-480p.webm</mount-name>
-        <username>emacsconf</username>
+        <mount-name>/{{ emacsconf_id }}/{{ track.id }}-480p.webm</mount-name>
+        <username>{{ icecast_emacsconf_user }}</username>
         <password>{{ icecast_emacsconf_password }}</password>
-        <stream-name>EmacsConf {{ emacsconf_year }} - {{ track.name }} track (480p)</stream-name>
-        <stream-description>The 480p livestream for the {{ track.name }} track of EmacsConf {{ emacsconf_year }}</stream-description>
-	      <stream-url>https://emacsconf.org/{{ emacsconf_year }}/watch/{{ track.id }}/</stream-url>
+        <stream-name>{{ emacsconf_name }} {{ emacsconf_year }} - {{ track.name }} track (480p)</stream-name>
+        <stream-description>The low-res livestream for the {{ track.name }} track of {{ emacsconf_name }} {{ emacsconf_year }}</stream-description>
+	      <stream-url>{{ track.watch_lowres }}</stream-url>
         <type>video/webm</type>
         <public>1</public>
     </mount>
diff --git a/roles/stream/templates/lowres.init.d b/roles/stream/templates/lowres.init.d
new file mode 100755
index 0000000..41da95c
--- /dev/null
+++ b/roles/stream/templates/lowres.init.d
@@ -0,0 +1,76 @@
+#!/bin/sh
+# {{ ansible_managed }}
+
+### BEGIN INIT INFO
+# Provides:          {{ emacsconf_id }}-lowres-{{ item.id }}
+# Required-Start:    $local_fs $remote_fs $network $syslog
+# Required-Stop:     $local_fs $remote_fs $network $syslog
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: 480p restreaming for {{ item.name }}
+# Description:       480p restreaming for {{ item.name }} using start-stop-daemon
+### END INIT INFO
+
+PATH="/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin"
+USER="{{ icecast_user }}"
+GROUP="{{ icecast_group }}"
+DESC="Low-res restream for {{ item.name }}"
+NAME="{{ emacsconf_id }}-lowres-{{ item.id }}"
+
+set -e
+
+. /lib/lsb/init-functions
+
+start() {
+  echo "Starting $DESC... "
+	start-stop-daemon --start --chuid "$USER:$GROUP" --background --make-pidfile --pidfile /var/run/$NAME.pid --exec "/usr/local/bin/{{ emacsconf_id }}-lowres-{{ item.id }}.sh http://localhost:{{ icecast_port }}/{{ item.id }}.webm http:// > /var/log/$NAME.log" || true
+  echo "done"
+}
+
+#We need this function to ensure the whole process tree will be killed
+killtree() {
+    local _pid=$1
+    local _sig=${2-TERM}
+    for _child in $(ps -o pid --no-headers --ppid ${_pid}); do
+        killtree ${_child} ${_sig}
+    done
+    kill -${_sig} ${_pid}
+}
+
+stop() {
+  echo "Stopping $DESC... "
+  if test -f /var/run/$NAME.pid; then
+    while test -d /proc/$(cat /var/run/$NAME.pid); do
+      killtree $(cat /var/run/$NAME.pid) 15
+      sleep 0.5
+    done
+    rm /var/run/$NAME.pid
+  fi
+  echo "done"
+}
+
+status() {
+  status_of_proc -p /var/run/$NAME.pid "" "$NAME" && exit 0 || exit $?
+}
+
+case "$1" in
+  start)
+	  start
+	  ;;
+  stop)
+    stop
+	  ;;
+  restart)
+	  stop
+	  start
+	  ;;
+  status)
+	  status
+	  ;;
+  *)
+	  echo "Usage: $NAME {start|stop|restart|status}" >&2
+	  exit 1
+	  ;;
+esac
+
+exit 0
diff --git a/roles/stream/templates/lowres.sh b/roles/stream/templates/lowres.sh
new file mode 100755
index 0000000..1f74215
--- /dev/null
+++ b/roles/stream/templates/lowres.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+# {{ ansible_managed }}
+sleep 10
+for i in 1 2 3 4 5; do
+    ffmpeg -loglevel 24 -f webm -reconnect_at_eof 1 -reconnect_streamed 1  -re -i "http://localhost:{{ icecast_port }}/{{ emacsconf_id }}/{{ item.id }}.webm" -vf scale="{{ icecast_lowres_scale }}" -f webm -c:a copy -b:v 500k -maxrate 1M -bufsize 1M -content_type video/webm -c:v libvpx "icecast://{{ icecast_emacsconf_user }}:{{ icecast_emacsconf_password }}@localhost:{{ icecast_port }}/{{ emacsconf_id }}/{{ item.id }}-480p.webm" >> {{ icecast_restream_dir }}/{{ emacsconf_id }}-lowres-{{ item.id }}.log || sleep 5
+done
diff --git a/roles/stream/templates/on-connect b/roles/stream/templates/on-connect
new file mode 100755
index 0000000..0f613bf
--- /dev/null
+++ b/roles/stream/templates/on-connect
@@ -0,0 +1,12 @@
+#!/bin/bash
+set -e
+PATH="/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin"
+PIDFILE="{{ icecast_restream_dir }}/{{ emacsconf_id }}-lowres-{{ item.id }}.pid"
+echo $(date) " on connect {{ item.id }} $*" >> {{ icecast_restream_dir }}/restream.log
+
+. /lib/lsb/init-functions
+
+start-stop-daemon --start --quiet --background \
+                  --make-pidfile --pidfile $PIDFILE \
+                  --oknodo \
+                  --exec /usr/local/bin/{{ emacsconf_id }}-lowres-{{ item.id }}.sh > /dev/null
diff --git a/roles/stream/templates/on-disconnect b/roles/stream/templates/on-disconnect
new file mode 100755
index 0000000..5847356
--- /dev/null
+++ b/roles/stream/templates/on-disconnect
@@ -0,0 +1,29 @@
+#!/bin/bash
+set -e
+PATH="/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin"
+PIDFILE="{{ icecast_restream_dir }}/{{ emacsconf_id }}-lowres-{{ item.id }}.pid"
+echo $(date) " on disconnect {{ item.id }} $*" >> {{ icecast_restream_dir }}/restream.log
+
+. /lib/lsb/init-functions
+
+#We need this function to ensure the whole process tree will be killed
+killtree() {
+    local _pid=$1
+    local _sig=${2-TERM}
+    for _child in $(ps -o pid --no-headers --ppid ${_pid}); do
+        killtree ${_child} ${_sig}
+    done
+    kill -${_sig} ${_pid}
+}
+
+stop() {
+  echo "Stopping $DESC... "
+  if test -f $PIDFILE; then
+    while test -d /proc/$(cat $PIDFILE); do
+      killtree $(cat $PIDFILE) 15
+      sleep 0.5
+    done
+    rm $PIDFILE
+  fi
+  echo "done"
+}
diff --git a/roles/wiki-publish/defaults/main.yml b/roles/wiki-publish/defaults/main.yml
index 863003f..4de7445 100644
--- a/roles/wiki-publish/defaults/main.yml
+++ b/roles/wiki-publish/defaults/main.yml
@@ -1,8 +1,9 @@
-emacsconf_year: 2022
 emacsconf_org_file: "{{ emacsconf_private_dir }}/conf.org"
 emacsconf_publishing_phase: schedule
-emacsconf_el_dir: ~/emacsconf-el
-emacsconf_edit_wiki_dir: ~/emacsconf-wiki
-emacsconf_private_dir: ~/emacsconf-2022-private
-emacs_config_dir: ~/.emacs.d
-emacs_package: emacs-snapshot-nox
+emacsconf_el_dir: ~{{ emacsconf_user }}/emacsconf-el
+emacsconf_edit_wiki_dir: ~{{ emacsconf_user }}/emacsconf-wiki
+emacsconf_private_dir: ~{{ emacsconf_user }}/emacsconf-2022-private
+emacs_config_dir: ~{{ emacsconf_user }}/.emacs.d
+emacs_version: 28.2
+emacs_build_parent: /usr/src/emacs
+emacs_build_dir: "{{ emacs_build_parent }}/emacs-{{ emacs_version }}"
diff --git a/roles/wiki-publish/tasks/emacs.yml b/roles/wiki-publish/tasks/emacs.yml
new file mode 100644
index 0000000..8d75979
--- /dev/null
+++ b/roles/wiki-publish/tasks/emacs.yml
@@ -0,0 +1,38 @@
+- name: Install dependencies
+  apt:
+    name:
+      - build-essential
+      - libncurses5-dev
+      - git
+      - gnutls-bin
+      - pkg-config
+      - libjansson-dev
+      - libgnutls28-dev
+      - automake
+      - texinfo
+    state: present
+- name: Create directory
+  file:
+    path: "{{ emacs_build_parent }}"
+    state: directory
+- name: Download Emacs
+  unarchive:
+    src: https://ftp.gnu.org/gnu/emacs/emacs-{{emacs_version}}.tar.gz
+    remote_src: yes
+    dest: "{{ emacs_build_parent }}"
+    creates: "{{ emacs_build_dir }}"
+- name: Configure
+  command: ./configure --with-x-toolkit=no 
+  args:
+    chdir: "{{ emacs_build_dir }}"
+    creates: '{{ emacs_build_dir }}/Makefile'
+- name: Build
+  command: make -j{{ cpus }}
+  args:
+    chdir: "{{ emacs_build_dir }}"
+    creates: '{{ emacs_build_dir }}/src/emacs'
+- name: Install
+  command: make install
+  args:
+    chdir: "{{ emacs_build_dir }}"
+    creates: '/usr/local/bin/emacs'
diff --git a/roles/wiki-publish/tasks/main.yml b/roles/wiki-publish/tasks/main.yml
index 960aa01..28566a5 100644
--- a/roles/wiki-publish/tasks/main.yml
+++ b/roles/wiki-publish/tasks/main.yml
@@ -1,45 +1,44 @@
 ---
 - name: Set up SSH directory
   ansible.builtin.file:
-    path: "/home/{{ emacsconf_publish_user }}/.ssh"
+    path: "/home/{{ emacsconf_user }}/.ssh"
     state: directory
     mode: '0700'
 - name: Install SSH key for EmacsConf wiki
   ansible.builtin.get_url:
     url: https://emacsconf.org/id_rsa_anon_git_emacsconf
-    dest: "/home/{{ emacsconf_publish_user }}/.ssh/id_rsa_anon_git_emacsconf"
+    dest: "/home/{{ emacsconf_user }}/.ssh/id_rsa_anon_git_emacsconf"
     mode: '0600'
-    owner: "{{ emacsconf_publish_user }}"
-- name: Set up packages
-  become: yes
-  when: emacs_package and emacs_package != "emacs"
-  block:
-    # This repository is currently not working
-    # - name: Add snapshot repository
-    #   ansible.builtin.apt_repository:
-    #     repo: deb http://emacs.ganneff.de/ buster main
-    - name: Remove old package
-      ansible.builtin.apt:
-        name: emacs
-        state: absent
-    - name: Install Emacs snapshot
-      ansible.builtin.apt:
-        name: "{{ emacs_package }}"
-        state: present
+- name: Install the SSH key for orga
+  template:
+    src: id_rsa
+    dest: "/home/{{ emacsconf_user }}/.ssh/id_rsa"
+    mode: '0600'
+- name: Change ownership of SSH directory
+  file:
+    path: "/home/{{ emacsconf_user }}/.ssh"
+    recurse: true
+    owner: "{{ emacsconf_user }}"
+
+- name: Set up Emacs
+  include: emacs.yml
+- name: Ensure configuration directory exists
+  file:
+    path: "{{ emacs_config_dir }}"
+    state: directory
 - name: Set up Emacs configuration for non-interactive publishing
-  become: true
-  become_user: "{{ emacsconf_publish_user }}"
   template:
     src: emacsconf-config.el
     dest: "{{ emacs_config_dir }}/emacsconf-config.el"
+
 - name: Check if Emacs base configuration already exists
   become: true
-  become_user: "{{ emacsconf_publish_user }}"
   lineinfile:
     dest: "{{ emacs_config_dir }}/init.el"
     state: present
     regexp: "emacsconf-config"
     line: "(load-file \"emacsconf-config.el\")"
+    create: yes
 - name: Set up or update repositories
   become: false
   tags: publish
@@ -71,6 +70,16 @@
         - private.failed
         - not 'Local modifications exist in repository' in private.msg
         - not 'Failed to checkout branch' in private.msg
+- name: Change ownership
+  file:
+    path: "{{ item }}"
+    owner: "{{ emacsconf_user }}"
+    recurse: true
+  with_items:
+    - "{{ emacsconf_config_dir }}"
+    - "{{ emacsconf_el_dir }}"
+    - "{{ emacsconf_private_dir }}"
+    - "{{ emacsconf_edit_wiki_dir }}"
 - name: Publish
   tags: publish
   block: