media, wiki-publish, caption updates

* common-playbook.yml: Start moving publishing setup to res
* group_vars/all.yml: New common variables.
* inventory.yml (prod): Number of threads, new hosts
* roles/caption/templates/process-captions.py: Add mp4, try to figure
out why script was failing, simplify
* roles/media/tasks/main.yml: New role for setting up
media.emacsconf.org for this year
* roles/prerec/templates/reencode.sh: Keep a copy of zaeph's script
* roles/stream/defaults/main.yml: Add more variables
* roles/stream: Restreaming lowres
* roles/wiki-publish/tasks/emacs.yml: Build Emacs from source

8 files changed, 175 insertions, 38 deletions

diff --git a/roles/stream/defaults/main.yml b/roles/stream/defaults/main.yml
index cd8984f..5ce637e 100644
--- a/roles/stream/defaults/main.yml
+++ b/roles/stream/defaults/main.yml
@@ -1,2 +1,6 @@
 icecast_admin_email: emacsconf-org@gnu.org
 icecast_port: 8001
+icecast_lowres_scale: "854:480"
+icecast_user: icecast2
+icecast_group: icecast
+icecast_restream_dir: /etc/icecast2/restream
diff --git a/roles/stream/tasks/main.yml b/roles/stream/tasks/main.yml
index 7cabb6b..76febe0 100644
--- a/roles/stream/tasks/main.yml
+++ b/roles/stream/tasks/main.yml
@@ -8,6 +8,33 @@
   template:
     src: icecast.xml
     dest: /etc/icecast2/icecast-emacsconf.xml
+- name: Create restream dir
+  file:
+    path: "{{ icecast_restream_dir }}"
+    owner: "{{ icecast_user }}"
+    state: directory
+- name: Debug
+  debug:
+    var: item.id
+  loop: "{{ emacsconf_tracks }}"
+- name: Copy the lowres restreaming shell script
+  template:
+    src: lowres.sh
+    dest: /usr/local/bin/{{ emacsconf_id }}-lowres-{{ item.id }}.sh
+    mode: 0755
+  loop: "{{ emacsconf_tracks }}"
+- name: Copy lowres on-connect
+  template:
+    src: on-connect
+    dest: /usr/local/bin/{{ emacsconf_id }}-lowres-{{ item.id }}-on-connect
+    mode: 0755
+  loop: "{{ emacsconf_tracks }}"
+- name: Copy lowres on-disconnect
+  template:
+    src: on-disconnect
+    dest: /usr/local/bin/{{ emacsconf_id }}-lowres-{{ item.id }}-on-disconnect
+    mode: 0755
+  loop: "{{ emacsconf_tracks }}"
 - name: Set up init file
   become: true
   template:
@@ -22,36 +49,21 @@
     mode: 0644
 - name: Enable icecast
   become: true
-  sysvinit:
+  service:
     name: emacsconf
-    enabled: yes
     state: started
+    enabled: yes
 - name: Check if icecast is listening
   wait_for:
     port: "{{ icecast_port }}"
     delay: 5
-    timeout: 10
+    timeout: 20
     msg: "Timeout waiting for {{ icecast_port }} to respond"
   register: port_check
   ignore_errors: yes
 - name: Try to restart icecast if not started
   service: name=emacsconf state=started enabled=yes
   when: port_check.failed == true
-- name: Test by streaming from a file
-  tags: test-stream-file, never
-  block:
-  - debug:
-      msg: ffmpeg -loglevel 32 -re -i {{ icecast_test }} -loop 1 -f webm -r 30 -ac 2 -cluster_size_limit 2M -cluster_time_limit 5100 -content_type video/webm -c:v libvpx -b:v 1M -crf 30 -g 125 -threads 4 -deadline realtime  icecast://emacsconf:{{ icecast_emacsconf_password }}@live0.emacsconf.org:{{ icecast_port }}/emacsconf/{{ icecast_test_track|d("gen") }}.webm
-  - local_action: shell ffmpeg -re -loglevel 32 -i {{ icecast_test }} -loop 1 -f webm -r 30 -ac 2 -cluster_size_limit 2M -cluster_time_limit 5100 -content_type video/webm -c:v libvpx -b:v 1M -crf 30 -g 125 -threads 4 -deadline realtime  icecast://emacsconf:{{ icecast_emacsconf_password }}@live0.emacsconf.org:{{ icecast_port }}/emacsconf/{{ icecast_test_track|d("gen") }}.webm
-  when: icecast_test is defined
-- name: Test by displaying the stream in mpv
-  tags: test-stream-mpv, never
-  local_action: shell mpv http://live0.emacsconf.org:{{ icecast_port }}/emacsconf/{{ icecast_test_track|d("gen") }}.webm
-  when: test is defined and test == 'mpv'
-- name: Test by streaming a test pattern
-  tags: test-stream-pattern
-  local_action: shell ffmpeg -loglevel 32 -f lavfi -i testsrc=duration=10:size=1280x720:rate=30 -f webm icecast://emacsconf:{{ icecast_emacsconf_password }}@live0.emacsconf.org:{{ icecast_port }}/emacsconf/{{ icecast_test_track|d("gen") }}.webm
-  when: test is defined and test == 'pattern'
 # - name: Set up restream script
 #   template:
 #     src: restream-yt.sh
diff --git a/roles/stream/templates/icecast-emacsconf.init.d b/roles/stream/templates/icecast-emacsconf.init.d
index b3ad491..3d880c1 100755
--- a/roles/stream/templates/icecast-emacsconf.init.d
+++ b/roles/stream/templates/icecast-emacsconf.init.d
@@ -30,14 +30,10 @@ test -x $DAEMON || exit 0
 
 # Defaults
 CONFIGFILE="/etc/icecast2/icecast-emacsconf.xml"
-CONFIGDEFAULTFILE="/etc/default/icecast2"
-USERID=icecast2
-GROUPID=icecast
+USERID={{ icecast_user }}
+GROUPID={{ icecast_group }}
 PIDFILE=/var/run/icecast-emacsconf
 
-# Reads config file (will override defaults above)
-[ -r "$CONFIGDEFAULTFILE" ] && . $CONFIGDEFAULTFILE
-
 set -e
 
 case "$1" in
diff --git a/roles/stream/templates/icecast.xml b/roles/stream/templates/icecast.xml
index d2bb89c..e09e6a0 100644
--- a/roles/stream/templates/icecast.xml
+++ b/roles/stream/templates/icecast.xml
@@ -190,33 +190,35 @@
     {% for track in emacsconf_tracks %}
     <mount type="normal">
         <mount-name>/emacsconf/{{ track.id }}.webm</mount-name>
-        <username>emacsconf</username>
+        <username>{{ icecast_emacsconf_user }}</username>
         <password>{{ icecast_emacsconf_password }}</password>
-	      <dump-file>/data/emacsconf-{{ emacsconf_year }}-{{ track.id }}_%Y-%m-%d_%H-%M-%S.webm</dump-file>
-        <stream-name>EmacsConf {{ emacsconf_year }} - {{ track.name }} track</stream-name>
-        <stream-description>The livestream for the {{ track.name }} track of EmacsConf {{ emacsconf_year }}</stream-description>
-	      <stream-url>https://emacsconf.org/{{ emacsconf_year }}/watch/{{ track.id }}/</stream-url>
+	      <dump-file>/data/{{ emacsconf_id }}-{{ emacsconf_year }}-{{ track.id }}_%Y-%m-%d_%H-%M-%S.webm</dump-file>
+        <stream-name>{{ emacsconf_name }} {{ emacsconf_year }} - {{ track.name }} track</stream-name>
+        <stream-description>The livestream for the {{ track.name }} track of {{ emacsconf_name }} {{ emacsconf_year }}</stream-description>
+	      <stream-url>{{ track.watch }}</stream-url>
         <type>video/webm</type>
         <public>1</public>
+        <on-connect>/usr/local/bin/{{ emacsconf_id }}-lowres-{{ track.id }}-on-connect</on-connect>
+        <on-disconnect>/usr/local/bin/{{ emacsconf_id }}-lowres-{{ track.id }}-on-disconnect</on-disconnect>
     </mount>
     <mount type="normal">
-        <mount-name>/emacsconf/{{ track.id }}-host.webm</mount-name>
-        <username>emacsconf</username>
+        <mount-name>/{{ emacsconf_id }}/{{ track.id }}-host.webm</mount-name>
+        <username>{{ icecast_emacsconf_user }}</username>
         <password>{{ icecast_emacsconf_password }}</password>
         <stream-name>EmacsConf {{ emacsconf_year }} - {{ track.name }} track</stream-name>
-        <stream-description>The host stream for the {{ track.name }} track of EmacsConf {{ emacsconf_year }}</stream-description>
-	      <stream-url>https://emacsconf.org/{{ emacsconf_year }}/watch/{{ track.id }}/</stream-url>
+        <stream-description>The host stream for the {{ track.name }} track of {{ emacsconf_name }} {{ emacsconf_year }}</stream-description>
+	      <stream-url>{{ track.watch }}</stream-url>
         <type>video/webm</type>
         <public>1</public>
         <hidden>1</hidden>
     </mount>
     <mount type="normal">
-        <mount-name>/emacsconf/{{ track.id }}-480p.webm</mount-name>
-        <username>emacsconf</username>
+        <mount-name>/{{ emacsconf_id }}/{{ track.id }}-480p.webm</mount-name>
+        <username>{{ icecast_emacsconf_user }}</username>
         <password>{{ icecast_emacsconf_password }}</password>
-        <stream-name>EmacsConf {{ emacsconf_year }} - {{ track.name }} track (480p)</stream-name>
-        <stream-description>The 480p livestream for the {{ track.name }} track of EmacsConf {{ emacsconf_year }}</stream-description>
-	      <stream-url>https://emacsconf.org/{{ emacsconf_year }}/watch/{{ track.id }}/</stream-url>
+        <stream-name>{{ emacsconf_name }} {{ emacsconf_year }} - {{ track.name }} track (480p)</stream-name>
+        <stream-description>The low-res livestream for the {{ track.name }} track of {{ emacsconf_name }} {{ emacsconf_year }}</stream-description>
+	      <stream-url>{{ track.watch_lowres }}</stream-url>
         <type>video/webm</type>
         <public>1</public>
     </mount>
diff --git a/roles/stream/templates/lowres.init.d b/roles/stream/templates/lowres.init.d
new file mode 100755
index 0000000..41da95c
--- /dev/null
+++ b/roles/stream/templates/lowres.init.d
@@ -0,0 +1,76 @@
+#!/bin/sh
+# {{ ansible_managed }}
+
+### BEGIN INIT INFO
+# Provides:          {{ emacsconf_id }}-lowres-{{ item.id }}
+# Required-Start:    $local_fs $remote_fs $network $syslog
+# Required-Stop:     $local_fs $remote_fs $network $syslog
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: 480p restreaming for {{ item.name }}
+# Description:       480p restreaming for {{ item.name }} using start-stop-daemon
+### END INIT INFO
+
+PATH="/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin"
+USER="{{ icecast_user }}"
+GROUP="{{ icecast_group }}"
+DESC="Low-res restream for {{ item.name }}"
+NAME="{{ emacsconf_id }}-lowres-{{ item.id }}"
+
+set -e
+
+. /lib/lsb/init-functions
+
+start() {
+  echo "Starting $DESC... "
+	start-stop-daemon --start --chuid "$USER:$GROUP" --background --make-pidfile --pidfile /var/run/$NAME.pid --exec "/usr/local/bin/{{ emacsconf_id }}-lowres-{{ item.id }}.sh http://localhost:{{ icecast_port }}/{{ item.id }}.webm http:// > /var/log/$NAME.log" || true
+  echo "done"
+}
+
+#We need this function to ensure the whole process tree will be killed
+killtree() {
+    local _pid=$1
+    local _sig=${2-TERM}
+    for _child in $(ps -o pid --no-headers --ppid ${_pid}); do
+        killtree ${_child} ${_sig}
+    done
+    kill -${_sig} ${_pid}
+}
+
+stop() {
+  echo "Stopping $DESC... "
+  if test -f /var/run/$NAME.pid; then
+    while test -d /proc/$(cat /var/run/$NAME.pid); do
+      killtree $(cat /var/run/$NAME.pid) 15
+      sleep 0.5
+    done
+    rm /var/run/$NAME.pid
+  fi
+  echo "done"
+}
+
+status() {
+  status_of_proc -p /var/run/$NAME.pid "" "$NAME" && exit 0 || exit $?
+}
+
+case "$1" in
+  start)
+	  start
+	  ;;
+  stop)
+    stop
+	  ;;
+  restart)
+	  stop
+	  start
+	  ;;
+  status)
+	  status
+	  ;;
+  *)
+	  echo "Usage: $NAME {start|stop|restart|status}" >&2
+	  exit 1
+	  ;;
+esac
+
+exit 0
diff --git a/roles/stream/templates/lowres.sh b/roles/stream/templates/lowres.sh
new file mode 100755
index 0000000..1f74215
--- /dev/null
+++ b/roles/stream/templates/lowres.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+# {{ ansible_managed }}
+sleep 10
+for i in 1 2 3 4 5; do
+    ffmpeg -loglevel 24 -f webm -reconnect_at_eof 1 -reconnect_streamed 1  -re -i "http://localhost:{{ icecast_port }}/{{ emacsconf_id }}/{{ item.id }}.webm" -vf scale="{{ icecast_lowres_scale }}" -f webm -c:a copy -b:v 500k -maxrate 1M -bufsize 1M -content_type video/webm -c:v libvpx "icecast://{{ icecast_emacsconf_user }}:{{ icecast_emacsconf_password }}@localhost:{{ icecast_port }}/{{ emacsconf_id }}/{{ item.id }}-480p.webm" >> {{ icecast_restream_dir }}/{{ emacsconf_id }}-lowres-{{ item.id }}.log || sleep 5
+done
diff --git a/roles/stream/templates/on-connect b/roles/stream/templates/on-connect
new file mode 100755
index 0000000..0f613bf
--- /dev/null
+++ b/roles/stream/templates/on-connect
@@ -0,0 +1,12 @@
+#!/bin/bash
+set -e
+PATH="/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin"
+PIDFILE="{{ icecast_restream_dir }}/{{ emacsconf_id }}-lowres-{{ item.id }}.pid"
+echo $(date) " on connect {{ item.id }} $*" >> {{ icecast_restream_dir }}/restream.log
+
+. /lib/lsb/init-functions
+
+start-stop-daemon --start --quiet --background \
+                  --make-pidfile --pidfile $PIDFILE \
+                  --oknodo \
+                  --exec /usr/local/bin/{{ emacsconf_id }}-lowres-{{ item.id }}.sh > /dev/null
diff --git a/roles/stream/templates/on-disconnect b/roles/stream/templates/on-disconnect
new file mode 100755
index 0000000..5847356
--- /dev/null
+++ b/roles/stream/templates/on-disconnect
@@ -0,0 +1,29 @@
+#!/bin/bash
+set -e
+PATH="/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin"
+PIDFILE="{{ icecast_restream_dir }}/{{ emacsconf_id }}-lowres-{{ item.id }}.pid"
+echo $(date) " on disconnect {{ item.id }} $*" >> {{ icecast_restream_dir }}/restream.log
+
+. /lib/lsb/init-functions
+
+#We need this function to ensure the whole process tree will be killed
+killtree() {
+    local _pid=$1
+    local _sig=${2-TERM}
+    for _child in $(ps -o pid --no-headers --ppid ${_pid}); do
+        killtree ${_child} ${_sig}
+    done
+    kill -${_sig} ${_pid}
+}
+
+stop() {
+  echo "Stopping $DESC... "
+  if test -f $PIDFILE; then
+    while test -d /proc/$(cat $PIDFILE); do
+      killtree $(cat $PIDFILE) 15
+      sleep 0.5
+    done
+    rm $PIDFILE
+  fi
+  echo "done"
+}