summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorSacha Chua <sacha@sachachua.com>2022-10-29 07:33:39 -0400
committerSacha Chua <sacha@sachachua.com>2022-10-29 07:33:39 -0400
commita6a373410bde6918f27992ff967660c0541f810a (patch)
treeab8fbd6a5e12f065a7af82027b98df818dfb9603
parentbff9ceae7f42723f939f620f9ef9faebca966742 (diff)
downloademacsconf-ansible-a6a373410bde6918f27992ff967660c0541f810a.tar.xz
emacsconf-ansible-a6a373410bde6918f27992ff967660c0541f810a.zip
media, wiki-publish, caption updates
* common-playbook.yml: Start moving publishing setup to res * group_vars/all.yml: New common variables. * inventory.yml (prod): Number of threads, new hosts * roles/caption/templates/process-captions.py: Add mp4, try to figure out why script was failing, simplify * roles/media/tasks/main.yml: New role for setting up media.emacsconf.org for this year * roles/prerec/templates/reencode.sh: Keep a copy of zaeph's script * roles/stream/defaults/main.yml: Add more variables * roles/stream: Restreaming lowres * roles/wiki-publish/tasks/emacs.yml: Build Emacs from source
-rw-r--r--common-playbook.yml23
-rw-r--r--group_vars/all.yml11
-rw-r--r--inventory.yml14
-rw-r--r--roles/caption/tasks/main.yml4
-rwxr-xr-xroles/caption/templates/process-captions.py84
-rw-r--r--roles/media/defaults/main.yml1
-rw-r--r--roles/media/tasks/main.yml34
-rw-r--r--roles/media/templates/nginx-include20
-rw-r--r--roles/prerec/templates/reencode.sh5
-rw-r--r--roles/stream/defaults/main.yml4
-rw-r--r--roles/stream/tasks/main.yml48
-rwxr-xr-xroles/stream/templates/icecast-emacsconf.init.d8
-rw-r--r--roles/stream/templates/icecast.xml30
-rwxr-xr-xroles/stream/templates/lowres.init.d76
-rwxr-xr-xroles/stream/templates/lowres.sh6
-rwxr-xr-xroles/stream/templates/on-connect12
-rwxr-xr-xroles/stream/templates/on-disconnect29
-rw-r--r--roles/wiki-publish/defaults/main.yml13
-rw-r--r--roles/wiki-publish/tasks/emacs.yml38
-rw-r--r--roles/wiki-publish/tasks/main.yml53
20 files changed, 390 insertions, 123 deletions
diff --git a/common-playbook.yml b/common-playbook.yml
index 388d84f..2adfdff 100644
--- a/common-playbook.yml
+++ b/common-playbook.yml
@@ -4,12 +4,12 @@
roles:
- wiki
- name: Set up wiki for publishing
- hosts: front
+ hosts: res
tags: wiki-publish
roles:
- wiki-publish
- name: Set up wiki for interactive editing
- hosts: front
+ hosts: res
tags: wiki-edit
roles:
- wiki-edit
@@ -28,14 +28,31 @@
tags: upload
roles:
- upload
-
- name: Set up icecast server for streaming
hosts: stream
tags: stream
roles:
- stream
+- name: Test the icecast server
+ hosts: res
+ tags: test, never
+ tasks:
+ - include_role:
+ name: stream
+ tasks_from: test
+ when: icecast_test is defined and icecast_test=='file'
- name: Set up captioning tools
hosts: res
tags: caption
roles:
- caption
+- name: Set up media
+ hosts: media
+ tags: media
+ roles:
+ - media
+- name: Set up OBS
+ hosts: obs
+ tags: obs
+ roles:
+ - obs
diff --git a/group_vars/all.yml b/group_vars/all.yml
index 8f86f46..f7bbbc4 100644
--- a/group_vars/all.yml
+++ b/group_vars/all.yml
@@ -1,7 +1,18 @@
docker: false
+res_x: 1280
+res_y: 720
+emacsconf_year: 2022
+emacsconf_name: EmacsConf
+emacsconf_id: emacsconf
emacsconf_user: orga
+emacsconf_group: orga
emacsconf_tracks:
- name: General
id: gen
+ watch: https://emacsconf.org/{{ emacsconf_year }}/watch/gen/
+ watch_lowres: https://live.emacsconf.org/{{ emacsconf_year }}/watch/gen-480p/
- name: Development
id: dev
+ watch: https://emacsconf.org/{{ emacsconf_year }}/watch/dev/
+ watch_lowres: https://live.emacsconf.org/{{ emacsconf_year }}/watch/dev-480p/
+icecast_emacsconf_user: emacsconf
diff --git a/inventory.yml b/inventory.yml
index 997e01f..f91434d 100644
--- a/inventory.yml
+++ b/inventory.yml
@@ -5,6 +5,7 @@ prod:
ansible_python_interpreter: /usr/bin/python3
ansible_become: true
emacsconf_group: org
+ cpus: 12
front:
ansible_host: front0.emacsconf.org
remote_user: orga
@@ -23,6 +24,13 @@ prod:
ansible_ssh_user: orga
ansible_python_interpreter: /usr/bin/python3
ansible_become: true
+ media:
+ ansible_host: media.emacsconf.org
+ remote_user: orga
+ ansible_ssh_user: orga
+ ansible_python_interpreter: /usr/bin/python3
+ ansible_become: true
+ host_name: media.emacsconf.org
upload:
ansible_host: media.emacsconf.org
remote_user: orga
@@ -35,6 +43,12 @@ prod:
ansible_ssh_user: orga
ansible_python_interpreter: /usr/bin/python3
ansible_become: true
+ obs:
+ ansible_host: obs
+ remote_user: sacha
+ ansible_ssh_user: sacha
+ ansible_python_interpreter: /usr/bin/python3
+ ansible_become: true
all:
hosts:
localhost:
diff --git a/roles/caption/tasks/main.yml b/roles/caption/tasks/main.yml
index 7fe1570..353b83b 100644
--- a/roles/caption/tasks/main.yml
+++ b/roles/caption/tasks/main.yml
@@ -31,7 +31,7 @@
path: "{{ emacsconf_caption_dir }}"
state: directory
- name: Copy the script for processing the files
- tags: process-captions, wip
+ tags: process-captions
template:
src: process-captions.py
dest: "{{ emacsconf_caption_dir }}/process-captions.py"
@@ -58,7 +58,6 @@
group: root
mode: 0755
- name: Change the group for all the files
- tags: wip
file:
dest: "{{ emacsconf_caption_dir }}"
group: "{{ emacsconf_group }}"
@@ -66,7 +65,6 @@
recurse: true
- name: Restart caption monitoring service
become: true
- tags: wip
service:
name: captions
enabled: true
diff --git a/roles/caption/templates/process-captions.py b/roles/caption/templates/process-captions.py
index 72e9ad2..66f39dd 100755
--- a/roles/caption/templates/process-captions.py
+++ b/roles/caption/templates/process-captions.py
@@ -32,16 +32,12 @@ import datetime
import sys
import webvtt
import xml.etree.ElementTree as ET
-from lhotse import RecordingSet, Recording, AudioSource, SupervisionSegment, SupervisionSet, create_cut_set_eager, align_with_torchaudio, CutSet, annotate_with_whisper
-from tqdm import tqdm
-import whisper
-import re
-import os
+from lhotse import RecordinRecording, AudioSource, SupervisionSegment, SupervisionSet, create_cut_set_e
import json
import torch
-THREADS = 12
-VIDEO_REGEXP = '\.(webm|mov)$'
+THREADS = {{ cpus }}
+VIDEO_REGEXP = '\.(webm|mov|mp4)$'
AUDIO_REGEXP = '\.(ogg|opus)$'
ALWAYS = False
TRIM_AUDIO = False
@@ -49,6 +45,8 @@ MODEL = os.environ.get('MODEL', 'large') # Set to tiny for testing
WORK_DIR = "{{ emacsconf_caption_dir }}"
JSON_FILE = os.path.join(WORK_DIR, 'talks.json')
+# ----------------------------------------------------------------
+
def get_slug_from_filename(filename):
m = re.search('emacsconf-[0-9]+-([a-z]+)--', filename)
if m:
@@ -155,41 +153,23 @@ def generate_srv2(work):
captions.append(SupervisionSegment(id=rec_id + '-sup' + '%05d' % i, channel=recs[0].channel_ids[0], recording_id=rec_id, start=to_sec(caption.start), duration=to_sec(caption.end) - to_sec(caption.start), text=caption.text, language='English'))
sups = SupervisionSet.from_segments(captions)
main = CutSet.from_manifests(recordings=recs, supervisions=sups)
- work['cuts'] = main.trim_to_supervisions(keep_overlapping=False,keep_all_channels=True)
+ work['cuts'] = main.trim_to_supervisions(keep_all_channels=True)
cuts_aligned = align_with_torchaudio(work['cuts'])
root = ET.Element("timedtext")
doc = ET.SubElement(root, "window")
for line, aligned in enumerate(cuts_aligned):
- # Numbers are weird
- words = re.split(' ', captions[line].text)
- tokenized_words = [re.sub('[^\'A-Z0-9]', '', w.upper()) for w in words]
- if len(aligned.supervisions) == 0:
- print(captions[line], aligned)
- continue
- aligned_words = list(aligned.supervisions[0].alignment['word'])
- aligned_index = 0
- aligned_len = len(aligned_words)
- word_index = 0
- word_len = len(words)
- while word_index < word_len and aligned_index < aligned_len:
- # log("Choosing %s %s" % (words[word_index], aligned_words[aligned_index].symbol))
- ET.SubElement(doc, 'text',
- t=str(float(aligned_words[aligned_index].start)*1000),
- d=str(float(aligned_words[aligned_index].duration)*1000),
- w="1",
- append="1").text = words[word_index]
- if tokenized_words[word_index] != aligned_words[aligned_index].symbol and word_index < word_len - 1:
- # Scan ahead for a word that maches the next word, but don't go too far
- cur_aligned = aligned_index
- while aligned_index < aligned_len and aligned_index < cur_aligned + 5 and aligned_words[aligned_index].symbol != tokenized_words[word_index + 1]:
- log("Sliding to match %s %d %s" % (tokenized_words[word_index + 1], aligned_index, aligned_words[aligned_index].symbol))
- aligned_index = aligned_index + 1
- if not aligned_words[aligned_index].symbol == tokenized_words[word_index + 1]:
- log("Resetting, couldn't find")
- aligned_index = cur_aligned + 1
- else:
- aligned_index = aligned_index + 1
- word_index = word_index + 1
+ if len(aligned.supervisions) > 0:
+ aligned_words = aligned.supervisions[0].alignment['word']
+ for w, word in enumerate(aligned_words):
+ el = ET.SubElement(doc, 'text',
+ t=str(float(word.start)*1000),
+ d=str(float(word.duration)*1000),
+ w="1",
+ append="1")
+ el.text = word.symbol
+ el.tail = "\n"
+ else:
+ print("No supervisions", aligned)
tree = ET.ElementTree(root)
work['srv2'] = work['base'] + '.srv2'
with open(work['srv2'], "w") as f:
@@ -218,18 +198,20 @@ directory = sys.argv[1] if len(sys.argv) > 1 else WORK_DIR
needs_work = get_files_to_work_on(directory)
if len(needs_work) > 0:
- if THREADS > 0:
- torch.set_num_threads(THREADS)
- for work in needs_work:
- log("Started processing %s" % work['base'])
- if work['audio']:
- if ALWAYS or not 'vtt' in work:
- work = generate_captions(work)
- if ALWAYS or not 'srv2' in work:
- work = generate_srv2(work)
- # print("Aligning words", audio_file, datetime.datetime.now())
- # word_cuts = align_words(cuts)
- # convert_cuts_to_word_timing(audio_file, word_cuts)
- log("Done %s" % str(work['base']))
+ while len(needs_work) > 0:
+ if THREADS > 0:
+ torch.set_num_threads(THREADS)
+ for work in needs_work:
+ log("Started processing %s" % work['base'])
+ if work['audio']:
+ if ALWAYS or not 'vtt' in work:
+ work = generate_captions(work)
+ if ALWAYS or not 'srv2' in work:
+ work = generate_srv2(work)
+ # print("Aligning words", audio_file, datetime.datetime.now())
+ # word_cuts = align_words(cuts)
+ # convert_cuts_to_word_timing(audio_file, word_cuts)
+ log("Done %s" % str(work['base']))
+ needs_work = get_files_to_work_on(directory)
else:
log("No work needed.")
diff --git a/roles/media/defaults/main.yml b/roles/media/defaults/main.yml
new file mode 100644
index 0000000..0d8046d
--- /dev/null
+++ b/roles/media/defaults/main.yml
@@ -0,0 +1 @@
+media_protect_root: false
diff --git a/roles/media/tasks/main.yml b/roles/media/tasks/main.yml
new file mode 100644
index 0000000..6eb3ab6
--- /dev/null
+++ b/roles/media/tasks/main.yml
@@ -0,0 +1,34 @@
+- name: Install package for setting htpasswd
+ package:
+ name: python3-passlib
+- name: Ensure web path exists
+ file:
+ path: /var/www/{{ host_name }}/{{ emacsconf_year }}/backstage
+ state: directory
+- name: Change ownership and permissions
+ file:
+ path: /var/www/{{ host_name }}/{{ emacsconf_year }}
+ owner: "{{ emacsconf_user }}"
+ group: "{{ emacsconf_group }}"
+ mode: "u=rwX,g=rwX,o=rX"
+ recurse: true
+- name: Create htpasswd entry
+ htpasswd:
+ create: yes
+ name: "{{ emacsconf_backstage_user }}"
+ password: "{{ emacsconf_backstage_password }}"
+ path: /etc/nginx/sites-available/{{ host_name }}-{{ emacsconf_year }}-htpasswd
+- name: Create Nginx include
+ template:
+ src: nginx-include
+ dest: /etc/nginx/sites-available/{{ host_name }}-{{ emacsconf_year }}-include
+- name: Include it in the main configuration
+ lineinfile:
+ path: /etc/nginx/sites-available/{{ host_name }}
+ regexp: "{{ host_name }}-{{ emacsconf_year }}-include"
+ line: " include sites-available/{{ host_name }}-{{ emacsconf_year }}-include;"
+ insertafter: root
+- name: Reload configuration
+ service:
+ name: nginx
+ state: reloaded
diff --git a/roles/media/templates/nginx-include b/roles/media/templates/nginx-include
new file mode 100644
index 0000000..3bfb58e
--- /dev/null
+++ b/roles/media/templates/nginx-include
@@ -0,0 +1,20 @@
+ location /{{ emacsconf_year }}/backstage {
+ auth_basic "Restricted";
+ auth_basic_user_file /etc/nginx/sites-available/{{ host_name }}-{{ emacsconf_year }}-htpasswd;
+ autoindex on;
+ }
+ {% if media_protect_root %}
+ location /{{ emacsconf_year }} {
+ auth_basic "Restricted";
+ auth_basic_user_file /etc/nginx/sites-available/{{ host_name }}-{{ emacsconf_year }}-htpasswd;
+ autoindex on;
+ }
+ {% endif %}
+ location /{{ emacsconf_year }}/emacsconf.ics {
+ auth_basic off;
+ }
+ {% for track in emacsconf_tracks %}
+ location /{{ emacsconf_year }}/emacsconf-{{ track.id }}.ics {
+ auth_basic off;
+ }
+ {% endfor %} \ No newline at end of file
diff --git a/roles/prerec/templates/reencode.sh b/roles/prerec/templates/reencode.sh
new file mode 100644
index 0000000..d0c98c3
--- /dev/null
+++ b/roles/prerec/templates/reencode.sh
@@ -0,0 +1,5 @@
+Q=32
+CPU=8
+TIME_LIMIT=''
+ffmpeg -y -i "$1" $TIME_LIMIT -c:v libvpx-vp9 -b:v 0 -crf $Q -an -row-mt 1 -tile-columns 2 -tile-rows 2 -cpu-used $CPU -g 240 -pass 1 -f webm -threads $CPU /dev/null &&
+ ffmpeg -y -i "$1" $TIME_LIMIT -c:v libvpx-vp9 -b:v 0 -crf $Q -c:a libopus -row-mt 1 -tile-columns 2 -tile-rows 2 -cpu-used $CPU -pass 2 -g 240 -threads $CPU "$2"
diff --git a/roles/stream/defaults/main.yml b/roles/stream/defaults/main.yml
index cd8984f..5ce637e 100644
--- a/roles/stream/defaults/main.yml
+++ b/roles/stream/defaults/main.yml
@@ -1,2 +1,6 @@
icecast_admin_email: emacsconf-org@gnu.org
icecast_port: 8001
+icecast_lowres_scale: "854:480"
+icecast_user: icecast2
+icecast_group: icecast
+icecast_restream_dir: /etc/icecast2/restream
diff --git a/roles/stream/tasks/main.yml b/roles/stream/tasks/main.yml
index 7cabb6b..76febe0 100644
--- a/roles/stream/tasks/main.yml
+++ b/roles/stream/tasks/main.yml
@@ -8,6 +8,33 @@
template:
src: icecast.xml
dest: /etc/icecast2/icecast-emacsconf.xml
+- name: Create restream dir
+ file:
+ path: "{{ icecast_restream_dir }}"
+ owner: "{{ icecast_user }}"
+ state: directory
+- name: Debug
+ debug:
+ var: item.id
+ loop: "{{ emacsconf_tracks }}"
+- name: Copy the lowres restreaming shell script
+ template:
+ src: lowres.sh
+ dest: /usr/local/bin/{{ emacsconf_id }}-lowres-{{ item.id }}.sh
+ mode: 0755
+ loop: "{{ emacsconf_tracks }}"
+- name: Copy lowres on-connect
+ template:
+ src: on-connect
+ dest: /usr/local/bin/{{ emacsconf_id }}-lowres-{{ item.id }}-on-connect
+ mode: 0755
+ loop: "{{ emacsconf_tracks }}"
+- name: Copy lowres on-disconnect
+ template:
+ src: on-disconnect
+ dest: /usr/local/bin/{{ emacsconf_id }}-lowres-{{ item.id }}-on-disconnect
+ mode: 0755
+ loop: "{{ emacsconf_tracks }}"
- name: Set up init file
become: true
template:
@@ -22,36 +49,21 @@
mode: 0644
- name: Enable icecast
become: true
- sysvinit:
+ service:
name: emacsconf
- enabled: yes
state: started
+ enabled: yes
- name: Check if icecast is listening
wait_for:
port: "{{ icecast_port }}"
delay: 5
- timeout: 10
+ timeout: 20
msg: "Timeout waiting for {{ icecast_port }} to respond"
register: port_check
ignore_errors: yes
- name: Try to restart icecast if not started
service: name=emacsconf state=started enabled=yes
when: port_check.failed == true
-- name: Test by streaming from a file
- tags: test-stream-file, never
- block:
- - debug:
- msg: ffmpeg -loglevel 32 -re -i {{ icecast_test }} -loop 1 -f webm -r 30 -ac 2 -cluster_size_limit 2M -cluster_time_limit 5100 -content_type video/webm -c:v libvpx -b:v 1M -crf 30 -g 125 -threads 4 -deadline realtime icecast://emacsconf:{{ icecast_emacsconf_password }}@live0.emacsconf.org:{{ icecast_port }}/emacsconf/{{ icecast_test_track|d("gen") }}.webm
- - local_action: shell ffmpeg -re -loglevel 32 -i {{ icecast_test }} -loop 1 -f webm -r 30 -ac 2 -cluster_size_limit 2M -cluster_time_limit 5100 -content_type video/webm -c:v libvpx -b:v 1M -crf 30 -g 125 -threads 4 -deadline realtime icecast://emacsconf:{{ icecast_emacsconf_password }}@live0.emacsconf.org:{{ icecast_port }}/emacsconf/{{ icecast_test_track|d("gen") }}.webm
- when: icecast_test is defined
-- name: Test by displaying the stream in mpv
- tags: test-stream-mpv, never
- local_action: shell mpv http://live0.emacsconf.org:{{ icecast_port }}/emacsconf/{{ icecast_test_track|d("gen") }}.webm
- when: test is defined and test == 'mpv'
-- name: Test by streaming a test pattern
- tags: test-stream-pattern
- local_action: shell ffmpeg -loglevel 32 -f lavfi -i testsrc=duration=10:size=1280x720:rate=30 -f webm icecast://emacsconf:{{ icecast_emacsconf_password }}@live0.emacsconf.org:{{ icecast_port }}/emacsconf/{{ icecast_test_track|d("gen") }}.webm
- when: test is defined and test == 'pattern'
# - name: Set up restream script
# template:
# src: restream-yt.sh
diff --git a/roles/stream/templates/icecast-emacsconf.init.d b/roles/stream/templates/icecast-emacsconf.init.d
index b3ad491..3d880c1 100755
--- a/roles/stream/templates/icecast-emacsconf.init.d
+++ b/roles/stream/templates/icecast-emacsconf.init.d
@@ -30,14 +30,10 @@ test -x $DAEMON || exit 0
# Defaults
CONFIGFILE="/etc/icecast2/icecast-emacsconf.xml"
-CONFIGDEFAULTFILE="/etc/default/icecast2"
-USERID=icecast2
-GROUPID=icecast
+USERID={{ icecast_user }}
+GROUPID={{ icecast_group }}
PIDFILE=/var/run/icecast-emacsconf
-# Reads config file (will override defaults above)
-[ -r "$CONFIGDEFAULTFILE" ] && . $CONFIGDEFAULTFILE
-
set -e
case "$1" in
diff --git a/roles/stream/templates/icecast.xml b/roles/stream/templates/icecast.xml
index d2bb89c..e09e6a0 100644
--- a/roles/stream/templates/icecast.xml
+++ b/roles/stream/templates/icecast.xml
@@ -190,33 +190,35 @@
{% for track in emacsconf_tracks %}
<mount type="normal">
<mount-name>/emacsconf/{{ track.id }}.webm</mount-name>
- <username>emacsconf</username>
+ <username>{{ icecast_emacsconf_user }}</username>
<password>{{ icecast_emacsconf_password }}</password>
- <dump-file>/data/emacsconf-{{ emacsconf_year }}-{{ track.id }}_%Y-%m-%d_%H-%M-%S.webm</dump-file>
- <stream-name>EmacsConf {{ emacsconf_year }} - {{ track.name }} track</stream-name>
- <stream-description>The livestream for the {{ track.name }} track of EmacsConf {{ emacsconf_year }}</stream-description>
- <stream-url>https://emacsconf.org/{{ emacsconf_year }}/watch/{{ track.id }}/</stream-url>
+ <dump-file>/data/{{ emacsconf_id }}-{{ emacsconf_year }}-{{ track.id }}_%Y-%m-%d_%H-%M-%S.webm</dump-file>
+ <stream-name>{{ emacsconf_name }} {{ emacsconf_year }} - {{ track.name }} track</stream-name>
+ <stream-description>The livestream for the {{ track.name }} track of {{ emacsconf_name }} {{ emacsconf_year }}</stream-description>
+ <stream-url>{{ track.watch }}</stream-url>
<type>video/webm</type>
<public>1</public>
+ <on-connect>/usr/local/bin/{{ emacsconf_id }}-lowres-{{ track.id }}-on-connect</on-connect>
+ <on-disconnect>/usr/local/bin/{{ emacsconf_id }}-lowres-{{ track.id }}-on-disconnect</on-disconnect>
</mount>
<mount type="normal">
- <mount-name>/emacsconf/{{ track.id }}-host.webm</mount-name>
- <username>emacsconf</username>
+ <mount-name>/{{ emacsconf_id }}/{{ track.id }}-host.webm</mount-name>
+ <username>{{ icecast_emacsconf_user }}</username>
<password>{{ icecast_emacsconf_password }}</password>
<stream-name>EmacsConf {{ emacsconf_year }} - {{ track.name }} track</stream-name>
- <stream-description>The host stream for the {{ track.name }} track of EmacsConf {{ emacsconf_year }}</stream-description>
- <stream-url>https://emacsconf.org/{{ emacsconf_year }}/watch/{{ track.id }}/</stream-url>
+ <stream-description>The host stream for the {{ track.name }} track of {{ emacsconf_name }} {{ emacsconf_year }}</stream-description>
+ <stream-url>{{ track.watch }}</stream-url>
<type>video/webm</type>
<public>1</public>
<hidden>1</hidden>
</mount>
<mount type="normal">
- <mount-name>/emacsconf/{{ track.id }}-480p.webm</mount-name>
- <username>emacsconf</username>
+ <mount-name>/{{ emacsconf_id }}/{{ track.id }}-480p.webm</mount-name>
+ <username>{{ icecast_emacsconf_user }}</username>
<password>{{ icecast_emacsconf_password }}</password>
- <stream-name>EmacsConf {{ emacsconf_year }} - {{ track.name }} track (480p)</stream-name>
- <stream-description>The 480p livestream for the {{ track.name }} track of EmacsConf {{ emacsconf_year }}</stream-description>
- <stream-url>https://emacsconf.org/{{ emacsconf_year }}/watch/{{ track.id }}/</stream-url>
+ <stream-name>{{ emacsconf_name }} {{ emacsconf_year }} - {{ track.name }} track (480p)</stream-name>
+ <stream-description>The low-res livestream for the {{ track.name }} track of {{ emacsconf_name }} {{ emacsconf_year }}</stream-description>
+ <stream-url>{{ track.watch_lowres }}</stream-url>
<type>video/webm</type>
<public>1</public>
</mount>
diff --git a/roles/stream/templates/lowres.init.d b/roles/stream/templates/lowres.init.d
new file mode 100755
index 0000000..41da95c
--- /dev/null
+++ b/roles/stream/templates/lowres.init.d
@@ -0,0 +1,76 @@
+#!/bin/sh
+# {{ ansible_managed }}
+
+### BEGIN INIT INFO
+# Provides: {{ emacsconf_id }}-lowres-{{ item.id }}
+# Required-Start: $local_fs $remote_fs $network $syslog
+# Required-Stop: $local_fs $remote_fs $network $syslog
+# Default-Start: 2 3 4 5
+# Default-Stop: 0 1 6
+# Short-Description: 480p restreaming for {{ item.name }}
+# Description: 480p restreaming for {{ item.name }} using start-stop-daemon
+### END INIT INFO
+
+PATH="/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin"
+USER="{{ icecast_user }}"
+GROUP="{{ icecast_group }}"
+DESC="Low-res restream for {{ item.name }}"
+NAME="{{ emacsconf_id }}-lowres-{{ item.id }}"
+
+set -e
+
+. /lib/lsb/init-functions
+
+start() {
+ echo "Starting $DESC... "
+ start-stop-daemon --start --chuid "$USER:$GROUP" --background --make-pidfile --pidfile /var/run/$NAME.pid --exec "/usr/local/bin/{{ emacsconf_id }}-lowres-{{ item.id }}.sh http://localhost:{{ icecast_port }}/{{ item.id }}.webm http:// > /var/log/$NAME.log" || true
+ echo "done"
+}
+
+#We need this function to ensure the whole process tree will be killed
+killtree() {
+ local _pid=$1
+ local _sig=${2-TERM}
+ for _child in $(ps -o pid --no-headers --ppid ${_pid}); do
+ killtree ${_child} ${_sig}
+ done
+ kill -${_sig} ${_pid}
+}
+
+stop() {
+ echo "Stopping $DESC... "
+ if test -f /var/run/$NAME.pid; then
+ while test -d /proc/$(cat /var/run/$NAME.pid); do
+ killtree $(cat /var/run/$NAME.pid) 15
+ sleep 0.5
+ done
+ rm /var/run/$NAME.pid
+ fi
+ echo "done"
+}
+
+status() {
+ status_of_proc -p /var/run/$NAME.pid "" "$NAME" && exit 0 || exit $?
+}
+
+case "$1" in
+ start)
+ start
+ ;;
+ stop)
+ stop
+ ;;
+ restart)
+ stop
+ start
+ ;;
+ status)
+ status
+ ;;
+ *)
+ echo "Usage: $NAME {start|stop|restart|status}" >&2
+ exit 1
+ ;;
+esac
+
+exit 0
diff --git a/roles/stream/templates/lowres.sh b/roles/stream/templates/lowres.sh
new file mode 100755
index 0000000..1f74215
--- /dev/null
+++ b/roles/stream/templates/lowres.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+# {{ ansible_managed }}
+sleep 10
+for i in 1 2 3 4 5; do
+ ffmpeg -loglevel 24 -f webm -reconnect_at_eof 1 -reconnect_streamed 1 -re -i "http://localhost:{{ icecast_port }}/{{ emacsconf_id }}/{{ item.id }}.webm" -vf scale="{{ icecast_lowres_scale }}" -f webm -c:a copy -b:v 500k -maxrate 1M -bufsize 1M -content_type video/webm -c:v libvpx "icecast://{{ icecast_emacsconf_user }}:{{ icecast_emacsconf_password }}@localhost:{{ icecast_port }}/{{ emacsconf_id }}/{{ item.id }}-480p.webm" >> {{ icecast_restream_dir }}/{{ emacsconf_id }}-lowres-{{ item.id }}.log || sleep 5
+done
diff --git a/roles/stream/templates/on-connect b/roles/stream/templates/on-connect
new file mode 100755
index 0000000..0f613bf
--- /dev/null
+++ b/roles/stream/templates/on-connect
@@ -0,0 +1,12 @@
+#!/bin/bash
+set -e
+PATH="/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin"
+PIDFILE="{{ icecast_restream_dir }}/{{ emacsconf_id }}-lowres-{{ item.id }}.pid"
+echo $(date) " on connect {{ item.id }} $*" >> {{ icecast_restream_dir }}/restream.log
+
+. /lib/lsb/init-functions
+
+start-stop-daemon --start --quiet --background \
+ --make-pidfile --pidfile $PIDFILE \
+ --oknodo \
+ --exec /usr/local/bin/{{ emacsconf_id }}-lowres-{{ item.id }}.sh > /dev/null
diff --git a/roles/stream/templates/on-disconnect b/roles/stream/templates/on-disconnect
new file mode 100755
index 0000000..5847356
--- /dev/null
+++ b/roles/stream/templates/on-disconnect
@@ -0,0 +1,29 @@
+#!/bin/bash
+set -e
+PATH="/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin"
+PIDFILE="{{ icecast_restream_dir }}/{{ emacsconf_id }}-lowres-{{ item.id }}.pid"
+echo $(date) " on disconnect {{ item.id }} $*" >> {{ icecast_restream_dir }}/restream.log
+
+. /lib/lsb/init-functions
+
+#We need this function to ensure the whole process tree will be killed
+killtree() {
+ local _pid=$1
+ local _sig=${2-TERM}
+ for _child in $(ps -o pid --no-headers --ppid ${_pid}); do
+ killtree ${_child} ${_sig}
+ done
+ kill -${_sig} ${_pid}
+}
+
+stop() {
+ echo "Stopping $DESC... "
+ if test -f $PIDFILE; then
+ while test -d /proc/$(cat $PIDFILE); do
+ killtree $(cat $PIDFILE) 15
+ sleep 0.5
+ done
+ rm $PIDFILE
+ fi
+ echo "done"
+}
diff --git a/roles/wiki-publish/defaults/main.yml b/roles/wiki-publish/defaults/main.yml
index 863003f..4de7445 100644
--- a/roles/wiki-publish/defaults/main.yml
+++ b/roles/wiki-publish/defaults/main.yml
@@ -1,8 +1,9 @@
-emacsconf_year: 2022
emacsconf_org_file: "{{ emacsconf_private_dir }}/conf.org"
emacsconf_publishing_phase: schedule
-emacsconf_el_dir: ~/emacsconf-el
-emacsconf_edit_wiki_dir: ~/emacsconf-wiki
-emacsconf_private_dir: ~/emacsconf-2022-private
-emacs_config_dir: ~/.emacs.d
-emacs_package: emacs-snapshot-nox
+emacsconf_el_dir: ~{{ emacsconf_user }}/emacsconf-el
+emacsconf_edit_wiki_dir: ~{{ emacsconf_user }}/emacsconf-wiki
+emacsconf_private_dir: ~{{ emacsconf_user }}/emacsconf-2022-private
+emacs_config_dir: ~{{ emacsconf_user }}/.emacs.d
+emacs_version: 28.2
+emacs_build_parent: /usr/src/emacs
+emacs_build_dir: "{{ emacs_build_parent }}/emacs-{{ emacs_version }}"
diff --git a/roles/wiki-publish/tasks/emacs.yml b/roles/wiki-publish/tasks/emacs.yml
new file mode 100644
index 0000000..8d75979
--- /dev/null
+++ b/roles/wiki-publish/tasks/emacs.yml
@@ -0,0 +1,38 @@
+- name: Install dependencies
+ apt:
+ name:
+ - build-essential
+ - libncurses5-dev
+ - git
+ - gnutls-bin
+ - pkg-config
+ - libjansson-dev
+ - libgnutls28-dev
+ - automake
+ - texinfo
+ state: present
+- name: Create directory
+ file:
+ path: "{{ emacs_build_parent }}"
+ state: directory
+- name: Download Emacs
+ unarchive:
+ src: https://ftp.gnu.org/gnu/emacs/emacs-{{emacs_version}}.tar.gz
+ remote_src: yes
+ dest: "{{ emacs_build_parent }}"
+ creates: "{{ emacs_build_dir }}"
+- name: Configure
+ command: ./configure --with-x-toolkit=no
+ args:
+ chdir: "{{ emacs_build_dir }}"
+ creates: '{{ emacs_build_dir }}/Makefile'
+- name: Build
+ command: make -j{{ cpus }}
+ args:
+ chdir: "{{ emacs_build_dir }}"
+ creates: '{{ emacs_build_dir }}/src/emacs'
+- name: Install
+ command: make install
+ args:
+ chdir: "{{ emacs_build_dir }}"
+ creates: '/usr/local/bin/emacs'
diff --git a/roles/wiki-publish/tasks/main.yml b/roles/wiki-publish/tasks/main.yml
index 960aa01..28566a5 100644
--- a/roles/wiki-publish/tasks/main.yml
+++ b/roles/wiki-publish/tasks/main.yml
@@ -1,45 +1,44 @@
---
- name: Set up SSH directory
ansible.builtin.file:
- path: "/home/{{ emacsconf_publish_user }}/.ssh"
+ path: "/home/{{ emacsconf_user }}/.ssh"
state: directory
mode: '0700'
- name: Install SSH key for EmacsConf wiki
ansible.builtin.get_url:
url: https://emacsconf.org/id_rsa_anon_git_emacsconf
- dest: "/home/{{ emacsconf_publish_user }}/.ssh/id_rsa_anon_git_emacsconf"
+ dest: "/home/{{ emacsconf_user }}/.ssh/id_rsa_anon_git_emacsconf"
mode: '0600'
- owner: "{{ emacsconf_publish_user }}"
-- name: Set up packages
- become: yes
- when: emacs_package and emacs_package != "emacs"
- block:
- # This repository is currently not working
- # - name: Add snapshot repository
- # ansible.builtin.apt_repository:
- # repo: deb http://emacs.ganneff.de/ buster main
- - name: Remove old package
- ansible.builtin.apt:
- name: emacs
- state: absent
- - name: Install Emacs snapshot
- ansible.builtin.apt:
- name: "{{ emacs_package }}"
- state: present
+- name: Install the SSH key for orga
+ template:
+ src: id_rsa
+ dest: "/home/{{ emacsconf_user }}/.ssh/id_rsa"
+ mode: '0600'
+- name: Change ownership of SSH directory
+ file:
+ path: "/home/{{ emacsconf_user }}/.ssh"
+ recurse: true
+ owner: "{{ emacsconf_user }}"
+
+- name: Set up Emacs
+ include: emacs.yml
+- name: Ensure configuration directory exists
+ file:
+ path: "{{ emacs_config_dir }}"
+ state: directory
- name: Set up Emacs configuration for non-interactive publishing
- become: true
- become_user: "{{ emacsconf_publish_user }}"
template:
src: emacsconf-config.el
dest: "{{ emacs_config_dir }}/emacsconf-config.el"
+
- name: Check if Emacs base configuration already exists
become: true
- become_user: "{{ emacsconf_publish_user }}"
lineinfile:
dest: "{{ emacs_config_dir }}/init.el"
state: present
regexp: "emacsconf-config"
line: "(load-file \"emacsconf-config.el\")"
+ create: yes
- name: Set up or update repositories
become: false
tags: publish
@@ -71,6 +70,16 @@
- private.failed
- not 'Local modifications exist in repository' in private.msg
- not 'Failed to checkout branch' in private.msg
+- name: Change ownership
+ file:
+ path: "{{ item }}"
+ owner: "{{ emacsconf_user }}"
+ recurse: true
+ with_items:
+ - "{{ emacsconf_config_dir }}"
+ - "{{ emacsconf_el_dir }}"
+ - "{{ emacsconf_private_dir }}"
+ - "{{ emacsconf_edit_wiki_dir }}"
- name: Publish
tags: publish
block: