diff options
Diffstat (limited to '')
| -rw-r--r-- | roles/prerec/templates/Makefile | 146 | ||||
| -rwxr-xr-x | roles/prerec/templates/copy-original.sh | 24 | ||||
| -rwxr-xr-x | roles/prerec/templates/get-file-prefix | 10 | ||||
| -rwxr-xr-x | roles/prerec/templates/process-prerec.sh | 6 | ||||
| -rwxr-xr-x | roles/prerec/templates/reencode-in-screen.sh | 20 | ||||
| -rwxr-xr-x | roles/prerec/templates/reencode.sh | 6 | ||||
| -rwxr-xr-x | roles/prerec/templates/talk | 3 | ||||
| -rwxr-xr-x | roles/prerec/templates/thumbnail.sh | 2 | ||||
| -rwxr-xr-x | roles/prerec/templates/upload.sh | 4 |
9 files changed, 205 insertions, 16 deletions
diff --git a/roles/prerec/templates/Makefile b/roles/prerec/templates/Makefile index 709a069..778ae3e 100644 --- a/roles/prerec/templates/Makefile +++ b/roles/prerec/templates/Makefile @@ -1,31 +1,62 @@ # {{ ansible_managed }} -VIDEO_EXTS = mp4 mkv webm mov +# To recreate this file, use ansible-playbook -i inventory.yml prod-playbook.yml --tags prerec-make + +.ONESHELL: +SHELL := /bin/bash +# --- Shell Function Definition --- +# This function checks a given file ($1) for an audio stream. +# It returns the string "audio" if found, or nothing if not found. +# This makes the conditional check cleaner. +define IS_AUDIO_PRESENT_SCRIPT + ffprobe -loglevel error -select_streams a:0 \ + -show_entries stream=codec_type \ + -of default=noprint_wrappers=1:nokey=1 "$1" +endef +IS_AUDIO_PRESENT = $(IS_AUDIO_PRESENT_SCRIPT) + +source_patterns = $(foreach ext,$(VIDEO_EXTS),$(1)--original.$(ext)) +ANSWER_EXTS = mp4 mkv mov mpv m4v source_patterns = $(foreach ext,$(VIDEO_EXTS),$(1)--original.$(ext)) +answer_patterns = $(foreach ext,$(ANSWER_EXTS),$(1)--answers.$(ext)) PRERECS_ORIGINAL := $(wildcard emacsconf-*--original.*) +ANSWERS_ORIGINAL := $(wildcard emacsconf-*--answers.*) +ANSWER_PREFIXES := $(shell for f in $(ANSWERS_ORIGINAL); do echo "$${f%--answers.*}"; done) PREFIXES := $(shell for f in $(PRERECS_ORIGINAL); do echo "$${f%--original.*}"; done) +ANSWERS_REENCODED := $(addsuffix --answers--reencoded.webm, $(ANSWER_PREFIXES)) PRERECS_REENCODED := $(addsuffix --reencoded.webm, $(PREFIXES)) PRERECS_OPUS := $(addsuffix --reencoded.opus, $(PREFIXES)) -PRERECS_NORMAL := $(addsuffix --normalized.opus, $(PREFIXES)) +PRERECS_MAIN_OPUS := $(addsuffix --main.opus, $(PREFIXES)) PRERECS_MAIN := $(addsuffix --main.webm, $(PREFIXES)) PRERECS_CAPTIONS := $(addsuffix --reencoded.vtt, $(PREFIXES)) PRERECS_FINAL := $(wildcard emacsconf-*--final.webm) +WAVEFORMS := $(addsuffix --backstage--waveform.png, $(PREFIXES)) +LINODE_BBB_ID := 67329098 .PHONY: all -all: reencoded opus normal main +all: reencoded opus main main-opus waveforms.png reencoded: $(PRERECS_REENCODED) opus: $(PRERECS_OPUS) -normal: $(PRERECS_NORMAL) - captions: $(PRERECS_CAPTIONS) main: $(PRERECS_MAIN) +main-opus: $(PRERECS_MAIN_OPUS) + +waveforms: $(WAVEFORMS) + +answers: $(ANSWERS_REENCODED) + +emacsconf-%--answers--reencoded.webm: SOURCES = $(call answer_patterns, emacsconf-$*) +emacsconf-%--answers--reencoded.webm: + $(eval SOURCE := $(lastword $(sort $(wildcard $(SOURCES))))) + @echo $(SOURCE) $@ + emacsconf-%--reencoded.webm: SOURCES = $(call source_patterns, emacsconf-$*) emacsconf-%--reencoded.webm: $(eval SOURCE := $(lastword $(sort $(wildcard $(SOURCES))))) @@ -35,13 +66,21 @@ emacsconf-%--reencoded.webm: exit 1; \ fi @echo "Using source: $(SOURCE)" - ./reencode-in-screen.sh "$(SOURCE)" + reencode-in-screen.sh "$(SOURCE)" emacsconf-%--reencoded.opus: emacsconf-%--reencoded.webm - ffmpeg -i "$<" -c:a copy "$@" + AUDIO_CHECK=$$( $(call IS_AUDIO_PRESENT, "$<") ) + if [ ! -z "$$AUDIO_CHECK" ]; then + ffmpeg -y -i "$<" -c:a copy "$@" + else + ffmpeg -y -f lavfi -i anullsrc=channel_layout=stereo:sample_rate=48000 -t 1 -c:a libopus "$@" + fi + +emacsconf-%--answers.opus: emacsconf-%--answers.webm + ffmpeg -y -i "$<" -c:a copy "$@" -emacsconf-%--normalized.opus: emacsconf-%--reencoded.opus - ffmpeg-normalize "$<" -ofmt opus -c:a libopus -o "$@" +# emacsconf-%--normalized.opus: emacsconf-%--reencoded.opus +# ffmpeg-normalize "$<" -ofmt opus -c:a libopus -o "$@" emacsconf-%--main.webm: emacsconf-%--reencoded.webm emacsconf-%--normalized.opus emacsconf-%--main.vtt ffmpeg -i emacsconf-$*--reencoded.webm -i emacsconf-$*--normalized.opus -i emacsconf-$*--main.vtt \ @@ -49,14 +88,34 @@ emacsconf-%--main.webm: emacsconf-%--reencoded.webm emacsconf-%--normalized.opus -map 2 -c:s webvtt -y \ $@ +emacsconf-%--main.webm: emacsconf-%--reencoded.webm emacsconf-%--normalized.opus + ffmpeg -i emacsconf-$*--reencoded.webm -i emacsconf-$*--normalized.opus \ + -map 0:v -map 1:a -c:v copy -c:a copy \ + -y $@ + emacsconf-%--main.webm: emacsconf-%--reencoded.webm cp "$<" "$@" +emacsconf-%--main.webm: emacsconf-%--reencoded.webm emacsconf-%--main.vtt + cp "$<" "$@" + ffmpeg -i emacsconf-$*--reencoded.webm -i emacsconf-$*--main.vtt \ + -c:v copy -c:a copy \ + -y $@ + emacsconf-%--final.webm: emacsconf-%--main.vtt mux-subs.sh "$@" "$<" +emacsconf-%--main.opus: emacsconf-%--main.webm + AUDIO_CHECK=$$( $(call IS_AUDIO_PRESENT, "$<") ) + if [ ! -z "$$AUDIO_CHECK" ]; then \ + ffmpeg -y -i "$<" -c:a copy "$@"; \ + fi + emacsconf-%--reencoded.vtt: emacsconf-%--reencoded.opus - whisperx --model large-v2 --align_model WAV2VEC2_ASR_LARGE_LV60K_960H --compute_type int8 --print_progress True --max_line_width 50 --segment_resolution chunk --max_line_count 1 --language en "$<" + whisperx --model large-v3 --align_model WAV2VEC2_ASR_LARGE_LV60K_960H --compute_type int8 --print_progress True --max_line_width 50 --segment_resolution chunk --max_line_count 1 --language en --initial_prompt "Transcribe this talk about Emacs. It may mention Emacs keywords such as Org Mode, Org Roam, Magit, gptel, or chatgpt-shell, or tech keywords such as LLMs. Format function names and keyboard shortcut sequences according to Emacs conventions using Markdown syntax. For example: control h becomes \`C-h\`." "$<" + +emacsconf-%--backstage--silences.csv: emacsconf-%--reencoded.opus + ffmpeg -i "$<" -af silencedetect=noise=-30dB:d=0.5 -f null - 2>&1 | awk '/silence_start/ {start=$$NF} /silence_end/ {print start "," (start + $$NF)}' > "$@" show-files: @echo "Original $(words $(PRERECS_ORIGINAL)):" @@ -68,5 +127,72 @@ show-files: @echo "Opus $(words $(PRERECS_OPUS)):" @echo "$(PRERECS_OPUS)" +bbb-testing: + @echo "Resizing BBB node to 8GB 4 core for testing" + linode-cli linodes resize $(LINODE_BBB_ID) --type g6-standard-4 --allow_auto_disk_resize false + sleep 2m + @echo "Booting up" + linode-cli linodes boot $(LINODE_BBB_ID) + ssh root@bbb.emacsverse.org "cd ~/greenlight-v3; docker compose restart" + +bbb-dormant: + @echo "Shutting down" + ssh root@bbb.emacsverse.org "/usr/sbin/shutdown -h now &" || true + sleep 30 + @echo "Powering off BBB node" + linode-cli linodes shutdown $(LINODE_BBB_ID) + sleep 30 + @echo "Resizing BBB node to nanode, dormant" + linode-cli linodes resize $(LINODE_BBB_ID) --type g6-nanode-1 --allow_auto_disk_resize false + +bbb-prod: + @echo "Resizing BBB node to production size" + linode-cli linodes resize $(LINODE_BBB_ID) --type g6-dedicated-8 --allow_auto_disk_resize true + sleep 2m + @echo "Booting up" + linode-cli linodes boot $(LINODE_BBB_ID) + +live-dormant: + @echo "Resizing live0 node to nanode, dormant" + linode-cli linodes resize 17921960 --type g6-nanode-1 --allow_auto_disk_resize false + sleep 120 + linode-cli linodes boot 17921960 + + rsync: rsync -avzue ssh {{ emacsconf_caption_dir }}/cache/ orga@media.emacsconf.org:/var/www/media.emacsconf.org/{{ emacsconf_year }}/backstage/ + + +WAVEFORM_HEIGHT := 240 +WAVEFORM_WIDTH := 400 +WAVEFORM_VOLUME := 3.0 +WAVEFORM_FILTER := "[0:a]aformat=channel_layouts=mono[b];[b]volume=$(WAVEFORM_VOLUME)[a];[a]compand=attacks=0:points=-90/-90|0/0,showwavespic=s=$(WAVEFORM_WIDTH)x$(WAVEFORM_HEIGHT):colors=\#9cf42f[fg];color=s=$(WAVEFORM_WIDTH)x$(WAVEFORM_HEIGHT):color=\#44582c,drawgrid=width=iw/10:height=ih/5:color=\#9cf42f@0.1[bg];[bg][fg]overlay=format=auto[comp];[comp]drawtext=text='$$ID':fontcolor=white:fontsize=24:x=10:y=10:box=1:boxcolor=\#44582c@0.5:boxborderw=5" + +emacsconf-%--backstage--waveform.png: emacsconf-%--normalized.opus + AUDIO_CHECK=$$(ffprobe -loglevel error -select_streams a:0 -show_entries stream=codec_type -of default=noprint_wrappers=1:nokey=1 "$<"); \ + ID=$$(echo "$<" | sed 's/--.*//'); \ + if [ ! -z "$$AUDIO_CHECK" ]; then \ + ffmpeg -i "$<" \ + -filter_complex $(WAVEFORM_FILTER) \ + -frames:v 1 \ + -c:v png \ + -update 1 \ + -y \ + "$@"; \ + fi + +emacsconf-%--backstage--waveform.png: emacsconf-%--reencoded.opus + AUDIO_CHECK=$$(ffprobe -loglevel error -select_streams a:0 -show_entries stream=codec_type -of default=noprint_wrappers=1:nokey=1 "$<"); \ + ID=$$(echo "$<" | sed 's/--.*//'); \ + if [ ! -z "$$AUDIO_CHECK" ]; then \ + ffmpeg -i "$<" \ + -filter_complex $(WAVEFORM_FILTER) \ + -frames:v 1 \ + -c:v png \ + -update 1 \ + -y \ + "$@"; \ + fi + +waveforms.png: $(WAVEFORMS) + convert $^ +append "$@" diff --git a/roles/prerec/templates/copy-original.sh b/roles/prerec/templates/copy-original.sh new file mode 100755 index 0000000..854bdb8 --- /dev/null +++ b/roles/prerec/templates/copy-original.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# {{ ansible_managed }} +# Usage: rename-original.sh $slug $file [$extra] +SLUG=$1 +FILE=$2 +EXTRA="" +if [ -z ${3-unset} ]; then + EXTRA="" +elif [ -n "$3" ]; then + EXTRA="$3" +elif echo "$FILE" | grep -e '\(webm\|mp4\|mov\)'; then + EXTRA="--original" +fi +filename=$(basename -- "$FILE") +extension="${filename##*.}" +filename="${filename%.*}" +FILE_PREFIX=$(get-file-prefix $SLUG) +if echo "$FILE" | grep -q \\. ; then + cp "$FILE" $FILE_PREFIX$EXTRA.$extension + echo $FILE_PREFIX$EXTRA.$extension +else + cp "$FILE" $FILE_PREFIX$EXTRA + echo $FILE_PREFIX$EXTRA +fi diff --git a/roles/prerec/templates/get-file-prefix b/roles/prerec/templates/get-file-prefix new file mode 100755 index 0000000..0240b99 --- /dev/null +++ b/roles/prerec/templates/get-file-prefix @@ -0,0 +1,10 @@ +#!/bin/bash +# {{ ansible_managed }} +# Usage: get-file-prefix $slug [talks.json] +# You can also set the TALKS_JSON environment variable + +SLUG=$1 +if [ -z "$TALKS_JSON" ]; then + TALKS_JSON=${2:-~orga/current/talks.json} +fi +jq -r '.talks[] | select(.slug=="'$SLUG'")["file-prefix"]' < $TALKS_JSON diff --git a/roles/prerec/templates/process-prerec.sh b/roles/prerec/templates/process-prerec.sh index d98337f..4347576 100755 --- a/roles/prerec/templates/process-prerec.sh +++ b/roles/prerec/templates/process-prerec.sh @@ -5,9 +5,11 @@ ORIGINAL=$1 REENCODED=$(echo "$ORIGINAL" | perl -pe 's/^(emacsconf-[0-9]*-.*?--.*?--.*?--).*/$1reencoded.webm/') SLUG=$(echo "$ORIGINAL" | perl -ne '/^emacsconf-[0-9]*-(.*?)--/ && print $1') MAIN=$(echo "$ORIGINAL" | perl -pe 's/^(emacsconf-[0-9]*-.*?--.*?--.*?--).*/$1main.webm/') +VTT=$(echo "$ORIGINAL" | perl -pe 's/^(emacsconf-[0-9]*-.*?--.*?--.*?--).*/$1reencoded.vtt/') # ( cd /data/emacsconf/cache; ./update-cache ) # /data/emacsconf/admin/{{ emacsconf_year }}/scripts/talk $SLUG "WAITING_FOR_PREREC" "PROCESSING" if [[ ! -f "$REENCODED" ]]; then - screen -dmS reencode-$SLUG /bin/bash -c "reencode.sh \"$ORIGINAL\" \"$REENCODED\" && upload.sh $REENCODED $MAIN && thumbnail.sh \"$MAIN\" && upload.sh $(echo "$MAIN" | sed s/webm$/png/); echo $(date -Iminutes) $SLUG reencoded >> ~/emacsconf.log && exec /bin/bash" & + screen -dmS reencode-$SLUG /bin/bash -c "reencode.sh \"$ORIGINAL\" \"$REENCODED\" && make $MAIN && upload.sh $REENCODED $MAIN && thumbnail.sh \"$MAIN\" && upload.sh $(echo "$MAIN" | sed s/webm$/png/); echo $(date -Iminutes) $SLUG reencoded >> ~/emacsconf.log && make $VTT; echo $(date -Iminutes) $SLUG captioned >> ~/emacsconf.log; exec /bin/bash" & +else + screen -dmS captions-$SLUG /bin/bash -c "make $VTT; echo $(date -Iminutes) $SLUG captioned >> ~/emacsconf.log; exec /bin/bash" fi -screen -dmS captions-$SLUG /bin/bash -c "/data/emacsconf/admin/{{ emacsconf_year }}/scripts/process-captions.py $(dirname $ORIGINAL); echo $(date -Iminutes) $SLUG captioned >> ~/emacsconf.log; exec /bin/bash" diff --git a/roles/prerec/templates/reencode-in-screen.sh b/roles/prerec/templates/reencode-in-screen.sh new file mode 100755 index 0000000..7119572 --- /dev/null +++ b/roles/prerec/templates/reencode-in-screen.sh @@ -0,0 +1,20 @@ +#!/bin/bash +ORIGINAL=$1 +BASE="${ORIGINAL%--original.*}" +if [ -n "$2" ]; then + REENCODED=$2 +else + REENCODED="${BASE}--reencoded.webm" +fi +SLUG=$(echo "$ORIGINAL" | perl -ne '/^emacsconf-[0-9]*-(.*?)--/ && print $1') +LOCK=".lock-$SLUG" + +if [ ! -f "$REENCODED" ]; then + if [ -f "$LOCK" ]; then + echo "$LOCK already exists, waiting for it" + else + touch "$LOCK" + screen -dmS reencode-$SLUG /bin/bash -c "reencode.sh \"$ORIGINAL\" \"$REENCODED\" && thumbnail.sh \"$MAIN\" && rm \"$LOCK\"" + echo "Processing $REENCODED in reencode-$SLUG" + fi +fi diff --git a/roles/prerec/templates/reencode.sh b/roles/prerec/templates/reencode.sh index 665fee2..ac5309a 100755 --- a/roles/prerec/templates/reencode.sh +++ b/roles/prerec/templates/reencode.sh @@ -10,6 +10,9 @@ print_only=false limit_resolution={{ res_y }} limit_fps={{ fps }} +output_final="${2:-$(echo $input | sed 's/--original.*/--reencoded.webm/')}" +output_temp="partial--${output_final}" # <-- New temporary name + while getopts :q:c:t:s OPT; do case $OPT in q|+q) @@ -47,7 +50,7 @@ ffmpeg -y -i "$input" $time_limit \ fps='$limit_fps'" \ -c:v libvpx-vp9 -b:v 0 -crf $q -c:a libopus \ -row-mt 1 -tile-columns 2 -tile-rows 2 -cpu-used $cpu \ - -pass 2 -threads $cpu -- "$output" + -pass 2 -threads $cpu -- "$output_temp" EOF )" @@ -55,4 +58,5 @@ if [ $print_only == true ]; then echo "$command" else eval "$command" + mv "$output_temp" "$output_final" fi diff --git a/roles/prerec/templates/talk b/roles/prerec/templates/talk index fa2e5f0..a9d4cb0 100755 --- a/roles/prerec/templates/talk +++ b/roles/prerec/templates/talk @@ -3,6 +3,9 @@ # How to use: talk slug from-status-regexp to-status # or talk slug to-status +# Statuses: +# PLAYING: + SLUG="$1" FROM_STATUS="$2" TO_STATUS="$3" diff --git a/roles/prerec/templates/thumbnail.sh b/roles/prerec/templates/thumbnail.sh index 436979d..1cf7372 100755 --- a/roles/prerec/templates/thumbnail.sh +++ b/roles/prerec/templates/thumbnail.sh @@ -1,3 +1,3 @@ #!/bin/bash # {{ ansible_managed }} -ffmpeg -i "$1" -vf "thumbnail,scale=400:225" -frames:v 1 $(echo "$1" | sed s/webm/png/ ) +ffmpeg -y -i "$1" -vf "thumbnail,scale=400:225" -frames:v 1 $(echo "$1" | sed s/webm/png/ ) diff --git a/roles/prerec/templates/upload.sh b/roles/prerec/templates/upload.sh index fbf3025..03cc66a 100755 --- a/roles/prerec/templates/upload.sh +++ b/roles/prerec/templates/upload.sh @@ -2,5 +2,5 @@ # {{ ansible_managed }} scp $* orga@media.emacsconf.org:~/backstage -/data/emacsconf/admin/{{ emacsconf_year }}/scripts/publish-backstage-index.sh -rsync -avze ssh orga@media.emacsconf.org:~/backstage/ /data/emacsconf/admin/cache/ +# /usr/local/bin/publish-backstage-index.sh +rsync -avze ssh orga@media.emacsconf.org:~/backstage/ /data/emacsconf/shared/{{ emacsconf_year }}/cache/ |