diff options
Diffstat (limited to 'roles/prerec/templates')
| -rw-r--r-- | roles/prerec/templates/Makefile | 115 | ||||
| -rwxr-xr-x | roles/prerec/templates/copy-original.sh | 24 | ||||
| -rwxr-xr-x | roles/prerec/templates/get-file-prefix | 10 | ||||
| -rwxr-xr-x | roles/prerec/templates/process-prerec.sh | 11 | ||||
| -rwxr-xr-x | roles/prerec/templates/reencode-in-screen.sh | 16 | ||||
| -rwxr-xr-x | roles/prerec/templates/reencode.sh | 2 | ||||
| -rwxr-xr-x | roles/prerec/templates/remux.sh | 41 | ||||
| -rwxr-xr-x | roles/prerec/templates/rename-original.sh | 11 | ||||
| -rwxr-xr-x | roles/prerec/templates/run-aeneas.sh | 2 | ||||
| -rwxr-xr-x | roles/prerec/templates/talk | 4 | ||||
| -rwxr-xr-x | roles/prerec/templates/upload.sh | 4 |
11 files changed, 205 insertions, 35 deletions
diff --git a/roles/prerec/templates/Makefile b/roles/prerec/templates/Makefile index 94aa412..8579965 100644 --- a/roles/prerec/templates/Makefile +++ b/roles/prerec/templates/Makefile @@ -1,16 +1,121 @@ # {{ ansible_managed }} -PRERECS_FINAL := $(wildcard {{ emacsconf_id }}-*--final.webm) -PRERECS_MAIN := $(patsubst %--final.webm, %--main.webm, $(PRERECS_FINAL)) +# To recreate this file, use ansible-playbook -i inventory.yml prod-playbook.yml --tags prerec-make + +VIDEO_EXTS = mp4 mkv webm mov +source_patterns = $(foreach ext,$(VIDEO_EXTS),$(1)--original.$(ext)) + +PRERECS_ORIGINAL := $(wildcard emacsconf-*--original.*) +PREFIXES := $(shell for f in $(PRERECS_ORIGINAL); do echo "$${f%--original.*}"; done) +PRERECS_REENCODED := $(addsuffix --reencoded.webm, $(PREFIXES)) +PRERECS_OPUS := $(addsuffix --reencoded.opus, $(PREFIXES)) +PRERECS_NORMAL := $(addsuffix --normalized.opus, $(PREFIXES)) +PRERECS_MAIN := $(addsuffix --main.webm, $(PREFIXES)) +PRERECS_CAPTIONS := $(addsuffix --reencoded.vtt, $(PREFIXES)) +PRERECS_FINAL := $(wildcard emacsconf-*--final.webm) +LINODE_BBB_ID := 67329098 .PHONY: all -all: $(PRERECS_MAIN) $(PRERECS_FINAL) -emacsconf-%--main.webm: {{ emacsconf_id }}-%--final.webm +all: reencoded opus normal main + +reencoded: $(PRERECS_REENCODED) + +opus: $(PRERECS_OPUS) + +normal: $(PRERECS_NORMAL) + +captions: $(PRERECS_CAPTIONS) + +main: $(PRERECS_MAIN) + +emacsconf-%--reencoded.webm: SOURCES = $(call source_patterns, emacsconf-$*) +emacsconf-%--reencoded.webm: + $(eval SOURCE := $(lastword $(sort $(wildcard $(SOURCES))))) + @if [ -z "$(SOURCE)" ]; then \ + echo "No source file found for $@"; \ + echo "Tried: $(SOURCES)"; \ + exit 1; \ + fi + @echo "Using source: $(SOURCE)" + ./reencode-in-screen.sh "$(SOURCE)" + +emacsconf-%--reencoded.opus: emacsconf-%--reencoded.webm + ffmpeg -y -i "$<" -c:a copy "$@" + +emacsconf-%--answers.opus: emacsconf-%--answers.webm + ffmpeg -y -i "$<" -c:a copy "$@" + +# emacsconf-%--normalized.opus: emacsconf-%--reencoded.opus +# ffmpeg-normalize "$<" -ofmt opus -c:a libopus -o "$@" + +emacsconf-%--main.webm: emacsconf-%--reencoded.webm emacsconf-%--normalized.opus emacsconf-%--main.vtt + ffmpeg -i emacsconf-$*--reencoded.webm -i emacsconf-$*--normalized.opus -i emacsconf-$*--main.vtt \ + -map 0:v -map 1:a -c:v copy -c:a copy \ + -map 2 -c:s webvtt -y \ + $@ + +emacsconf-%--main.webm: emacsconf-%--reencoded.webm emacsconf-%--normalized.opus + ffmpeg -i emacsconf-$*--reencoded.webm -i emacsconf-$*--normalized.opus \ + -map 0:v -map 1:a -c:v copy -c:a copy \ + -y $@ + +emacsconf-%--main.webm: emacsconf-%--reencoded.webm cp "$<" "$@" -emacsconf-%--final.webm: {{ emacsconf_id }}-%--main.vtt +emacsconf-%--final.webm: emacsconf-%--main.vtt mux-subs.sh "$@" "$<" +emacsconf-%--main.opus: emacsconf-%--main.webm + ffmpeg -y -i "$<" -c:a copy "$@" + +emacsconf-%--reencoded.vtt: emacsconf-%--reencoded.opus + whisperx --model large-v2 --align_model WAV2VEC2_ASR_LARGE_LV60K_960H --compute_type int8 --print_progress True --max_line_width 50 --segment_resolution chunk --max_line_count 1 --language en "$<" + +emacsconf-%--backstage--silences.csv: emacsconf-%--reencoded.opus + ffmpeg -i "$<" -af silencedetect=noise=-30dB:d=0.5 -f null - 2>&1 | awk '/silence_start/ {start=$$NF} /silence_end/ {print start "," (start + $$NF)}' > "$@" + +show-files: + @echo "Original $(words $(PRERECS_ORIGINAL)):" + @echo "$(PRERECS_ORIGINAL)" + @echo "Prefixes $(words $(PREFIXES)):" + @echo "$(PREFIXES)" + @echo "Reencoded $(words $(PRERECS_REENCODED)):" + @echo "$(PRERECS_REENCODED)" + @echo "Opus $(words $(PRERECS_OPUS)):" + @echo "$(PRERECS_OPUS)" + +bbb-testing: + @echo "Resizing BBB node to 8GB 4 core for testing" + linode-cli linodes resize $(LINODE_BBB_ID) --type g6-standard-4 --allow_auto_disk_resize false + sleep 2m + @echo "Booting up" + linode-cli linodes boot $(LINODE_BBB_ID) + ssh root@bbb.emacsverse.org "cd ~/greenlight-v3; docker compose restart" + +bbb-dormant: + @echo "Shutting down" + ssh root@bbb.emacsverse.org "/usr/sbin/shutdown -h now &" || true + sleep 30 + @echo "Powering off BBB node" + linode-cli linodes shutdown $(LINODE_BBB_ID) + sleep 30 + @echo "Resizing BBB node to nanode, dormant" + linode-cli linodes resize $(LINODE_BBB_ID) --type g6-nanode-1 --allow_auto_disk_resize false + +bbb-prod: + @echo "Resizing BBB node to production size" + linode-cli linodes resize $(LINODE_BBB_ID) --type g6-dedicated-8 --allow_auto_disk_resize true + sleep 2m + @echo "Booting up" + linode-cli linodes boot $(LINODE_BBB_ID) + +live-dormant: + @echo "Resizing live0 node to nanode, dormant" + linode-cli linodes resize 17921960 --type g6-nanode-1 --allow_auto_disk_resize false + sleep 120 + linode-cli linodes boot 17921960 + + rsync: rsync -avzue ssh {{ emacsconf_caption_dir }}/cache/ orga@media.emacsconf.org:/var/www/media.emacsconf.org/{{ emacsconf_year }}/backstage/ diff --git a/roles/prerec/templates/copy-original.sh b/roles/prerec/templates/copy-original.sh new file mode 100755 index 0000000..854bdb8 --- /dev/null +++ b/roles/prerec/templates/copy-original.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# {{ ansible_managed }} +# Usage: rename-original.sh $slug $file [$extra] +SLUG=$1 +FILE=$2 +EXTRA="" +if [ -z ${3-unset} ]; then + EXTRA="" +elif [ -n "$3" ]; then + EXTRA="$3" +elif echo "$FILE" | grep -e '\(webm\|mp4\|mov\)'; then + EXTRA="--original" +fi +filename=$(basename -- "$FILE") +extension="${filename##*.}" +filename="${filename%.*}" +FILE_PREFIX=$(get-file-prefix $SLUG) +if echo "$FILE" | grep -q \\. ; then + cp "$FILE" $FILE_PREFIX$EXTRA.$extension + echo $FILE_PREFIX$EXTRA.$extension +else + cp "$FILE" $FILE_PREFIX$EXTRA + echo $FILE_PREFIX$EXTRA +fi diff --git a/roles/prerec/templates/get-file-prefix b/roles/prerec/templates/get-file-prefix new file mode 100755 index 0000000..0240b99 --- /dev/null +++ b/roles/prerec/templates/get-file-prefix @@ -0,0 +1,10 @@ +#!/bin/bash +# {{ ansible_managed }} +# Usage: get-file-prefix $slug [talks.json] +# You can also set the TALKS_JSON environment variable + +SLUG=$1 +if [ -z "$TALKS_JSON" ]; then + TALKS_JSON=${2:-~orga/current/talks.json} +fi +jq -r '.talks[] | select(.slug=="'$SLUG'")["file-prefix"]' < $TALKS_JSON diff --git a/roles/prerec/templates/process-prerec.sh b/roles/prerec/templates/process-prerec.sh index 795753d..d98337f 100755 --- a/roles/prerec/templates/process-prerec.sh +++ b/roles/prerec/templates/process-prerec.sh @@ -5,14 +5,9 @@ ORIGINAL=$1 REENCODED=$(echo "$ORIGINAL" | perl -pe 's/^(emacsconf-[0-9]*-.*?--.*?--.*?--).*/$1reencoded.webm/') SLUG=$(echo "$ORIGINAL" | perl -ne '/^emacsconf-[0-9]*-(.*?)--/ && print $1') MAIN=$(echo "$ORIGINAL" | perl -pe 's/^(emacsconf-[0-9]*-.*?--.*?--.*?--).*/$1main.webm/') -SCREEN=reencode-$SLUG -if ! ( screen -ls | grep -q $SLUG ); then - screen -dmS $SCREEN -fi # ( cd /data/emacsconf/cache; ./update-cache ) -# /data/emacsconf/{{ emacsconf_year }}/scripts/talk $SLUG "WAITING_FOR_PREREC" "PROCESSING" +# /data/emacsconf/admin/{{ emacsconf_year }}/scripts/talk $SLUG "WAITING_FOR_PREREC" "PROCESSING" if [[ ! -f "$REENCODED" ]]; then - screen -S $SCREEN -X screen -t reencode-$SLUG /bin/bash -c "/data/emacsconf/{{ emacsconf_year }}/scripts/reencode.sh \"$ORIGINAL\" \"$REENCODED\" && /data/emacsconf/{{ emacsconf_year }}/scripts/upload.sh $REENCODED $MAIN && /data/emacsconf/{{ emacsconf_year }}/scripts/thumbnail.sh \"$MAIN\" && /data/emacsconf/{{ emacsconf_year }}/scripts/upload.sh $(echo "$MAIN" | sed s/webm$/png/) exec /bin/bash" & + screen -dmS reencode-$SLUG /bin/bash -c "reencode.sh \"$ORIGINAL\" \"$REENCODED\" && upload.sh $REENCODED $MAIN && thumbnail.sh \"$MAIN\" && upload.sh $(echo "$MAIN" | sed s/webm$/png/); echo $(date -Iminutes) $SLUG reencoded >> ~/emacsconf.log && exec /bin/bash" & fi -screen -S $SCREEN -X screen -t captions-$SLUG /bin/bash -c "/data/emacsconf/{{ emacsconf_year }}/scripts/process-captions.py $(dirname $ORIGINAL); exec /bin/bash" -screen -x $SCREEN +screen -dmS captions-$SLUG /bin/bash -c "/data/emacsconf/admin/{{ emacsconf_year }}/scripts/process-captions.py $(dirname $ORIGINAL); echo $(date -Iminutes) $SLUG captioned >> ~/emacsconf.log; exec /bin/bash" diff --git a/roles/prerec/templates/reencode-in-screen.sh b/roles/prerec/templates/reencode-in-screen.sh new file mode 100755 index 0000000..5c12203 --- /dev/null +++ b/roles/prerec/templates/reencode-in-screen.sh @@ -0,0 +1,16 @@ +#!/bin/bash +ORIGINAL=$1 +BASE="${ORIGINAL%--original.*}" +REENCODED="${BASE}--reencoded.webm" +SLUG=$(echo "$ORIGINAL" | perl -ne '/^emacsconf-[0-9]*-(.*?)--/ && print $1') +LOCK=".lock-$SLUG" + +if [ ! -f "$REENCODED" ]; then + if [ -f "$LOCK" ]; then + echo "$LOCK already exists, waiting for it" + else + touch "$LOCK" + screen -dmS reencode-$SLUG /bin/bash -c "reencode.sh \"$ORIGINAL\" \"$REENCODED\" && thumbnail.sh \"$MAIN\" && rm \"$LOCK\"" + echo "Processing $REENCODED in reencode-$SLUG" + fi +fi diff --git a/roles/prerec/templates/reencode.sh b/roles/prerec/templates/reencode.sh index b0bdfc7..665fee2 100755 --- a/roles/prerec/templates/reencode.sh +++ b/roles/prerec/templates/reencode.sh @@ -33,7 +33,7 @@ shift `expr $OPTIND - 1` OPTIND=1 input="$1" -output="$2" +output="${2:-$(echo $input | sed 's/--original.*/--reencoded.webm/')}" command="$(cat<<EOF ffmpeg -y -i "$input" $time_limit \ diff --git a/roles/prerec/templates/remux.sh b/roles/prerec/templates/remux.sh index c378133..2d2320b 100755 --- a/roles/prerec/templates/remux.sh +++ b/roles/prerec/templates/remux.sh @@ -1,20 +1,29 @@ #!/usr/bin/env bash # {{ ansible_managed }} # Mix in the normalized audio -# Usage: remux.sh $input_video +# Usage: remux.sh $input_video_or_slug with_suffix() { - echo "$input_video" | sed "s/--\(reencoded\|original\).webm\$/--$1/" + echo "$input_video" | sed "s/--\(main\|reencoded\|original\).webm\$/--$1/" } input_video="$1" +if [ ! -f $input_video ]; then + # treat it as a slug + input_video=$(get-file-prefix $1)--reencoded.webm +fi + input_audio="$(with_suffix "normalized.opus")" output_video="$(with_suffix "final.webm")" main_video="$(with_suffix "main.webm")" main_subs="$(with_suffix "main.vtt")" +if cat $main_subs | head -1 | grep captioned; then + $subs = "-i $main_subs" +fi + command="$(cat<<EOF -ffmpeg -i "$input_video" -i "$input_audio" -c:v copy -c:a copy -map 0:v:0 -map 1:a:0 "$output_video" && +ffmpeg -y -i "$input_video" -i "$input_audio" $subs -c:v copy -c:a copy -map 0:v:0 -map 1:a:0 "$output_video" && cp "$output_video" "$main_video" && if [[ -f "$main_subs" ]]; then touch -m "$main_subs"; fi EOF @@ -23,14 +32,18 @@ EOF printf "input: %s\ncomputed output: %s\nrelated main: %s\n\n" "$input_video" "$output_video" "$main_video" printf "Produced incantation:\n%s\n\n" "$command" -while true; -do - read -r -p "Run it? y/n " -n 1 -r response - if [[ $response =~ ^([yY])$ ]]; then - eval "$command" - exit 0 - else - printf "\nExiting\n" - exit 3 - fi -done +if [ -z "$CONFIRMED" ]; then + eval "$command" +else + while true; + do + read -r -p "Run it? y/n " -n 1 -r response + if [[ $response =~ ^([yY])$ ]]; then + eval "$command" + exit 0 + else + printf "\nExiting\n" + exit 3 + fi + done +fi diff --git a/roles/prerec/templates/rename-original.sh b/roles/prerec/templates/rename-original.sh index 0499ae8..ed085ed 100755 --- a/roles/prerec/templates/rename-original.sh +++ b/roles/prerec/templates/rename-original.sh @@ -8,7 +8,7 @@ EXTRA="" if [ -z ${3-unset} ]; then EXTRA="" elif [ -n "$3" ]; then - EXTRA="--$3" + EXTRA="$3" elif echo "$FILE" | grep -e '\(webm\|mp4\|mov\)'; then EXTRA="--original" fi @@ -16,8 +16,13 @@ filename=$(basename -- "$FILE") extension="${filename##*.}" filename="${filename%.*}" FILE_PREFIX=$(jq -r '.talks[] | select(.slug=="'$SLUG'")["file-prefix"]' < $TALKS_JSON) -mv "$FILE" $FILE_PREFIX$EXTRA.$extension -echo $FILE_PREFIX$EXTRA.$extension +if echo "$FILE" | grep -q \\. ; then + mv "$FILE" $FILE_PREFIX$EXTRA.$extension + echo $FILE_PREFIX$EXTRA.$extension +else + mv "$FILE" $FILE_PREFIX$EXTRA + echo $FILE_PREFIX$EXTRA +fi # Copy to original if needed if [ -f $FILE_PREFIX--original.webm ] && [ ! -f $FILE_PREFIX--main.$extension ]; then cp $FILE_PREFIX--original.$extension $FILE_PREFIX--main.webm diff --git a/roles/prerec/templates/run-aeneas.sh b/roles/prerec/templates/run-aeneas.sh index 8246c85..a9e27f4 100755 --- a/roles/prerec/templates/run-aeneas.sh +++ b/roles/prerec/templates/run-aeneas.sh @@ -11,4 +11,4 @@ if [ ! -f $BASE--whisper.vtt ]; then fi python3 -m aeneas.tools.execute_task $AUDIO *.txt "task_language=eng|os_task_file_format=vtt|is_text_type=plain" ${BASE}--aeneas.vtt cp ${BASE}--aeneas.vtt ${BASE}--main.vtt -/data/emacsconf/{{ emacsconf_year }}/scripts/upload.sh ${BASE}--main.vtt +/data/emacsconf/admin/{{ emacsconf_year }}/scripts/upload.sh ${BASE}--main.vtt diff --git a/roles/prerec/templates/talk b/roles/prerec/templates/talk index 0531cda..fa2e5f0 100755 --- a/roles/prerec/templates/talk +++ b/roles/prerec/templates/talk @@ -6,6 +6,8 @@ SLUG="$1" FROM_STATUS="$2" TO_STATUS="$3" +XDG_RUNTIME_DIR=/run/user/{{ getent_passwd[emacsconf_user].1 }} + if [ "x$TO_STATUS" == "x" ]; then FROM_STATUS=. TO_STATUS="$2" @@ -14,7 +16,7 @@ cd {{ emacsconf_private_dir }} #echo "Pulling conf.org..." #git pull echo "Updating status..." -emacsclient --eval "(emacsconf-with-todo-hooks (emacsconf-update-talk-status \"$SLUG\" \"$FROM_STATUS\" \"$TO_STATUS\"))" -a emacs +XDG_RUNTIME_DIR=$XDG_RUNTIME_DIR emacsclient --eval "(emacsconf-with-todo-hooks (emacsconf-update-talk-status \"$SLUG\" \"$FROM_STATUS\" \"$TO_STATUS\"))" -a emacs #echo "Committing and pushing in the background" #git commit -m "Update task status for $SLUG from $FROM_STATUS to $TO_STATUS" conf.org #git push & diff --git a/roles/prerec/templates/upload.sh b/roles/prerec/templates/upload.sh index f3dc9c5..09d55d9 100755 --- a/roles/prerec/templates/upload.sh +++ b/roles/prerec/templates/upload.sh @@ -2,5 +2,5 @@ # {{ ansible_managed }} scp $* orga@media.emacsconf.org:~/backstage -/data/emacsconf/{{ emacsconf_year }}/scripts/publish-backstage-index.sh -rsync -avze ssh orga@media.emacsconf.org:~/backstage/ /data/emacsconf/cache/ +/usr/local/bin/publish-backstage-index.sh +rsync -avze ssh orga@media.emacsconf.org:~/backstage/ /data/emacsconf/shared/{{ emacsconf_year }}/cache/ |