From 4ef6627aa71a2c94a2ae7615d5e93d6689ac8b5a Mon Sep 17 00:00:00 2001 From: Sacha Chua Date: Tue, 10 Oct 2023 10:21:30 -0400 Subject: copy scripts --- roles/caption/tasks/main.yml | 11 ++++++++ roles/caption/templates/process-captions.py | 16 ++++++----- roles/media/tasks/main.yml | 19 +++++++++++++ roles/media/templates/nginx-include | 44 ++++++++++++++--------------- roles/prerec/tasks/main.yml | 26 +++++++++++++---- roles/prerec/templates/process-prerec.sh | 8 +++--- 6 files changed, 86 insertions(+), 38 deletions(-) diff --git a/roles/caption/tasks/main.yml b/roles/caption/tasks/main.yml index b34a67d..fea78f4 100644 --- a/roles/caption/tasks/main.yml +++ b/roles/caption/tasks/main.yml @@ -34,6 +34,17 @@ state: directory owner: "{{ emacsconf_user }}" group: "{{ emacsconf_group }}" +- name: Copy scripts for processing + tags: process-prerec + template: + src: "{{ item }}" + dest: "{{ emacsconf_caption_dir }}/scripts/{{ item }}" + owner: "{{ emacsconf_user }}" + group: "{{ emacsconf_group }}" + mode: 0775 + loop: + - process-captions.py + - split-captions.py # - name: Copy the inotify script # tags: process-captions # template: diff --git a/roles/caption/templates/process-captions.py b/roles/caption/templates/process-captions.py index dc7ef29..a42439b 100755 --- a/roles/caption/templates/process-captions.py +++ b/roles/caption/templates/process-captions.py @@ -93,7 +93,7 @@ def get_files_to_work_on(directory): talk = next(filter(lambda talk: talk['slug'] == val['slug'], talks), None) if talk: val['base'] = os.path.join(os.path.dirname(val['video'] or val['audio']), - base_name(talk['video-slug'])) + base_name(talk['file-prefix'])) else: val['base'] = os.path.join(os.path.dirname(val['video'] or val['audio']), base_name(val['video'] or val['audio'])) @@ -114,7 +114,8 @@ def extract_audio(work): log("Extracting audio from %s acodec %s" % (work['video'], acodec)) output = subprocess.check_output(['ffmpeg', '-y', '-i', work['video'], '-acodec', acodec, '-vn', new_file], stderr=subprocess.STDOUT) work['audio'] = new_file - subprocess.call(["/data/emacsconf/{{ emacsconf_year }}/scripts/upload.sh", work['audio']]) + if os.path.isfile("/data/emacsconf/{{ emacsconf_year }}/scripts/upload.sh"): + subprocess.call(["/data/emacsconf/{{ emacsconf_year }}/scripts/upload.sh", work['audio']]) return work def to_sec(time_str): @@ -143,13 +144,14 @@ def generate_captions(work): audio = whisper.pad_or_trim(audio) result = model.transcribe(audio, verbose=True, language="en") result = clean_up_timestamps(result) - with open(new_file, 'w') as vtt: - whisper.utils.write_vtt(result['segments'], file=vtt) - with open(work['base'] + '.txt', 'w') as txt: - whisper.utils.write_txt(result['segments'], file=txt) + vtt_writer = whisper.utils.get_writer('vtt', os.path.dirname(new_file)) + txt_writer = whisper.utils.get_writer('txt', os.path.dirname(new_file)) + vtt_writer(result, work['audio'], {'max_line_width': 60, 'max_line_count': None, 'highlight_words': None}) + txt_writer(result, work['audio'], {'max_line_width': 60, 'max_line_count': None, 'highlight_words': None}) work['vtt'] = new_file work['txt'] = work['base'] + '.txt' - subprocess.call(["/data/emacsconf/{{ emacsconf_year }}/scripts/upload.sh", work['vtt'], work['txt']]) + if os.path.isfile("/data/emacsconf/{{ emacsconf_year }}/scripts/upload.sh"): + subprocess.call(["/data/emacsconf/{{ emacsconf_year }}/scripts/upload.sh", work['vtt'], work['txt']]) if 'srv2' in work: del work['srv2'] return work diff --git a/roles/media/tasks/main.yml b/roles/media/tasks/main.yml index 6eb3ab6..44374f5 100644 --- a/roles/media/tasks/main.yml +++ b/roles/media/tasks/main.yml @@ -1,3 +1,9 @@ +- name: Set up packages + ansible.builtin.apt: + update_cache: true + pkg: + - nginx + state: present - name: Install package for setting htpasswd package: name: python3-passlib @@ -22,6 +28,19 @@ template: src: nginx-include dest: /etc/nginx/sites-available/{{ host_name }}-{{ emacsconf_year }}-include +- name: Create main configuration if needed + template: + src: nginx-site-config + dest: /etc/nginx/sites-available/{{ host_name }} + force: no +- name: Make sure main configuration is enabled + file: + src: /etc/nginx/sites-available/{{ host_name }} + dest: /etc/nginx/sites-enabled/{{ host_name }} + owner: "{{ emacsconf_user }}" + group: "{{ emacsconf_group }}" + force: no + state: link - name: Include it in the main configuration lineinfile: path: /etc/nginx/sites-available/{{ host_name }} diff --git a/roles/media/templates/nginx-include b/roles/media/templates/nginx-include index 03a7bfa..0f74c60 100644 --- a/roles/media/templates/nginx-include +++ b/roles/media/templates/nginx-include @@ -1,26 +1,14 @@ rewrite ^/current/bbb-open.html$ {{ bbb_open_url }} redirect; location /{{ emacsconf_year }}/backstage { -auth_basic "Restricted"; -auth_basic_user_file /etc/nginx/sites-available/{{ host_name }}-{{ emacsconf_year }}-htpasswd; -autoindex on; -rewrite ^/{{ emacsconf_year }}/backstage/current/pad/([^/]*)$ https://{{ etherpad_server_name }}/{{ emacsconf_year }}-$1 redirect; -rewrite ^/{{ emacsconf_year }}/backstage/current/room/([^/]*)$ https://{{ media_server_name }}/{{ emacsconf_year }}/backstage/assets/redirects/open/bbb-$1.html redirect; -rewrite ^/{{ emacsconf_year }}/backstage/current/([^/]*)/pad/?$ https://{{ etherpad_server_name }}/{{ emacsconf_year }}-$1 redirect; -rewrite ^/{{ emacsconf_year }}/backstage/current/([^/]*)/room/?$ https://{{ media_server_name }}/{{ emacsconf_year }}/backstage/assets/redirects/open/bbb-$1.html redirect; -} - {% if media_protect_root|bool %} - location /{{ emacsconf_year }} { - # TODO: Figure out how to make this properly controlled by the variable - # auth_basic "Restricted"; - # auth_basic_user_file /etc/nginx/sites-available/{{ host_name }}-{{ emacsconf_year }}-htpasswd; - autoindex on; - } - {% else %} - location /{{ emacsconf_year }} { - autoindex on; - } - {% endif %} - location /{{ emacsconf_year }}/{{ emacsconf_id }}.ics { + auth_basic "Restricted"; + auth_basic_user_file /etc/nginx/sites-available/{{ host_name }}-{{ emacsconf_year }}-htpasswd; + autoindex on; + rewrite ^/{{ emacsconf_year }}/backstage/current/pad/([^/]*)$ https://{{ etherpad_server_name }}/{{ emacsconf_year }}-$1 redirect; + rewrite ^/{{ emacsconf_year }}/backstage/current/room/([^/]*)$ https://{{ media_server_name }}/{{ emacsconf_year }}/backstage/assets/redirects/open/bbb-$1.html redirect; + rewrite ^/{{ emacsconf_year }}/backstage/current/([^/]*)/pad/?$ https://{{ etherpad_server_name }}/{{ emacsconf_year }}-$1 redirect; + rewrite ^/{{ emacsconf_year }}/backstage/current/([^/]*)/room/?$ https://{{ media_server_name }}/{{ emacsconf_year }}/backstage/assets/redirects/open/bbb-$1.html redirect; + } + location /{{ emacsconf_year }}/{{ emacsconf_id }}.ics { auth_basic off; } location /{{ emacsconf_year }}/schedules/ { @@ -30,4 +18,16 @@ rewrite ^/{{ emacsconf_year }}/backstage/current/([^/]*)/room/?$ https://{{ medi location /{{ emacsconf_year }}/{{ emacsconf_id }}-{{ track.id }}.ics { auth_basic off; } - {% endfor %} \ No newline at end of file + {% endfor %} +{% if media_protect_root == 'true' %} + location /{{ emacsconf_year }} { + # TODO: Figure out how to make this properly controlled by the variable + auth_basic_user_file /etc/nginx/sites-available/{{ host_name }}-{{ emacsconf_year }}-htpasswd; + auth_basic "Restricted"; + autoindex on; + } +{% else %} + location /{{ emacsconf_year }} { + autoindex on; + } +{% endif %} diff --git a/roles/prerec/tasks/main.yml b/roles/prerec/tasks/main.yml index 43c9bd5..da62340 100644 --- a/roles/prerec/tasks/main.yml +++ b/roles/prerec/tasks/main.yml @@ -1,13 +1,28 @@ - name: Ensure the directory exists file: - path: "{{ emacsconf_caption_dir }}/scripts" + path: "{{ emacsconf_caption_dir }}/{{ item }}" state: directory owner: "{{ emacsconf_user }}" group: "{{ emacsconf_group }}" -- name: Create the symlink + loop: + - scripts + - cache + - files +- name: Create the symlinks file: - src: "{{ emacsconf_caption_dir }}/scripts" - dest: "/home/{{ emacsconf_user }}/scripts" + src: "{{ emacsconf_caption_dir }}/{{ item }}" + dest: "/home/{{ emacsconf_user }}/{{ item }}" + owner: "{{ emacsconf_user }}" + group: "{{ emacsconf_group }}" + state: link + loop: + - scripts + - cache + - files +- name: Create the symlinks + file: + src: "{{ emacsconf_caption_dir }}" + dest: "/home/{{ emacsconf_user }}/current" owner: "{{ emacsconf_user }}" group: "{{ emacsconf_group }}" state: link @@ -27,7 +42,7 @@ backup: yes mode: 0775 - name: Copy scripts for processing - tags: process-prerec, wip + tags: process-prerec template: src: "{{ item }}" dest: "/usr/local/bin/{{ item }}" @@ -43,3 +58,4 @@ - thumbnail.sh - reencode.sh - run-aeneas.sh + - rename-original.sh diff --git a/roles/prerec/templates/process-prerec.sh b/roles/prerec/templates/process-prerec.sh index 69787f2..795753d 100755 --- a/roles/prerec/templates/process-prerec.sh +++ b/roles/prerec/templates/process-prerec.sh @@ -9,10 +9,10 @@ SCREEN=reencode-$SLUG if ! ( screen -ls | grep -q $SLUG ); then screen -dmS $SCREEN fi -( cd /data/emacsconf/cache; ./update-cache ) +# ( cd /data/emacsconf/cache; ./update-cache ) # /data/emacsconf/{{ emacsconf_year }}/scripts/talk $SLUG "WAITING_FOR_PREREC" "PROCESSING" -#if [[ ! -f "$REENCODED" ]]; then -screen -S $SCREEN -X screen -t reencode-$SLUG /bin/bash -c "/data/emacsconf/{{ emacsconf_year }}/scripts/reencode.sh \"$ORIGINAL\" \"$REENCODED\" && /data/emacsconf/{{ emacsconf_year }}/scripts/upload.sh $REENCODED $MAIN && /data/emacsconf/{{ emacsconf_year }}/scripts/thumbnail.sh \"$MAIN\" && /data/emacsconf/{{ emacsconf_year }}/scripts/upload.sh $(echo "$MAIN" | sed s/webm$/png/) exec /bin/bash" & -#fi +if [[ ! -f "$REENCODED" ]]; then + screen -S $SCREEN -X screen -t reencode-$SLUG /bin/bash -c "/data/emacsconf/{{ emacsconf_year }}/scripts/reencode.sh \"$ORIGINAL\" \"$REENCODED\" && /data/emacsconf/{{ emacsconf_year }}/scripts/upload.sh $REENCODED $MAIN && /data/emacsconf/{{ emacsconf_year }}/scripts/thumbnail.sh \"$MAIN\" && /data/emacsconf/{{ emacsconf_year }}/scripts/upload.sh $(echo "$MAIN" | sed s/webm$/png/) exec /bin/bash" & +fi screen -S $SCREEN -X screen -t captions-$SLUG /bin/bash -c "/data/emacsconf/{{ emacsconf_year }}/scripts/process-captions.py $(dirname $ORIGINAL); exec /bin/bash" screen -x $SCREEN -- cgit v1.2.3