Add more documentation

1 files changed, 153 insertions, 90 deletions

diff --git a/2020/organizers-notebook.org b/2020/organizers-notebook.org
index 549ca142..19ad9138 100644
--- a/2020/organizers-notebook.org
+++ b/2020/organizers-notebook.org
@@ -1,4 +1,5 @@
 #+todo: TODO(t) INPROGRESS(i) | DONE(d) CANCELLED(c)
+#+PROPERTY: header-args    :results silent
 
 * Tasks
 ** DONE bandali: Copy compressed files from front0:/var/www/media.emacsconf.org/2020 to CSC mirror and update links
@@ -160,30 +161,111 @@ O1 - main organizer, O2 - secondary organizer or volunteer
 - If can't be easily resolved, play pre-recorded talk early and try again later (or follow up)
 - Stream a technical issues slide to the end point
 
+* During the conference
+** ffmpeg: mirror main stream to low-resolution stream
+
+ Needs the =$main480p= environment variable set to somethnig of the form =icecast://username:password@site:port/mount-point.webm=.
+
+ #+begin_src sh :eval no
+ while true; do ffmpeg -f webm -reconnect_at_eof 1 -reconnect_streamed 1  -re -i http://localhost:8000/main.webm -vf scale=854:480 -f webm -c:a copy -b:v 500k -maxrate 1M -bufsize 1M  -content_type video/webm -c:v libvpx  $main480p done
+ #+end_src
+
+** ERC
+*** Load data
+
+ #+begin_src emacs-lisp :var rooms=rooms :results silent
+   (defvar conf/collaborative-pad "https://etherpad.wikimedia.org/p/emacsconf-2020" "URL of collaborative pad.")
+   (defvar conf/topic-templates nil "List of (channel topic-template) entries for mass-setting channel topics.")
+   (defvar conf/streaming-nick "bandali" "IRC nick of main organizer in charge of streaming.")
+   "List of (code join-url) entries, one for each meeting room.")  ;; Set this in a private file
+   (defvar conf/channels nil "List of IRC channels for broadcasts.")
+   (defvar conf/info nil "List of plists with the following keys: `:talk-id', `:name', `:speakers', and other info.") ; Set from submissions.org
+   (setq conf/topic-templates 
+   '(("#emacsconf" "EmacsConf 2020 is over, thanks for joining! | Subscribe to https://lists.gnu.org/mailman/listinfo/emacsconf-discuss for updates")
+     ("#emacsconf-accessible" "EmacsConf 2020 is over. Thanks for making it more accessible! | Subscribe to https://lists.gnu.org/mailman/listinfo/emacsconf-discuss for updates")
+     ("#emacsconf-org" "EmacsConf2020 is over, thanks for joining! | Dedicated channel for EmacsConf organizers and speakers | this is intended as an internal, low-traffic channel; for main discussion around EmacsConf, please join #emacsconf | Subscribe to https://lists.gnu.org/mailman/listinfo/emacsconf-discuss for updates")))
+   (defvar conf/rooms '(("A" "http://example.org?room=a")
+                        ("B" "http://example.org?room=b")
+                        ("C" "http://example.org?room=c")) 
+     "List of (code join-url) entries. Room codes should be uppercase.") ; set from organizers' wiki index.org
+ #+end_src
+
+*** Announce topics
+
+ #+begin_src emacs-lisp
+   (defmacro conf/erc-with-channels (channel-list &rest forms)
+     (declare (indent 1) (debug (form form body)))
+     `(mapcar (lambda (channel)
+                (with-current-buffer (erc-get-buffer channel)
+                  ,@forms))
+              ,channel-list))
+
+   (defun conf/get-room (room)
+     (cadr (assoc (upcase room) conf/rooms)))
+
+   (defun erc-cmd-CONFTOPIC (&rest message)
+     "Set the topic to MESSAGE | template in the conference channels.
+   If MESSAGE is not specified, reset the topic to the template."
+     (mapc (lambda (template) 
+             (with-current-buffer (erc-get-buffer (car template))
+               (erc-cmd-TOPIC (if message (concat (if (stringp message) message (s-join " " message)) " | " (cadr template))
+                                (cadr template)))))
+           conf/topic-templates))
+
+   (defun erc-cmd-CHECKIN (room nick)
+     "Send instructions for ROOM and `conf/collaborative-pad' to NICK."
+     (let ((room-url (conf/get-room room)))
+       (unless room-url (error "Please specify nick and room name"))
+       (erc-send-message (format "%s: Thanks for checking in! I'll send you some private messages with the instructions for room %s, so please check there. (Let me know if you don't get them!)" nick
+       (upcase room)))
+       (erc-message "PRIVMSG" (format "%s You can use this BBB room for your presentation: %s . I'll join you there shortly to set up the room and do the last-minute tech check." nick room-url))
+       (erc-message "PRIVMSG" (format "%s The collaborative pad we'll be using for questions is at %s . We'll collect questions from #emacsconf and put them there. If you'd like to jump to your part of the document, you might be able to keep an eye on questions. Alternatively, we can read questions to you." nick conf/collaborative-pad))
+       (erc-message "PRIVMSG" (format "%s Amin will join when it's time for your presentation, and he will give you the go-ahead when it's time to present. See you in the BBB room!" nick))))
+
+   (defun erc-cmd-READY (code &rest filter)
+     "Notify #emacsconf-org and `conf/streaming-nick' that CODE is ready for the talk specified by FILTER.
+   FILTER can be the talk ID or strings to match against the title or speaker names."
+     (let ((room-url (conf/get-room code))
+           (talk (conf/find-talk filter)))
+       (unless room-url (error "Could not find room"))
+       (unless talk (error "Could not find talk"))
+       (with-current-buffer (erc-get-buffer "#emacsconf-org")
+         (erc-send-message (format "Ready in Room %s: %s (%s)"
+                                   (upcase code)
+                                   (plist-get talk :title)
+                                   (plist-get talk :speakers))))
+       (erc-message "PRIVMSG" 
+                    (format "%s Ready in Room %s ( %s ): %s (%s)"
+                            conf/streaming-nick
+                            (upcase code)
+                            room-url
+                            (plist-get talk :title)
+                            (plist-get talk :speakers)))))
+
+   (defun erc-cmd-ANNOUNCE (&rest filter)
+     "Set the channel topics to announce the talk specified by FILTER.
+   FILTER can be the talk ID or strings to match against the title or speaker names."
+     (let ((info (conf/find-talk filter)) message)
+       (unless info (error "Could not find talk."))
+       (erc-cmd-CONFTOPIC (format "talk%s: %s (%s)"
+                                  (plist-get info :talk-id)
+                                  (plist-get info :title)
+                                  (plist-get info :speakers)))))
+
+   (defun erc-cmd-BROADCAST (&rest message)
+     "Say MESSAGE in all the conference channels."
+     (conf/erc-with-channels conf/channels
+                             (erc-send-message (s-join " " message))))
+ #+end_src
+
+* After the conference
 * Splitting up the stream recording into individual files
 
-** Timing
-
-** Day 1
-| 00 | Opening remarks                                                              |   13:04 |   20:08 |
-| 02 | An Emacs Developer Story: From User to Package Maintainer                    |   27:24 |   51:39 |
-| 03 | Q&A Idea to Novel Superstructure: Emacs for Writing                          | 1:00:10 | 1:09:21 |
-| 05 | Q&A Bard Bivou(m)acs - Building a bandcamp-like page for an album of music   | 1:41:30 | 1:55:12 |
-| 06 | Trivial Emacs Kits                                                           | 1:57:25 | 2:11:04 |
-| 07 | Q&A Beyond Vim and Emacs: A Scalable UI Paradigm                             | 2:32:02 | 2:36:34 |
-| 08 | Building reproducible Emacs                                                  | 2:38:03 | 2:55:21 |
-| 09 | Q&A Orgmode - your life in plain text                                        | 4:22:00 | 4:27:48 |
-| 12 | One Big-ass Org File or multiple tiny ones?  Finally, the End of the debate! | 4:55:23 | 5:11:38 |
-| 13 | Experience Report: Steps to "Emacs Hyper Notebooks"                          | 5:13:07 | 5:25:11 |
-| 15 | Moving from Jekyll to OrgMode, an experience report                          | 5:47:45 | 6:04:16 |
-| 16 | Org-roam: Presentation, Demonstration, and What's on the Horizon             | 6:06:05 | 6:27:55 |
-| 17 | Org-mode and Org-Roam for Scholars and Researchers                           | 6:29:00 | 6:50:14 |
-| 18 | Org-roam: Technical Presentation                                             | 6:51:20 | 7:21:45 |
-| 40 | Closing remarks part 1                                                       | 7:24:01 | 7:27:23 |
-| 20 | OMG Macros                                                                   | 7:28:00 | 7:50:36 |
-| 40 | Closing remarks part 2                                                       | 7:51:48 | 8:04:25 |
-
-** Commands used for splitting
+NOTE: ffmpeg has a hard time splitting with -c:v copy unless it's on a
+keyframe boundary. If it isn't on a keyframe, then you'll have a few
+seconds of black video until the next keyframe kicks in.
+
+Here's an example of what we had for splitting.
 
 #+begin_src sh :eval no
 ffmpeg -y -i main.webm-2020-11-28--08-48.webm -ss   13:04 -to   20:08    -c:a copy -c:v copy emacsconf-2020--00-opening-remarks.webm
@@ -211,7 +293,7 @@ ffmpeg -y -i main.webm-2020-11-29--08-44.webm -ss 7:55:44 -to 8:02:02    -c:a co
 ffmpeg -y -i main.webm-2020-11-29--08-44.webm -ss 8:03:32 -to 8:40:01.10 -c:a copy -c:v copy emacsconf-2020--42-closing-remarks.webm
 #+end_src
 
-Run this code before a copy of the ffmpeg scripts (adjusting the value of adjust as needed) and it will show only the lines that need tweaking.
+This fiddles with the =-ss= to make it divisible by 4. Run this code before a copy of the ffmpeg scripts (adjusting the value of adjust as needed) and it will show only the lines that need tweaking.
 
 #+begin_src emacs-lisp :eval no
   (save-excursion
@@ -235,93 +317,74 @@ Run this code before a copy of the ffmpeg scripts (adjusting the value of adjust
           (replace-match "")))))
 #+end_src
 
-ffmpeg -y -i main.webm-2020-11-29--08-44.webm -ss 23116 -to 6:31:04.90 -c:a copy -c:v copy emacsconf-2020--31-lakota-language-and-emacs--questions--grant-shangreaux.webm
-
 Thanks to SirVolta and bandali for figuring out keyframe issue!
-*** Additional notes
-I've decided to use a etherpad document for this to make it possible for everyone to contribute and fix mistakes. Hope that's ok for everybody interested.
-And hopefully I won't destroy too much by accidentally hitting Emacs keys :')
-
-Feel free to edit at will. The goal of this document is to learn as much as possible about video manipulation
-using ffmpeg.
-
-
-**** video compression: keyframes and deltaframes
-   https://blog.video.ibm.com/streaming-video-tips/keyframes-interframe-video-compression/#keyframe
-   The problem is actually really simple. Most video contains a lot of non changing background images. It would be wasteful to re-encode all that every single frame. This is solved by only sometimes encoding the entire frame (keyframe, also called interframe or i-frame) and use the difference between the video in that frame and the current frame most of the time (deltaframe). The exact ratio between keyframes and deltaframes is a configuration option of the encoder. But this means that if you grab just a single frame, chances are high that it is a deltaframe and you would need at least one more keyframe or, depending on the codec, even all of the frames before, after or in between to restore the original frame content. This is also why going frame by frame backwards in MPV is so slow, and VLC doesn't even support it. It has to go all the way backwards to the last keyframe and reconstruct from there.
-
-   This means that when copying data, you can only start and stop on keyframes as deltaframes need the data before and possibly after it to work.
-
-   The VP8 codec used appears to only look backwards to a previous keyframe. 
-   This explains why all endings were perfect.
-
-   To be reliable with all codecs, I consider the data after keyframes as important as the data before. 
-
-**** Possible solution
-   I'm thinking about splitting the rendering process in three pieces:
-   1. cut the time before the first keyframe and after the last keyframe using re-encoding
-   2. from that point (so between the first and last keyframe) cut using copy codec
-   3. merge the three
-   This way there is the least possible amount of re-encoding done while still having frame perfect timing.
-   It would mean figuring out a way to coax ffmpeg into splitting exactly on iframes and knowing where this is.
-
-   I think it's okay to cut to the keyframe boundary before the start timestamp. Up to three extra seconds of content never hurt anyone. =)
-
-   I'm currently testing a proof-of-concept script to do exactly this.
-
-   Command to extract all keyframe times:
-   AV_LOG_FORCE_NOCOLOR=1 ffmpeg -i input -vf select='eq(pict_type\,PICT_TYPE_I)' -f null NUL -loglevel debug 2>&1 | grep -F 'select_out:0' | awk '{print $4 " " $6}'
-   It should be a lot more efficient (or more like less inefficient) using start and end times.
-
-   Using this data, it should be possible to glue everything together in a script. Although it is going to be annoying dealing with time in seconds and HH:MM:SS.ms format. time<->string conversions. Yaay...
-   Or just go with seconds everywhere. Human readable formats are nice but kinda pointless now you can't edit them without causing keyframe issues.
-
-**** resources
-   (all links work with LibreJS enabled)
-   https://blog.video.ibm.com/streaming-video-tips/keyframes-interframe-video-compression/
-   https://blog.streamspot.com/blog/compression-codecs-keyframes-and-the-basics-of-stream-quality
-   https://blog.superuser.com/2012/02/24/ffmpeg-the-ultimate-video-and-audio-manipulation-tool/
-   https://superuser.com/questions/138331/using-ffmpeg-to-cut-up-video
-   http://blog.webmproject.org/2010/05/inside-webm-technology-vp8-alternate.html
-
-
 
+Further reading:
 
+(all links work with LibreJS enabled)
+https://blog.video.ibm.com/streaming-video-tips/keyframes-interframe-video-compression/
+https://blog.streamspot.com/blog/compression-codecs-keyframes-and-the-basics-of-stream-quality
+https://blog.superuser.com/2012/02/24/ffmpeg-the-ultimate-video-and-audio-manipulation-tool/
+https://superuser.com/questions/138331/using-ffmpeg-to-cut-up-video
+http://blog.webmproject.org/2010/05/inside-webm-technology-vp8-alternate.html
 
 
 * Compressing video
 
-Usage: compress-video input-filename.webm output-filename.webm
-Thanks to ArneBab for this incantation!
+Thanks to ArneBab for this ffmpeg script which is now documented in [[https://www.draketo.de/software/ffmpeg-compression-vp9-av1][Extreme compression of Video with VP9 (webm) using ffmpeg]]. We modified it to keep the original audio.
+
+Usage: compress-video.sh input-filename.webm output-filename.webm
 
 #+begin_src sh :tangle compress-video.sh
 Q=56
 nice ffmpeg -y -i $1 -c:v libvpx-vp9 -b:v 0 -crf $Q -aq-mode 2 -an -tile-columns 0 -tile-rows 0 -frame-parallel 0 -cpu-used 8 -auto-alt-ref 1 -lag-in-frames 25 -g 999 -pass 1 -f webm -threads 8 /dev/null &&
-nice ffmpeg -y -i $1 -c:v libvpx-vp9 -b:v 0 -crf $Q -aq-mode 2 -c:a libopus -b:a 12k -tile-columns 2 -tile-rows 2 -frame-parallel 0 -cpu-used -5 -auto-alt-ref 1 -lag-in-frames 25 -pass 2 -g 999 -threads 8 $2
+nice ffmpeg -y -i $1 -c:v libvpx-vp9 -b:v 0 -crf $Q -c:a copy -tile-columns 2 -tile-rows 2 -frame-parallel 0 -cpu-used -5 -auto-alt-ref 1 -lag-in-frames 25 -pass 2 -g 999 -threads 8 $2
 #+end_src
 
-Usage: compress-video-original-audio input-filename.webm output-filename.webm
+Here's the original version which compresses audio too. Usage: compress-video-compressed-audio.sh input-filename.webm output-filename.webm
 
-#+begin_src sh :tangle compress-video-original-audio.sh
+#+begin_src sh :tangle compress-video-compressed-audio.sh
 Q=56
 nice ffmpeg -y -i $1 -c:v libvpx-vp9 -b:v 0 -crf $Q -aq-mode 2 -an -tile-columns 0 -tile-rows 0 -frame-parallel 0 -cpu-used 8 -auto-alt-ref 1 -lag-in-frames 25 -g 999 -pass 1 -f webm -threads 8 /dev/null &&
-nice ffmpeg -y -i $1 -c:v libvpx-vp9 -b:v 0 -crf $Q -aq-mode 2 -c:a copy -tile-columns 2 -tile-rows 2 -frame-parallel 0 -cpu-used -5 -auto-alt-ref 1 -lag-in-frames 25 -pass 2 -g 999 -threads 8 $2
+nice ffmpeg -y -i $1 -c:v libvpx-vp9 -b:v 0 -crf $Q -aq-mode 2 -c:a libopus -b:a 12k -tile-columns 2 -tile-rows 2 -frame-parallel 0 -cpu-used -5 -auto-alt-ref 1 -lag-in-frames 25 -pass 2 -g 999 -threads 8 $2
 #+end_src
-* Code sachac used to move sbv files
+
+* Code sachac used to move sbv files from the Downloads directory
+
+Autogenerated captions can save a bit of time when setting up
+captions. This code renames a downloaded file to match the current
+file's naming scheme and moves it to the right directory.
 
 #+begin_src emacs-lisp
-(defun my/latest-file (path &optional filter)
-  (car (sort (seq-remove #'file-directory-p (directory-files path 'full filter t)) #'file-newer-than-file-p)))
-(defun my/rename-latest-download ()
-  (interactive)
-  (let* ((file (my/latest-file "~/Downloads"))
-         (new-file (expand-file-name (concat (file-name-base (dired-get-filename)) "-autogen." (file-name-extension file)) "~/vendor/emacsconf-wiki/2020/subtitles/")))
-    (rename-file file new-file t)
-    (message "%s" new-file)))
+  (defvar conf/subtitle-directory (expand-file-name "subtitles" default-directory) "Directory where subtitles will be kept.")
+  (defvar conf/download-directory "~/Downloads" "Directory where downloaded files are saved.")
+
+  (defun my/latest-file (path &optional filter)
+    "Returns the newest file in PATH.
+  If FILTER is specified, files should match this regex."
+    (car (sort (seq-remove #'file-directory-p (directory-files path 'full filter t)) #'file-newer-than-file-p)))
+  (defun my/rename-latest-download-as-subtitle-file ()
+    "Rename the most recent downloaded file to match the current file and move it to `conf/subtitle-directory'.
+  To use this, open a Dired buffer with a list of the correctly-named
+  videos. Move your cursor to the line for the video that you have just
+  downloaded captions for, then call `my/rename-latest-download-as-subtitle-file.'"
+    (interactive)
+    (let* ((file (my/latest-file conf/download-directory))
+           (new-file (expand-file-name (concat (file-name-base (dired-get-filename)) "-autogen." (file-name-extension file)) conf/subtitle-directory)))
+      (rename-file file new-file t)
+      (message "%s" new-file)))
+  ;; Ex: (local-set-key [f5] 'my/rename-latest-download)
 #+end_src
 
-* Code sachac used to play a video file back over a virtual microphone for passing to dictation
+To convert from SBV to VTT (used for the HTML5 video player) and fix
+timestamps so that they're not overlapping, install =python3-webtt=
+and run [[file:subtitles/fix.py]] like this: =fix.py
+emacsconf-2020--04-music-in-plain-text--jonathan-gregory.sbv=.
+
+* Restarting ikiwiki manually
+
+This is needed when you change the template or if the ikiwiki process gets stuck on something.
+
 #+begin_src sh :eval no
-sudo modprobe snd-aloop
-ffmpeg -re -i emacsconf-2020--10-lead-your-future-with-org--andrea.webm -f alsa -ar 44100 hw:1,1
+ssh front 'sudo -iu ikiwiki ikiwiki --setup ~ikiwiki/emacsconf.setup'
 #+end_src