more extraction updates

4 files changed, 482 insertions, 177 deletions

diff --git a/emacsconf-extract.el b/emacsconf-extract.el
index 2ef7d74..3635338 100644
--- a/emacsconf-extract.el
+++ b/emacsconf-extract.el
@@ -753,6 +753,7 @@ Files should be downloaded to `emacsconf-extract-bbb-raw-dir'."
 				 recording-start
 				 recording-stop
 				 recording-spans
+				 stream-start
 				 chat
 				 talking
 				 participants
@@ -1092,7 +1093,9 @@ Strategies:
 			 ("authorization-code-function" . emacsconf-extract-oauth-browse-and-prompt)
 			 ("authorization-endpoint" . "https://accounts.google.com/o/oauth2/v2/auth")
 			 ("authorization-extra-arguments" .
-				(("redirect_uri" . "http://localhost:8080")))
+				(("redirect_uri" . "http://localhost:8080")
+				 ("access_type" . "offline")
+				 ("prompt" . "consent")))
 			 ("access-token-endpoint" . "https://oauth2.googleapis.com/token")
 			 ("scope" . "https://www.googleapis.com/auth/youtube")
 			 ("client-secret-method" . prompt))))
@@ -1389,6 +1392,55 @@ If QA is non-nil, treat it as a Q&A video."
 	(expect (emacsconf-extract-youtube-format-duration "PT1H9M22S") :to-equal "1:09:22")
 	(expect (emacsconf-extract-youtube-format-duration "PT9M22S") :to-equal "9:22"))
 
+(defun emacsconf-extract-youtube-publish-video-drafts-with-spookfox ()
+	"Look for drafts and publish them."
+	(while (not (eq (spookfox-js-injection-eval-in-active-tab
+									 "document.querySelector('.edit-draft-button div') != null" t) :false))
+		(progn
+			(spookfox-js-injection-eval-in-active-tab
+			 "document.querySelector('.edit-draft-button div').click()" t)
+			(sleep-for 2)
+			(spookfox-js-injection-eval-in-active-tab
+			 "document.querySelector('#step-title-3').click()" t)
+			(when (spookfox-js-injection-eval-in-active-tab
+						 "document.querySelector('tp-yt-paper-radio-button[name=\"PUBLIC\"] #radioLabel').click()" t)
+				(spookfox-js-injection-eval-in-active-tab
+				 "document.querySelector('#done-button').click()" t)
+				(while (not (eq  (spookfox-js-injection-eval-in-active-tab
+													"document.querySelector('#close-button .label') == null" t)
+												 :false))
+					(sleep-for 1))
+
+				(spookfox-js-injection-eval-in-active-tab
+				 "document.querySelector('#close-button .label').click()" t)
+				(sleep-for 1)))))
+
+(defun emacsconf-extract-youtube-store-url (&optional prefix)
+	(interactive "p")
+	(let* ((desc (spookfox-js-injection-eval-in-active-tab
+							 "document.querySelector('#description').innerHTML" t))
+				 (slug (if (and desc
+												(string-match (rx (literal emacsconf-base-url) (literal emacsconf-year) "/talks/"
+																		(group (1+ (not (or " " "/" "\"")))))
+																desc))
+									(match-string 1 desc)
+								(emacsconf-complete-slug)))
+				(url (spookfox-js-injection-eval-in-active-tab "window.location.href" t))
+				(qa (or (> (or prefix 1)
+									 1)
+								(string-match "Q&A" (or desc ""))))
+				(field (if qa
+									 "QA_YOUTUBE_URL"
+								 "YOUTUBE_URL")))
+		(save-window-excursion
+					(emacsconf-with-talk-heading slug
+						(org-entry-put (point)
+													 field
+													 url)
+						(message "Updating %s %s %s"
+										 slug
+										 field
+										 url)))))
 
 ;; (setq emacsconf-extract-youtube-api-video-details (emacsconf-extract-youtube-get-video-details emacsconf-extract-youtube-api-playlist-items))
 
@@ -1445,60 +1497,6 @@ If QA is non-nil, treat it as a Q&A video."
 													:as #'json-read))
 				 nil)))
 
-(defun emacsconf-extract-youtube-store-url (&optional prefix)
-	(interactive "p")
-	(let* ((desc (spookfox-js-injection-eval-in-active-tab
-							 "document.querySelector('#description').innerHTML" t))
-				 (slug (if (and desc
-												(string-match (rx (literal emacsconf-base-url) (literal emacsconf-year) "/talks/"
-																		(group (1+ (not (or " " "/" "\"")))))
-																desc))
-									(match-string 1 desc)
-								(emacsconf-complete-slug)))
-				(url (spookfox-js-injection-eval-in-active-tab "window.location.href" t))
-				(qa (or (> (or prefix 1)
-									 1)
-								(string-match "Q&A" (or desc ""))))
-				(field (if qa
-									 "QA_YOUTUBE_URL"
-								 "YOUTUBE_URL")))
-		(save-window-excursion
-					(emacsconf-with-talk-heading slug
-						(org-entry-put (point)
-													 field
-													 url)
-						(message "Updating %s %s %s"
-										 slug
-										 field
-										 url)))))
-
-(defun emacsconf-extract-toobnix-store-url (&optional prefix)
-	(interactive "p")
-	(let* ((desc (spookfox-js-injection-eval-in-active-tab
-							 "document.querySelector('.video-info-description').innerHTML" t))
-				 (slug (if (and desc
-												(string-match (rx (literal emacsconf-base-url) (literal emacsconf-year) "/talks/"
-																		(group (1+ (not (or " " "/" "\"")))))
-																desc))
-									(match-string 1 desc)
-								(emacsconf-complete-slug)))
-				(url (spookfox-js-injection-eval-in-active-tab "window.location.href" t))
-				(qa (or (> (or prefix 1)
-									 1)
-								(string-match "Q&A" (or desc ""))))
-				(field (if qa
-									 "QA_TOOBNIX_URL"
-								 "TOOBNIX_URL")))
-		(save-window-excursion
-					(emacsconf-with-talk-heading slug
-						(org-entry-put (point)
-													 field
-													 url)
-						(message "Updating %s %s %s"
-										 slug
-										 field
-										 url)))))
-
 (defun emacsconf-extract-toobnix-publish-video-from-edit-page ()
 	"Messy hack to set a video to public and store the URL."
 	(interactive)
@@ -1507,7 +1505,7 @@ If QA is non-nil, treat it as a Q&A video."
 	(spookfox-js-injection-eval-in-active-tab "document.querySelector('span[title=\"Anyone can see this video\"]').click()" t)
 	(sit-for 1)
 	(spookfox-js-injection-eval-in-active-tab "document.querySelector('button.orange-button').click()" t)(sit-for 3)
-	(emacsconf-extract-toobnix-store-url)
+	(emacsconf-extract-store-url)
 	(shell-command "xdotool key Alt+Tab sleep 1 key Ctrl+w Alt+Tab"))
 
 (defun emacsconf-extract-toobnix-set-up-playlist ()
@@ -1528,6 +1526,10 @@ If QA is non-nil, treat it as a Q&A video."
 			 (read-key "press a key when saved to playlist")))
 	 (emacsconf-publish-prepare-for-display (emacsconf-get-talk-info))))
 
+(defun emacsconf-extract-toobnix-view-qa (talk)
+	(interactive (list (emacsconf-complete-talk-info)))
+	(browse-url (plist-get (emacsconf-resolve-talk talk) :qa-toobnix-url)))
+
 (defun emacsconf-extract-youtube-spookfox-add-playlist-numbers ()
 	"Number the playlist for easier checking.
 Related: `emacsconf-extract-check-playlists'."
@@ -1575,5 +1577,43 @@ Related: `emacsconf-extract-check-playlists'."
 				 (org-todo "DONE"))))
 	 (emacsconf-publish-prepare-for-display (emacsconf-get-talk-info))))
 
+(defun emacsconf-extract-store-url (&optional qa)
+	"Store the URL for the currently-displayed field.
+Call with a prefix arg to store the URL as Q&A."
+	(interactive (list current-prefix-arg))
+	(let* ((url (spookfox-js-injection-eval-in-active-tab "window.location.href" t))
+				 (platform (if (string-match "toobnix" url)
+											 'toobnix
+										 'youtube))
+				 (desc (spookfox-js-injection-eval-in-active-tab
+								(format "document.querySelector('%s').innerHTML"
+												(if (eq platform 'toobnix)
+														".video-info-description"
+													"#description"))
+								t))
+				 (slug (if (and desc
+												(string-match (rx (literal emacsconf-base-url) (literal emacsconf-year) "/talks/"
+																					(group (1+ (not (or " " "/" "\"")))))
+																			desc))
+									 (match-string 1 desc)
+								 (emacsconf-complete-slug)))
+				 (qa (or qa (string-match "Q&A" (or desc ""))))
+				 (field
+					(concat
+					 (if qa "QA_" "")
+					 (if (eq platform 'toobnix) "TOOBNIX" "YOUTUBE")
+					 "_URL"
+					 )))
+		(save-window-excursion
+			(emacsconf-with-talk-heading slug
+				(org-entry-put (point)
+											 field
+											 url)
+				(message "Updating %s %s %s"
+								 slug
+								 field
+								 url)))))
+
+
 (provide 'emacsconf-extract)
 ;;; emacsconf-extract.el ends here
diff --git a/emacsconf-mail.el b/emacsconf-mail.el
index 2411522..190ca4b 100644
--- a/emacsconf-mail.el
+++ b/emacsconf-mail.el
@@ -968,7 +968,7 @@ ${captions}
       (if (string= (plist-get talk :captioner) "sachac")
           ""
         (format "%s: Thank you for editing the captions!\n\n" (assoc-default "NAME_SHORT" captioner-info)))
-      :captions (mapconcat (lambda (sub) (concat (emacsconf-surround "\n" (elt sub 4) "" "") (elt sub 3))) (subed-parse-file captions) "\n")))
+      :captions (mapconcat (lambda (sub) (concat (emacsconf-surround "\nNOTE " (elt sub 4) "\n\n" "") (elt sub 3))) (subed-parse-file captions) "\n")))
     (mml-attach-file captions "text/vtt" "Subtitles" "attachment")))
 
 (defun emacsconf-mail-upload-and-backstage-info (group)
@@ -1620,12 +1620,186 @@ Sacha")
      :base-url emacsconf-base-url
      :conf-name emacsconf-name
      :email (car group)
-		:user-email user-mail-address
+		 :user-email user-mail-address
      :speakers-short (plist-get (cadr group) :speakers-short)
      :url (mapconcat (lambda (o) (concat emacsconf-base-url (plist-get o :url)))
                      (cdr group) " , ")
 		 :email-notes (emacsconf-surround "ZZZ: " (plist-get (cadr group) :email-notes) "\n\n" ""))))
 
+(defun emacsconf-mail-template-speakers-thanks-after-conferences ()
+	(interactive)
+	(let* ((log-note "sent thanks to speaker after conference")
+				 (groups
+					(emacsconf-mail-groups
+					 (emacsconf-filter-talks-by-logbook
+						log-note
+						(seq-filter (lambda (o) (plist-get o :email))
+												(emacsconf-publish-prepare-for-display (emacsconf-get-talk-info)))))))
+		(dolist (group groups)
+			(emacsconf-mail-prepare
+			 (list
+				:subject "Thanks for speaking at ${conf-name} ${year}!"
+				:reply-to "emacsconf-submit@gnu.org, ${email}, ${user-email}"
+				:mail-followup-to "emacsconf-submit@gnu.org, ${email}, ${user-email}"
+				:log-note log-note
+				:body
+				"${email-notes}Hi, ${speakers-short}!
+
+Thank you so much for being part of ${conf-name} ${year}! Hundreds of people
+enjoyed it, and I'm sure even more will come across the videos in the
+days to follow.
+
+Your videos are available on the talk page at ${talk-urls} , and
+we've added the questions and comments that we've collected from
+IRC/BBB/Etherpad. For your convenience, I've also included them below.
+
+Your videos are also available on YouTube and Toobnix at:
+
+${video-urls}
+
+If you want to reupload the video to your own channel, feel free
+to do so.  If you let me know where you've uploaded it, I can
+switch our playlist to include your version of the video
+instead. That way, it might be easier for you to respond to
+comments on videos.
+
+If you would like to share more resources or add more answers to
+any of the questions, you can add them to the talk page or reply
+to this and we can add them for you.
+
+Thanks again for speaking at ${conf-name} ${year}!
+
+${signature}
+----
+${feedback}
+"
+				)
+			 (car group)
+			 (list
+				:email-notes (emacsconf-surround "ZZZ: " (string-join (seq-uniq (seq-map (lambda (talk) (plist-get talk :email-notes)) (cdr group)))
+																															", ") "\n\n" "")
+				:speakers-short (plist-get (cadr group) :speakers-short)
+				:conf-name emacsconf-name
+				:year emacsconf-year
+				:talk-urls
+				(mapconcat
+				 (lambda (talk)
+					 (plist-get talk :absolute-url))
+				 (cdr group)
+				 " , ")
+				:video-urls
+				(mapconcat
+				 (lambda (talk)
+					 (concat
+						(plist-get talk :title) "\n"
+						(emacsconf-surround "YouTube - talk: " (plist-get talk :youtube-url) "\n" "")
+						(emacsconf-surround "YouTube - Q&A: " (plist-get talk :qa-youtube-url) "\n" "")
+						(emacsconf-surround "Toobnix - talk: " (plist-get talk :toobnix-url) "\n" "")
+						(emacsconf-surround "Toobnix - Q&A: " (plist-get talk :qa-toobnix-url) "\n" "")))
+				 (cdr group)
+				 "\n")
+				:signature user-full-name
+				:email (car group)
+				:user-email user-mail-address
+				:feedback
+				(mapconcat
+				 (lambda (talk)
+					 (plist-get talk :absolute-url)
+					 (with-temp-buffer
+						 (insert (emacsconf-talk-markdown-from-wiki (plist-get talk :slug)))
+						 (goto-char (point-min))
+						 (re-search-forward "^# Discussion" nil t)
+						 (buffer-substring (point) (point-max))))
+				 (cdr group)
+				 "\n---------------------------\n")
+				)))))
+
+(defun emacsconf-mail-template-qa-permission (group)
+	"Ask for permission to post more of the Q&A."
+	(interactive (list (emacsconf-mail-complete-email-group
+                      (seq-filter
+                       (lambda (o)
+												 (and
+													(or
+                           (emacsconf-talk-file o "--answers--original.vtt")
+													 (emacsconf-talk-file o "--original.vtt"))
+													(not (string-match "Asked for permission regarding the rest of the Q&A"
+																						 (plist-get o :logbook)))))
+                       (emacsconf-get-talk-info)))))
+	(emacsconf-mail-prepare
+	 (list
+		:subject "${conf-name} ${year}: May we post the rest of the Q&A?"
+		:reply-to "emacsconf-submit@gnu.org, ${email}, ${user-email}"
+		:mail-followup-to "emacsconf-submit@gnu.org, ${email}, ${user-email}"
+		:log-note "Asked for permission regarding the rest of the Q&A"
+		:body
+		"${email-notes}Hi, ${speakers-short}!
+
+We're experimenting with a new harvesting workflow for live and
+Q&A videos this year to make things more predictable for speakers
+and participants. Sometimes people have so much fun chatting
+after the talk that they might forget that the recording for the
+session Q&A will be posted for other people to learn from.
+
+I've trimmed your online videos to roughly when the host left the
+BigBlueButton room. There was lots of great discussion
+afterwards, though, so I'd love to include the rest of it if
+that's okay with you. To make it easier for you to review that
+part or reuse what you shared in the Q&A session, I've included
+an automatically-generated transcript for the whole Q&A
+session. I've indicated the section that got trimmed out of the
+published recording with \"NOTE Start of section to review\" in
+the transcript. You can watch the session at ${bbb-recording-url} .
+
+- Option A: We could post the rest of the Q&A as is, which lets
+  people listen to the conversation and learn from it
+
+- Option B: We can keep the published Q&A video to just the part
+  that the host was in, and either you or I can go over the
+  transcript to pull out interesting notes for the summary or for
+  other posts
+
+What do you think?
+
+${signature}
+----
+${transcript}
+")
+	 (car group)
+	 (list
+		:email-notes (emacsconf-surround "ZZZ: " (string-join (seq-uniq (seq-map (lambda (talk) (plist-get talk :email-notes)) (cdr group)))
+																													", ") "\n\n" "")
+		:speakers-short (plist-get (cadr group) :speakers-short)
+		:conf-name emacsconf-name
+		:year emacsconf-year
+		:bbb-recording-url
+		(mapconcat
+		 (lambda (talk)
+			 (plist-get talk :bbb-rec))
+		 (cdr group)
+		 " , ")
+		:signature user-full-name
+		:email (car group)
+		:transcript
+		(mapconcat
+		 (lambda (talk)
+			 (concat
+				(plist-get talk :title) "\n\n"
+				(mapconcat
+				 (lambda (sub)
+					 (concat (emacsconf-surround "\nNOTE " (elt sub 4) "\n\n" "")
+									 (elt sub 3)))
+				 (subed-parse-file (or (emacsconf-talk-file talk "--answers--original.vtt")
+															 (emacsconf-talk-file talk "--original.vtt"))) "\n")))
+		 (cdr group)
+		 "----")
+		:user-email user-mail-address))
+	(dolist (talk (cdr group))
+		(mml-attach-file (or (emacsconf-talk-file talk "--answers--original.vtt")
+												 (emacsconf-talk-file talk "--original.vtt"))
+										 "text/vtt"
+										 (concat "Automatic captions for " (plist-get talk :title))
+										 "attachment")))
 
 ;;; Other mail functions
 
diff --git a/emacsconf-publish.el b/emacsconf-publish.el
index d4d4dd6..acb1992 100644
--- a/emacsconf-publish.el
+++ b/emacsconf-publish.el
@@ -346,25 +346,27 @@
                           (file-name-nondirectory video-file))
                 (file-name-nondirectory video-file)))
             :captions
-            (and (stringp video-file)
-                 (or (plist-get talk :captions-edited)
-										 (and
-											(emacsconf-talk-file talk "--main.vtt")
-											(emacsconf-captions-edited-p
-											 (expand-file-name (emacsconf-talk-file talk "--main.vtt") emacsconf-cache-dir))))
-                 (let ((tracks
-                        (emacsconf-video-subtitle-tracks
-												 (or (plist-get talk :caption-file)
-														 (concat (replace-regexp-in-string "reencoded\\|original" "main"
-																															 video-base)
-																		 ".vtt"))
-                         (or (plist-get talk :track-base-url)
-                             (plist-get talk :base-url))
-                         (plist-get talk :files))))
-                   (cond
-                    ((zerop (length tracks)) "")
-                    ((eq (plist-get talk :format) 'wiki) (format "captions=\"\"\"%s\"\"\"" tracks))
-                    (t tracks))))
+						(or
+						 (and (stringp video-file)
+									(or (plist-get talk :captions-edited)
+											(and
+											 (plist-get talk :caption-file)
+											 (emacsconf-captions-edited-p
+												(expand-file-name (plist-get talk :caption-file) emacsconf-cache-dir))))
+									(let ((tracks
+                         (emacsconf-video-subtitle-tracks
+													(or (plist-get talk :caption-file)
+															(concat (replace-regexp-in-string "reencoded\\|original" "main"
+																																video-base)
+																			".vtt"))
+													(or (plist-get talk :track-base-url)
+															(plist-get talk :base-url))
+													(plist-get talk :files))))
+										(cond
+                     ((zerop (length tracks)) "")
+                     ((eq (plist-get talk :format) 'wiki) (format "captions=\"\"\"%s\"\"\"" tracks))
+                     (t tracks))))
+						 "")
             :chapter-track (or (plist-get chapter-info :track) "")
             :chapter-list
             (if chapter-info
@@ -380,9 +382,11 @@
 						:links
 						(concat
 						 (emacsconf-surround "<li><a href=\""
-																 (if (plist-get talk :backstage)
-																		 (emacsconf-backstage-url (plist-get talk :pad-url))
-																	 (plist-get talk :pad-url)) "\">Open Etherpad</a></li>" "")
+																 (unless (eq emacsconf-publishing-phase 'resources)
+																	 (if (plist-get talk :backstage)
+																			 (emacsconf-backstage-url (plist-get talk :pad-url))
+																		 (plist-get talk :pad-url)))
+																 "\">Open Etherpad</a></li>" "")
 						 (emacsconf-surround "<li><a href=\""
 																 (and (plist-get talk :backstage)
 																			(plist-get talk :bbb-backstage))
@@ -392,7 +396,7 @@
 																			(plist-get talk :qa-url))
 																 "\">Open public Q&A</a></li>" "")
 						 (emacsconf-surround "<li><a href=\""
-																 (plist-get talk :bbb-rec)
+																 (and (not (eq emacsconf-publishing-phase 'resources)) (plist-get talk :bbb-rec))
 																 "\">Play recording from BigBlueButton</a></li>" ""))
             :other-files
             (mapconcat
@@ -416,7 +420,7 @@
      :video
      (emacsconf-replace-plist-in-string
       info
-			(if (stringp video-file)
+			(if (and (stringp video-file) (string-match "webm$" video-file))
 					"<video controls preload=\"none\" id=\"${video-id}\"><source src=\"${source-src}\" />${captions}${chapter-track}<p><em>Your browser does not support the video tag. Please download the video instead.</em></p></video>${chapter-list}"
 				(or (plist-get talk :video-note) "")))
 		 :audio
@@ -526,21 +530,27 @@ resources."
                   o))
   (concat
    (if (plist-get o :qa-public) "# Talk\n\n" "")
-   (emacsconf-publish-index-card o)
+   (emacsconf-publish-index-card
+		(append o
+						(list
+						 :caption-file (emacsconf-talk-file o "--main.vtt")
+						 :files (seq-remove (lambda (f) (string-match "--answers" f))
+																(emacsconf-publish-filter-public-files o)))))
    (if (plist-get o :qa-public)
        (concat "\n\n# Q&A\n\n"
                (emacsconf-publish-index-card (append
-                                      (list
-                                       :public 1
-                                       :video-id (concat (plist-get o :slug) "-qanda")
-                                       :toobnix-url nil
-																			 :captions-edited (plist-get o :qa-captions-edited)
-                                       :video-file (emacsconf-talk-file o "--answers.webm")
-																			 :audio-file (emacsconf-talk-file o "--answers.opus")
-																			 :chapter-file (emacsconf-talk-file o "--answers--chapters.vtt"))
-
-                                      o)
-                                     (list "--answers.webm" "--answers.vtt" "--answers--chapters.vtt" "--answers.opus")))
+                             (list
+                              :public 1
+                              :video-id (concat (plist-get o :slug) "-qanda")
+                              :toobnix-url nil
+															:captions-edited (plist-get o :qa-captions-edited)
+															:caption-file (emacsconf-talk-file o "--answers.vtt")
+                              :video-file (emacsconf-talk-file o "--answers.webm")
+															:video-duration (plist-get o :qa-video-duration)
+															:audio-file (emacsconf-talk-file o "--answers.opus")
+															:chapter-file (emacsconf-talk-file o "--answers--chapters.vtt")
+															:files (emacsconf-publish-filter-public-files o "answers"))
+                             o)))
      "")))
 
 (defun emacsconf-publish-webchat-link (o)
@@ -610,19 +620,18 @@ resources."
                      (plist-get (emacsconf-get-track (plist-get o :track)) :id)))
                 "")))
      "[[!toc  ]]
-Format: ${format}
-${pad-info}${irc-info}${status-info}${schedule-info}\n
+Format: ${format}  \n${pad-info}${irc-info}${status-info}${schedule-info}\n
 ${alternate-apac-info}\n")))
 
 (defun emacsconf-publish-format-email-questions-and-comments (talk)
 	"Invite people to e-mail either the public contact for TALK or the private list."
-  (format "Questions or comments? Please e-mail %s"
-          (emacsconf-publish-format-public-email talk
-                                         (or
-                                          (and (string= (plist-get talk :public-email) "t")
-                                               (plist-get talk :email))
-                                          (plist-get talk :public-email)
-                                          "emacsconf-org-private@gnu.org"))))
+	(format "Questions or comments? Please e-mail %s"
+					(emacsconf-publish-format-public-email talk
+																(or
+																 (and (string= (plist-get talk :public-email) "t")
+																			(plist-get talk :email))
+																 (plist-get talk :public-email)
+																 "emacsconf-org-private@gnu.org"))))
 
 (defun emacsconf-publish-captions-in-wiki (talk)
   "Copy the captions file."
@@ -707,7 +716,7 @@ This includes the intro note, the schedule, and talk resources."
      (let ((msecs (elt sub 1)))
 			 (concat
 				(if (and (elt sub 4) (not (string= (elt sub 4) "")))
-						(format "\n[[!template new=\"1\" text=\"\"\"%s\"\"\" video=\"%s\" id=\"subtitle\"%s]]\n\n"
+						(format "\n[[!template new=\"1\" text=\"\"\"%s\"\"\" start=\"%s\" video=\"%s\" id=\"subtitle\"%s]]\n\n"
 										(string-trim (replace-regexp-in-string "^NOTE[ \n]" "" (elt sub 4)))
 										(concat (format-seconds "%02h:%02m:%02s" (/ (floor msecs) 1000))
 														"." (format "%03d" (mod (floor msecs) 1000)))
@@ -1169,14 +1178,15 @@ You can also get this schedule as iCalendar files: ${icals}. Importing that into
 														:endutc (format-time-string "%FT%T%z" (plist-get o :end-time) t)
 														:start (format-time-string "%-l:%M" (plist-get o :start-time) emacsconf-timezone)
 														:end (format-time-string "%-l:%M" (plist-get o :end-time) emacsconf-timezone)))
-													('resources
+													((or 'harvest 'resources)
 													 (list
 														:pad nil
 														:channel nil
 														:resources
 														(concat
 														 (emacsconf-surround "<li><a href=\""
-																								 (plist-get o :bbb-rec)
+																								 (and (not (eq emacsconf-publishing-phase 'resources))
+																											(plist-get o :bbb-rec))
 																								 "\">Play recording from BigBlueButton</a></li>" "")
 														 (mapconcat
 															(lambda (s) (concat "<li>" s "</li>"))
@@ -1201,10 +1211,14 @@ You can also get this schedule as iCalendar files: ${icals}. Importing that into
                                  (when (and (plist-get o :public)
                                             (or (plist-get o :toobnix-url)
                                                 (plist-get o :video-file)))
-                                   "video posted")))
+                                   "video posted")
+																 (emacsconf-surround "video: " (plist-get o :video-duration) "" nil)
+																 (emacsconf-surround "answers: " (and (plist-get o :qa-public)
+																																			(plist-get o :qa-video-duration))
+																										 "" nil))
+																)
                           ", ")
-												 )
-												)))
+												 ))))
             (while attrs
               (let ((field (pop attrs))
                     (val (pop attrs)))
@@ -1342,7 +1356,7 @@ If MODIFY-FUNC is specified, use it to modify the talk."
 						 (svg-print img)
 						 (buffer-string)))
 				 "</p>"
-				 (if (eq emacsconf-backstage-phase 'prerec)
+				 (if (member emacsconf-publishing-phase '(program schedule conference))
 						 (format "<p>Waiting for %d talks (~%d minutes) out of %d total</p>"
 										 (length (assoc-default "WAITING_FOR_PREREC" by-status))
 										 (emacsconf-sum :time (assoc-default "WAITING_FOR_PREREC" by-status))
@@ -1363,9 +1377,11 @@ If MODIFY-FUNC is specified, use it to modify the talk."
 																									 (emacsconf-surround " (" (plist-get o :video-duration) ")" "")))
                                (assoc-default status by-status)
                                ", ")))
-					(pcase emacsconf-backstage-phase
-						('prerec '("WAITING_FOR_PREREC" "PROCESSING" "TO_ASSIGN" "TO_CAPTION" "TO_CHECK" "TO_STREAM"))
-						('harvest '("TO_ARCHIVE" "TO_REVIEW_QA" "TO_INDEX_QA" "TO_CAPTION_QA")))
+					(pcase emacsconf-publishing-phase
+						((or 'program 'schedule 'conference)
+						 '("WAITING_FOR_PREREC" "PROCESSING" "TO_ASSIGN" "TO_CAPTION" "TO_CHECK" "TO_STREAM"))
+						((or 'harvest 'resources)
+						 '("TO_ARCHIVE" "TO_REVIEW_QA" "TO_INDEX_QA" "TO_CAPTION_QA" "DONE")))
 					"")
          "</ul>"
 				 ;; alphabetical index
@@ -1377,8 +1393,8 @@ If MODIFY-FUNC is specified, use it to modify the talk."
 										(sort talks (lambda (a b) (string< (plist-get a :slug) (plist-get b :slug))))
 										", ")
 				 "</div>"
-				 (pcase emacsconf-backstage-phase
-					 ('prerec
+				 (pcase emacsconf-publishing-phase
+					 ((or 'program 'schedule 'conference)
 						(concat
 						 (emacsconf-publish-backstage-list
 							(append
@@ -1402,7 +1418,7 @@ If MODIFY-FUNC is specified, use it to modify the talk."
 							(assoc-default "WAITING_FOR_PREREC" by-status) files
 							"we're waiting for"
 							"Speakers might submit these, do them live, or cancel the talks.")))
-					 ('harvest
+					 ((or 'harvest 'resources)
 						(let ((stages
 									 '(("TO_REVIEW_QA" .
 											"Please review the --bbb-webcams.webm file and/or the --bbb-webcams.vtt and tell us (emacsconf-submit@gnu.org) if a Q&amp;A session can be published or if it needs to be trimmed (lots of silence at the end of the recording, accidentally included sensitive information, etc.).")
@@ -1478,6 +1494,7 @@ answers without needing to listen to everything again. You can see <a href=\"htt
 
 (defun emacsconf-publish-filter-public-files (talk &optional selector files)
 	"Return files that are okay to post publicly for TALK."
+	(setq files (or files (emacsconf-publish-talk-files talk)))
   (and (plist-get talk :file-prefix)
 			 (seq-filter
 				(lambda (f)
@@ -1490,13 +1507,18 @@ answers without needing to listen to everything again. You can see <a href=\"htt
 							 (or (plist-get talk :captions-edited)
 									 (emacsconf-captions-edited-p (expand-file-name f emacsconf-cache-dir))))
 							((rx (seq "--"
-												(or "original" "reencoded" "normalized")
-												"."
-												(1+ (syntax word))
-												string-end))
+												(or "reencoded" "normalized" "final" "old" "bbb")))
 							 nil)
+							((rx "--original")
+							 ;; include original only if --main or --answers does not exist
+							 (not (member (concat (plist-get talk :file-prefix)
+																		(if (string-match "--answers-original" f)
+																				"--answers.webm"
+																			"--main.webm"))
+														files)))
+							((rx (or "--main.txt" "--after-zaeph")) nil)
 							(_ t))))
-				(or files (emacsconf-publish-talk-files talk)))))
+				files)))
 
 (defun emacsconf-publish-public-index-for-talk (o files)
 	(format "<li><div class=\"title\"><a name=\"%s\" href=\"%s\">%s</a></div></div><div class=\"speakers\">%s</div>%s</li>%s"
@@ -1513,17 +1535,20 @@ answers without needing to listen to everything again. You can see <a href=\"htt
 												 :links
 												 (concat
 													(emacsconf-surround "<li><a href=\""
-																							(if (plist-get o :backstage)
-																									(emacsconf-backstage-url (plist-get o :pad-url))
-																								(plist-get o :pad-url)) "\">Open Etherpad</a></li>" "")
+																							(unless (eq emacsconf-publishing-phase 'resources)
+																								(if (plist-get o :backstage)
+																										(emacsconf-backstage-url (plist-get o :pad-url))
+																									(plist-get o :pad-url)))
+																							"\">Open Etherpad</a></li>" "")
 													(emacsconf-surround "<li><a href=\""
 																							(and (member emacsconf-publishing-phase '(schedule conference))
 																									 (plist-get o :qa-url))
 																							"\">Open public Q&A</a></li>" "")
-													(emacsconf-surround "<li><a href=\""
-																							(plist-get o :bbb-rec)
-																							"\">Play recording from BigBlueButton</a></li>" "")))
-                   o))
+													(unless (eq emacsconf-publishing-phase 'resources)
+														(emacsconf-surround "<li><a href=\""
+																								(plist-get o :bbb-rec)
+																								"\">Play recording from BigBlueButton</a></li>" ""))))
+									 o))
           (if (or (emacsconf-talk-file o "--answers.webm")
 									(emacsconf-talk-file o "--answers.opus"))
               (format "<li><div class=\"title\"><a href=\"%s\">Q&amp;A for %s</a></div>%s</li>"
@@ -1541,7 +1566,7 @@ answers without needing to listen to everything again. You can see <a href=\"htt
 												 :audio-file (emacsconf-talk-file o "--answers.opus")
                          :files (emacsconf-publish-filter-public-files
                                  o
-                                 "answers"
+                                 "--answers"
 																 files))
                         o)))
             "")))
@@ -1599,13 +1624,15 @@ ${include}
                                      :track-base-url
                                      (format "/%s/captions/" (plist-get f :conf-year))
 																		 :links
-																			(emacsconf-surround "<li><a href=\""
-																													(plist-get o :bbb-rec)
-																													"\">Play recording from BigBlueButton</a></li>" "")
-                                     :files
+																		 (unless (eq emacsconf-publishing-phase 'resources)
+																			 (emacsconf-surround "<li><a href=\""
+																													 (plist-get o :bbb-rec)
+																													 "\">Play recording from BigBlueButton</a></li>" ""))
+
+																		 :files
 																		 (seq-remove (lambda (f) (string-match "--answers" f))
 																								 (emacsconf-publish-filter-public-files f files)))
-                               f))
+																		 f))
                     "")
                   (if (plist-get f :qa-public)
                       (emacsconf-publish-index-card
@@ -1660,7 +1687,7 @@ ${include}
 (defun emacsconf-video-subtitle-tracks (filename track-base-url &optional files)
   (setq files (or files (directory-files emacsconf-cache-dir)))
   (concat
-   (if (member filename files)
+   (if (member (file-name-nondirectory filename) files)
        (format "<track label=\"English\" kind=\"captions\" srclang=\"en\" src=\"%s\" default />"
                (concat (or track-base-url "") (file-name-nondirectory filename)))
      "")
@@ -1732,7 +1759,7 @@ ${include}
 					(when (and dir (file-directory-p dir))
 						(with-temp-file (expand-file-name "talks.json" dir)
 							(insert (emacsconf-publish-talks-json)))))
-				(list emacsconf-res-dir emacsconf-ansible-directory)))
+				(list emacsconf-res-dir emacsconf-ansible-directory emacsconf-public-media-directory)))
 
 (defun emacsconf-talks-csv ()
   "Make a CSV of the talks.
@@ -2024,30 +2051,43 @@ This video is available under the terms of the Creative Commons Attribution-Shar
              duration)
         (unless (file-exists-p main)
           (setq main video-file-name))
-        (when video-file
-          (org-entry-put (point) "VIDEO_FILE" (file-name-nondirectory video-file))
-          (org-entry-put (point) "VIDEO_FILE_SIZE" (file-size-human-readable (file-attribute-size (file-attributes video-file))))
-          (unless (plist-get talk :captions-edited)
-            (let ((caption-file (expand-file-name
-                                 (concat (plist-get talk :file-prefix)
-                                         "--main.vtt")
-                                 emacsconf-cache-dir)))
-              (when (emacsconf-captions-edited-p caption-file)
-                (org-entry-put (point) "CAPTIONS_EDITED" "1"))))
-          (setq duration (/ (compile-media-get-file-duration-ms video-file) 1000))
-          (org-entry-put (point) "VIDEO_DURATION" (format-seconds "%h:%z%.2m:%.2s" duration))
-          (org-entry-put (point) "VIDEO_TIME" (number-to-string (ceiling (/ duration 60)))))
-				(when qa-file
-          (org-entry-put (point) "QA_VIDEO_FILE" (file-name-nondirectory qa-file))
-          (org-entry-put (point) "QA_VIDEO_FILE_SIZE" (file-size-human-readable (file-attribute-size (file-attributes qa-file))))
-          (unless (plist-get talk :qa-captions-edited)
-            (let ((caption-file (emacsconf-talk-file talk "--answers.vtt")))
-              (when (emacsconf-captions-edited-p caption-file)
-                (org-entry-put (point) "QA_CAPTIONS_EDITED" "1"))))
-          (unless (plist-get talk :qa-video-duration)
-            (setq duration (/ (compile-media-get-file-duration-ms qa-file) 1000))
-            (org-entry-put (point) "QA_VIDEO_DURATION" (format-seconds "%h:%z%.2m:%.2s" duration))
-            (org-entry-put (point) "QA_VIDEO_TIME" (number-to-string (ceiling (/ duration 60))))))
+        (if video-file
+						(progn
+							(org-entry-put (point) "VIDEO_FILE" (file-name-nondirectory video-file))
+							(org-entry-put (point) "VIDEO_FILE_SIZE" (file-size-human-readable (file-attribute-size (file-attributes video-file))))
+							(unless (plist-get talk :captions-edited)
+								(let ((caption-file (expand-file-name
+																		 (concat (plist-get talk :file-prefix)
+																						 "--main.vtt")
+																		 emacsconf-cache-dir)))
+									(if (emacsconf-captions-edited-p caption-file)
+											(org-entry-put (point) "CAPTIONS_EDITED" "1")
+										(org-entry-delete (point) "CAPTIONS_EDITED"))))
+							(setq duration (/ (compile-media-get-file-duration-ms video-file) 1000))
+							(org-entry-put (point) "VIDEO_DURATION" (format-seconds "%h:%z%.2m:%.2s" duration))
+							(org-entry-put (point) "VIDEO_TIME" (number-to-string (ceiling (/ duration 60)))))
+					(org-entry-delete (point) "VIDEO_FILE")
+					(org-entry-delete (point) "VIDEO_FILE_SIZE")
+					(org-entry-delete (point) "VIDEO_DURATION")
+					(org-entry-delete (point) "VIDEO_TIME")
+					(org-entry-delete (point) "CAPTIONS_EDITED"))
+				(if qa-file
+						(progn
+							(org-entry-put (point) "QA_VIDEO_FILE" (file-name-nondirectory qa-file))
+							(org-entry-put (point) "QA_VIDEO_FILE_SIZE" (file-size-human-readable (file-attribute-size (file-attributes qa-file))))
+							(unless (plist-get talk :qa-captions-edited)
+								(let ((caption-file (emacsconf-talk-file talk "--answers.vtt")))
+									(if (emacsconf-captions-edited-p caption-file)
+											(org-entry-put (point) "QA_CAPTIONS_EDITED" "1")
+										(org-entry-delete (point) "QA_CAPTIONS_EDITED"))))
+							(setq duration (/ (compile-media-get-file-duration-ms qa-file) 1000))
+							(org-entry-put (point) "QA_VIDEO_DURATION" (format-seconds "%h:%z%.2m:%.2s" duration))
+							(org-entry-put (point) "QA_VIDEO_TIME" (number-to-string (ceiling (/ duration 60)))) )
+					(org-entry-delete (point) "QA_VIDEO_FILE")
+					(org-entry-delete (point) "QA_VIDEO_FILE_SIZE")
+					(org-entry-delete (point) "QA_VIDEO_DURATION")
+					(org-entry-delete (point) "QA_VIDEO_TIME")
+					(org-entry-delete (point) "QA_CAPTIONS_EDITED"))
 				(when (file-exists-p intro-file)
 					(org-entry-put
 					 (point) "INTRO_TIME"
@@ -2579,10 +2619,24 @@ This video is available under the terms of the Creative Commons Attribution-Shar
 			(buffer-string))))
 ;; YouTube
 
+(defun emacsconf-publish-spookfox-update-youtube-video ()
+	(interactive)
+	(require 'spookfox)
+	;; Figure out which video this is
+	(let* ((filename (spookfox-js-injection-eval-in-active-tab "document.querySelector('#original-filename').textContent.trim()" t))
+				 (slug (emacsconf-get-slug-from-string filename))
+				 (talk (emacsconf-resolve-talk slug))
+				 (properties (emacsconf-publish-talk-video-properties talk 'youtube)))
+		(kill-new (plist-get properties :title))
+		(shell-command "xdotool search --name \"Channel content\" windowactivate sleep 1 key Ctrl+Shift+v sleep 2")
+		)
+
+	)
 (defvar emacsconf-publish-youtube-upload-command '("python3" "/home/sacha/vendor/youtube-upload/bin/youtube-upload"))
 
-
 (defun emacsconf-publish-upload-to-youtube (properties)
+	"Use youtube-upload to upload the talk based on PROPERTIES.
+Tends to be quota-limited, though."
 	(let ((arguments (append
 										(cdr emacsconf-publish-youtube-upload-command)
 										(when (plist-get properties :title)
@@ -2683,13 +2737,35 @@ This video is available under the terms of the Creative Commons Attribution-Shar
 (defvar emacsconf-publish-talk-video-tags (format "emacs,%s,%s%s" emacsconf-id emacsconf-id emacsconf-year)
 	"Comma-separated tags to add to the talk videos.")
 
+(defun emacsconf-publish-talk-video-properties (talk platform)
+	(let ((title (concat emacsconf-name " " emacsconf-year ": "
+											 (plist-get talk :title) " - " (plist-get talk :speakers))))
+		(list
+		 :file (emacsconf-talk-file talk "--main.webm")
+		 :tags emacsconf-publish-talk-video-tags
+		 :playlist (concat emacsconf-name " " emacsconf-year)
+		 :date (plist-get talk :start-time)
+		 :privacy (if (plist-get talk :public) "public" "unlisted")
+		 :title (if (< (length title) 100) title (concat (substring title 0 97) "..."))
+		 :description (emacsconf-publish-talk-description talk platform))))
+
+(defun emacsconf-publish-answers-video-properties (talk platform)
+	(let ((title (concat emacsconf-name " " emacsconf-year " Q&A: "
+											 (plist-get talk :title) " - " (plist-get talk :speakers))))
+		(list
+		 :file (emacsconf-talk-file talk "--answers.webm")
+		 :tags emacsconf-publish-talk-video-tags
+		 :playlist (concat emacsconf-name " " emacsconf-year)
+		 :date (plist-get talk :start-time)
+		 :privacy (if (plist-get talk :public) "public" "unlisted")
+		 :title (if (< (length title) 100) title (concat (substring title 0 97) "..."))
+		 :description (emacsconf-publish-answers-description talk platform))))
+
 (defun emacsconf-publish-upload-talk (talk platform)
 	(interactive
 	 (list (emacsconf-complete-talk-info)
 				 (intern (completing-read "Platform: " '("youtube" "toobnix")))))
 	(let ((file (emacsconf-talk-file talk "--main.webm"))
-				(title (concat emacsconf-name " " emacsconf-year ": "
-											 (plist-get talk :title) " - " (plist-get talk :speakers)))
 				output)
 		(when (and file (not (plist-get talk (if (eq platform 'toobnix) :toobnix-url :youtube-url))))
 			(setq output
@@ -2697,14 +2773,7 @@ This video is available under the terms of the Creative Commons Attribution-Shar
 						 (if (eq platform 'toobnix)
 								 #'emacsconf-publish-upload-to-toobnix
 							 #'emacsconf-publish-upload-to-youtube)
-						 (list
-							:file file
-							:tags emacsconf-publish-talk-video-tags
-							:playlist (concat emacsconf-name " " emacsconf-year)
-							:date (plist-get talk :start-time)
-							:privacy (if (plist-get talk :public) "public" "unlisted")
-							:title (if (< (length title) 100) title (concat (substring title 0 97) "..."))
-							:description (emacsconf-publish-talk-description talk platform))))
+						 (emacsconf-publish-talk-video-properties talk platform)))
 			(when (and (string-match "Video URL: \\(.*+\\)" output) (eq platform 'youtube))
 				(setq output (match-string 1 output))
 				(save-window-excursion
@@ -2713,6 +2782,8 @@ This video is available under the terms of the Creative Commons Attribution-Shar
 			output)))
 
 (defun emacsconf-publish-upload-answers (talk platform)
+	(interactive (list (emacsconf-complete-talk-info)
+										 (intern (completing-read "Platform: " '("youtube" "toobnix")))))
 	(let ((file (emacsconf-talk-file talk "--answers.webm"))
 				(title (concat emacsconf-name " " emacsconf-year " Q&A: " (plist-get talk :title)))
 				output)
diff --git a/emacsconf.el b/emacsconf.el
index 48b52c8..df54a78 100644
--- a/emacsconf.el
+++ b/emacsconf.el
@@ -75,14 +75,16 @@
 'program - don't include times
 'schedule - include times; use this leading up to the conference
 'conference - show IRC and watching info
-'resources - after EmacsConf, don't need status"
+'harvest - after EmacsConf, starting to process
+'resources - after EmacsConf, publish all the stuff"
   :group 'emacsconf
   :type '(choice
           (const :tag "CFP: include invitation" cfp)
           (const :tag "Program: Don't include times" program)
           (const :tag "Schedule: Include detailed times" schedule)
 					(const :tag "Conference: Show IRC and watching info" conference)
-          (const :tag "Resources: Don't include status" resources)))
+          (const :tag "Harvest: Extracting info" conference)
+          (const :tag "Resources: Don't include status, publish all Q&A" resources)))
 
 (defcustom emacsconf-backstage-phase 'prerec
 	"Contros what information to include backstage.
@@ -698,6 +700,8 @@ The subheading should match `emacsconf-abstract-heading-regexp'."
 				 (time-less-p (plist-get o :start-time)
 											(current-time)))
     (plist-put o :public t))
+	(when (eq emacsconf-publishing-phase 'resource)
+		(plist-put o :qa-public t))
   o)
 
 (defun emacsconf-talk-live-p (talk)
@@ -746,7 +750,8 @@ The subheading should match `emacsconf-abstract-heading-regexp'."
     emacsconf-add-checkin-time
     emacsconf-add-timezone-conversions
     emacsconf-add-speakers-with-pronouns
-    emacsconf-add-live-info)
+    emacsconf-add-live-info
+		emacsconf-add-video-info)
 	"Functions to collect information.")
 
 (defun emacsconf-add-speakers-with-pronouns (o)
@@ -793,6 +798,18 @@ The subheading should match `emacsconf-abstract-heading-regexp'."
   o)
 
 (require 'emacsconf-pad)
+
+(defun emacsconf-add-video-info (o)
+	(mapc (lambda (field)
+					(when
+							(and (plist-get o (intern (concat ":" field "-url")))
+									 (string-match "\\(?:watch\\?v=\\|https://youtu\\.be/\\|/w/\\)\\(.+\\)\\(?:[?&]\\|$\\)"
+																 (plist-get o (intern (concat ":" field "-url")))))
+						(plist-put o (intern (concat ":" field "-id"))
+											 (match-string 1 (plist-get o (intern (concat ":" field "-url")))) )))
+				(list "youtube" "qa-youtube" "toobnix" "qa-toobnix"))
+	o)
+
 (defun emacsconf-add-live-info (o)
   (plist-put o :absolute-url (concat emacsconf-base-url (plist-get o :url)))
   (plist-put o :in-between-url (format "%s%s/in-between/%s.png"
@@ -804,6 +821,8 @@ The subheading should match `emacsconf-abstract-heading-regexp'."
                                      emacsconf-year
                                      (plist-get o :slug)))
 	(plist-put o :intro-expanded (emacsconf-pad-expand-intro o))
+	(when (string-match "meetingId=\\(.+\\)" (or (plist-get o :bbb-rec) ""))
+		(plist-put o :bbb-meeting-id (match-string 1 (plist-get o :bbb-rec))))
   (let ((track (seq-find (lambda (track) (string= (plist-get o :track) (plist-get track :name)))
 												 emacsconf-tracks)))
     (when track
@@ -1716,7 +1735,7 @@ tracks with the ID in the cdr of that list."
   (find-file (expand-file-name filename emacsconf-cache-dir)))
 
 (defun emacsconf-format-seconds (seconds)
-	(concat (format-seconds "%.2m:%.2s" (floor seconds))
+	(concat (format-seconds "%.2h:%z%.2m:%.2s" (floor seconds))
 					"." (format "%03d" (% (floor (* 1000 seconds)) 1000))))
 
 (defun emacsconf-insert-time-for-speaker (talk)
@@ -1737,6 +1756,7 @@ tracks with the ID in the cdr of that list."
 	(mapcar (lambda (o) (plist-get o prop)) list))
 
 (defun emacsconf-talk-file (talk suffix &optional always source)
+	(setq talk (emacsconf-resolve-talk talk))
 	(let ((wiki-filename
 				 (expand-file-name (concat (plist-get talk :file-prefix) suffix)
 													 (expand-file-name "captions"
@@ -1746,9 +1766,9 @@ tracks with the ID in the cdr of that list."
 				 (expand-file-name (concat (plist-get talk :file-prefix) suffix)
 													 emacsconf-cache-dir)))
 		(cond
+		 (always cache-filename)
 		 ((and (file-exists-p wiki-filename) (not (eq source 'cache))) wiki-filename)
-		 ((and (file-exists-p cache-filename) (not (eq source 'wiki-captions))) cache-filename)
-		 (always cache-filename))))
+		 ((and (file-exists-p cache-filename) (not (eq source 'wiki-captions))) cache-filename))))
 
 (with-eval-after-load 'org
 	(defun emacsconf-el-complete ()