From 38a7562df8daadfee638ec34a1a0adc8f56899ce Mon Sep 17 00:00:00 2001 From: Sacha Chua Date: Tue, 20 Dec 2022 13:07:29 -0500 Subject: extraction --- emacsconf-extract.el | 289 +++++++++++++++++++----- emacsconf-publish.el | 609 ++++++++++++++++++++++++++++++++------------------- emacsconf-subed.el | 2 +- emacsconf.el | 28 ++- 4 files changed, 635 insertions(+), 293 deletions(-) diff --git a/emacsconf-extract.el b/emacsconf-extract.el index be28033..6f5f6b1 100644 --- a/emacsconf-extract.el +++ b/emacsconf-extract.el @@ -22,7 +22,7 @@ ;;; Code: -(defun emacsconf-extract-extract-chat (filename) +(defun emacsconf-extract-chat (filename) (when (file-exists-p filename) (message "%s" filename) (mapcar @@ -416,22 +416,55 @@ ;; Some speech-to-text systems can do speaker diarization, which also tries to identify speakers ;; huh, is the StartRecordingEvent timestamp reliable? Am I misreading it? -(defvar emacsconf-extract-irc-speaker-nick nil) -(defun emacsconf-extract-irc-copy-line-to-other-window-as-list-item () +(defvar emacsconf-extract-irc-speaker-nick nil "*Nick for the speaker.") + +(defun emacsconf-extract-selected-irc () + "Copy the lines that start with -." + (interactive) + (let ((results "")) + (save-excursion + (goto-char (point-min)) + (while (re-search-forward "^\\( *- \\([QA]: \\)?\\)\\[[0-9:]+\\] <.*?> \\(.*\n\\)" nil t) + (setq results (concat results (match-string 1) (match-string 3))) + (replace-match "" nil t nil 1)) + (kill-new results)))) + +(defun emacsconf-extract-irc-backward-by-nick () + (interactive) + (goto-char (line-beginning-position)) + (when (looking-at "\\[[0-9:]+\\] <\\(.*?\\)> \\([^ :]+?\\)?[ :]\\(.+\\)$") + (save-excursion + (let ((nick (match-string 2))) + (while (and (re-search-backward (concat "\\[[0-9:]+\\] <" (regexp-quote nick) ">") nil t) + (y-or-n-p "Continue? ")) + ;; keep going backwards + ))))) +(defun emacsconf-extract-irc-copy-line-to-other-window-as-list-item (&optional prefix indent) (interactive) (goto-char (line-beginning-position)) (when (looking-at "\\[[0-9:]+\\] <\\(.*?\\)> \\([^ ]+?:\\)?\\(.+\\)$") - (let ((line (string-trim (match-string 3)))) + (let ((line (string-trim (match-string 3))) + (prefix (or + prefix + (and (string= (or emacsconf-extract-irc-speaker-nick "") + (match-string 1)) + "A: ") + ""))) (setq line - (if (string= (or emacsconf-extract-irc-speaker-nick "") - (match-string 1)) - (concat " - A: " line "\n") - (concat "- " line "\n"))) + (concat + (if (or (string= prefix "A: ") indent) " " "") + "- " + prefix + line "\n")) (other-window 1) (insert line) (other-window 1) (forward-line 1)))) +(defun emacsconf-extract-irc-copy-line-to-other-window-as-question () + (interactive) + (emacsconf-extract-irc-copy-line-to-other-window-as-list-item "Q: ")) + (defvar emacsconf-extract-irc-map (make-sparse-keymap)) (defalias 'emacsconf-extract-irc-other-window #'other-window) (defalias 'emacsconf-extract-irc-next-line #'next-line) @@ -441,61 +474,201 @@ (other-window 1) (emacsconf-edit-wiki-page talk)) -(define-key emacsconf-extract-irc-map "c" #'emacsconf-extract-irc-copy-line-to-other-window-as-list-item) -(define-key emacsconf-extract-irc-map "o" #'emacsconf-extract-irc-other-window) -(define-key emacsconf-extract-irc-map "t" #'emacsconf-extract-irc-open-talk-in-other-window) -(define-key emacsconf-extract-irc-map "n" #'emacsconf-extract-irc-next-line) -(define-key emacsconf-extract-irc-map "p" #'emacsconf-extract-irc-previous-line) -(mapc (lambda (sym) - (put sym 'repeat-map 'emacsconf-extract-irc-map)) - '(emacsconf-extract-irc-copy-line-to-other-window-as-list-item - emacsconf-extract-irc-other-window - emacsconf-extract-irc-next-line - emacsconf-extract-irc-previous-line)) - -;; (local-set-key (kbd "C-c C-c") emacsconf-extract-irc-map) - -(defun emacsconf-extract-publish-qa (talk &optional note) - (interactive (list (emacsconf-complete-talk-info))) - (setq talk (emacsconf-resolve-talk talk)) - (let ((large-file-warning-threshold nil)) - ;; Copy the files - (unless (file-exists-p (expand-file-name (concat (plist-get talk :video-slug) "--answers.webm" emacsconf-cache-dir))) - (if (file-exists-p (expand-file-name (concat (plist-get talk :video-slug) "--bbb-deskshare.webm") emacsconf-cache-dir)) - ;; use the screenshare if available - (call-process "ffmpeg" nil (get-buffer-create "*ffmpeg*") t - "-y" - "-i" (expand-file-name (concat (plist-get talk :video-slug) "--bbb-deskshare.webm") emacsconf-cache-dir) - "-i" (expand-file-name (concat (plist-get talk :video-slug) "--bbb-webcams.opus") emacsconf-cache-dir) - "-c" "copy" - (expand-file-name (concat (plist-get talk :video-slug) "--answers.webm") emacsconf-cache-dir)) - (copy-file - (expand-file-name (concat (plist-get talk :video-slug) "--bbb-webcams.webm") emacsconf-cache-dir) - (expand-file-name (concat (plist-get talk :video-slug) "--answers.webm") emacsconf-cache-dir) - t))) - (unless (file-exists-p (expand-file-name (concat (plist-get talk :video-slug) "--answers.opus") emacsconf-cache-dir)) +(require 'hydra) +(defhydra emacsconf-extract-irc () + "Make it easy to extract lines from IRC" + ("c" emacsconf-extract-irc-copy-line-to-other-window-as-list-item "copy") + ("q" (emacsconf-extract-irc-copy-line-to-other-window-as-list-item "Q: ") "question") + ("o" other-window "other") + ("t" emacsconf-extract-irc-open-talk-in-other-window "talk") + ("n" next-line "next") + ("p" previous-line "previous") + ("N" move-line-down "move down") + ("P" move-line-up "move up") + ("" (progn (goto-char (line-beginning-position)) (insert " ")) "indent") + ("" (progn (goto-char (line-beginning-position)) (delete-char 2)) "dedent") + ("" scroll-down-command) + ("" scroll-up-command) + ("a" (emacsconf-extract-irc-copy-line-to-other-window-as-list-item "A: ") "answer") + ("l" (save-window-excursion (other-window 1) (consult-line)) "check line") + ("r" (when (string-match )) (re-search-backward nil t)) + (" " pop-to-mark-command) + ) + +(defun emacsconf-extract-irc-anonymize-log (beg end speakers) + (interactive "r\nMNick(s): ") + (when (stringp speakers) (setq speakers (split-string speakers))) + (let ((text (buffer-substring beg end)) + nicks) + (with-temp-buffer + (insert text) + (goto-char (point-min)) + ;; make a list of nicks + (while (re-search-forward "^\\[[0-9:]+\\] <\\(.*?\\)>" nil t) + (unless (member (match-string 1) speakers) + (add-to-list 'nicks (match-string 1)))) + (goto-char (point-min)) + (while (re-search-forward "^\\[[0-9:]+\\] <\\(.*?\\)> \\(.+\\)" nil t) + (replace-match + (if (member (match-string 1) speakers) + (concat " - A: " (match-string 2)) + (format "- {{%d}} %s" + (seq-position nicks (match-string 1)) + (propertize (match-string 2) + 'nick (match-string 1)))))) + (goto-char (point-min)) + (perform-replace (regexp-opt nicks) (lambda ())) + (setq text (buffer-string)) + (other-window 1) + (insert text)))) + +(defun emacsconf-private-qa (&optional info) + (seq-remove (lambda (o) + (or (null (emacsconf-talk-file o "--bbb-webcams.webm")) + (plist-get o :qa-public))) + (or info (emacsconf-get-talk-info)))) +;; sqlite detached localizing +(defun emacsconf-extract-review-qa (talk) + (interactive (list (emacsconf-complete-talk-info (emacsconf-private-qa)))) + (find-file (emacsconf-talk-file talk "--bbb-webcams.vtt"))) + +(defun emacsconf-extract-publish-qa (talk &optional time) + (interactive (list (emacsconf-complete-talk-info (unless current-prefix-arg (emacsconf-private-qa))) + (when current-prefix-arg + (read-string "Time: ")))) + (when (stringp talk) (setq talk (emacsconf-resolve-talk talk))) + (let ((buff (get-buffer-create "*ffmpeg*")) + (large-file-warning-threshold nil)) + (cond + ((emacsconf-talk-file talk "--bbb-deskshare.webm") + (apply 'call-process "ffmpeg" nil buff nil + (append + (list + "-y" + "-i" + (expand-file-name + (concat + (plist-get talk :video-slug) + "--bbb-deskshare.webm") + emacsconf-cache-dir)) + (when time (list "-to" time)) + (list + "-i" + (expand-file-name + (concat + (plist-get talk :video-slug) + "--bbb-webcams.opus") + emacsconf-cache-dir)) + (when time (list "-to" time)) + (list + "-c" + "copy" + (expand-file-name + (concat + (plist-get talk :video-slug) + "--answers.webm") + emacsconf-cache-dir))))) + (time + (apply 'call-process "ffmpeg" nil buff nil + (append + (list + "-y" + "-i" + (expand-file-name + (concat + (plist-get talk :video-slug) + "--bbb-webcams.webm") + emacsconf-cache-dir)) + (when time (list "-to" time)) + (list + "-c" + "copy" + (expand-file-name + (concat + (plist-get talk :video-slug) + "--answers.webm") + emacsconf-cache-dir))))) + (t + (copy-file + (expand-file-name + (concat + (plist-get talk :video-slug) + "--bbb-webcams.webm") + emacsconf-cache-dir) + (expand-file-name + (concat + (plist-get talk :video-slug) + "--answers.webm") + emacsconf-cache-dir) + t))) + (call-process "ffmpeg" nil buff nil "-y" "-i" + (emacsconf-talk-file talk "--answers.webm") + "-c" "copy" + (emacsconf-talk-file talk "--answers.opus" t)) + (dolist (suffix '("opus" "webm")) (copy-file - (expand-file-name (concat (plist-get talk :video-slug) "--bbb-webcams.opus") emacsconf-cache-dir) - (expand-file-name (concat (plist-get talk :video-slug) "--answers.opus") emacsconf-cache-dir) + (expand-file-name + (concat + (plist-get talk :video-slug) + "--answers." suffix) + emacsconf-cache-dir) + (expand-file-name + (concat + (plist-get talk :video-slug) + "--answers." suffix) + emacsconf-backstage-dir) + t) + (copy-file + (expand-file-name + (concat + (plist-get talk :video-slug) + "--answers." suffix) + emacsconf-backstage-dir) + (expand-file-name + (concat + (plist-get talk :video-slug) + "--answers." suffix) + emacsconf-public-media-directory) t)) - (dolist (suffix '("--answers.webm" "--answers.opus")) - (unless (file-exists-p (expand-file-name (concat (plist-get talk :video-slug) suffix) emacsconf-backstage-dir)) - (copy-file - (expand-file-name (concat (plist-get talk :video-slug) suffix) emacsconf-cache-dir) - (expand-file-name (concat (plist-get talk :video-slug) suffix) emacsconf-backstage-dir) - t)) - (unless (file-exists-p (expand-file-name (concat (plist-get talk :video-slug) suffix) emacsconf-public-media-directory)) - (copy-file - (expand-file-name (concat (plist-get talk :video-slug) suffix) emacsconf-backstage-dir) - (expand-file-name (concat (plist-get talk :video-slug) suffix) emacsconf-public-media-directory) - t))) - ;; Update the org entry (save-window-excursion (emacsconf-go-to-talk talk) - (org-entry-put (point) "QA_PUBLIC" "t") - (org-entry-put (point) "QA_NOTE" (or note (concat "Q&A posted publicly." (emacsconf-surround " " (plist-get talk :qa-note) "" ""))))))) -;; (kill-new (mapconcat #'emacsconf-extract-bbb-events-xml (emacsconf-get-talk-info) "")) + (org-entry-put + (point) + "QA_PUBLIC" "t") + (unless (string-match "Q&A posted publicly." (or (org-entry-get (point) "QA_NOTE") "")) + (org-entry-put + (point) + "QA_NOTE" + (concat "Q&A posted publicly." + (emacsconf-surround " " + (org-entry-get (point) "QA_NOTE") + "" ""))))))) +;; (emacsconf-extract-publish-qa "workflows" "13:56.000") (emacsconf-extract-publish-qa "journalism") (emacsconf-extract-publish-qa "handwritten" "28:36.240") + ;; (kill-new (mapconcat #'emacsconf-extract-bbb-events-xml (emacsconf-get-talk-info) "")) ;; (dolist (slug '("haskell" "hyperorg" "health" "jupyter" "workflows" "wayland" "mail" "meetups" "orgsuperlinks" "rde" "science")) ;; (emacsconf-extract-publish-qa slug)) + +(defun emacsconf-extract-add-help-index-qa (talk) + (interactive (list (emacsconf-complete-talk-info))) + (if (stringp talk) (setq talk (emacsconf-resolve-talk talk))) + (when (and (emacsconf-talk-file talk "--answers.vtt") + (not (emacsconf-talk-file talk "--answers--chapters.vtt"))) + (with-current-buffer (find-file-noselect (expand-file-name (concat (plist-get talk :slug) ".md") (expand-file-name "talks" (expand-file-name emacsconf-year emacsconf-directory)))) + (goto-char (point-min)) + (unless (re-search-forward "help_with_chapter_markers" nil t) + (when (re-search-forward (concat (plist-get talk :slug) "-before)") nil t) + (forward-line 1) + (insert (format "[[!template id=\"help\" +volunteer=\"\" +summary=\"Q&A could be indexed with chapter markers\" +tags=\"help_with_chapter_markers\" +message=\"\"\"The Q&A session for this talk does not have chapter markers yet. +Would you like to help? See [[help_with_chapter_markers]] for more details. You can use the vidid=\"%s-qanda\" if adding the markers to this wiki page, or e-mail your chapter notes to .\"\"\"]] + +" (plist-get talk :slug))) + (save-buffer)))))) + +;; (mapc #'emacsconf-extract-add-help-index-qa (emacsconf-prepare-for-display (emacsconf-get-talk-info))) + (provide 'emacsconf-extract) ;;; emacsconf-extract.el ends here diff --git a/emacsconf-publish.el b/emacsconf-publish.el index 33d9b90..412696d 100644 --- a/emacsconf-publish.el +++ b/emacsconf-publish.el @@ -30,7 +30,7 @@ :type 'string :group 'emacsconf) -(defcustom emacsconf-main-extensions '(".webm" "--main.webm" "--main.org" ".org" ".odp" ".pdf" ".el" "--compressed56.webm" "--main.vtt" "--main_fr.vtt" "--main_ja.vtt" "--main_es.vtt" "--chapters.vtt" "--main--chapters.vtt" "--script.fountain" "--main.pdf" "--slides.pdf") +(defcustom emacsconf-main-extensions '("--main.webm" "--main.opus" "--main.org" ".org" ".odp" ".pdf" ".el" "--compressed56.webm" "--main.vtt" "--main_fr.vtt" "--main_ja.vtt" "--main_es.vtt" "--main--chapters.vtt" "--script.fountain" "--main.pdf" "--slides.pdf") "Extensions to list on public pages." :type '(repeat string) :group 'emacsconf) @@ -137,11 +137,7 @@ (defun emacsconf-index-card (talk &optional extensions) "Format an HTML card for TALK, linking the files in EXTENSIONS." (let* ((video-slug (plist-get talk :video-slug)) - (video-file - (or (plist-get talk :video-file) - (and video-slug - (emacsconf-get-preferred-video (plist-get talk :video-slug) - (plist-get talk :files))))) + (video-file (plist-get talk :video-file)) (video (and video-slug (emacsconf-index-card-video (or (plist-get talk :video-id) @@ -153,15 +149,14 @@ talk (list :video-html (or (plist-get video :video) "") + :audio-html (or (plist-get video :audio) "") :chapter-list (or (plist-get video :chapter-list) "") :resources (or (plist-get video :resources) "") :extra (or (plist-get talk :extra) "") :speaker-info (or (plist-get talk :speakers-with-pronouns) "")))) - (if (eq (plist-get talk :format) 'wiki) - (plist-get talk :video-html) - (emacsconf-replace-plist-in-string - talk - "
${video-html}
${extra}
${resources}${chapter-list}
")))) + (emacsconf-replace-plist-in-string + talk + "
${video-html}${audio-html}
${extra}
${resources}${chapter-list}
"))) ;; (emacsconf-publish-format-track-as-org (car emacsconf-tracks) "US/Eastern") ;; (emacsconf-get-talk-info) @@ -260,7 +255,7 @@ info)) (result (concat - " + "
" (with-temp-buffer (svg-print (emacsconf-schedule-svg 800 300 info)) @@ -312,8 +307,10 @@ (plist-get talk :captions-edited) (let ((tracks (emacsconf-video-subtitle-tracks - (concat (replace-regexp-in-string "reencoded\\|original" "main" video-base) - ".vtt") + (or (plist-get talk :caption-file) + (concat (replace-regexp-in-string "reencoded\\|original" "main" + video-base) + ".vtt")) (or (plist-get talk :track-base-url) (plist-get talk :base-url)) (plist-get talk :files)))) @@ -336,16 +333,12 @@ :other-files (mapconcat (lambda (s) - (if (eq (plist-get talk :format) 'wiki) - (concat s " \n") - (concat "
  • " s "
    • "))) + (concat "
  • " s "
    • ")) (emacsconf-link-file-formats-as-list talk (or extensions emacsconf-main-extensions)) "") :toobnix-info (if (plist-get talk :toobnix-url) (format - (if (eq (plist-get talk :format) 'wiki) - "[View on Toobnix](%s) \n" - "
  • View on Toobnix
    • ") + "
  • View on Toobnix
    • " (plist-get talk :toobnix-url)) "") :transcript-link @@ -359,25 +352,22 @@ :video (emacsconf-replace-plist-in-string info - (if (stringp video-file) - (if (eq (plist-get talk :format) 'wiki) - "[[!template id=\"vid\" vidid=\"${video-id}\" src=\"${source-src}\" poster=\"${poster}\" ${captions} -size=\"${video-file-size}\" duration=\"${video-duration}\" other_resources=\"\"\"${other-files}${transcript-link}${toobnix-info}\"\"\"]] -${chapter-list} -" - "${chapter-list}") - (or (plist-get talk :video-note) - "The video for \"${title}\" will be posted here when available. You can also subscribe to the emacsconf-discuss mailing list for updates."))) + (if (stringp video-file) + "${chapter-list}" + (or (plist-get talk :video-note) ""))) + :audio + (if (and (plist-get talk :audio-file) (plist-get talk :public)) + (format "
    Listen to just the audio:
    " + video-id + (if (plist-get talk :public) + (format "%s%s/%s" emacsconf-media-base-url (plist-get talk :conf-year) + (file-name-nondirectory (plist-get talk :audio-file))) + (file-name-nondirectory (plist-get talk :audio-file)))) + "") :resources (emacsconf-replace-plist-in-string - (append info - (list :video-download - (if (stringp video-file) - (emacsconf-replace-plist-in-string - info - "
  • Download video (${video-duration}, ${video-file-size}B)
    • ") - ""))) - "
      ${video-download}${other-files}${toobnix-info}
    ")))) + info + "
      ${other-files}${toobnix-info}
    ")))) (defun emacsconf-format-public-email (o &optional email) (format "[%s](mailto:%s?subject=%s)" @@ -469,11 +459,10 @@ resources." :video-id (concat (plist-get o :slug) "-qanda") :toobnix-url nil :captions-edited (plist-get o :qa-captions-edited) - :video-file (expand-file-name - (concat (file-name-sans-extension (plist-get o :video-slug)) "--answers.webm") - emacsconf-cache-dir)) + :video-file (emacsconf-talk-file o "--answers.webm") + :audio-file (emacsconf-talk-file o "--answers.opus")) o) - (list "--answers.vtt" "--answers--chapters.vtt" "--answers--compressed32.webm"))) + (list "--answers.webm" "--answers.vtt" "--answers--chapters.vtt" "--answers.opus"))) ""))) (defun emacsconf-publish-webchat-link (o) @@ -536,9 +525,7 @@ ${pad-info}${irc-info}${status-info}${schedule-info}\n" (if (plist-get o :alternate-apac) (format "[[!inline pages=\"internal(%s/inline-alternate)\" raw=\"yes\"]] \n" emacsconf-year) "") - "\n" - (if (plist-get o :public) (emacsconf-wiki-talk-resources o) "") - "\n# Description\n")))) + "\n")))) (defun emacsconf-format-email-questions-and-comments (talk) (format "Questions or comments? Please e-mail %s" @@ -568,6 +555,41 @@ ${pad-info}${irc-info}${status-info}${schedule-info}\n" (shell-command (concat "git add " (shell-quote-argument filename)))))) (seq-filter (lambda (o) (string-match "vtt$" o)) emacsconf-main-extensions)))) +(defun emacsconf-publish-format-talk-page-schedule (talk info) + (concat + "\nThe following image shows where the talk is in the schedule for " + (format-time-string "%a %Y-%m-%d" (plist-get talk :start-time) emacsconf-timezone) ". Solid lines show talks with Q&A via BigBlueButton. Dashed lines show talks with Q&A via IRC or Etherpad." + (format "
    \n" (plist-get talk :slug)) + (let* ((width 800) (height 150) + (talk-date (format-time-string "%Y-%m-%d" (plist-get talk :start-time) emacsconf-timezone)) + (start (date-to-time (concat talk-date "T" emacsconf-schedule-start-time emacsconf-timezone-offset))) + (end (date-to-time (concat talk-date "T" emacsconf-schedule-end-time emacsconf-timezone-offset))) + svg) + (with-temp-buffer + (setq svg (emacsconf-schedule-svg-day + (svg-create width height) + (format-time-string "%A" (plist-get talk :start-time) emacsconf-timezone) + width height + start end + (emacsconf-by-track + (seq-filter (lambda (o) (string= (format-time-string "%Y-%m-%d" (plist-get o :start-time) emacsconf-timezone) + talk-date)) + info)))) + (mapc (lambda (node) + (let ((rect (car (dom-by-tag node 'rect)))) + (if (string= (dom-attr node 'data-slug) (plist-get talk :slug)) + (progn + (dom-set-attribute rect 'opacity "0.8") + (dom-set-attribute rect 'stroke-width "3") + (dom-set-attribute (car (dom-by-tag node 'text)) 'font-weight "bold")) + (dom-set-attribute rect 'opacity "0.5")))) + (dom-by-tag svg 'a)) + (svg-print svg) + (buffer-string))) + "\n
    \n" + "\n" + (emacsconf-format-talk-schedule-info talk) "\n\n")) + (defun emacsconf-publish-before-page (talk &optional info) "Info included before the abstract." (interactive (list (emacsconf-complete-talk-info))) @@ -579,43 +601,15 @@ ${pad-info}${irc-info}${status-info}${schedule-info}\n" (insert (emacsconf-surround "" (plist-get talk :intro-note) "\n\n" "")) (let ((is-live (emacsconf-talk-live-p talk))) (when is-live (emacsconf-publish-captions-in-wiki talk)) - (when (eq emacsconf-publishing-phase 'schedule) - (insert "\nThe following image shows where the talk is in the schedule for " - (format-time-string "%a %Y-%m-%d" (plist-get talk :start-time) emacsconf-timezone) ". Solid lines show talks with Q&A via BigBlueButton. Dashed lines show talks with Q&A via IRC or Etherpad." - (format "
    \n" (plist-get talk :slug)) - (let* ((width 800) (height 150) - (talk-date (format-time-string "%Y-%m-%d" (plist-get talk :start-time) emacsconf-timezone)) - (start (date-to-time (concat talk-date "T" emacsconf-schedule-start-time emacsconf-timezone-offset))) - (end (date-to-time (concat talk-date "T" emacsconf-schedule-end-time emacsconf-timezone-offset))) - svg) - (with-temp-buffer - (setq svg (emacsconf-schedule-svg-day - (svg-create width height) - (format-time-string "%A" (plist-get talk :start-time) emacsconf-timezone) - width height - start end - (emacsconf-by-track - (seq-filter (lambda (o) (string= (format-time-string "%Y-%m-%d" (plist-get o :start-time) emacsconf-timezone) - talk-date)) - info)))) - (mapc (lambda (node) - (let ((rect (car (dom-by-tag node 'rect)))) - (if (string= (dom-attr node 'data-slug) (plist-get talk :slug)) - (progn - (dom-set-attribute rect 'opacity "0.8") - (dom-set-attribute rect 'stroke-width "3") - (dom-set-attribute (car (dom-by-tag node 'text)) 'font-weight "bold")) - (dom-set-attribute rect 'opacity "0.5")))) - (dom-by-tag svg 'a)) - (svg-print svg) - (buffer-string))) - "\n
    \n" - "\n" - (emacsconf-format-talk-schedule-info talk) "\n\n" - ))) + (when (eq emacsconf-publishing-phase 'schedule) + (insert (emacsconf-publish-format-talk-page-schedule talk info))) + (insert + (if (plist-get talk :public) (emacsconf-wiki-talk-resources talk) "") + "\n# Description\n")) ;; Contact information ;; (insert "\n\n" (emacsconf-format-email-questions-and-comments talk) "\n") (insert ""))) +;; (emacsconf-publish-before-page (emacsconf-resolve-talk "rms")) (defun emacsconf-format-transcript-from-list (subtitles paragraphs video-id &optional lang) "Return subtitle directives for SUBTITLES split by PARAGRAPHS." @@ -682,18 +676,20 @@ ${pad-info}${irc-info}${status-info}${schedule-info}\n" (insert "\n" "\n\n" + ;; main transcript (if (plist-get talk :public) - (let ((transcripts (mapconcat - (lambda (lang) - (let ((filename (expand-file-name - (concat (plist-get talk :video-slug) - (emacsconf-surround "--main_" lang ".vtt" "--main.vtt")) - emacsconf-cache-dir))) - (if (emacsconf-captions-edited-p filename) ; todo: cache this somewhere - (emacsconf-publish-format-transcript talk "mainVideo" lang) - ""))) - (cons nil (mapcar 'car emacsconf-publish-subtitle-languages)) - ""))) + (let ((transcripts + (mapconcat + (lambda (lang) + (let ((filename (expand-file-name + (concat (plist-get talk :video-slug) + (emacsconf-surround "--main_" lang ".vtt" "--main.vtt")) + emacsconf-cache-dir))) + (if (emacsconf-captions-edited-p filename) ; todo: cache this somewhere + (emacsconf-publish-format-transcript talk "mainVideo" lang) + ""))) + (cons nil (mapcar 'car emacsconf-publish-subtitle-languages)) + ""))) (if (> (length transcripts) 0) (concat transcripts (emacsconf-surround @@ -966,15 +962,42 @@ Entries are sorted chronologically, with different tracks interleaved." received))))) (defun emacsconf-publish-sched-directive (o) - (format "[[!template id=sched%s%s]]" + (format "[[!template id=sched%s]]" (let ((result "") (attrs (append - (list + (pcase emacsconf-publishing-phase + ('program nil) + ('schedule (list + :status (pcase (plist-get o :status) + ("CAPTIONED" "captioned") + ("PREREC_RECEIVED" "received") + ("DONE" "done") + ("STARTED" "now playing") + (_ nil)) + :time (plist-get o :time) + :q-and-a (plist-get o :qa-link) + :pad (plist-get o :pad-url) + :startutc (format-time-string "%FT%T%z" (plist-get o :start-time) t) + :endutc (format-time-string "%FT%T%z" (plist-get o :end-time) t) + :start (format-time-string "%-l:%M" (plist-get o :start-time) emacsconf-timezone) + :end (format-time-string "%-l:%M" (plist-get o :end-time) emacsconf-timezone))) + ('resources + (list + :pad nil + :channel nil + :resources (mapconcat (lambda (s) (concat "
  • " s "
    • ")) + (emacsconf-link-file-formats-as-list + (append o + (list :base-url (format "%s%s/" emacsconf-media-base-url emacsconf-year))) + (append emacsconf-main-extensions (list "--answers.webm" "--answers.opus" "--answers.vtt"))) + "")))) + (list :title (plist-get o :title) :url (concat "/" (plist-get o :url)) :speakers (plist-get o :speakers) - :q-and-a (plist-get o :qa-link) + :track (plist-get o :track) :watch (plist-get o :watch-url) + :slug (plist-get o :slug) :note (string-join (delq nil @@ -986,37 +1009,15 @@ Entries are sorted chronologically, with different tracks interleaved." (plist-get o :video-file))) "video posted"))) ", ") - :pad (plist-get o :pad-url)) - (unless (eq emacsconf-publishing-phase 'program) - (list - :track (plist-get o :track) - :slug (plist-get o :slug) - :status (pcase (plist-get o :status) - ("CAPTIONED" "captioned") - ("PREREC_RECEIVED" "received") - ("DONE" "done") - ("STARTED" "now playing") - (_ nil)) - :time (plist-get o :time) - :startutc (format-time-string "%FT%T%z" (plist-get o :start-time) t) - :endutc (format-time-string "%FT%T%z" (plist-get o :end-time) t) - :start (format-time-string "%-l:%M" (plist-get o :start-time) emacsconf-timezone) - :end (format-time-string "%-l:%M" (plist-get o :end-time) emacsconf-timezone)))))) + ) + ))) (while attrs (let ((field (pop attrs)) (val (pop attrs))) (when val (setq result (concat result " " (substring (symbol-name field) 1) "=\"\"\"" val "\"\"\""))))) result) - (if (eq emacsconf-publishing-phase 'resources) - (format" resources=\"\"\"\n%s\n\"\"\"" - (mapconcat (lambda (s) (concat "
  • " s "
    • ")) - (emacsconf-link-file-formats-as-list - (append o - (list :base-url (format "%s%s/" emacsconf-media-base-url emacsconf-year))) - (append emacsconf-main-extensions '("--main.webm"))) - "")) - ""))) + )) (defun emacsconf-format-main-schedule (info) (mapconcat #'emacsconf-publish-sched-directive (emacsconf-active-talks info) "\n")) @@ -1035,7 +1036,7 @@ Entries are sorted chronologically, with different tracks interleaved." (emacsconf-with-talk-heading talk (when (and (member org-state '("PROCESSING" "TO_ASSIGN")) (not (plist-get talk :video-time))) - (emacsconf-cache-video-data talk)) + (emacsconf-publish-cache-video-data talk)) (when (member org-state '("TO_CAPTION")) (unless (or noninteractive (org-entry-get (point) "CAPTIONER")) (org-entry-put (point) "CAPTIONER" @@ -1197,19 +1198,10 @@ Entries are sorted chronologically, with different tracks interleaved." '(("TO_REVIEW_QA" . "Please review the --bbb-webcams.webm file and/or the --bbb-webcams.vtt and tell us (emacsconf-submit@gnu.org) if a Q&A session can be published or if it needs to be trimmed (lots of silence at the end of the recording, accidentally included sensitive information, etc.).") ("TO_INDEX_QA" . - "Please review the --bbb-webcams.webm file, the --bbb-webcams.vtt file, and possibly the --bbb-deskshare.webm file to make chapter markers so that people can jump to specific parts of the Q&A session. The harvesting page on the wiki has some notes on the process. That way, it's easier for people to see the questions and -answers without needing to listen to everything again, and the -speakers can see any questions they've missed and all the -wonderful feedback people have shared. You can see the talk pages -at https://emacsconf.org/2021/talks/ -for examples of a Discussion section with sections for Q&A and -feedback (ex: frownies, -freedom).") + "Please review the --answers.webm and --answers.vtt files to make chapter markers so that people can jump to specific parts of the Q&A session. The harvesting page on the wiki has some notes on the process. That way, it's easier for people to see the questions and +answers without needing to listen to everything again. You can see asmblox for an example of the Q&A chapter markers.") ("TO_CAPTION_QA" . - "Please edit the --bbb-webcams.vtt for the Q&A talk you're interested in, correcting misrecognized words and cleaning it up so that it's nice to use as closed captions. See the QA note to find out which ones have large-model captions that might be more pleasant to edit.")))) + "Please edit the --answers.vtt for the Q&A talk you're interested in, correcting misrecognized words and cleaning it up so that it's nice to use as closed captions. All talks should now have large-model VTTs to make it easier to edit.")))) (mapconcat (lambda (stage) (let ((status (car stage))) @@ -1283,60 +1275,69 @@ href=\"https://emacsconf.org/2021/talks/freedom/\">freedom).") (string-match (concat (regexp-quote (plist-get talk :video-slug)) (if selector (concat "--" selector)) ".*" - (regexp-opt extensions)) + (regexp-opt extensions) + "$") f)) files))) +(defun emacsconf-publish-public-index-for-talk (o files) + (format "
  • %s
    • %s
    %s%s" + (plist-get o :slug) + (plist-get o :absolute-url) + (plist-get o :title) + (plist-get o :speakers) + (emacsconf-index-card + (append (list :files + (emacsconf-publish-filter-files o files emacsconf-main-extensions) + :audio-file + (emacsconf-talk-file o "--main.opus")) + o) + '(".org" ".pdf" "--main.vtt" "--compressed56.webm")) + (if (or (emacsconf-talk-file o "--answers.webm") + (emacsconf-talk-file o "--answers.opus")) + (format "
  • Q&A for %s
    %s
    • " + (plist-get o :absolute-url) + (plist-get o :title) + (emacsconf-index-card + (append + (list + :public 1 + :video-id (concat "qanda-" (plist-get o :slug)) + :toobnix-url nil + :video-duration (plist-get o :qa-video-duration) + :video-file-size (plist-get o :qa-video-file-size) + :video-file (plist-get o :qa-video-file) + :audio-file (emacsconf-talk-file o "--answers.opus") + :files (emacsconf-publish-filter-files + o + files emacsconf-main-extensions + "answers")) + o) + (list "--answers.webm" "--answers.opus" "--answers.vtt" "--answers--chapters.vtt"))) + ""))) +;; (emacsconf-publish-public-index-for-talk (emacsconf-resolve-talk "rms") (directory-files emacsconf-cache-dir)) + (defun emacsconf-publish-public-index (&optional filename) (interactive (list (expand-file-name "index.html" emacsconf-public-media-directory))) (setq filename (or filename (expand-file-name "index.html" emacsconf-public-media-directory))) (let ((files (directory-files emacsconf-public-media-directory)) (info (emacsconf-public-talks (emacsconf-prepare-for-display (emacsconf-get-talk-info))))) (emacsconf-publish-playlist (expand-file-name "index.m3u" emacsconf-public-media-directory) - (concat emacsconf-name emacsconf-year) - info - (format "https://media.emacsconf.org/%s/" emacsconf-year)) + (concat emacsconf-name emacsconf-year) + info + (format "https://media.emacsconf.org/%s/" emacsconf-year)) (with-temp-file filename (insert - "" + "" "

    " emacsconf-name " " emacsconf-year "

    " "
    M3U playlist for playing in MPV and other players
    " + "
    Quick links: " (mapconcat (lambda (o) (format "%s" + (plist-get o :slug) + (plist-get o :slug))) + info ", ") + "
    " "
      " - (mapconcat (lambda (o) - (format "
    1. %s
      %s
      %s
      2. %s" - (plist-get o :absolute-url) - (plist-get o :title) - (plist-get o :speakers) - (emacsconf-index-card - (append (list :files - (emacsconf-publish-filter-files o files emacsconf-main-extensions)) - o) - '(".org" ".pdf" "--main.vtt" "--compressed56.webm")) - (if (member (concat (plist-get o :video-slug) - "--answers.webm") - files) - (format "
    2. Q&A for %s
      %s
      3. " - (plist-get o :title) - (emacsconf-index-card - (append - (list - :public 1 - :video-id (concat "qanda-" (plist-get o :slug)) - :toobnix-url nil - :video-file - (expand-file-name - (concat (file-name-sans-extension (plist-get o :video-slug)) - "--answers.webm") - emacsconf-public-media-directory) - :files (emacsconf-publish-filter-files - o - files emacsconf-main-extensions - "answers")) - o) - (list "--answers.webm" "--answers.vtt" "--answers--chapters.vtt"))) - ""))) - info - "\n") + (mapconcat (lambda (o) (emacsconf-publish-public-index-for-talk o files)) info "\n") "
    " (if (file-exists-p (expand-file-name "include-in-public-index.html" emacsconf-cache-dir)) (with-temp-buffer (insert-file-contents (expand-file-name "include-in-public-index.html" emacsconf-cache-dir)) (buffer-string)) @@ -1345,16 +1346,15 @@ href=\"https://emacsconf.org/2021/talks/freedom/\">freedom).") (defun emacsconf-publish-public-index-on-wiki () (interactive) - (let ((info (seq-filter (lambda (o) - (not (string= (plist-get o :status) "CANCELLED"))) - (emacsconf-filter-talks (emacsconf-get-talk-info)))) + (let ((info (emacsconf-prepare-for-display (emacsconf-filter-talks (emacsconf-get-talk-info)))) (files (directory-files emacsconf-public-media-directory))) - (with-temp-file (expand-file-name "all-include.md" (expand-file-name emacsconf-year emacsconf-directory)) + (with-temp-file (expand-file-name "all-include.md" (expand-file-name emacsconf-year emacsconf-directory)) (insert "
      " (mapconcat (lambda (f) - (format "
    1. %s
      %s
      %s%s
      2. " + (format "
    2. %s
      %s
      %s%s
      3. " + (plist-get f :slug) (plist-get f :absolute-url) (plist-get f :title) (or (plist-get f :speakers) "") @@ -1382,7 +1382,7 @@ href=\"https://emacsconf.org/2021/talks/freedom/\">freedom).") (concat (file-name-sans-extension (plist-get f :video-slug)) "--answers.webm") emacsconf-cache-dir)) f) - (list "--answers.vtt" "--answers--chapters.vtt")) + (list "--answers.vtt" "--answers--chapters.vtt" "--answers.webm")) ""))) info "\n")) "
    "))) @@ -1395,7 +1395,9 @@ href=\"https://emacsconf.org/2021/talks/freedom/\">freedom).") (concat (or track-base-url "") (file-name-nondirectory filename))) :md (mapconcat (lambda (chapter) (concat - (format-seconds "%.2h:%z%.2m:%.2s" (floor (/ (elt chapter 1) 1000))) + (if (= (elt chapter 1) 0) + "00:00" + (format-seconds "%.2h:%z%.2m:%.2s" (floor (/ (elt chapter 1) 1000)))) (format ".%03d" (mod (elt chapter 1) 1000)) " " (elt chapter 3) @@ -1406,8 +1408,10 @@ href=\"https://emacsconf.org/2021/talks/freedom/\">freedom).") (or target "") (mapconcat (lambda (chapter) - (format "%s.%03d %s" - (format-seconds "%.2h:%z%.2m:%.2s" (floor (/ (elt chapter 1) 1000))) + (format "%s.%03d %s" + (if (= (elt chapter 1) 0) + "00:00" + (format-seconds "%.2h:%z%.2m:%.2s" (floor (/ (elt chapter 1) 1000)))) (mod (elt chapter 1) 1000) (elt chapter 3))) chapters @@ -1439,17 +1443,13 @@ href=\"https://emacsconf.org/2021/talks/freedom/\">freedom).") (defun emacsconf-link-file-formats-as-list (talk extensions) (if (plist-get talk :files) - (seq-map (lambda (file) - (if (eq (plist-get talk :format) 'wiki) - (format "[Download %s](%s%s)" - (replace-regexp-in-string (concat "^" (regexp-quote (plist-get talk :video-slug))) "" file) - (or (plist-get talk :base-url) "") - file) - (format "Download %s" - (or (plist-get talk :base-url) "") - file - (replace-regexp-in-string (concat "^" (regexp-quote (plist-get talk :video-slug))) "" file)))) - (plist-get talk :files)) + (seq-map + (lambda (file) + (format "Download %s" + (or (plist-get talk :base-url) "") + file + (replace-regexp-in-string (concat "^" (regexp-quote (plist-get talk :video-slug))) "" file))) + (plist-get talk :files)) (let ((video-slug (plist-get talk :video-slug))) (delq nil (seq-map (lambda (ext) (let ((file (expand-file-name @@ -1461,17 +1461,11 @@ href=\"https://emacsconf.org/2021/talks/freedom/\">freedom).") (if (> (file-attribute-size (file-attributes file)) 1000000) (format " (%sB)" (file-size-human-readable (file-attribute-size (file-attributes file)))) "")) - (if (eq (plist-get talk :format) 'wiki) - (format "[Download %s%s](%s%s)" - ext - size - (or (plist-get talk :base-url) "") - (concat video-slug ext)) - (format "Download %s%s" - (or (plist-get talk :base-url) "") - (concat video-slug ext) - ext - size))))) + (format "Download %s%s" + (or (plist-get talk :base-url) "") + (concat video-slug ext) + ext + size)))) extensions))))) (defun emacsconf-talks-csv () @@ -1619,15 +1613,22 @@ href=\"https://emacsconf.org/2021/talks/freedom/\">freedom).") playlist-name playlist-name (mapconcat (lambda (talk) - (let* ((slug (plist-get talk :video-slug)) - (filename (concat (plist-get talk :video-slug) "--main.webm"))) - (if (and slug (file-exists-p (expand-file-name filename emacsconf-cache-dir))) - (format "#EXTINF:-1,%s - %s\n%s%s\n" - (plist-get talk :title) - (plist-get talk :speakers) - base-url - filename) - ""))) + (concat + (if (emacsconf-talk-file talk "--main.webm") + (format "#EXTINF:-1,%s - %s\n%s%s\n" + (plist-get talk :title) + (plist-get talk :speakers) + base-url + (file-name-nondirectory (emacsconf-talk-file talk "--main.webm"))) + "") + (if (emacsconf-talk-file talk "--answers.webm") + (format "#EXTINF:-1,Q&A for %s - %s\n%s%s\n" + (plist-get talk :title) + (plist-get talk :speakers) + base-url + (file-name-nondirectory (emacsconf-talk-file talk "--answers.webm"))) + "") + )) talks ""))))) @@ -1720,18 +1721,19 @@ This video is available under the terms of the Creative Commons Attribution-Shar (lambda (talk) (when (and (plist-get talk :video-slug) (or force (null (plist-get talk :video-file-size)))) - (emacsconf-cache-video-data talk))) + (emacsconf-publish-cache-video-data talk))) (emacsconf-get-talk-info))) ;; (emacsconf-cache-all-video-data t) (defvar emacsconf-cache-dir (expand-file-name "cache" (file-name-directory emacsconf-org-file))) -(defun emacsconf-cache-video-data (talk) +(defun emacsconf-publish-cache-video-data (talk) (interactive (list (emacsconf-complete-talk-info))) (let ((main (expand-file-name (concat (plist-get talk :video-slug) "--main.webm") emacsconf-cache-dir))) (emacsconf-with-talk-heading talk (let* ((video-file-name (emacsconf-get-preferred-video (plist-get talk :video-slug))) (video-file (and video-file-name (expand-file-name video-file-name emacsconf-cache-dir))) + (qa-file (emacsconf-talk-file talk "--answers.webm")) (intro-file (expand-file-name (concat (plist-get talk :slug) ".webm") (expand-file-name "intros" emacsconf-stream-asset-dir))) duration) @@ -1749,8 +1751,19 @@ This video is available under the terms of the Creative Commons Attribution-Shar (org-entry-put (point) "CAPTIONS_EDITED" "1")))) (unless (plist-get talk :video-duration) (setq duration (/ (compile-media-get-file-duration-ms video-file) 1000)) - (org-entry-put (point) "VIDEO_DURATION" (format-seconds "%m:%.2s" duration)) + (org-entry-put (point) "VIDEO_DURATION" (format-seconds "%h:%z%.2m:%.2s" duration)) (org-entry-put (point) "VIDEO_TIME" (number-to-string (ceiling (/ duration 60)))))) + (when qa-file + (org-entry-put (point) "QA_VIDEO_FILE" (file-name-nondirectory qa-file)) + (org-entry-put (point) "QA_VIDEO_FILE_SIZE" (file-size-human-readable (file-attribute-size (file-attributes qa-file)))) + (unless (plist-get talk :qa-captions-edited) + (let ((caption-file (emacsconf-talk-file talk "--answers.vtt"))) + (when (emacsconf-captions-edited-p caption-file) + (org-entry-put (point) "QA_CAPTIONS_EDITED" "1")))) + (unless (plist-get talk :qa-video-duration) + (setq duration (/ (compile-media-get-file-duration-ms qa-file) 1000)) + (org-entry-put (point) "QA_VIDEO_DURATION" (format-seconds "%h:%z%.2m:%.2s" duration)) + (org-entry-put (point) "QA_VIDEO_TIME" (number-to-string (ceiling (/ duration 60)))))) (when (file-exists-p intro-file) (org-entry-put (point) "INTRO_TIME" @@ -1868,7 +1881,7 @@ This video is available under the terms of the Creative Commons Attribution-Shar (emacsconf-replace-plist-in-string (append talk (list :start-info (emacsconf-surround "" (plist-get talk :start) "" "") - :end-info(emacsconf-surround " - " (plist-get talk :end) "" "") + :end-info (emacsconf-surround " - " (plist-get talk :end) "" "") :track-info (emacsconf-surround (format " " (or (plist-get talk :track) "")) (plist-get talk :track) "" "") :q-info (emacsconf-surround " Q&A: " (plist-get talk :qa-link) "; " "") :pad-info (emacsconf-surround "