From 9aecc040619a714c6f9af69680c46a2e8aad8e04 Mon Sep 17 00:00:00 2001 From: Sacha Chua Date: Tue, 7 Jan 2025 12:08:26 -0500 Subject: add youtube comment things --- emacsconf-extract.el | 325 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 244 insertions(+), 81 deletions(-) diff --git a/emacsconf-extract.el b/emacsconf-extract.el index efd2ade..5600494 100644 --- a/emacsconf-extract.el +++ b/emacsconf-extract.el @@ -247,6 +247,11 @@ (gethash "sentences" data))))) ;; (emacsconf-extract-qa-from-assemblyai-sentences "~/proj/emacsconf/rms/sentences") +(defun emacsconf-extract-unescape (s) + (replace-regexp-in-string + "\\\\\\(['\"]\\)" + "\\1" s)) + ;;;###autoload (defun emacsconf-extract-copy-pad-to-wiki () "Copy the notes and questions from the current file to the wiki page for this talk." @@ -268,12 +273,12 @@ nil (re-search-forward "-after)" nil t) (forward-line -1) - (insert "# Discussion\n\n")) - (save-excursion - (unless (string= (or questions "") "") - (insert "## Questions and answers\n\n" questions "\n\n")) - (unless (string= (or notes "") "") - (insert "## Notes\n\n" notes "\n\n"))))) + (insert "# Discussion\n\n") + (save-excursion + (unless (string= (or questions "") "") + (insert "## Questions and answers\n\n" (emacsconf-extract-unescape questions) "\n\n")) + (unless (string= (or notes "") "") + (insert "## Notes\n\n" (emacsconf-extract-unescape notes) "\n\n")))))) (defun emacsconf-extract-question-headings (slug) (with-temp-buffer @@ -308,7 +313,7 @@ "Question: " (emacsconf-extract-question-headings (emacsconf-get-slug-from-string (file-name-base (buffer-file-name))))))) - (insert "NOTE " question "\n\n")) + (subed-set-subtitle-comment (concat "Q: " question))) (defun emacsconf-extract-wget-bbb (o) (when (plist-get o :bbb-playback) @@ -355,7 +360,8 @@ (date-to-time (dom-text (dom-by-tag - (dom-elements dom 'eventname "StopRecordingEvent") + (or (dom-elements dom 'eventname "StopRecordingEvent") + (dom-elements dom 'eventname "EndAndKickAllEvent")) 'date)))) (setq start-ms (* 1000 (time-to-seconds start-recording)) stop-ms (* 1000 (time-to-seconds stop-recording))) @@ -428,11 +434,28 @@ (let ((results "")) (save-excursion (goto-char (point-min)) - (while (re-search-forward "^\\( *- \\([QA]: \\)?\\)\\[[0-9:]+\\] <.*?> \\(.*\n\\)" nil t) - (setq results (concat results (match-string 1) (match-string 3))) + (while (re-search-forward "^\\([qna] *\\| *- +\\([QA]: \\)?\\)\\[[0-9:]+\\] <.*?> \\(.*\n\\)" nil t) + (setq results (concat results + (save-match-data + (pcase (match-string 1) + ((rx "q") "- Q: ") + ((rx "a") "- A: ") + ((rx "n") "- ") + (_ "- "))) + (match-string 3))) (replace-match "" nil t nil 1)) (kill-new results)))) +(defvar-keymap emacsconf-extract-irc-log-map + "" #'forward-line + "" #'previous-line + "" (lambda () (interactive) (insert "-") (forward-line)) + "" #'forward-line) + +(defun emacsconf-extract-irc-log () + (interactive) + (set-transient-map emacsconf-extract-irc-log-map t)) + (defun emacsconf-extract-irc-backward-by-nick () (interactive) (goto-char (line-beginning-position)) @@ -503,29 +526,12 @@ (defun emacsconf-extract-irc-anonymize-log (beg end speakers) (interactive "r\nMNick(s): ") (when (stringp speakers) (setq speakers (split-string speakers))) - (let ((text (buffer-substring beg end)) - nicks) - (with-temp-buffer - (insert text) - (goto-char (point-min)) - ;; make a list of nicks - (while (re-search-forward "^\\[[0-9:]+\\] <\\(.*?\\)>" nil t) - (unless (member (match-string 1) speakers) - (add-to-list 'nicks (match-string 1)))) - (goto-char (point-min)) - (while (re-search-forward "^\\[[0-9:]+\\] <\\(.*?\\)> \\(.+\\)" nil t) - (replace-match - (if (member (match-string 1) speakers) - (concat " - A: " (match-string 2)) - (format "- {{%d}} %s" - (seq-position nicks (match-string 1)) - (propertize (match-string 2) - 'nick (match-string 1)))))) - (goto-char (point-min)) - (perform-replace (regexp-opt nicks) (lambda ())) - (setq text (buffer-string)) - (other-window 1) - (insert text)))) + (save-excursion + (goto-char beg) + (while (re-search-forward "^\\[[0-9:]+\\] <\\(.*?\\)> \\(.+\\)" end t) + (if (member (match-string 1) speakers) + (replace-match (concat "- " (match-string 1) ": " (match-string 2)) t t) + (replace-match (concat "- " (match-string 2)) t t))))) (defun emacsconf-private-qa (&optional info) (seq-remove (lambda (o) @@ -720,7 +726,7 @@ Would you like to help? See [[help_with_chapter_markers]] for more details. You (setq talk (emacsconf-resolve-talk talk)) (expand-file-name "events.xml" (expand-file-name (plist-get talk :bbb-meeting-id) emacsconf-extract-bbb-raw-dir))) -(defun emacsconf-extract-bbb-report () +(defun emacsconf-extract-bbb-report (&optional event-xml-files) (let* ((max 0) (participant-count 0) (meeting-count 0) @@ -730,39 +736,41 @@ Would you like to help? See [[help_with_chapter_markers]] for more details. You (meeting-events (sort (seq-mapcat - (lambda (talk) - (when (plist-get talk :bbb-meeting-id) - (let ((dom (xml-parse-file (emacsconf-extract-bbb-raw-events-file-name talk))) - participants talking meeting-events) - (mapc (lambda (o) - (pcase (dom-attr o 'eventname) - ("ParticipantJoinEvent" - (cl-pushnew (cons (dom-text (dom-by-tag o 'userId)) - (dom-text (dom-by-tag o 'name))) - participants) - (push (cons (string-to-number (dom-text (dom-by-tag o 'timestampUTC))) - (dom-attr o 'eventname)) - meeting-events)) - ("ParticipantLeftEvent" - (when (string= (dom-attr o 'module) "PARTICIPANT") - (push (cons (string-to-number (dom-text (dom-by-tag o 'timestampUTC))) - (dom-attr o 'eventname)) - meeting-events))) - ("ParticipantTalkingEvent" - (cl-pushnew (assoc-default (dom-text (dom-by-tag o 'participant)) participants) talking)) - ((or - "CreatePresentationPodEvent" - "EndAndKickAllEvent") + (lambda (file) + (let ((dom (xml-parse-file file)) + participants talking meeting-events) + (mapc (lambda (o) + (pcase (dom-attr o 'eventname) + ("ParticipantJoinEvent" + (cl-pushnew (cons (dom-text (dom-by-tag o 'userId)) + (dom-text (dom-by-tag o 'name))) + participants) + (push (cons (string-to-number (dom-text (dom-by-tag o 'timestampUTC))) + (dom-attr o 'eventname)) + meeting-events)) + ("ParticipantLeftEvent" + (when (string= (dom-attr o 'module) "PARTICIPANT") (push (cons (string-to-number (dom-text (dom-by-tag o 'timestampUTC))) (dom-attr o 'eventname)) - meeting-events)))) - (dom-search dom (lambda (o) (dom-attr o 'eventname)))) - (cl-pushnew (list :slug (plist-get talk :slug) - :participants participants - :talking talking) - meeting-participants) - meeting-events))) - (emacsconf-get-talk-info)) + meeting-events))) + ("ParticipantTalkingEvent" + (cl-pushnew (assoc-default (dom-text (dom-by-tag o 'participant)) participants) talking)) + ((or + "CreatePresentationPodEvent" + "EndAndKickAllEvent") + (push (cons (string-to-number (dom-text (dom-by-tag o 'timestampUTC))) + (dom-attr o 'eventname)) + meeting-events)))) + (dom-search dom (lambda (o) (dom-attr o 'eventname)))) + (cl-pushnew (list ;; :slug (plist-get talk :slug) + :participants participants + :talking talking) + meeting-participants) + meeting-events)) + (or event-xml-files + (mapcar #'emacsconf-extract-bbb-raw-events-file-name + (seq-filter (lambda (talk) (plist-get talk :bbb-meeting-id)) + (emacsconf-get-talk-info))))) (lambda (a b) (< (car a) (car b)))))) (dolist (event meeting-events) (pcase (cdr event) @@ -794,13 +802,8 @@ Would you like to help? See [[help_with_chapter_markers]] for more details. You (expand-file-name (plist-get talk :bbb-meeting-id) emacsconf-extract-bbb-published-dir)))) -(defun emacsconf-extract-bbb-parse-events (talk) - "Parse events TALK from raw recordings. -This works with the events.xml from /var/bigbluebutton/raw. -Files should be downloaded to `emacsconf-extract-bbb-raw-dir'." - (setq talk (emacsconf-resolve-talk talk)) - (let* ((xml-file (emacsconf-extract-bbb-raw-events-file-name talk)) - (dom (xml-parse-file xml-file)) +(defun emacsconf-extract-bbb-parse-events (xml-file) + (let* ((dom (xml-parse-file xml-file)) (meeting-name (dom-attr (dom-by-tag dom 'metadata) 'meetingName)) (meeting-id (dom-attr dom 'meeting_id)) (conf-joined (dom-search dom (lambda (o) (and (string= (dom-tag o) "name") (string= (dom-text o) emacsconf-extract-conference-username))))) @@ -886,6 +889,37 @@ Files should be downloaded to `emacsconf-extract-bbb-raw-dir'." (talking . ,(nreverse talking)) (chat . ,(nreverse chat))))) +(defun emacsconf-extract-bbb-talking-report (meeting-xml) + (let ((data (emacsconf-extract-bbb-parse-events meeting-xml))) + (unless (string= "" (alist-get 'meeting-date data)) + (format "- %s %s: %s\n" + (alist-get 'name data) + (format-time-string "%a %I:%M %p" + (date-to-time + (alist-get 'meeting-date data))) + (mapconcat + (lambda (person) + (format "%s (%s)" + (car person) + (/ (cdr person) 60000))) + (sort + (mapcar + (lambda (group) + (cons + (car group) + (apply '+ (mapcar (lambda (o) (- (elt o 2) (elt o 1))) (cdr group))))) + (seq-group-by 'car (alist-get 'talking data))) + :key 'cdr + :reverse t) + ", "))))) + +(defun emacsconf-extract-bbb-parse-events-for-talk (talk) + "Parse events TALK from raw recordings. +This works with the events.xml from /var/bigbluebutton/raw. +Files should be downloaded to `emacsconf-extract-bbb-raw-dir'." + (setq talk (emacsconf-resolve-talk talk)) + (emacsconf-extract-bbb-parse-events (emacsconf-extract-bbb-raw-events-file-name talk))) + (defun emacsconf-extract-bbb-format-chat () (mapconcat (lambda (events) @@ -1127,6 +1161,10 @@ Strategies: ;; To avoid being prompted for the client secret, it's helpful to have a line in ~/.authinfo or ~/.authinfo.gpg with ;; machine https://oauth2.googleapis.com/token username CLIENT_ID password CLIENT_SECRET +;; reset: +;; (setq url-http-oauth--interposed nil url-http-oauth--interposed-regexp nil) +;; and remove the token from ~/.authinfo + (defvar emacsconf-extract-google-client-identifier nil) (defvar emacsconf-extract-youtube-api-channels nil) (defvar emacsconf-extract-youtube-api-categories nil) @@ -1154,7 +1192,7 @@ Strategies: ("access_type" . "offline") ("prompt" . "consent"))) ("access-token-endpoint" . "https://oauth2.googleapis.com/token") - ("scope" . "https://www.googleapis.com/auth/youtube") + ("scope" . "https://www.googleapis.com/auth/youtube https://www.googleapis.com/auth/youtube.force-ssl https://www.googleapis.com/auth/youtube.upload") ("client-secret-method" . prompt)))) (setq emacsconf-extract-youtube-api-channels (plz 'get "https://youtube.googleapis.com/youtube/v3/channels?part=contentDetails&mine=true" @@ -1180,6 +1218,106 @@ Strategies: (string-match (regexp-quote emacsconf-year) (let-alist item .snippet.title)))))) +(defun emacsconf-extract-youtube-comment-list () + (seq-mapcat + (lambda (item) + (append + (if (alist-get 'topLevelComment (alist-get 'snippet item)) + (list (alist-get 'topLevelComment (alist-get 'snippet item)))) + (alist-get 'comments (alist-get 'replies item)))) + (alist-get + 'items + (or emacsconf-extract-youtube-comments (emacsconf-extract-youtube-get-channel-comments))))) + +(defun emacsconf-extract-youtube-get-talk-for-video-id (video-id) + (seq-find (lambda (o) + (or (string-match (regexp-quote video-id) (or (plist-get o :youtube-url) "")) + (string-match (regexp-quote video-id) (or (plist-get o :qa-youtube-url) "")))) + (emacsconf-get-talk-info))) + +(defun emacsconf-extract-youtube-comments-after (date) + (interactive (list (org-read-date nil t nil "On or after date: "))) + (when (stringp date) + (setq date (org-read-date nil t date))) + (seq-filter + (lambda (entry) + (time-less-p + date + (date-to-time + (alist-get + 'publishedAt + (alist-get 'snippet entry))))) + (emacsconf-extract-youtube-comment-list))) + +(defun emacsconf-extract-youtube-format-talk-comments (videos) + (mapconcat + (lambda (video) + (format + "- https://youtu.be/%s\n%s\n" + (car video) + (mapconcat + (lambda (comment) + (let-alist comment + (format + " - %s: %s\n" + .snippet.authorDisplayName + (replace-regexp-in-string "\n" "\n " .snippet.textOriginal)))) + (cdr video) + ""))) + videos + "")) + +(defun emacsconf-extract-youtube-comments-by-talk (&optional comments) + (interactive (list + (if current-prefix-arg (emacsconf-extract-youtube-comments-after (org-read-date nil nil nil "Date: "))))) + (setq comments (or comments (emacsconf-extract-youtube-comment-list))) + (let ((by-talk + (seq-group-by + (lambda (group) + (plist-get (emacsconf-extract-youtube-get-talk-for-video-id (car group)) :slug)) + (seq-group-by (lambda (o) + (alist-get 'videoId (alist-get 'snippet o))) + comments)))) + (when (called-interactively-p 'any) + (with-current-buffer (get-buffer-create "*comments*") + (erase-buffer) + (org-mode) + (dolist (group by-talk) + (when (car group) + (insert (format + "* %s\n\n%s\n\n" + (org-link-make-string + (concat "file:" + (expand-file-name + (concat + (car group) ".md") + (expand-file-name + "talks" + (expand-file-name + emacsconf-year + emacsconf-directory)))) + (car group)) + (emacsconf-extract-youtube-format-talk-comments (cdr group)))))) + (display-buffer (current-buffer)))) + by-talk)) + + +;; (emacsconf-extract-youtube-comment-list) + +;; (emacsconf-extract-youtube-comments-after "-2mon") + +(defvar emacsconf-extract-youtube-comments nil) +(defun emacsconf-extract-youtube-get-channel-comments (&optional no-cache) + (setq + emacsconf-extract-youtube-comments + (or (and emacsconf-extract-youtube-comments (not no-cache)) + (plz 'get + (format + "https://youtube.googleapis.com/youtube/v3/commentThreads?part=snippet,replies&allThreadsRelatedToChannelId=%s&maxResults=100" + (alist-get 'id (car (alist-get 'items emacsconf-extract-youtube-api-channels)))) + :headers `(("Authorization" . ,(url-oauth-auth "https://youtube.googleapis.com/youtube/v3/"))) + :as #'json-read)))) + (defvar emacsconf-extract-youtube-tags '("emacs" "emacsconf")) (defun emacsconf-extract-youtube-object (video-id talk &optional privacy-status qa) "Format the video object for VIDEO-ID using TALK details. @@ -1206,19 +1344,27 @@ If QA is non-nil, treat it as a Q&A video." (let-alist video-object (cond ;; not yet renamed - ((string-match (rx (literal emacsconf-id) " " (literal emacsconf-year) " " - (group (1+ (or (syntax word) "-"))) - " ") - .snippet.title) + ((and .snippet.title (string-match (rx (literal emacsconf-id) " " (literal emacsconf-year) " " + (group (1+ (or (syntax word) "-"))) + " ") + .snippet.title)) (match-string 1 .snippet.title)) ;; renamed, match the description instead - ((string-match (rx (literal emacsconf-base-url) (literal emacsconf-year) "/talks/" - (group (1+ (or (syntax word) "-")))) - .snippet.description) + ((and .snippet.description + (string-match (rx (literal emacsconf-base-url) (literal emacsconf-year) "/talks/" + (group (1+ (or (syntax word) "-")))) + .snippet.description)) (match-string 1 .snippet.description)) (t (plist-get - (seq-find (lambda (o) (string-match (regexp-quote .snippet.resourceId.videoId) (or (plist-get o :youtube-url) ""))) + (seq-find (lambda (o) + (or + (string-match (regexp-quote (or .snippet.videoId + .snippet.resourceId.videoId)) + (or (plist-get o :youtube-url) "")) + (string-match (regexp-quote (or .snippet.videoId + .snippet.resourceId.videoId)) + (or (plist-get o :qa-youtube-url) "")))) (emacsconf-get-talk-info)) :slug))))) @@ -1261,6 +1407,7 @@ If QA is non-nil, treat it as a Q&A video." (if (<= num-pages 0) (setq url null)))) result)) + (defun emacsconf-extract-youtube-api-update-video (video-object &optional qa) "Update VIDEO-OBJECT. If QA is non-nil, treat it as a Q&A video." @@ -1419,6 +1566,8 @@ If QA is non-nil, treat it as a Q&A video." :as #'json-read)) nil))))) + + (defun emacsconf-extract-youtube-duration-msecs (video) (let-alist video (when-let ((duration .contentDetails.duration)) @@ -1672,5 +1821,19 @@ Call with a prefix arg to store the URL as Q&A." url))))) +(defun emacsconf-extract-subed-copy-section-text () + (interactive) + (save-excursion + (subed-copy-region-text + (unless (looking-at "^NOTE") + (if (re-search-backward "^NOTE" nil t) + (point) + (point-min))) + (progn + (forward-line) + (if (re-search-forward "^NOTE" nil t) + (match-beginning 0) + (point-max)))))) + (provide 'emacsconf-extract) ;;; emacsconf-extract.el ends here -- cgit v1.2.3