Subed reflowing, status in backstage

3 files changed, 182 insertions, 25 deletions

diff --git a/emacsconf-publish.el b/emacsconf-publish.el
index d31371d..7dd3f9f 100644
--- a/emacsconf-publish.el
+++ b/emacsconf-publish.el
@@ -922,9 +922,13 @@ Entries are sorted chronologically, with different tracks interleaved."
   (interactive)
   (setq filename (or filename (expand-file-name "index.html" emacsconf-backstage-dir)))
   (setq emacsconf-info (emacsconf-get-talk-info))
+  (let ((emacsconf-schedule-svg-modify-functions '(emacsconf-schedule-svg-color-by-status)))
+    (with-temp-file (expand-file-name "schedule.svg" emacsconf-backstage-dir)
+      (svg-print (emacsconf-schedule-svg 800 200 emacsconf-info))))
+
   (with-temp-file filename
     (let* ((talks (seq-filter (lambda (o) (plist-get o :speakers))
-			      (emacsconf-active-talks (emacsconf-filter-talks emacsconf-info))))
+			                        (emacsconf-active-talks (emacsconf-filter-talks emacsconf-info))))
            (by-status (seq-group-by (lambda (o) (plist-get o :status)) talks))
            (files (directory-files emacsconf-backstage-dir)))
       (insert
@@ -935,6 +939,18 @@ Entries are sorted chronologically, with different tracks interleaved."
                (length (assoc-default "WAITING_FOR_PREREC" by-status))
                (emacsconf-sum :time (assoc-default "WAITING_FOR_PREREC" by-status))
                (length talks))
+       "<ul>"
+       (mapconcat
+        (lambda (status)
+          (concat "<li>" status ": " 
+                  (mapconcat (lambda (o) (format "<a href=\"#%s\">%s</a>"
+                                                 (plist-get o :slug)
+                                                 (plist-get o :slug)))
+                             (assoc-default status by-status)
+                             ", ")
+                  "</li>"))
+        '("TO_PROCESS" "PROCESSING" "TO_ASSIGN" "TO_CAPTION" "TO_STREAM"))
+       "</ul>"
        (let ((list (append
 		                (assoc-default "TO_ASSIGN" by-status)
 		                (assoc-default "TO_PROCESS" by-status)
@@ -1061,8 +1077,8 @@ Entries are sorted chronologically, with different tracks interleaved."
                   (emacsconf-public-talks (emacsconf-get-talk-info))
                   "\n")
        "</ol>"
-       (if (file-exists-p (expand-file-name "include-in-index.html" emacsconf-cache-dir))
-           (with-temp-buffer (insert-file-contents (expand-file-name "include-in-index.html" emacsconf-cache-dir)) (buffer-string))
+       (if (file-exists-p (expand-file-name "include-in-public-index.html" emacsconf-cache-dir))
+           (with-temp-buffer (insert-file-contents (expand-file-name "include-in-public-index.html" emacsconf-cache-dir)) (buffer-string))
          "")
        "</body></html>"))))
 
@@ -1119,6 +1135,7 @@ Entries are sorted chronologically, with different tracks interleaved."
        :md (mapconcat (lambda (chapter)
                         (concat
                          (format-seconds "%.2h:%z%.2m:%.2s" (floor (/ (elt chapter 1) 1000)))
+                         (format ".%03d" (mod (elt chapter 1) 1000))
                          " "
                          (elt chapter 3)
                          "\n"))
@@ -1128,8 +1145,9 @@ Entries are sorted chronologically, with different tracks interleaved."
                      (or target "")
                      (mapconcat
                       (lambda (chapter)
-                        (format "%s %s"
+                        (format "%s.%03d %s"
                                 (format-seconds "%.2h:%z%.2m:%.2s" (floor (/ (elt chapter 1) 1000)))
+                                (mod (elt chapter 1) 1000)
                                 (elt chapter 3)))
                       chapters
                       "\n"))))))
@@ -1394,25 +1412,33 @@ Entries are sorted chronologically, with different tracks interleaved."
 
 ;;; Video services
 
-(defun emacsconf-cache-all-video-data ()
-  (interactive)
-  (org-map-entries (lambda () (when (and (org-entry-get (point) "VIDEO_SLUG") (null (org-entry-get (point) "VIDEO_FILE_SIZE"))) (emacsconf-cache-video-data-for-entry)))))
+(defun emacsconf-cache-all-video-data (&optional force)
+  (interactive (list current-prefix-arg))
+  (mapc
+   (lambda (talk)
+     (when (and (plist-get talk :video-slug)
+                (or force (null (plist-get talk :video-file-size))))
+       (emacsconf-cache-video-data talk)))
+   (emacsconf-get-talk-info)))
+;; (emacsconf-cache-all-video-data t)
 (defvar emacsconf-cache-dir (expand-file-name "cache" (file-name-directory emacsconf-org-file)))
 (defun emacsconf-cache-video-data (talk)
   (interactive (list (emacsconf-complete-talk-info)))
-  (let ((main (expand-file-name (concat (plist-get talk :video-slug) "--main.webm") emacsconf-backstage-dir)))
+  (let ((main (expand-file-name (concat (plist-get talk :video-slug) "--main.webm")
+                                emacsconf-cache-dir)))
     (emacsconf-with-talk-heading talk
       (let* ((video-file-name (emacsconf-get-preferred-video (org-entry-get (point) "VIDEO_SLUG")))
-             (video-file (expand-file-name video-file-name emacsconf-cache-dir))
+             (video-file (and video-file-name (expand-file-name video-file-name emacsconf-cache-dir)))
              duration)
         (unless (file-exists-p main)
           (setq main video-file-name))
-        (org-entry-put (point) "VIDEO_FILE" (file-name-nondirectory video-file))
-        (org-entry-put (point) "VIDEO_FILE_SIZE" (file-size-human-readable (file-attribute-size (file-attributes video-file))))
-        (unless (plist-get talk :video-time)
-          (setq duration (/ (compile-media-get-file-duration-ms video-file) 1000))
-          (org-entry-put (point) "VIDEO_DURATION" (format-seconds "%m:%.2s" duration))
-          (org-entry-put (point) "VIDEO_TIME" (number-to-string (ceiling (/ duration 60)))))))))
+        (when video-file
+          (org-entry-put (point) "VIDEO_FILE" (file-name-nondirectory video-file))
+          (org-entry-put (point) "VIDEO_FILE_SIZE" (file-size-human-readable (file-attribute-size (file-attributes video-file))))
+          (unless (plist-get talk :video-time)
+            (setq duration (/ (compile-media-get-file-duration-ms video-file) 1000))
+            (org-entry-put (point) "VIDEO_DURATION" (format-seconds "%m:%.2s" duration))
+            (org-entry-put (point) "VIDEO_TIME" (number-to-string (ceiling (/ duration 60))))))))))
 
 (defvar emacsconf-youtube-channel-id "UCwuyodzTl_KdEKNuJmeo99A")
 (defun emacsconf-youtube-edit ()
diff --git a/emacsconf-subed.el b/emacsconf-subed.el
index 9aca0e0..b219a6a 100644
--- a/emacsconf-subed.el
+++ b/emacsconf-subed.el
@@ -29,7 +29,7 @@
 
 (require 'subed)
 
-(defcustom emacsconf-subed-subtitle-max-length 50
+(defcustom emacsconf-subed-subtitle-max-length 60
   "Target number of characters."
   :group 'emacsconf
   :type 'integer)
@@ -39,16 +39,136 @@
   :group 'emacsconf
   :type 'integer)
 
+(defun emacsconf-combine-close-subtitles (subtitles &optional auto-space-msecs)
+  "Combine closely-spaced SUBTITLES for analysis.
+If a subtitle ends within AUTO-SPACE-MSECS of the previous one, concatenate them."
+  (setq auto-space-msecs (or auto-space-msecs 1000))
+  (let ((current (seq-copy (car subtitles)))
+        results)
+    (setq subtitles (cdr subtitles))
+    ;; if the subtitle ends within auto-space-msecs of the current one, combine it
+    (while subtitles
+      (if (< (elt (car subtitles) 1) (+ (elt current 2) auto-space-msecs))
+          (progn
+            (setf (elt current 2) (elt (car subtitles) 2))
+            (setf (elt current 3) (concat (elt current 3) " "
+                                          (propertize (elt (car subtitles) 3)
+                                                      :start (elt (car subtitles) 1)))))
+        (setq results (cons current results))
+        (setq current (seq-copy (car subtitles))))
+      (setq subtitles (cdr subtitles)))
+    (nreverse (cons current results))))
 
-(defun emacsconf-subed-fix-timestamps ()
-  "Change overlapping timestamps to the start of the next subtitle."
+;; (emacsconf-combine-close-subtitles (subed-parse-file "/home/sacha/proj/emacsconf/cache/emacsconf-2022-health--health-data-journaling-and-visualization-with-org-mode-and-gnuplot--david-otoole--main.vtt") 700)
+
+(defun emacsconf-last-string-match-before (regexp string before &optional after type)
+  "Find the last match of REGEXP in STRING before BEFORE position (exclusive).
+If AFTER is specified, start the search from there.
+TYPE can be 'end if you want the match end instead of the beginning."
+  (let (pos (found (string-match regexp string after)) result)
+    (while (and found (< found before))
+      (setq result (if (eq type 'end) (match-end 0) (match-beginning 0)))
+      (if (string-match regexp string (1+ found))
+          (setq found (match-end 0))
+        (setq found nil)))
+    result))
+
+;; (assert (= (emacsconf-last-string-match-before " " "012345 789 12345 7890" 12) 10))
+
+(defun emacsconf-split-text-based-on-heuristics (text limit)
+  "Split subtitle TEXT based on simple heuristics so that it fits in LIMIT."
+  (let ((position 0)
+        (current "")
+        (reversed (string-reverse text))
+        (case-fold-search t)
+        result new-position (next-limit limit))
+    ;; Treat sub as a bag of words
+    (while (< position (length text))
+      (let ((heuristic (delq
+                        nil
+                        (list
+                         (emacsconf-last-string-match-before "[,\\.\\?;]+ "
+                                                             text next-limit (1+ position) 'end)
+                         (emacsconf-last-string-match-before
+                          (concat "\\<" (regexp-opt (list "or how"
+                                                          "or that"
+                                                          "now that"
+                                                          "then"
+                                                          "but that"
+                                                          "now"
+                                                          "which"
+                                                          "when"
+                                                          "using"
+                                                          "of"
+                                                          "by"
+                                                          "and" "that" "so" "so in" "because" "but"
+                                                          "is" "how" "in" "or" "--" "than" "with" "to"
+                                                          "I"))
+                                  "\\>")
+                          text next-limit (1+ position))))))
+        (setq
+         new-position
+         (or
+          (and (< (length text) next-limit) (length text))
+          (and heuristic (apply #'max heuristic))
+          (emacsconf-last-string-match-before
+           " " text next-limit (1+ position) 'end)
+          (length text))))
+      (setq result (cons (string-trim (substring text position new-position))
+                         result))
+      (setq position new-position)
+      (setq next-limit (+ new-position limit)))
+    (setq result (cons (string-trim current) result))
+    (nreverse result)))
+
+(defun emacsconf-reflow-automatically ()
+  (interactive)
+  (let* ((subtitle-text-limit emacsconf-subed-subtitle-max-length)
+         (auto-space-msecs 700)
+         (subtitles
+          (mapconcat
+           (lambda (sub)
+             (string-join 
+              (emacsconf-split-text-based-on-heuristics (elt sub 3) subtitle-text-limit)
+              "\n"))
+           (emacsconf-combine-close-subtitles (subed-subtitle-list))
+           "\n")))
+    (find-file (concat (file-name-sans-extension (buffer-file-name)) ".txt"))
+    (erase-buffer)
+    (insert subtitles)
+    (goto-char (point-min))
+    (set-fill-column subtitle-text-limit)
+    (display-fill-column-indicator-mode 1)))
+
+(defun emacsconf-subed-make-chapter-file-based-on-comments ()
+  "Create a chapter file based on NOTE comments."
   (interactive)
-  (goto-char (point-max))
-  (let ((timestamp (subed-subtitle-msecs-start)))
-    (while (subed-backward-subtitle-time-start)
-      (when (> (subed-subtitle-msecs-stop) timestamp)
-        (subed-set-subtitle-time-stop timestamp))
-      (setq timestamp (subed-subtitle-msecs-start)))))
+  (let ((new-filename (concat (file-name-sans-extension (buffer-file-name)) "--chapters.vtt"))
+        (subtitles (subed-subtitle-list))
+        (subed-auto-play-media nil))
+    (when (or (not (file-exists-p new-filename))
+              (yes-or-no-p (format "%s exists. Overwrite? " new-filename)))
+      (subed-create-file
+       new-filename 
+       (emacsconf-subed-list-chapter-markers-based-on-comments
+        subtitles)
+       t))))
+
+(defun emacsconf-subed-list-chapter-markers-based-on-comments (subtitles)
+  "Make a list of subtitles based on which SUBTITLES have comments."
+  (let (result current)
+    (mapc
+     (lambda (o)
+       (if (elt o 4)
+           (progn
+             (when current (setq result (cons current result)))
+             (setq current
+                   (list nil (elt o 1) (elt o 2)
+                         (string-trim (replace-regexp-in-string "^NOTE[ \n]+" " " (elt o 4))))))
+         ;; update the end time to include the current subtitle
+         (when current (setf (elt current 2) (elt o 2)))))
+     subtitles)
+    (nreverse (cons current result))))
 
 (defun emacsconf-subed-mark-chapter (chapter-name)
   (interactive "MChapter: ")
@@ -126,7 +246,7 @@
     (emacsconf-subed-download-captions url new-file)
     (with-current-buffer (find-file new-file)
       (goto-char (point-min))
-      (emacsconf-subed-fix-timestamps)
+      (subed-trim-overlaps)
       (save-buffer))))
 
 (defun emacsconf-subed-copy-downloaded-captions ()
diff --git a/emacsconf.el b/emacsconf.el
index 855e2f8..f32c47c 100644
--- a/emacsconf.el
+++ b/emacsconf.el
@@ -1271,5 +1271,16 @@ tracks with the ID in the cdr of that list."
   (interactive)
   (mapc #'load-library '("emacsconf" "emacsconf-erc" "emacsconf-publish" "emacsconf-stream" "emacsconf-pad")))
 
+(defun emacsconf-find-talk-file-in-cache (talk filename)
+  (interactive (let ((talk (emacsconf-complete-talk-info)))
+                 (list
+                  talk
+                  (completing-read "File: " (directory-files emacsconf-cache-dir t (plist-get talk :video-slug))))))
+  (find-file filename))
+
+(defun emacsconf-cache-find-file (filename)
+  (interactive (list (read-file-name "File: " (expand-file-name "./" emacsconf-cache-dir) nil t)))
+  (find-file (expand-file-name filename emacsconf-cache-dir)))
+
 (provide 'emacsconf)
 ;;; emacsconf.el ends here