refactor(main.py): update segment setup and preloading to improve video handling and performance

croppa/main.py

@@ -182,12 +182,12 @@ class VideoEditor:
         self.video_path = video_path
         
         # Try different backends for better performance
-        # Order of preference: DirectShow (Windows), FFmpeg, any available
+        # Order of preference: FFmpeg (best for video files), DirectShow (cameras), any available
         backends_to_try = []
-        if hasattr(cv2, 'CAP_DSHOW'):  # Windows DirectShow
-            backends_to_try.append(cv2.CAP_DSHOW)
-        if hasattr(cv2, 'CAP_FFMPEG'):  # FFmpeg
+        if hasattr(cv2, 'CAP_FFMPEG'):  # FFmpeg - best for video files
             backends_to_try.append(cv2.CAP_FFMPEG)
+        if hasattr(cv2, 'CAP_DSHOW'):  # DirectShow - usually for cameras
+            backends_to_try.append(cv2.CAP_DSHOW)
         backends_to_try.append(cv2.CAP_ANY)  # Fallback
         
         self.cap = None

main.py

@@ -43,7 +43,7 @@ class MediaGrader:
     CTRL_SEEK_MULTIPLIER = 10  # CTRL + A/D multiplier
     
     # Multi-segment mode configuration
-    SEGMENT_COUNT = 16  # Number of video segments (2x2 grid)
+    SEGMENT_COUNT = 16  # Number of video segments (4x4 grid)
     SEGMENT_OVERLAP_PERCENT = 10  # Percentage overlap between segments
 
     def __init__(
@@ -203,13 +203,13 @@ class MediaGrader:
 
         if self.is_video(file_path):
             # Try different backends for better performance
-            # Order of preference: DirectShow (Windows), FFmpeg, any available
+            # For video files: FFmpeg is usually best, DirectShow is for cameras
             backends_to_try = []
-            if hasattr(cv2, 'CAP_DSHOW'):  # Windows DirectShow
-                backends_to_try.append(cv2.CAP_DSHOW)
-            if hasattr(cv2, 'CAP_FFMPEG'):  # FFmpeg
+            if hasattr(cv2, 'CAP_FFMPEG'):  # FFmpeg - best for video files
                 backends_to_try.append(cv2.CAP_FFMPEG)
-            backends_to_try.append(cv2.CAP_ANY)  # Fallback
+            if hasattr(cv2, 'CAP_DSHOW'):  # DirectShow - usually for cameras, but try as fallback
+                backends_to_try.append(cv2.CAP_DSHOW)
+            backends_to_try.append(cv2.CAP_ANY)  # Final fallback
             
             self.current_cap = None
             for backend in backends_to_try:
@@ -488,13 +488,22 @@ class MediaGrader:
         self.multi_segment_mode = not self.multi_segment_mode
         
         if self.multi_segment_mode:
-            print(f"Enabled multi-segment mode ({self.segment_count} segments)")
-            self.setup_segment_captures()
+            print(f"Enabling multi-segment mode ({self.segment_count} segments)...")
+            try:
+                self.setup_segment_captures()
+                print("Multi-segment mode enabled successfully")
+            except Exception as e:
+                print(f"Failed to setup multi-segment mode: {e}")
+                import traceback
+                traceback.print_exc()
+                self.multi_segment_mode = False
+                return False
         else:
-            print("Disabled multi-segment mode")
+            print("Disabling multi-segment mode...")
             self.cleanup_segment_captures()
             # Reload single video
             self.load_media(self.media_files[self.current_index])
+            print("Multi-segment mode disabled")
         
         return True
 
@@ -509,120 +518,162 @@ class MediaGrader:
             return
             
         start_time = time.time()
-        print(f"Setting up {self.segment_count} segments...")
+        print(f"Setting up {self.segment_count} segments with video preloading...")
         
-        # Clean up existing segment captures
-        self.cleanup_segment_captures()
+        try:
+            # Clean up existing segment captures
+            print("Cleaning up existing captures...")
+            self.cleanup_segment_captures()
             
-        current_file = self.media_files[self.current_index]
+            current_file = self.media_files[self.current_index]
+            print(f"Working with file: {current_file}")
             
-        # Initialize arrays
-        self.segment_caps = [None] * self.segment_count
-        self.segment_frames = [None] * self.segment_count
-        self.segment_positions = []
+            # Initialize arrays
+            print("Initializing arrays...")
+            self.segment_caps = [None] * self.segment_count  # Keep for compatibility
+            self.segment_frames = [None] * self.segment_count
+            self.segment_positions = []
+            self.segment_current_frames = [0] * self.segment_count  # Track current frame for each segment
             
-        # Calculate target positions
-        for i in range(self.segment_count):
-            position_ratio = i / max(1, self.segment_count - 1)
-            start_frame = int(position_ratio * (self.total_frames - 1))
-            self.segment_positions.append(start_frame)
+            # Calculate target positions
+            print("Calculating segment positions...")
+            if self.total_frames <= 1:
+                print("Error: Video has insufficient frames for multi-segment mode")
+                return
             
-        load_start = time.time()
+            # Use conservative frame range
+            safe_frame_count = max(1, int(self.total_frames * 0.6))
+            print(f"Using safe frame count: {safe_frame_count} (60% of reported {self.total_frames})")
                 
-        shared_cap_start = time.time()
-        
-        # Use same backend optimization as main capture
-        backends_to_try = []
-        if hasattr(cv2, 'CAP_DSHOW'):  # Windows DirectShow
-            backends_to_try.append(cv2.CAP_DSHOW)
-        if hasattr(cv2, 'CAP_FFMPEG'):  # FFmpeg
-            backends_to_try.append(cv2.CAP_FFMPEG)
-        backends_to_try.append(cv2.CAP_ANY)  # Fallback
-        
-        shared_cap = None
-        for backend in backends_to_try:
-            try:
-                shared_cap = cv2.VideoCapture(str(current_file), backend)
-                if shared_cap.isOpened():
-                    shared_cap.set(cv2.CAP_PROP_BUFFERSIZE, 1)
-                    break
-                shared_cap.release()
-            except:
-                continue
-        
-        if not shared_cap:
-            shared_cap = cv2.VideoCapture(str(current_file))  # Fallback
-            
-        shared_cap_create_time = (time.time() - shared_cap_start) * 1000
-        print(f"Capture creation: {shared_cap_create_time:.1f}ms")
-        
-        if shared_cap.isOpened():
-            frames_start = time.time()
-            
-            # Strategy: Read a much smaller subset and interpolate/approximate
-            # Only read 4-6 key frames and generate the rest through approximation
-            key_frames_to_read = min(6, self.segment_count)
-            frames_read = 0
-            
-            for i in range(key_frames_to_read):
-                target_frame = self.segment_positions[i * (self.segment_count // key_frames_to_read)]
-                
-                seek_start = time.time()
-                shared_cap.set(cv2.CAP_PROP_POS_FRAMES, target_frame)
-                seek_time = (time.time() - seek_start) * 1000
-                
-                read_start = time.time()
-                ret, frame = shared_cap.read()
-                read_time = (time.time() - read_start) * 1000
-                
-                if ret:
-                    # Use this frame for multiple segments (approximation)
-                    segments_per_key = self.segment_count // key_frames_to_read
-                    start_seg = i * segments_per_key
-                    end_seg = min(start_seg + segments_per_key, self.segment_count)
-                    
-                    for seg_idx in range(start_seg, end_seg):
-                        self.segment_frames[seg_idx] = frame.copy()
-                    
-                    frames_read += 1
-                    print(f"Key frame {i}: Frame {target_frame} -> Segments {start_seg}-{end_seg-1} ({seek_time:.1f}ms + {read_time:.1f}ms)")
+            for i in range(self.segment_count):
+                if self.segment_count <= 1:
+                    position_ratio = 0
                 else:
-                    print(f"Failed to read key frame {i} at position {target_frame}")
+                    position_ratio = i / (self.segment_count - 1)
+                start_frame = int(position_ratio * (safe_frame_count - 1))
+                start_frame = max(0, min(start_frame, safe_frame_count - 1))
+                self.segment_positions.append(start_frame)
+                self.segment_current_frames[i] = start_frame  # Start each segment at its position
+            print(f"Segment positions: {self.segment_positions}")
             
-            # Fill any remaining segments with the last valid frame
-            last_valid_frame = None
-            for frame in self.segment_frames:
-                if frame is not None:
-                    last_valid_frame = frame
-                    break
+            # Preload the entire video into memory
+            print("Preloading entire video into memory...")
+            preload_start = time.time()
             
-            if last_valid_frame is not None:
-                for i in range(len(self.segment_frames)):
-                    if self.segment_frames[i] is None:
-                        self.segment_frames[i] = last_valid_frame.copy()
+            self.video_frame_cache = []  # Array to hold all frames
             
-            frames_time = (time.time() - frames_start) * 1000
-            print(f"Smart frame reading: {frames_time:.1f}ms ({frames_read} key frames for {self.segment_count} segments)")
+            if self.current_cap and self.current_cap.isOpened():
+                # Reset to beginning
+                self.current_cap.set(cv2.CAP_PROP_POS_FRAMES, 0)
                 
-            shared_cap.release()
-        else:
-            print("Failed to create shared capture!")
+                frame_count = 0
+                while frame_count < safe_frame_count:
+                    ret, frame = self.current_cap.read()
+                    if ret and frame is not None:
+                        self.video_frame_cache.append(frame.copy())
+                        frame_count += 1
+                        
+                        # Show progress for large videos
+                        if frame_count % 50 == 0:
+                            print(f"Preloaded {frame_count}/{safe_frame_count} frames...")
+                    else:
+                        print(f"Reached end of video at frame {frame_count}")
+                        break
+                
+                # Reset main capture to original position
+                self.current_cap.set(cv2.CAP_PROP_POS_FRAMES, self.current_frame)
+            
+            preload_time = (time.time() - preload_start) * 1000
+            print(f"Video preloading: {preload_time:.1f}ms ({len(self.video_frame_cache)} frames)")
+            
+            # Initialize segment frames from the preloaded cache
+            print("Initializing segment frames...")
+            for i in range(self.segment_count):
+                if self.segment_current_frames[i] < len(self.video_frame_cache):
+                    self.segment_frames[i] = self.video_frame_cache[self.segment_current_frames[i]].copy()
+            
+        except Exception as e:
+            print(f"Error in setup: {e}")
+            import traceback
+            traceback.print_exc()
+            return
         
         total_time = time.time() - start_time
         print(f"Total setup time: {total_time * 1000:.1f}ms")
         
         # Report success
         successful_segments = sum(1 for frame in self.segment_frames if frame is not None)
-        print(f"Successfully approximated {successful_segments}/{self.segment_count} segments")
+        print(f"Successfully preloaded video with {successful_segments}/{self.segment_count} active segments")
+
+    def _create_segment_parallel(self, segment_index: int, file_path: str, start_frame: int):
+        """Create a single segment capture and load its initial frame (runs in thread)"""
+        try:
+            # Create optimized capture for this segment
+            cap = None
+            backends_to_try = []
+            if hasattr(cv2, 'CAP_FFMPEG'):
+                backends_to_try.append(cv2.CAP_FFMPEG)
+            if hasattr(cv2, 'CAP_DSHOW'):
+                backends_to_try.append(cv2.CAP_DSHOW)
+            backends_to_try.append(cv2.CAP_ANY)
+            
+            for backend in backends_to_try:
+                try:
+                    cap = cv2.VideoCapture(file_path, backend)
+                    if cap.isOpened():
+                        cap.set(cv2.CAP_PROP_BUFFERSIZE, 1)
+                        break
+                    cap.release()
+                except:
+                    continue
+            
+            if not cap or not cap.isOpened():
+                cap = cv2.VideoCapture(file_path)  # Fallback
+                
+            if not cap.isOpened():
+                return None, None
+            
+            # Try seeking to the requested frame
+            cap.set(cv2.CAP_PROP_POS_FRAMES, start_frame)
+            ret, frame = cap.read()
+            
+            if ret and frame is not None:
+                # Reset to start position for future reads
+                cap.set(cv2.CAP_PROP_POS_FRAMES, start_frame)
+                return cap, frame.copy()
+            else:
+                # Frame doesn't exist - try seeking to a safer position
+                # Try progressively earlier frames
+                for fallback_frame in [start_frame // 2, start_frame // 4, 0]:
+                    cap.set(cv2.CAP_PROP_POS_FRAMES, fallback_frame)
+                    ret, frame = cap.read()
+                    if ret and frame is not None:
+                        print(f"Segment {segment_index}: fell back from frame {start_frame} to {fallback_frame}")
+                        cap.set(cv2.CAP_PROP_POS_FRAMES, fallback_frame)
+                        return cap, frame.copy()
+                
+                # If all fallbacks failed, give up
+                cap.release()
+                return None, None
+                
+        except Exception as e:
+            print(f"Error creating segment {segment_index}: {e}")
+            if cap:
+                cap.release()
+            return None, None
 
     def cleanup_segment_captures(self):
-        """Clean up all segment video captures"""
+        """Clean up all segment video captures and preloaded cache"""
         for cap in self.segment_caps:
             if cap:
                 cap.release()
         self.segment_caps = []
         self.segment_frames = []
         self.segment_positions = []
+        if hasattr(self, 'video_frame_cache'):
+            self.video_frame_cache = []  # Clear preloaded video cache
+        if hasattr(self, 'segment_current_frames'):
+            self.segment_current_frames = []  # Clear frame tracking
         # Clear frame cache
         self.frame_cache.clear()
 
@@ -642,11 +693,11 @@ class MediaGrader:
             # Use optimized backend for temporary capture too
             temp_cap = None
             backends_to_try = []
-            if hasattr(cv2, 'CAP_DSHOW'):
-                backends_to_try.append(cv2.CAP_DSHOW)
-            if hasattr(cv2, 'CAP_FFMPEG'):
+            if hasattr(cv2, 'CAP_FFMPEG'):  # FFmpeg - best for video files
                 backends_to_try.append(cv2.CAP_FFMPEG)
-            backends_to_try.append(cv2.CAP_ANY)
+            if hasattr(cv2, 'CAP_DSHOW'):  # DirectShow - fallback
+                backends_to_try.append(cv2.CAP_DSHOW)
+            backends_to_try.append(cv2.CAP_ANY)  # Final fallback
             
             for backend in backends_to_try:
                 try:
@@ -690,11 +741,11 @@ class MediaGrader:
                 # Use optimized backend for segment capture
                 cap = None
                 backends_to_try = []
-                if hasattr(cv2, 'CAP_DSHOW'):
-                    backends_to_try.append(cv2.CAP_DSHOW)
-                if hasattr(cv2, 'CAP_FFMPEG'):
+                if hasattr(cv2, 'CAP_FFMPEG'):  # FFmpeg - best for video files
                     backends_to_try.append(cv2.CAP_FFMPEG)
-                backends_to_try.append(cv2.CAP_ANY)
+                if hasattr(cv2, 'CAP_DSHOW'):  # DirectShow - fallback
+                    backends_to_try.append(cv2.CAP_DSHOW)
+                backends_to_try.append(cv2.CAP_ANY)  # Final fallback
                 
                 for backend in backends_to_try:
                     try:
@@ -740,45 +791,22 @@ class MediaGrader:
             return segment_index, None
 
     def update_segment_frames(self):
-        """Update frames for all segments during playback with parallel processing"""
-        if not self.multi_segment_mode or not self.segment_frames:
+        """Update frames for segments using the preloaded video array - smooth playback!"""
+        if not self.multi_segment_mode or not self.segment_frames or not hasattr(self, 'video_frame_cache'):
             return
         
-        # Only update segments that have valid frames loaded
-        active_segments = [i for i, frame in enumerate(self.segment_frames) if frame is not None]
+        # Each segment advances through the video at its own pace
+        for i in range(len(self.segment_frames)):
+            if self.segment_frames[i] is not None and self.video_frame_cache:
+                # Advance this segment's current frame
+                self.segment_current_frames[i] += 1
                 
-        if not active_segments:
-            return
+                # Loop back to start if we reach the end
+                if self.segment_current_frames[i] >= len(self.video_frame_cache):
+                    self.segment_current_frames[i] = 0
                 
-        # Use thread pool for parallel frame updates (but limit to avoid overwhelming)
-        if len(active_segments) <= 4:
-            # For small numbers, use parallel processing
-            futures = []
-            for i in active_segments:
-                future = self.thread_pool.submit(self.update_segment_frame_parallel, i)
-                futures.append(future)
-            
-            # Collect results
-            for future in futures:
-                segment_index, frame = future.result()
-                if frame is not None:
-                    self.segment_frames[segment_index] = frame
-        else:
-            # For larger numbers, process in smaller batches to avoid resource exhaustion
-            batch_size = 4
-            for batch_start in range(0, len(active_segments), batch_size):
-                batch = active_segments[batch_start:batch_start + batch_size]
-                futures = []
-                
-                for i in batch:
-                    future = self.thread_pool.submit(self.update_segment_frame_parallel, i)
-                    futures.append(future)
-                
-                # Collect batch results
-                for future in futures:
-                    segment_index, frame = future.result()
-                    if frame is not None:
-                        self.segment_frames[segment_index] = frame
+                # Update the segment frame from the cache
+                self.segment_frames[i] = self.video_frame_cache[self.segment_current_frames[i]].copy()
 
     def reposition_segments_around_frame(self, center_frame: int):
         """Reposition all segments around a center frame while maintaining spacing"""