Hallucinate up zoom and crop keyframes

2025-12-23 13:02:26 +01:00
parent 43feae622e
commit bd8066c471
1 changed files with 276 additions and 26 deletions
--- a/croppa/main.py
+++ b/croppa/main.py
@@ -160,6 +160,7 @@ class VideoEditor:
        self.crop_drag_mode = None  # 'expand' or 'contract'
        self.mouse_left_down = False
        self.mouse_right_down = False
+        self.crop_zoom_keyframes = {}  # {frame_index: {'crop_rect': (x, y, w, h), 'zoom_factor': float}}

        # Zoom settings
        self.zoom_factor = 1.0
@@ -206,6 +207,7 @@ class VideoEditor:
        # Display optimization - track when redraw is needed
        self.display_needs_update = True
        self.last_display_state = None
+        self.show_help_overlay = False
        
        # Cached transformations for performance
        self.cached_transformed_frame = None
@@ -308,7 +310,14 @@ class VideoEditor:
                'templates': [{
                    'start_frame': start_frame,
                    'region': region
-                } for start_frame, region, template_image in self.templates]
+                } for start_frame, region, template_image in self.templates],
+                'crop_zoom_keyframes': {
+                    str(frame): {
+                        'crop_rect': list(data['crop_rect']),
+                        'zoom_factor': data['zoom_factor'],
+                    }
+                    for frame, data in self.crop_zoom_keyframes.items()
+                },
            }

            with open(state_file, 'w') as f:
@@ -401,6 +410,27 @@ class VideoEditor:
            if 'feature_tracker' in state:
                self.feature_tracker.load_state_dict(state['feature_tracker'])
                print(f"Loaded feature tracker state")
+
+            # Load crop/zoom keyframes
+            if 'crop_zoom_keyframes' in state and isinstance(state['crop_zoom_keyframes'], dict):
+                loaded_keyframes = {}
+                for k, v in state['crop_zoom_keyframes'].items():
+                    try:
+                        frame_index = int(k)
+                    except ValueError:
+                        continue
+                    if not isinstance(v, dict):
+                        continue
+                    if 'crop_rect' not in v or 'zoom_factor' not in v:
+                        continue
+                    crop_rect = tuple(v['crop_rect']) if v['crop_rect'] is not None else None
+                    zoom_value = v['zoom_factor']
+                    loaded_keyframes[frame_index] = {
+                        'crop_rect': crop_rect,
+                        'zoom_factor': zoom_value,
+                    }
+                self.crop_zoom_keyframes = loaded_keyframes
+                print(f"Loaded crop/zoom keyframes: {len(self.crop_zoom_keyframes)}")
            
            # Load template matching state
            if 'template_matching_full_frame' in state:
@@ -1079,10 +1109,13 @@ class VideoEditor:
        if frame is None:
            return None

+        frame_number = getattr(self, 'current_frame', 0)
+        base_crop_rect, effective_zoom = self.get_interpolated_crop_zoom(frame_number)
+
        # Create a hash of the transformation parameters for caching
        transform_hash = hash((
-            self.crop_rect,
-            self.zoom_factor,
+            tuple(base_crop_rect),
+            effective_zoom,
            self.rotation_angle,
            self.brightness,
            self.contrast,
@@ -1107,7 +1140,7 @@ class VideoEditor:
            processed_frame = self.apply_rotation(processed_frame)

        # Apply crop (interpreted in rotated frame coordinates) using EFFECTIVE rect
-        eff_x, eff_y, eff_w, eff_h = self._get_effective_crop_rect_for_frame(getattr(self, 'current_frame', 0))
+        eff_x, eff_y, eff_w, eff_h = self._get_effective_crop_rect_for_frame(frame_number, base_crop_rect=base_crop_rect)
        if eff_w > 0 and eff_h > 0:
            eff_x = max(0, min(eff_x, processed_frame.shape[1] - 1))
            eff_y = max(0, min(eff_y, processed_frame.shape[0] - 1))
@@ -1116,10 +1149,10 @@ class VideoEditor:
            processed_frame = processed_frame[eff_y : eff_y + eff_h, eff_x : eff_x + eff_w]

        # Apply zoom
-        if self.zoom_factor != 1.0:
+        if effective_zoom != 1.0:
            height, width = processed_frame.shape[:2]
-            new_width = int(width * self.zoom_factor)
-            new_height = int(height * self.zoom_factor)
+            new_width = int(width * effective_zoom)
+            new_height = int(height * effective_zoom)
            processed_frame = cv2.resize(
                processed_frame, (new_width, new_height), interpolation=cv2.INTER_LINEAR
            )
@@ -1206,9 +1239,9 @@ class VideoEditor:
            print(f"Error calculating frame difference: {e}")
            return 0.0

-    # --- Motion tracking helpers ---
-    def _get_effective_crop_rect_for_frame(self, frame_number):
-        """Return EFFECTIVE crop_rect in ROTATED frame coords for this frame (applies tracking follow)."""
+    # --- Motion tracking and crop/zoom helpers ---
+    def _get_base_crop_rect_for_frame(self, frame_number):
+        """Return base crop_rect in rotated frame coords for this frame (without tracking follow)."""
        # Rotated base dims
        if self.rotation_angle in (90, 270):
            rot_w, rot_h = self.frame_height, self.frame_width
@@ -1218,6 +1251,27 @@ class VideoEditor:
        if not self.crop_rect:
            return (0, 0, rot_w, rot_h)
        x, y, w, h = map(int, self.crop_rect)
+        # Clamp in rotated space
+        x = max(0, min(x, rot_w - 1))
+        y = max(0, min(y, rot_h - 1))
+        w = min(w, rot_w - x)
+        h = min(h, rot_h - y)
+        return (x, y, w, h)
+
+    def _get_effective_crop_rect_for_frame(self, frame_number, base_crop_rect=None):
+        """Return EFFECTIVE crop_rect in ROTATED frame coords for this frame (applies tracking follow)."""
+        # Rotated base dims
+        if self.rotation_angle in (90, 270):
+            rot_w, rot_h = self.frame_height, self.frame_width
+        else:
+            rot_w, rot_h = self.frame_width, self.frame_height
+        # Default full-frame or provided base rect
+        if base_crop_rect is not None:
+            x, y, w, h = map(int, base_crop_rect)
+        elif not self.crop_rect:
+            return (0, 0, rot_w, rot_h)
+        else:
+            x, y, w, h = map(int, self.crop_rect)
        # Tracking follow: center crop on interpolated rotated position
        if self.tracking_enabled:
            pos = self._get_interpolated_tracking_position(frame_number)
@@ -1232,6 +1286,157 @@ class VideoEditor:
        h = min(h, rot_h - y)
        return (x, y, w, h)

+    def _get_crop_zoom_keyframe_neighbors(self, frame_index):
+        """Return (prev_frame, prev_data, next_frame, next_data) for crop/zoom keyframes around frame_index."""
+        if not self.crop_zoom_keyframes:
+            return None, None, None, None
+        frames = sorted(self.crop_zoom_keyframes.keys())
+        prev_frame = None
+        next_frame = None
+        for f in frames:
+            if f <= frame_index:
+                prev_frame = f
+            if f >= frame_index and next_frame is None:
+                next_frame = f
+            if f > frame_index and prev_frame is not None and next_frame is not None:
+                break
+        prev_data = self.crop_zoom_keyframes.get(prev_frame) if prev_frame is not None else None
+        next_data = self.crop_zoom_keyframes.get(next_frame) if next_frame is not None else None
+        return prev_frame, prev_data, next_frame, next_data
+
+    def get_interpolated_crop_zoom(self, frame_index):
+        """Return (crop_rect, zoom_factor) for the given frame, using keyframe interpolation when available."""
+        if not self.crop_zoom_keyframes:
+            base_rect = self._get_base_crop_rect_for_frame(frame_index)
+            return base_rect, self.zoom_factor
+
+        prev_frame, prev_data, next_frame, next_data = self._get_crop_zoom_keyframe_neighbors(frame_index)
+
+        # Outside keyframe range: use nearest keyframe
+        if next_data is None and prev_data is not None:
+            return tuple(prev_data['crop_rect']), prev_data['zoom_factor']
+        if prev_data is None and next_data is not None:
+            return tuple(next_data['crop_rect']), next_data['zoom_factor']
+
+        # Exact keyframe
+        if prev_frame == next_frame and prev_data is not None:
+            return tuple(prev_data['crop_rect']), prev_data['zoom_factor']
+
+        # Interpolate between neighbors
+        if prev_data is None or next_data is None or prev_frame == next_frame:
+            base_rect = self._get_base_crop_rect_for_frame(frame_index)
+            return base_rect, self.zoom_factor
+
+        span = max(1, next_frame - prev_frame)
+        t = (frame_index - prev_frame) / span
+
+        px, py, pw, ph = prev_data['crop_rect']
+        nx, ny, nw, nh = next_data['crop_rect']
+        zx0 = prev_data['zoom_factor']
+        zx1 = next_data['zoom_factor']
+
+        def lerp(a, b, t_value):
+            return a + (b - a) * t_value
+
+        ix = int(round(lerp(px, nx, t)))
+        iy = int(round(lerp(py, ny, t)))
+        iw = int(round(lerp(pw, nw, t)))
+        ih = int(round(lerp(ph, nh, t)))
+        iz = lerp(zx0, zx1, t)
+
+        return (ix, iy, iw, ih), iz
+
+    def set_crop_zoom_keyframe_at_current_frame(self):
+        """Create or update crop/zoom keyframe at current frame from current crop/zoom settings."""
+        frame_index = getattr(self, 'current_frame', 0)
+        base_rect = self._get_base_crop_rect_for_frame(frame_index)
+        self.crop_zoom_keyframes[frame_index] = {
+            'crop_rect': base_rect,
+            'zoom_factor': self.zoom_factor,
+        }
+        self.crop_rect = base_rect
+        self.clear_transformation_cache()
+        self.display_needs_update = True
+        self.show_feedback_message("Crop/zoom keyframe set")
+
+    def delete_crop_zoom_keyframe_at_current_frame(self):
+        """Delete crop/zoom keyframe at current frame if it exists."""
+        frame_index = getattr(self, 'current_frame', 0)
+        if frame_index in self.crop_zoom_keyframes:
+            del self.crop_zoom_keyframes[frame_index]
+            self.clear_transformation_cache()
+            self.display_needs_update = True
+            self.show_feedback_message("Crop/zoom keyframe deleted")
+        else:
+            self.show_feedback_message("No crop/zoom keyframe at this frame")
+
+    def clone_previous_crop_zoom_keyframe_to_current(self):
+        """Clone the nearest previous crop/zoom keyframe to the current frame."""
+        if not self.crop_zoom_keyframes:
+            self.show_feedback_message("No crop/zoom keyframes to clone")
+            return
+        frame_index = getattr(self, 'current_frame', 0)
+        frames = sorted(self.crop_zoom_keyframes.keys())
+        prev_frame = None
+        for f in frames:
+            if f <= frame_index:
+                prev_frame = f
+            if f > frame_index:
+                break
+        if prev_frame is None:
+            self.show_feedback_message("No previous crop/zoom keyframe to clone")
+            return
+        prev_data = self.crop_zoom_keyframes[prev_frame]
+        base_rect = tuple(prev_data['crop_rect'])
+        zoom_value = prev_data['zoom_factor']
+        self.crop_zoom_keyframes[frame_index] = {
+            'crop_rect': base_rect,
+            'zoom_factor': zoom_value,
+        }
+        self.crop_rect = base_rect
+        self.zoom_factor = zoom_value
+        self.clear_transformation_cache()
+        self.display_needs_update = True
+        self.show_feedback_message("Cloned crop/zoom keyframe from previous")
+
+    def jump_to_previous_crop_zoom_keyframe(self):
+        """Seek to previous crop/zoom keyframe before current frame."""
+        if not self.crop_zoom_keyframes:
+            self.show_feedback_message("No crop/zoom keyframes")
+            return
+        frame_index = getattr(self, 'current_frame', 0)
+        frames = sorted(self.crop_zoom_keyframes.keys())
+        candidates = [f for f in frames if f < frame_index]
+        if not candidates:
+            self.show_feedback_message("No previous crop/zoom keyframe")
+            return
+        target = max(candidates)
+        data = self.crop_zoom_keyframes[target]
+        self.crop_rect = tuple(data['crop_rect'])
+        self.zoom_factor = data['zoom_factor']
+        self.clear_transformation_cache()
+        self.display_needs_update = True
+        self.seek_to_frame(target)
+
+    def jump_to_next_crop_zoom_keyframe(self):
+        """Seek to next crop/zoom keyframe after current frame."""
+        if not self.crop_zoom_keyframes:
+            self.show_feedback_message("No crop/zoom keyframes")
+            return
+        frame_index = getattr(self, 'current_frame', 0)
+        frames = sorted(self.crop_zoom_keyframes.keys())
+        candidates = [f for f in frames if f > frame_index]
+        if not candidates:
+            self.show_feedback_message("No next crop/zoom keyframe")
+            return
+        target = min(candidates)
+        data = self.crop_zoom_keyframes[target]
+        self.crop_rect = tuple(data['crop_rect'])
+        self.zoom_factor = data['zoom_factor']
+        self.clear_transformation_cache()
+        self.display_needs_update = True
+        self.seek_to_frame(target)
+
    
    def toggle_interesting_region_selection(self):
        """Toggle region selection mode for interesting point detection"""
@@ -1541,10 +1746,12 @@ class VideoEditor:

    def _get_display_params(self):
        """Unified display transform parameters for current frame in rotated space."""
-        eff_x, eff_y, eff_w, eff_h = self._get_effective_crop_rect_for_frame(getattr(self, 'current_frame', 0))
-        new_w = int(eff_w * self.zoom_factor)
-        new_h = int(eff_h * self.zoom_factor)
-        cropped_due_to_zoom = (self.zoom_factor != 1.0) and (new_w > self.window_width or new_h > self.window_height)
+        frame_number = getattr(self, 'current_frame', 0)
+        base_crop_rect, effective_zoom = self.get_interpolated_crop_zoom(frame_number)
+        eff_x, eff_y, eff_w, eff_h = self._get_effective_crop_rect_for_frame(frame_number, base_crop_rect=base_crop_rect)
+        new_w = int(eff_w * effective_zoom)
+        new_h = int(eff_h * effective_zoom)
+        cropped_due_to_zoom = (effective_zoom != 1.0) and (new_w > self.window_width or new_h > self.window_height)
        if cropped_due_to_zoom:
            offx_max = max(0, new_w - self.window_width)
            offy_max = max(0, new_h - self.window_height)
@@ -1576,13 +1783,15 @@ class VideoEditor:
    def _map_rotated_to_screen(self, rx, ry):
        """Map a point in ROTATED frame coords to canvas screen coords (post-crop)."""
        # Subtract crop offset in rotated space (EFFECTIVE crop at current frame)
-        cx, cy, cw, ch = self._get_effective_crop_rect_for_frame(getattr(self, 'current_frame', 0))
+        frame_number = getattr(self, 'current_frame', 0)
+        base_crop_rect, effective_zoom = self.get_interpolated_crop_zoom(frame_number)
+        cx, cy, cw, ch = self._get_effective_crop_rect_for_frame(frame_number, base_crop_rect=base_crop_rect)
        rx2 = rx - cx
        ry2 = ry - cy
        # Zoomed dimensions of cropped-rotated frame
-        new_w = int(cw * self.zoom_factor)
-        new_h = int(ch * self.zoom_factor)
-        cropped_due_to_zoom = (self.zoom_factor != 1.0) and (new_w > self.window_width or new_h > self.window_height)
+        new_w = int(cw * effective_zoom)
+        new_h = int(ch * effective_zoom)
+        cropped_due_to_zoom = (effective_zoom != 1.0) and (new_w > self.window_width or new_h > self.window_height)
        if cropped_due_to_zoom:
            offx_max = max(0, new_w - self.window_width)
            offy_max = max(0, new_h - self.window_height)
@@ -1591,8 +1800,8 @@ class VideoEditor:
        else:
            offx = 0
            offy = 0
-        zx = rx2 * self.zoom_factor - offx
-        zy = ry2 * self.zoom_factor - offy
+        zx = rx2 * effective_zoom - offx
+        zy = ry2 * effective_zoom - offy
        visible_w = new_w if not cropped_due_to_zoom else min(new_w, self.window_width)
        visible_h = new_h if not cropped_due_to_zoom else min(new_h, self.window_height)
        available_height = self.window_height - (0 if self.is_image_mode else self.TIMELINE_HEIGHT)
@@ -1616,8 +1825,10 @@ class VideoEditor:
        zx += params['offx']
        zy += params['offy']
        # Reverse zoom
-        rx = zx / max(1e-6, self.zoom_factor)
-        ry = zy / max(1e-6, self.zoom_factor)
+        frame_number = getattr(self, 'current_frame', 0)
+        _, effective_zoom = self.get_interpolated_crop_zoom(frame_number)
+        rx = zx / max(1e-6, effective_zoom)
+        ry = zy / max(1e-6, effective_zoom)
        # Unapply current EFFECTIVE crop to get PRE-crop rotated coords
        rx = rx + params['eff_x']
        ry = ry + params['eff_y']
@@ -2763,6 +2974,13 @@ class VideoEditor:
                canvas, (int(x), int(y)), (int(x + w), int(y + h)), (0, 255, 0), 2
            )

+        # Prepare effective crop/zoom for overlays
+        effective_frame_number = self.current_frame
+        base_crop_rect, effective_zoom = self.get_interpolated_crop_zoom(effective_frame_number)
+        eff_crop_x, eff_crop_y, eff_crop_w, eff_crop_h = self._get_effective_crop_rect_for_frame(
+            effective_frame_number, base_crop_rect=base_crop_rect
+        )
+
        # Add info overlay
        rotation_text = (
            f" | Rotation: {self.rotation_angle}°" if self.rotation_angle != 0 else ""
@@ -2795,9 +3013,9 @@ class VideoEditor:
            f" | Loop: ON" if self.looping_between_markers else ""
        )
        if self.is_image_mode:
-            info_text = f"Image | Zoom: {self.zoom_factor:.1f}x{rotation_text}{brightness_text}{contrast_text}{motion_text}{feature_text}{template_text}"
+            info_text = f"Image | Zoom: {effective_zoom:.1f}x{rotation_text}{brightness_text}{contrast_text}{motion_text}{feature_text}{template_text}"
        else:
-            info_text = f"Frame: {self.current_frame}/{self.total_frames} | Speed: {self.playback_speed:.1f}x | Zoom: {self.zoom_factor:.1f}x{seek_multiplier_text}{rotation_text}{brightness_text}{contrast_text}{motion_text}{feature_text}{template_text}{autorepeat_text} | {'Playing' if self.is_playing else 'Paused'}"
+            info_text = f"Frame: {self.current_frame}/{self.total_frames} | Speed: {self.playback_speed:.1f}x | Zoom: {effective_zoom:.1f}x{seek_multiplier_text}{rotation_text}{brightness_text}{contrast_text}{motion_text}{feature_text}{template_text}{autorepeat_text} | {'Playing' if self.is_playing else 'Paused'}"
        cv2.putText(
            canvas,
            info_text,
@@ -2837,8 +3055,8 @@ class VideoEditor:
            y_offset = 60

        # Add crop info
-        if self.crop_rect:
-            crop_text = f"Crop: {int(self.crop_rect[0])},{int(self.crop_rect[1])} {int(self.crop_rect[2])}x{int(self.crop_rect[3])}"
+        if self.crop_rect or self.crop_zoom_keyframes:
+            crop_text = f"Crop: {int(eff_crop_x)},{int(eff_crop_y)} {int(eff_crop_w)}x{int(eff_crop_h)}"
            cv2.putText(
                canvas,
                crop_text,
@@ -2932,6 +3150,22 @@ class VideoEditor:
        if self.template_selection_rect:
            x, y, w, h = self.template_selection_rect
            cv2.rectangle(canvas, (x, y), (x + w, y + h), (255, 0, 255), 2)  # Magenta for template selection
+
+        # Help overlay for keybindings
+        if self.show_help_overlay:
+            help_lines = [
+                "Crop/Zoom Keyframes:",
+                "  y - set/update keyframe at current frame",
+                "  U - delete keyframe at current frame",
+                "  i - clone previous keyframe to current frame",
+                "  [ / ] - prev/next keyframe",
+                "  ? - toggle this help",
+            ]
+            base_y = self.window_height - self.TIMELINE_HEIGHT - 10
+            for idx, text in enumerate(help_lines):
+                y_pos = base_y - 18 * (len(help_lines) - idx)
+                cv2.putText(canvas, text, (10, y_pos), cv2.FONT_HERSHEY_SIMPLEX, self.FONT_SCALE_SMALL, (255, 255, 255), 2)
+                cv2.putText(canvas, text, (10, y_pos), cv2.FONT_HERSHEY_SIMPLEX, self.FONT_SCALE_SMALL, (0, 0, 0), 1)
        
        # Draw previous and next tracking points with motion path visualization
        if not self.is_image_mode and self.tracking_points:
@@ -4537,6 +4771,7 @@ class VideoEditor:
                    self.crop_history.append(self.crop_rect)
                self.crop_rect = None
                self.zoom_factor = 1.0
+                self.crop_zoom_keyframes.clear()
                self.clear_transformation_cache()
                self.save_state()  # Save state when crop is cleared
            elif key == ord("C"):
@@ -4796,6 +5031,21 @@ class VideoEditor:
                    print("Render cancellation requested")
                else:
                    print("No render operation to cancel")
+            elif key == ord("?"):
+                self.show_help_overlay = not self.show_help_overlay
+                self.display_needs_update = True
+
+            # Crop/zoom keyframe controls
+            elif key == ord("y"):
+                self.set_crop_zoom_keyframe_at_current_frame()
+            elif key == ord("U"):
+                self.delete_crop_zoom_keyframe_at_current_frame()
+            elif key == ord("i"):
+                self.clone_previous_crop_zoom_keyframe_to_current()
+            elif key == ord("["):
+                self.jump_to_previous_crop_zoom_keyframe()
+            elif key == ord("]"):
+                self.jump_to_next_crop_zoom_keyframe()

            # Individual direction controls using shift combinations we can detect
            elif key == ord("J"):  # Shift+i - expand up