From 5f03abe6cda7bf3896424fcf7b935bae908a894d Mon Sep 17 00:00:00 2001 From: Adam Letts Date: Wed, 30 Mar 2022 16:36:29 -0400 Subject: [PATCH] Adds video_init_frame_continuity parameter. Removes some extraneous code from do_3d_step() --- Disco_Diffusion.ipynb | 46 ++++++++++++++++++++----------------------- disco.py | 46 ++++++++++++++++++++----------------------- 2 files changed, 42 insertions(+), 50 deletions(-) diff --git a/Disco_Diffusion.ipynb b/Disco_Diffusion.ipynb index b32bfe8..e6faf4a 100644 --- a/Disco_Diffusion.ipynb +++ b/Disco_Diffusion.ipynb @@ -1009,8 +1009,6 @@ "stop_on_next_loop = False # Make sure GPU memory doesn't get corrupted from cancelling the run mid-way through, allow a full frame to complete\n", "\n", "def do_3d_step(img_filepath, frame_num, midas_model, midas_transform):\n", - " global seed\n", - "\n", " if args.key_frames:\n", " translation_x = args.translation_x_series[frame_num]\n", " translation_y = args.translation_y_series[frame_num]\n", @@ -1027,25 +1025,19 @@ " f'rotation_3d_z: {rotation_3d_z}',\n", " )\n", "\n", - " if frame_num > 0:\n", - " seed = seed + 1\n", - " if resume_run and frame_num == start_frame:\n", - " img_filepath = batchFolder+f\"/{batch_name}({batchNum})_{start_frame-1:04}.png\"\n", - " else:\n", - " img_filepath = '/content/prevFrame.png' if is_colab else 'prevFrame.png'\n", - " trans_scale = 1.0/200.0\n", - " translate_xyz = [-translation_x*trans_scale, translation_y*trans_scale, -translation_z*trans_scale]\n", - " rotate_xyz_degrees = [rotation_3d_x, rotation_3d_y, rotation_3d_z]\n", - " print('translation:',translate_xyz)\n", - " print('rotation:',rotate_xyz_degrees)\n", - " rotate_xyz = [math.radians(rotate_xyz_degrees[0]), math.radians(rotate_xyz_degrees[1]), math.radians(rotate_xyz_degrees[2])]\n", - " rot_mat = p3dT.euler_angles_to_matrix(torch.tensor(rotate_xyz, device=device), \"XYZ\").unsqueeze(0)\n", - " print(\"rot_mat: \" + str(rot_mat))\n", - " next_step_pil = dxf.transform_image_3d(img_filepath, midas_model, midas_transform, DEVICE,\n", - " rot_mat, translate_xyz, args.near_plane, args.far_plane,\n", - " args.fov, padding_mode=args.padding_mode,\n", - " sampling_mode=args.sampling_mode, midas_weight=args.midas_weight)\n", - " return next_step_pil\n", + " trans_scale = 1.0/200.0\n", + " translate_xyz = [-translation_x*trans_scale, translation_y*trans_scale, -translation_z*trans_scale]\n", + " rotate_xyz_degrees = [rotation_3d_x, rotation_3d_y, rotation_3d_z]\n", + " print('translation:',translate_xyz)\n", + " print('rotation:',rotate_xyz_degrees)\n", + " rotate_xyz = [math.radians(rotate_xyz_degrees[0]), math.radians(rotate_xyz_degrees[1]), math.radians(rotate_xyz_degrees[2])]\n", + " rot_mat = p3dT.euler_angles_to_matrix(torch.tensor(rotate_xyz, device=device), \"XYZ\").unsqueeze(0)\n", + " print(\"rot_mat: \" + str(rot_mat))\n", + " next_step_pil = dxf.transform_image_3d(img_filepath, midas_model, midas_transform, DEVICE,\n", + " rot_mat, translate_xyz, args.near_plane, args.far_plane,\n", + " args.fov, padding_mode=args.padding_mode,\n", + " sampling_mode=args.sampling_mode, midas_weight=args.midas_weight)\n", + " return next_step_pil\n", "\n", "def do_run():\n", " seed = args.seed\n", @@ -1089,7 +1081,7 @@ " )\n", " \n", " if frame_num > 0:\n", - " seed = seed + 1 \n", + " seed += 1\n", " if resume_run and frame_num == start_frame:\n", " img_0 = cv2.imread(batchFolder+f\"/{batch_name}({batchNum})_{start_frame-1:04}.png\")\n", " else:\n", @@ -1119,7 +1111,7 @@ " if frame_num == 0:\n", " turbo_blend = False\n", " else:\n", - " seed = seed + 1 \n", + " seed += 1 \n", " if resume_run and frame_num == start_frame:\n", " img_filepath = batchFolder+f\"/{batch_name}({batchNum})_{start_frame-1:04}.png\"\n", " if turbo_mode and frame_num > turbo_preroll:\n", @@ -1163,7 +1155,8 @@ " skip_steps = args.calc_frames_skip_steps\n", "\n", " if args.animation_mode == \"Video Input\":\n", - " seed = seed + 1 \n", + " if not video_init_seed_continuity:\n", + " seed += 1\n", " init_image = f'{videoFramesFolder}/{frame_num+1:04}.jpg'\n", " init_scale = args.frames_scale\n", " skip_steps = args.calc_frames_skip_steps\n", @@ -1519,6 +1512,7 @@ " 'sampling_mode': sampling_mode,\n", " 'video_init_path':video_init_path,\n", " 'extract_nth_frame':extract_nth_frame,\n", + " 'video_init_seed_continuity': video_init_seed_continuity,\n", " 'turbo_mode':turbo_mode,\n", " 'turbo_steps':turbo_steps,\n", " 'turbo_preroll':turbo_preroll,\n", @@ -2520,7 +2514,8 @@ " video_init_path = \"/content/training.mp4\" #@param {type: 'string'}\n", "else:\n", " video_init_path = \"training.mp4\" #@param {type: 'string'}\n", - "extract_nth_frame = 2 #@param {type:\"number\"} \n", + "extract_nth_frame = 2 #@param {type: 'number'}\n", + "video_init_seed_continuity = True #@param {type: 'boolean'}\n", "\n", "if animation_mode == \"Video Input\":\n", " if is_colab:\n", @@ -3077,6 +3072,7 @@ " 'animation_mode': animation_mode,\n", " 'video_init_path': video_init_path,\n", " 'extract_nth_frame': extract_nth_frame,\n", + " 'video_init_seed_continuity': video_init_seed_continuity,\n", " 'key_frames': key_frames,\n", " 'max_frames': max_frames if animation_mode != \"None\" else 1,\n", " 'interp_spline': interp_spline,\n", diff --git a/disco.py b/disco.py index 16caaf0..155930f 100644 --- a/disco.py +++ b/disco.py @@ -916,8 +916,6 @@ def range_loss(input): stop_on_next_loop = False # Make sure GPU memory doesn't get corrupted from cancelling the run mid-way through, allow a full frame to complete def do_3d_step(img_filepath, frame_num, midas_model, midas_transform): - global seed - if args.key_frames: translation_x = args.translation_x_series[frame_num] translation_y = args.translation_y_series[frame_num] @@ -934,25 +932,19 @@ def do_3d_step(img_filepath, frame_num, midas_model, midas_transform): f'rotation_3d_z: {rotation_3d_z}', ) - if frame_num > 0: - seed = seed + 1 - if resume_run and frame_num == start_frame: - img_filepath = batchFolder+f"/{batch_name}({batchNum})_{start_frame-1:04}.png" - else: - img_filepath = '/content/prevFrame.png' if is_colab else 'prevFrame.png' - trans_scale = 1.0/200.0 - translate_xyz = [-translation_x*trans_scale, translation_y*trans_scale, -translation_z*trans_scale] - rotate_xyz_degrees = [rotation_3d_x, rotation_3d_y, rotation_3d_z] - print('translation:',translate_xyz) - print('rotation:',rotate_xyz_degrees) - rotate_xyz = [math.radians(rotate_xyz_degrees[0]), math.radians(rotate_xyz_degrees[1]), math.radians(rotate_xyz_degrees[2])] - rot_mat = p3dT.euler_angles_to_matrix(torch.tensor(rotate_xyz, device=device), "XYZ").unsqueeze(0) - print("rot_mat: " + str(rot_mat)) - next_step_pil = dxf.transform_image_3d(img_filepath, midas_model, midas_transform, DEVICE, - rot_mat, translate_xyz, args.near_plane, args.far_plane, - args.fov, padding_mode=args.padding_mode, - sampling_mode=args.sampling_mode, midas_weight=args.midas_weight) - return next_step_pil + trans_scale = 1.0/200.0 + translate_xyz = [-translation_x*trans_scale, translation_y*trans_scale, -translation_z*trans_scale] + rotate_xyz_degrees = [rotation_3d_x, rotation_3d_y, rotation_3d_z] + print('translation:',translate_xyz) + print('rotation:',rotate_xyz_degrees) + rotate_xyz = [math.radians(rotate_xyz_degrees[0]), math.radians(rotate_xyz_degrees[1]), math.radians(rotate_xyz_degrees[2])] + rot_mat = p3dT.euler_angles_to_matrix(torch.tensor(rotate_xyz, device=device), "XYZ").unsqueeze(0) + print("rot_mat: " + str(rot_mat)) + next_step_pil = dxf.transform_image_3d(img_filepath, midas_model, midas_transform, DEVICE, + rot_mat, translate_xyz, args.near_plane, args.far_plane, + args.fov, padding_mode=args.padding_mode, + sampling_mode=args.sampling_mode, midas_weight=args.midas_weight) + return next_step_pil def do_run(): seed = args.seed @@ -996,7 +988,7 @@ def do_run(): ) if frame_num > 0: - seed = seed + 1 + seed += 1 if resume_run and frame_num == start_frame: img_0 = cv2.imread(batchFolder+f"/{batch_name}({batchNum})_{start_frame-1:04}.png") else: @@ -1026,7 +1018,7 @@ def do_run(): if frame_num == 0: turbo_blend = False else: - seed = seed + 1 + seed += 1 if resume_run and frame_num == start_frame: img_filepath = batchFolder+f"/{batch_name}({batchNum})_{start_frame-1:04}.png" if turbo_mode and frame_num > turbo_preroll: @@ -1070,7 +1062,8 @@ def do_run(): skip_steps = args.calc_frames_skip_steps if args.animation_mode == "Video Input": - seed = seed + 1 + if not video_init_seed_continuity: + seed += 1 init_image = f'{videoFramesFolder}/{frame_num+1:04}.jpg' init_scale = args.frames_scale skip_steps = args.calc_frames_skip_steps @@ -1426,6 +1419,7 @@ def save_settings(): 'sampling_mode': sampling_mode, 'video_init_path':video_init_path, 'extract_nth_frame':extract_nth_frame, + 'video_init_seed_continuity': video_init_seed_continuity, 'turbo_mode':turbo_mode, 'turbo_steps':turbo_steps, 'turbo_preroll':turbo_preroll, @@ -2376,7 +2370,8 @@ if is_colab: video_init_path = "/content/training.mp4" #@param {type: 'string'} else: video_init_path = "training.mp4" #@param {type: 'string'} -extract_nth_frame = 2 #@param {type:"number"} +extract_nth_frame = 2 #@param {type: 'number'} +video_init_seed_continuity = True #@param {type: 'boolean'} if animation_mode == "Video Input": if is_colab: @@ -2900,6 +2895,7 @@ args = { 'animation_mode': animation_mode, 'video_init_path': video_init_path, 'extract_nth_frame': extract_nth_frame, + 'video_init_seed_continuity': video_init_seed_continuity, 'key_frames': key_frames, 'max_frames': max_frames if animation_mode != "None" else 1, 'interp_spline': interp_spline,