inference_video_OpenCV.py

import os
import cv2
import torch
import importlib.util
import sys
import argparse
import numpy as np
from torch.nn import functional as F
import warnings
import _thread
from queue import Queue, Empty
from model.pytorch_msssim import ssim_matlab
import time

warnings.filterwarnings("ignore")
loglevel = os.environ.get("RIFE_LOGLEVEL", "error")

os.environ["SDL_AUDIODRIVER"] = "dummy"
os.environ["ALSA_CONFIG_PATH"] = "/dev/null"

# =============== CUSTOM PROGRESS TRACKER ===============
def format_time(seconds: float) -> str:
    """Format seconds to H:MM:SS or MM:SS."""
    m, s = divmod(int(seconds), 60)
    h, m = divmod(m, 60)
    if h > 0:
        return f"{h}:{m:02}:{s:02}"
    else:
        return f"{m:02}:{s:02}"

class VideoProgressTracker:
    def __init__(self, total_frames):
        self.total_frames = total_frames
        self.current_frame = 0
        self.start_time = time.time()

    def update(self, frame_num=None):
        if frame_num is not None:
            self.current_frame = frame_num
        else:
            self.current_frame += 1
        self.display_progress()

    def display_progress(self):
        elapsed = time.time() - self.start_time
        progress_fraction = self.current_frame / self.total_frames if self.total_frames > 0 else 0
        fps = self.current_frame / elapsed if elapsed > 0 else 0
        eta = (elapsed / progress_fraction - elapsed) if progress_fraction > 0 else 0
        percent = int(progress_fraction * 100)

        info = (f"Interpolating: {percent:3}% "
               f"Frame {self.current_frame}/{self.total_frames} | "
               f"Elapsed: {format_time(elapsed)} | ETA: {format_time(eta)} | {fps:.1f} frame/s")

        sys.stdout.write('\r' + info)
        sys.stdout.flush()

    def finish(self):
        """Finalize progress display"""
        # Show 100% completion before finishing
        self.current_frame = self.total_frames
        self.display_progress()
        sys.stdout.write('\n')
        sys.stdout.flush()

def transferAudio(sourceVideo, targetVideo):
    import shutil
    import moviepy.editor
    tempAudioFileName = "./temp/audio.mkv"
    if True:
        if os.path.isdir("temp"):
            shutil.rmtree("temp")
        os.makedirs("temp")
        os.system('ffmpeg -hide_banner -loglevel {} -y -i "{}" -c:a copy -vn {}'.format(loglevel, sourceVideo, tempAudioFileName))

    targetNoAudio = os.path.splitext(targetVideo)[0] + "_noaudio" + os.path.splitext(targetVideo)[1]
    os.rename(targetVideo, targetNoAudio)
    os.system('ffmpeg -hide_banner -loglevel {} -y -i "{}" -i {} -c copy "{}"'.format(loglevel, targetNoAudio, tempAudioFileName, targetVideo))

    if os.path.getsize(targetVideo) == 0:
        tempAudioFileName = "./temp/audio.m4a"
        os.system('ffmpeg -hide_banner -loglevel {} -y -i "{}" -c:a aac -b:a 160k -vn {}'.format(loglevel, sourceVideo, tempAudioFileName))
        os.system('ffmpeg -hide_banner -loglevel {} -y -i "{}" -i {} -c copy "{}"'.format(loglevel, targetNoAudio, tempAudioFileName, targetVideo))
        if (os.path.getsize(targetVideo) == 0):
            os.rename(targetNoAudio, targetVideo)
            print("Audio transfer failed. Interpolated video will have no audio")
        else:
            print("Lossless audio transfer failed. Audio was transcoded to AAC (M4A) instead.")
            os.remove(targetNoAudio)
    else:
        os.remove(targetNoAudio)
    shutil.rmtree("temp")

parser = argparse.ArgumentParser(description='Interpolation for a pair of images')
parser.add_argument('--video', dest='video', type=str, default=None)
parser.add_argument('--output', dest='output', type=str, default=None)
parser.add_argument('--img', dest='img', type=str, default=None)
parser.add_argument('--montage', dest='montage', action='store_true', help='montage origin video')
parser.add_argument('--model', dest='modelDir', type=str, default='train_log', help='directory with trained model files')
parser.add_argument('--interpolation_factor', type=int, default=2, help="How many total frames between two input frames")
parser.add_argument('--mode', type=str, choices=['fast', 'slow'], default='slow', help="Interpolation mode: 'fast uses multi' (simple split) or 'slow uses exp' (recursive)")
parser.add_argument('--UHD', dest='UHD', action='store_true', help='support 4k video')
parser.add_argument('--scale', dest='scale', type=float, default=1.0, help='Try scale=0.5 for 4k video')
parser.add_argument('--skip', dest='skip', action='store_true', help='whether to remove static frames before processing')
parser.add_argument('--fps', dest='fps', type=int, default=None)
parser.add_argument('--png', dest='png', action='store_true', help='whether to vid_out png format vid_outs')
parser.add_argument('--ext', dest='ext', type=str, default='mp4', help='vid_out video extension')
args = parser.parse_args()

args.multi = args.interpolation_factor
assert (not args.video is None or not args.img is None)
if args.skip:
    print("skip flag is abandoned, please refer to issue #207.")
if args.UHD and args.scale==1.0:
    args.scale = 0.5
assert args.scale in [0.25, 0.5, 1.0, 2.0, 4.0]
if not args.img is None:
    args.png = True

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
torch.set_grad_enabled(False)
if torch.cuda.is_available():
    torch.backends.cudnn.enabled = True
    torch.backends.cudnn.benchmark = True

# Set Path to Practical-RIFE
args.modelDir = os.path.join("/content/Practical-RIFE", args.modelDir)
args.modelDir = os.path.abspath(args.modelDir)
sys.path.insert(0, args.modelDir)
model_path = os.path.join(args.modelDir, 'RIFE_HDv3.py')
spec = importlib.util.spec_from_file_location("RIFE_HDv3", model_path)
RIFE_module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(RIFE_module)
Model = RIFE_module.Model
model = Model()
if not hasattr(model, 'version'):
    model.version = 0
model.load_model(args.modelDir, -1)
model.eval()
model.device()
model_name = os.path.basename(os.path.abspath(args.modelDir))

if not args.video is None:
    videoCapture = cv2.VideoCapture(args.video)
    fps = videoCapture.get(cv2.CAP_PROP_FPS)
    tot_frame = videoCapture.get(cv2.CAP_PROP_FRAME_COUNT)
    videoCapture.release()
    if args.fps is None:
        fpsNotAssigned = True
        args.fps = fps * args.interpolation_factor
    else:
        fpsNotAssigned = False
    videoCapture = cv2.VideoCapture(args.video)
    success, lastframe = videoCapture.read()
    if success:
        lastframe = cv2.cvtColor(lastframe, cv2.COLOR_BGR2RGB)
    videogen = []
    while success:
        videogen.append(lastframe)
        success, lastframe = videoCapture.read()
        if success:
            lastframe = cv2.cvtColor(lastframe,cv2.COLOR_BGR2RGB)
    videoCapture.release()
    lastframe = videogen.pop(0)
    fourcc = cv2.VideoWriter_fourcc('m', 'p', '4', 'v')
    video_path_wo_ext, ext = os.path.splitext(args.video)
    file_name = os.path.basename(args.video)
    print(f"{tot_frame} frames in total, {fps}FPS to {fps * args.interpolation_factor}FPS\n")
    if args.png == False and fpsNotAssigned == True:
        pass
    else:
        pass
else:
    videogen = []
    for f in os.listdir(args.img):
        if 'png' in f:
            videogen.append(f)
    tot_frame = len(videogen)
    videogen.sort(key= lambda x:int(x[:-4]))
    lastframe = cv2.imread(os.path.join(args.img, videogen[0]), cv2.IMREAD_UNCHANGED)[:, :, ::-1].copy()
    videogen = videogen[1:]
    folder_path = os.path.abspath(args.img)
    print(f"{len(videogen)} PNG frames found.\n")
h, w, _ = lastframe.shape
vid_out_name = None
vid_out = None
if args.png:
    if not os.path.exists('vid_out'):
        os.mkdir('vid_out')
else:
    if args.output is not None:
        vid_out_name = args.output
    else:
        vid_out_name = '{}_{}X_{}fps.{}'.format(video_path_wo_ext, args.interpolation_factor, int(np.round(args.fps)), args.ext)
    vid_out = cv2.VideoWriter(vid_out_name, fourcc, args.fps, (w, h))

def clear_write_buffer(user_args, write_buffer):
    cnt = 0
    while True:
        item = write_buffer.get()
        if item is None:
            break
        if user_args.png:
            cv2.imwrite('vid_out/{:0>7d}.png'.format(cnt), item[:, :, ::-1])
            cnt += 1
        else:
            vid_out.write(item[:, :, ::-1])

def build_read_buffer(user_args, read_buffer, videogen):
    try:
        for frame in videogen:
            if not user_args.img is None:
                frame = cv2.imread(os.path.join(user_args.img, frame), cv2.IMREAD_UNCHANGED)[:, :, ::-1].copy()
            if user_args.montage:
                frame = frame[:, left: left + w]
            read_buffer.put(frame)
    except:
        pass
    read_buffer.put(None)

#OriginalLogic
def make_inference(I0, I1, n):
    global model
    if args.mode == "slow":
        if n == 1:
            middle = model.inference(I0, I1, 0.5, args.scale)
            return [middle]
        middle = model.inference(I0, I1, 0.5, args.scale)
        left_half = make_inference(I0, middle, n // 2)
        right_half = make_inference(middle, I1, n // 2)
        if n % 2:
            return [*left_half, middle, *right_half]
        else:
            return [*left_half, *right_half]
    else:
        outputs = []
        for i in range(n):
            timestep = (i + 1) / (n + 1)
            middle = model.inference(I0, I1, timestep, args.scale)
            outputs.append(middle)
        return outputs

def pad_image(img, padding):
    if any(padding):
        img = F.pad(img, padding)
    return img

if args.montage:
    left = w // 4
    w = w // 2
tmp = max(128, int(128 / args.scale))
ph = ((h - 1) // tmp + 1) * tmp
pw = ((w - 1) // tmp + 1) * tmp
padding = (0, pw - w, 0, ph - h)

# Initialize custom progress tracker
progress = VideoProgressTracker(int(tot_frame))

if args.montage:
    lastframe = lastframe[:, left: left + w]
write_buffer = Queue(maxsize=500)
read_buffer = Queue(maxsize=500)
_thread.start_new_thread(build_read_buffer, (args, read_buffer, videogen))
_thread.start_new_thread(clear_write_buffer, (args, write_buffer))

I1 = torch.from_numpy(np.transpose(lastframe, (2,0,1))).to(device, non_blocking=True).unsqueeze(0).float() / 255.
I1 = pad_image(I1, padding)
temp = None

while True:
    if temp is not None:
        frame = temp
        temp = None
    else:
        frame = read_buffer.get()
    if frame is None:
        break
    I0 = I1
    I1 = torch.from_numpy(np.transpose(frame, (2,0,1))).to(device, non_blocking=True).unsqueeze(0).float() / 255.
    I1 = pad_image(I1, padding)
    I0_small = F.interpolate(I0, (32, 32), mode='bilinear', align_corners=False)
    I1_small = F.interpolate(I1, (32, 32), mode='bilinear', align_corners=False)
    ssim = ssim_matlab(I0_small[:, :3], I1_small[:, :3])

    break_flag = False
    if ssim > 0.996:
        frame = read_buffer.get()
        if frame is None:
            break_flag = True
            frame = lastframe
        else:
            temp = frame
        I1 = torch.from_numpy(np.transpose(frame, (2,0,1))).to(device, non_blocking=True).unsqueeze(0).float() / 255.
        I1 = pad_image(I1, padding)
        I1 = model.inference(I0, I1, scale=args.scale)
        I1_small = F.interpolate(I1, (32, 32), mode='bilinear', align_corners=False)
        ssim = ssim_matlab(I0_small[:, :3], I1_small[:, :3])
        frame = (I1[0] * 255).byte().cpu().numpy().transpose(1, 2, 0)[:h, :w]
        
    if ssim < 0.2:
        output = []
        for i in range(args.interpolation_factor - 1):
            output.append(I0)
    else:
        output = make_inference(I0, I1, args.interpolation_factor - 1)

    if args.montage:
        write_buffer.put(np.concatenate((lastframe, lastframe), 1))
        for mid in output:
            mid = (((mid[0] * 255.).byte().cpu().numpy().transpose(1, 2, 0)))
            write_buffer.put(np.concatenate((lastframe, mid[:h, :w]), 1))
    else:
        write_buffer.put(lastframe)
        for mid in output:
            mid = (((mid[0] * 255.).byte().cpu().numpy().transpose(1, 2, 0)))
            write_buffer.put(mid[:h, :w])

    progress.update()
    lastframe = frame
    if break_flag:
        break

if args.montage:
    write_buffer.put(np.concatenate((lastframe, lastframe), 1))
else:
    write_buffer.put(lastframe)
progress.update()  # Add progress update for final frame
write_buffer.put(None)

while(not write_buffer.empty()):
    time.sleep(0.1)

progress.finish()

if not vid_out is None:
    vid_out.release()

if args.png == False and fpsNotAssigned == True and not args.video is None:
    try:
        transferAudio(args.video, vid_out_name)
    except:
        print("Audio transfer failed. Interpolated video will have no audio")
        targetNoAudio = os.path.splitext(vid_out_name)[0] + "_noaudio" + os.path.splitext(vid_out_name)[1]
        os.rename(targetNoAudio, vid_out_name)