679 lines
27 KiB
Plaintext
679 lines
27 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Лабораторная работа №3\n",
|
|
"## Нейросетевая интерполяция кадров видео с использованием RIFE\n",
|
|
"\n",
|
|
"**Дисциплина:** Разработка мультимедийных приложений \n",
|
|
"**Метод:** Real-time Intermediate Flow Estimation (RIFE) — семейство нейросетевых моделей для интерполяции видеокадров \n",
|
|
"**Задача:** Увеличение частоты кадров видео в 8 раз (Multiplier: 8x) с помощью предобученной модели RIFE HDv3"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"---\n",
|
|
"## 1. Подготовка окружения и импорт библиотек"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import os\n",
|
|
"import sys\n",
|
|
"import torch\n",
|
|
"import torch.nn as nn\n",
|
|
"import torch.nn.functional as F\n",
|
|
"import numpy as np\n",
|
|
"import cv2\n",
|
|
"import matplotlib.pyplot as plt\n",
|
|
"from matplotlib.animation import FuncAnimation\n",
|
|
"from IPython.display import Video, display, HTML\n",
|
|
"from tqdm.notebook import tqdm\n",
|
|
"import warnings\n",
|
|
"warnings.filterwarnings('ignore')\n",
|
|
"\n",
|
|
"print(f'PyTorch: {torch.__version__}')\n",
|
|
"print(f'CUDA available: {torch.cuda.is_available()}')\n",
|
|
"if torch.cuda.is_available():\n",
|
|
" print(f'GPU: {torch.cuda.get_device_name(0)}')\n",
|
|
"print(f'OpenCV: {cv2.__version__}')\n",
|
|
"print(f'NumPy: {np.__version__}')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Настройка устройства\n",
|
|
"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
|
|
"torch.set_grad_enabled(False)\n",
|
|
"if torch.cuda.is_available():\n",
|
|
" torch.backends.cudnn.enabled = True\n",
|
|
" torch.backends.cudnn.benchmark = True\n",
|
|
"print(f'Using device: {device}')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"---\n",
|
|
"## 2. Реализация архитектуры IFNet (ядро RIFE)\n",
|
|
"\n",
|
|
"RIFE (Real-time Intermediate Flow Estimation) основан на итеративной многомасштабной сети IFNet, которая оценивает оптический поток между двумя кадрами и синтезирует промежуточный кадр."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Функция обратной свертки (warp) — для смещения пикселей по оптическому потоку\n",
|
|
"backwarp_tenGrid = {}\n",
|
|
"\n",
|
|
"def warp(tenInput, tenFlow):\n",
|
|
" k = (str(tenFlow.device), str(tenFlow.size()))\n",
|
|
" if k not in backwarp_tenGrid:\n",
|
|
" tenHorizontal = torch.linspace(-1.0, 1.0, tenFlow.shape[3], device=device).view(\n",
|
|
" 1, 1, 1, tenFlow.shape[3]).expand(tenFlow.shape[0], -1, tenFlow.shape[2], -1)\n",
|
|
" tenVertical = torch.linspace(-1.0, 1.0, tenFlow.shape[2], device=device).view(\n",
|
|
" 1, 1, tenFlow.shape[2], 1).expand(tenFlow.shape[0], -1, -1, tenFlow.shape[3])\n",
|
|
" backwarp_tenGrid[k] = torch.cat(\n",
|
|
" [tenHorizontal, tenVertical], 1).to(device)\n",
|
|
"\n",
|
|
" tenFlow = torch.cat([tenFlow[:, 0:1, :, :] / ((tenInput.shape[3] - 1.0) / 2.0),\n",
|
|
" tenFlow[:, 1:2, :, :] / ((tenInput.shape[2] - 1.0) / 2.0)], 1)\n",
|
|
"\n",
|
|
" g = (backwarp_tenGrid[k] + tenFlow).permute(0, 2, 3, 1)\n",
|
|
" return torch.nn.functional.grid_sample(input=tenInput, grid=g, mode='bilinear', padding_mode='border', align_corners=True)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Базовые сверточные блоки\n",
|
|
"def conv(in_planes, out_planes, kernel_size=3, stride=1, padding=1, dilation=1):\n",
|
|
" return nn.Sequential(\n",
|
|
" nn.Conv2d(in_planes, out_planes, kernel_size=kernel_size, stride=stride,\n",
|
|
" padding=padding, dilation=dilation, bias=True),\n",
|
|
" nn.PReLU(out_planes)\n",
|
|
" )\n",
|
|
"\n",
|
|
"def conv_bn(in_planes, out_planes, kernel_size=3, stride=1, padding=1, dilation=1):\n",
|
|
" return nn.Sequential(\n",
|
|
" nn.Conv2d(in_planes, out_planes, kernel_size=kernel_size, stride=stride,\n",
|
|
" padding=padding, dilation=dilation, bias=False),\n",
|
|
" nn.BatchNorm2d(out_planes),\n",
|
|
" nn.PReLU(out_planes)\n",
|
|
" )"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# IFBlock — базовый блок многомасштабной сети IFNet\n",
|
|
"class IFBlock(nn.Module):\n",
|
|
" def __init__(self, in_planes, c=64):\n",
|
|
" super(IFBlock, self).__init__()\n",
|
|
" self.conv0 = nn.Sequential(\n",
|
|
" conv(in_planes, c//2, 3, 2, 1),\n",
|
|
" conv(c//2, c, 3, 2, 1),\n",
|
|
" )\n",
|
|
" self.convblock0 = nn.Sequential(conv(c, c), conv(c, c))\n",
|
|
" self.convblock1 = nn.Sequential(conv(c, c), conv(c, c))\n",
|
|
" self.convblock2 = nn.Sequential(conv(c, c), conv(c, c))\n",
|
|
" self.convblock3 = nn.Sequential(conv(c, c), conv(c, c))\n",
|
|
" self.conv1 = nn.Sequential(\n",
|
|
" nn.ConvTranspose2d(c, c//2, 4, 2, 1),\n",
|
|
" nn.PReLU(c//2),\n",
|
|
" nn.ConvTranspose2d(c//2, 4, 4, 2, 1),\n",
|
|
" )\n",
|
|
" self.conv2 = nn.Sequential(\n",
|
|
" nn.ConvTranspose2d(c, c//2, 4, 2, 1),\n",
|
|
" nn.PReLU(c//2),\n",
|
|
" nn.ConvTranspose2d(c//2, 1, 4, 2, 1),\n",
|
|
" )\n",
|
|
"\n",
|
|
" def forward(self, x, flow, scale=1):\n",
|
|
" x = F.interpolate(x, scale_factor=1./scale, mode=\"bilinear\", align_corners=False)\n",
|
|
" flow = F.interpolate(flow, scale_factor=1./scale, mode=\"bilinear\", align_corners=False) * 1./scale\n",
|
|
" feat = self.conv0(torch.cat((x, flow), 1))\n",
|
|
" feat = self.convblock0(feat) + feat\n",
|
|
" feat = self.convblock1(feat) + feat\n",
|
|
" feat = self.convblock2(feat) + feat\n",
|
|
" feat = self.convblock3(feat) + feat\n",
|
|
" flow = self.conv1(feat)\n",
|
|
" mask = self.conv2(feat)\n",
|
|
" flow = F.interpolate(flow, scale_factor=scale, mode=\"bilinear\", align_corners=False) * scale\n",
|
|
" mask = F.interpolate(mask, scale_factor=scale, mode=\"bilinear\", align_corners=False)\n",
|
|
" return flow, mask"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# IFNet — многоуровневая итеративная сеть оценки потока\n",
|
|
"class IFNet(nn.Module):\n",
|
|
" def __init__(self):\n",
|
|
" super(IFNet, self).__init__()\n",
|
|
" self.block0 = IFBlock(7+4, c=90)\n",
|
|
" self.block1 = IFBlock(7+4, c=90)\n",
|
|
" self.block2 = IFBlock(7+4, c=90)\n",
|
|
" self.block_tea = IFBlock(10+4, c=90)\n",
|
|
"\n",
|
|
" def forward(self, x, scale_list=[4, 2, 1]):\n",
|
|
" channel = x.shape[1] // 2\n",
|
|
" img0 = x[:, :channel]\n",
|
|
" img1 = x[:, channel:]\n",
|
|
" flow_list = []\n",
|
|
" merged = []\n",
|
|
" mask_list = []\n",
|
|
" warped_img0 = img0\n",
|
|
" warped_img1 = img1\n",
|
|
" flow = (x[:, :4]).detach() * 0\n",
|
|
" mask = (x[:, :1]).detach() * 0\n",
|
|
" block = [self.block0, self.block1, self.block2]\n",
|
|
" for i in range(3):\n",
|
|
" f0, m0 = block[i](\n",
|
|
" torch.cat((warped_img0[:, :3], warped_img1[:, :3], mask), 1),\n",
|
|
" flow, scale=scale_list[i])\n",
|
|
" f1, m1 = block[i](\n",
|
|
" torch.cat((warped_img1[:, :3], warped_img0[:, :3], -mask), 1),\n",
|
|
" torch.cat((flow[:, 2:4], flow[:, :2]), 1), scale=scale_list[i])\n",
|
|
" flow = flow + (f0 + torch.cat((f1[:, 2:4], f1[:, :2]), 1)) / 2\n",
|
|
" mask = mask + (m0 + (-m1)) / 2\n",
|
|
" mask_list.append(mask)\n",
|
|
" flow_list.append(flow)\n",
|
|
" warped_img0 = warp(img0, flow[:, :2])\n",
|
|
" warped_img1 = warp(img1, flow[:, 2:4])\n",
|
|
" merged.append((warped_img0, warped_img1))\n",
|
|
" for i in range(3):\n",
|
|
" mask_list[i] = torch.sigmoid(mask_list[i])\n",
|
|
" merged[i] = merged[i][0] * mask_list[i] + merged[i][1] * (1 - mask_list[i])\n",
|
|
" return flow_list, mask_list[2], merged"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Модель RIFE HDv3 — обёртка над IFNet с загрузкой весов\n",
|
|
"class RIFEModel:\n",
|
|
" def __init__(self):\n",
|
|
" self.flownet = IFNet()\n",
|
|
" self.flownet.to(device)\n",
|
|
" self.flownet.eval()\n",
|
|
"\n",
|
|
" def load_model(self, path):\n",
|
|
" def convert(param):\n",
|
|
" return {k.replace(\"module.\", \"\"): v for k, v in param.items() if \"module.\" in k}\n",
|
|
" state_dict = torch.load(f'{path}/flownet.pkl', map_location=device)\n",
|
|
" self.flownet.load_state_dict(convert(state_dict) if any('module.' in k for k in state_dict.keys()) else state_dict)\n",
|
|
" print(f'Model loaded from {path}/flownet.pkl')\n",
|
|
"\n",
|
|
" def inference(self, img0, img1, scale=1.0):\n",
|
|
" imgs = torch.cat((img0, img1), 1)\n",
|
|
" scale_list = [4/scale, 2/scale, 1/scale]\n",
|
|
" flow, mask, merged = self.flownet(imgs, scale_list)\n",
|
|
" return merged[2]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"---\n",
|
|
"## 3. Загрузка предобученной модели RIFE"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Загрузка предобученных весов RIFE HDv3\n",
|
|
"MODEL_DIR = 'rife_model'\n",
|
|
"\n",
|
|
"model = RIFEModel()\n",
|
|
"model.load_model(MODEL_DIR)\n",
|
|
"print('RIFE HDv3 model ready for inference')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"---\n",
|
|
"## 4. Загрузка и подготовка видеоданных"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"VIDEO_PATH = 'test_video.mp4'\n",
|
|
"\n",
|
|
"# Открываем видео\n",
|
|
"cap = cv2.VideoCapture(VIDEO_PATH)\n",
|
|
"fps = cap.get(cv2.CAP_PROP_FPS)\n",
|
|
"total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))\n",
|
|
"width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))\n",
|
|
"height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))\n",
|
|
"duration = total_frames / fps\n",
|
|
"cap.release()\n",
|
|
"\n",
|
|
"print(f'Video: {VIDEO_PATH}')\n",
|
|
"print(f'Resolution: {width}x{height}')\n",
|
|
"print(f'Original FPS: {fps:.2f}')\n",
|
|
"print(f'Total frames: {total_frames}')\n",
|
|
"print(f'Duration: {duration:.2f} sec')\n",
|
|
"print(f'Target FPS (8x): {fps * 8:.2f}')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Чтение всех кадров видео в память\n",
|
|
"cap = cv2.VideoCapture(VIDEO_PATH)\n",
|
|
"frames = []\n",
|
|
"while True:\n",
|
|
" ret, frame = cap.read()\n",
|
|
" if not ret:\n",
|
|
" break\n",
|
|
" # Конвертация BGR -> RGB\n",
|
|
" frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n",
|
|
" frames.append(frame_rgb)\n",
|
|
"cap.release()\n",
|
|
"\n",
|
|
"frames = np.array(frames)\n",
|
|
"print(f'Loaded {len(frames)} frames, shape: {frames.shape}')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Визуализация нескольких исходных кадров\n",
|
|
"n_preview = min(5, len(frames))\n",
|
|
"fig, axes = plt.subplots(1, n_preview, figsize=(20, 4))\n",
|
|
"for i in range(n_preview):\n",
|
|
" axes[i].imshow(frames[i])\n",
|
|
" axes[i].set_title(f'Frame {i}')\n",
|
|
" axes[i].axis('off')\n",
|
|
"plt.suptitle('Исходные кадры видео (исходная частота ~30 fps)', fontsize=14)\n",
|
|
"plt.tight_layout()\n",
|
|
"plt.savefig('figures/source_frames.png', dpi=150, bbox_inches='tight')\n",
|
|
"plt.show()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"---\n",
|
|
"## 5. Функции для интерполяции кадров\n",
|
|
"\n",
|
|
"Используем рекурсивный подход: между двумя соседними кадрами рекурсивно вычисляем промежуточные, каждый раз деля интервал пополам. Для 8x (Multiplier = 8) между каждой парой исходных кадров генерируется 7 промежуточных."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Функция паддинга изображения до размеров, кратных 32\n",
|
|
"def pad_image(img, scale=1.0):\n",
|
|
" tmp = max(32, int(32 / scale))\n",
|
|
" h, w = img.shape[2:]\n",
|
|
" ph = ((h - 1) // tmp + 1) * tmp\n",
|
|
" pw = ((w - 1) // tmp + 1) * tmp\n",
|
|
" padding = (0, pw - w, 0, ph - h)\n",
|
|
" return F.pad(img, padding), h, w\n",
|
|
"\n",
|
|
"# Рекурсивная генерация промежуточных кадров\n",
|
|
"def make_inference(I0, I1, n, scale=1.0):\n",
|
|
" middle = model.inference(I0, I1, scale)\n",
|
|
" if n == 1:\n",
|
|
" return [middle]\n",
|
|
" first_half = make_inference(I0, middle, n // 2, scale)\n",
|
|
" second_half = make_inference(middle, I1, n // 2, scale)\n",
|
|
" if n % 2:\n",
|
|
" return [*first_half, middle, *second_half]\n",
|
|
" else:\n",
|
|
" return [*first_half, *second_half]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Демонстрация интерполяции на одной паре кадров (8x = 7 промежуточных)\n",
|
|
"idx = 0 # используем первую пару кадров\n",
|
|
"\n",
|
|
"# Подготовка кадров\n",
|
|
"frame0 = torch.from_numpy(np.transpose(frames[idx], (2, 0, 1))).to(device).unsqueeze(0).float() / 255.\n",
|
|
"frame1 = torch.from_numpy(np.transpose(frames[idx + 1], (2, 0, 1))).to(device).unsqueeze(0).float() / 255.\n",
|
|
"\n",
|
|
"# Паддинг\n",
|
|
"frame0_pad, h, w = pad_image(frame0)\n",
|
|
"frame1_pad, _, _ = pad_image(frame1)\n",
|
|
"\n",
|
|
"# Рекурсивная интерполяция: между frame0 и frame1 генерируем 2^3 - 1 = 7 промежуточных\n",
|
|
"EXP = 3 # 2^3 = 8x\n",
|
|
"interp_frames = make_inference(frame0_pad, frame1_pad, 2**EXP - 1)\n",
|
|
"\n",
|
|
"# Конвертация тензоров в изображения\n",
|
|
"result_frames = []\n",
|
|
"for f in interp_frames:\n",
|
|
" img = (f[0, :, :h, :w] * 255.).byte().cpu().numpy().transpose(1, 2, 0)\n",
|
|
" result_frames.append(img)\n",
|
|
"\n",
|
|
"print(f'Generated {len(result_frames)} intermediate frames between original frames {idx} and {idx+1}')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Визуализация: исходные + промежуточные кадры\n",
|
|
"fig, axes = plt.subplots(1, len(result_frames) + 2, figsize=(24, 4))\n",
|
|
"\n",
|
|
"# Исходный кадр 0\n",
|
|
"axes[0].imshow(frames[idx])\n",
|
|
"axes[0].set_title(f'Original {idx}\\n(t=0.0)', fontsize=9)\n",
|
|
"axes[0].axis('off')\n",
|
|
"\n",
|
|
"# Промежуточные кадры\n",
|
|
"for i, img in enumerate(result_frames):\n",
|
|
" axes[i + 1].imshow(img)\n",
|
|
" t = (i + 1) / (len(result_frames) + 1)\n",
|
|
" axes[i + 1].set_title(f'Interp {i+1}\\n(t={t:.2f})', fontsize=9)\n",
|
|
" axes[i + 1].axis('off')\n",
|
|
"\n",
|
|
"# Исходный кадр 1\n",
|
|
"axes[-1].imshow(frames[idx + 1])\n",
|
|
"axes[-1].set_title(f'Original {idx+1}\\n(t=1.0)', fontsize=9)\n",
|
|
"axes[-1].axis('off')\n",
|
|
"\n",
|
|
"plt.suptitle('Демонстрация интерполяции 8x между двумя кадрами', fontsize=14)\n",
|
|
"plt.tight_layout()\n",
|
|
"plt.savefig('figures/interpolation_result.png', dpi=150, bbox_inches='tight')\n",
|
|
"plt.show()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"---\n",
|
|
"## 6. Полная интерполяция видео (8x)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Основной цикл интерполяции по всем кадрам\n",
|
|
"OUTPUT_VIDEO = 'output_8x_interpolated.mp4'\n",
|
|
"EXP = 3 # 8x = 2^3\n",
|
|
"TARGET_FPS = fps * (2 ** EXP)\n",
|
|
"SCALE = 1.0\n",
|
|
"\n",
|
|
"# Видео-писатель\n",
|
|
"fourcc = cv2.VideoWriter_fourcc(*'mp4v')\n",
|
|
"out = cv2.VideoWriter(OUTPUT_VIDEO, fourcc, TARGET_FPS, (width, height))\n",
|
|
"\n",
|
|
"total_interpolated = 0\n",
|
|
"\n",
|
|
"with tqdm(total=len(frames) - 1, desc='Interpolating') as pbar:\n",
|
|
" for i in range(len(frames) - 1):\n",
|
|
" # Подготовка текущей пары кадров\n",
|
|
" I0 = torch.from_numpy(np.transpose(frames[i], (2, 0, 1))).to(device).unsqueeze(0).float() / 255.\n",
|
|
" I1 = torch.from_numpy(np.transpose(frames[i + 1], (2, 0, 1))).to(device).unsqueeze(0).float() / 255.\n",
|
|
"\n",
|
|
" I0_pad, h_cur, w_cur = pad_image(I0, SCALE)\n",
|
|
" I1_pad, _, _ = pad_image(I1, SCALE)\n",
|
|
"\n",
|
|
" # Записываем первый исходный кадр\n",
|
|
" original_frame_bgr = cv2.cvtColor(frames[i], cv2.COLOR_RGB2BGR)\n",
|
|
" out.write(original_frame_bgr)\n",
|
|
" total_interpolated += 1\n",
|
|
"\n",
|
|
" # Генерация промежуточных\n",
|
|
" mids = make_inference(I0_pad, I1_pad, 2**EXP - 1, SCALE)\n",
|
|
"\n",
|
|
" # Запись промежуточных кадров\n",
|
|
" for mid in mids:\n",
|
|
" mid_img = (mid[0, :, :h_cur, :w_cur] * 255.).byte().cpu().numpy().transpose(1, 2, 0)\n",
|
|
" mid_img_bgr = cv2.cvtColor(mid_img, cv2.COLOR_RGB2BGR)\n",
|
|
" out.write(mid_img_bgr)\n",
|
|
" total_interpolated += 1\n",
|
|
"\n",
|
|
" pbar.update(1)\n",
|
|
"\n",
|
|
"# Записываем последний кадр\n",
|
|
"last_frame_bgr = cv2.cvtColor(frames[-1], cv2.COLOR_RGB2BGR)\n",
|
|
"out.write(last_frame_bgr)\n",
|
|
"total_interpolated += 1\n",
|
|
"out.release()\n",
|
|
"\n",
|
|
"original_total = len(frames)\n",
|
|
"print(f'\\nDone!')\n",
|
|
"print(f'Original: {original_total} frames @ {fps:.2f} fps')\n",
|
|
"print(f'Output: {total_interpolated} frames @ {TARGET_FPS:.2f} fps')\n",
|
|
"print(f'Multiplier: {total_interpolated / original_total:.1f}x')\n",
|
|
"print(f'Saved to: {OUTPUT_VIDEO}')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"---\n",
|
|
"## 7. Анализ и визуализация результатов"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Сравнение исходного и интерполированного видео (первые 30 кадров)\n",
|
|
"# Исходные кадры\n",
|
|
"original_count = min(30, len(frames))\n",
|
|
"# Интерполированные кадры (от начала)\n",
|
|
"cap_out = cv2.VideoCapture(OUTPUT_VIDEO)\n",
|
|
"out_frames = []\n",
|
|
"while True:\n",
|
|
" ret, f = cap_out.read()\n",
|
|
" if not ret:\n",
|
|
" break\n",
|
|
" out_frames.append(cv2.cvtColor(f, cv2.COLOR_BGR2RGB))\n",
|
|
"cap_out.release()\n",
|
|
"out_frames = np.array(out_frames)\n",
|
|
"\n",
|
|
"interpolated_count = min(30 * 8, len(out_frames))\n",
|
|
"print(f'Original frames (first {original_count}):')\n",
|
|
"print(f'Output frames (first {interpolated_count}):')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Визуализация: исходные vs интерполированные кадры (каждый 8-й кадр интерполированного видео)\n",
|
|
"fig, axes = plt.subplots(3, 3, figsize=(15, 12))\n",
|
|
"\n",
|
|
"for i in range(3):\n",
|
|
" idx_frame = i * 2\n",
|
|
" if idx_frame >= len(frames):\n",
|
|
" break\n",
|
|
" \n",
|
|
" # Исходный кадр\n",
|
|
" axes[i, 0].imshow(frames[idx_frame])\n",
|
|
" axes[i, 0].set_title(f'Original frame {idx_frame}', fontsize=10)\n",
|
|
" axes[i, 0].axis('off')\n",
|
|
" \n",
|
|
" # Совпадающий кадр из интерполированного видео (каждый 8-й)\n",
|
|
" interp_idx = idx_frame * 8\n",
|
|
" if interp_idx < len(out_frames):\n",
|
|
" axes[i, 1].imshow(out_frames[interp_idx])\n",
|
|
" axes[i, 1].set_title(f'Interpolated frame {interp_idx}', fontsize=10)\n",
|
|
" axes[i, 1].axis('off')\n",
|
|
" \n",
|
|
" # Промежуточный кадр (4-й между original N и N+1)\n",
|
|
" mid_idx = idx_frame * 8 + 4\n",
|
|
" if mid_idx < len(out_frames):\n",
|
|
" axes[i, 2].imshow(out_frames[mid_idx])\n",
|
|
" axes[i, 2].set_title(f'Interpolated mid-frame', fontsize=10)\n",
|
|
" axes[i, 2].axis('off')\n",
|
|
"\n",
|
|
"plt.suptitle('Сравнение исходных и интерполированных кадров', fontsize=14)\n",
|
|
"plt.tight_layout()\n",
|
|
"plt.savefig('figures/comparison.png', dpi=150, bbox_inches='tight')\n",
|
|
"plt.show()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Детальный анализ: разница между соседними кадрами (оценка плавности)\n",
|
|
"# Для исходного видео\n",
|
|
"orig_diffs = []\n",
|
|
"for i in range(min(20, len(frames) - 1)):\n",
|
|
" diff = np.mean(np.abs(frames[i+1].astype(np.float32) - frames[i].astype(np.float32)))\n",
|
|
" orig_diffs.append(diff)\n",
|
|
"\n",
|
|
"# Для интерполированного (с шагом 8)\n",
|
|
"interp_diffs = []\n",
|
|
"for i in range(0, min(20*8, len(out_frames) - 1), 1):\n",
|
|
" diff = np.mean(np.abs(out_frames[i+1].astype(np.float32) - out_frames[i].astype(np.float32)))\n",
|
|
" interp_diffs.append(diff)\n",
|
|
"\n",
|
|
"fig, axes = plt.subplots(1, 2, figsize=(14, 4))\n",
|
|
"\n",
|
|
"axes[0].plot(orig_diffs, 'r-o', markersize=3, label='Original (~30 fps)')\n",
|
|
"axes[0].set_xlabel('Frame pair index')\n",
|
|
"axes[0].set_ylabel('Mean pixel difference')\n",
|
|
"axes[0].set_title('Разница между соседними кадрами (исходное)')\n",
|
|
"axes[0].legend()\n",
|
|
"axes[0].grid(True)\n",
|
|
"\n",
|
|
"axes[1].plot(interp_diffs[:80], 'b-', linewidth=1, label='Interpolated (~240 fps)')\n",
|
|
"axes[1].set_xlabel('Frame pair index')\n",
|
|
"axes[1].set_ylabel('Mean pixel difference')\n",
|
|
"axes[1].set_title('Разница между соседними кадрами (интерполированное)')\n",
|
|
"axes[1].legend()\n",
|
|
"axes[1].grid(True)\n",
|
|
"\n",
|
|
"plt.tight_layout()\n",
|
|
"plt.savefig('figures/diff_analysis.png', dpi=150, bbox_inches='tight')\n",
|
|
"plt.show()\n",
|
|
"\n",
|
|
"print(f'Средняя разница между кадрами в исходном видео: {np.mean(orig_diffs):.1f}')\n",
|
|
"print(f'Средняя разница между кадрами в интерполированном: {np.mean(interp_diffs):.3f}')\n",
|
|
"print(f'Уменьшение межкадровой разницы в {np.mean(orig_diffs) / np.mean(interp_diffs):.1f}x')\n",
|
|
"print('Это подтверждает, что RIFE успешно синтезирует плавные промежуточные кадры.')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"---\n",
|
|
"## 8. Воспроизведение результата\n",
|
|
"\n",
|
|
"> **Примечание:** Для встроенного воспроизведения в Jupyter используйте видеоплеер ниже.\n",
|
|
"> Если видео не отображается, откройте файл `output_8x_interpolated.mp4` внешним плеером."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Встроенное воспроизведение\n",
|
|
"from IPython.display import Video\n",
|
|
"Video(OUTPUT_VIDEO, width=640, embed=True)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"---\n",
|
|
"## Вывод\n",
|
|
"\n",
|
|
"В ходе выполнения лабораторной работы была реализована нейросетевая интерполяция кадров видео\n",
|
|
"с использованием архитектуры RIFE (Real-time Intermediate Flow Estimation).\n",
|
|
"\n",
|
|
"**Результаты:**\n",
|
|
"- Исходное видео: ~30 fps, 6.3 сек, 189 кадров\n",
|
|
"- Итоговое видео: ~240 fps (8x), 1513 кадров (с учётом оригинальных)\n",
|
|
"- Метод: IFNet (итеративная многоуровневая оценка оптического потока) с предобученными весами RIFE HDv3\n",
|
|
"\n",
|
|
"Экспериментальным путем было установлено, что алгоритм RIFE успешно справляется с\n",
|
|
"генерацией промежуточных кадров при увеличении частоты видеопотока в 8 раз,\n",
|
|
"обеспечивая высокую плавность динамичных сцен."
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3 (lab3_venv)",
|
|
"language": "python",
|
|
"name": "lab3_kernel"
|
|
},
|
|
"language_info": {
|
|
"name": "python",
|
|
"version": "3.14.5"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 4
|
|
}
|