Add NVIDIA support for FFmpeg in Docker and enhance chat rendering functionality
All checks were successful
Publish Twitch Archive Container / publish (push) Successful in 7m36s

- Introduced a new docker-compose.nvidia.yml for NVIDIA GPU support.
- Updated dockerstart.bat to allow optional NVIDIA runtime.
- Enhanced ContentDownloader to manage chat rendering status and font settings.
- Improved hardware acceleration detection in utils.py.
- Added tests for hardware acceleration and chat rendering behavior.

Co-authored-by: Copilot <copilot@github.com>
This commit is contained in:
MaddoScientisto 2026-04-25 12:28:59 +02:00
commit ec44981a9d
8 changed files with 226 additions and 18 deletions

View file

@ -86,6 +86,17 @@ Or use the Windows helper:
That batch launcher mirrors the old pattern and expands to a compose `run` command, so you can test any streamer manually.
If the host has the NVIDIA Container Toolkit installed and you want FFmpeg/NVENC inside the container, use the optional NVIDIA override:
```powershell
.\dockerrebuild.bat
.\dockerstart.bat --nvidia vinesauce --verbose
```
The image built by `.\dockerrebuild.bat` already includes the NVIDIA-capable FFmpeg/container toolchain. The optional [docker-compose.nvidia.yml](docker-compose.nvidia.yml) layer is only for runtime GPU passthrough: it requests `gpus: all` and sets `NVIDIA_VISIBLE_DEVICES` plus `NVIDIA_DRIVER_CAPABILITIES=compute,utility,video` for the container.
On systems without NVIDIA support, keep using the normal command without `--nvidia`; the image still builds the same way, it just runs without GPU passthrough.
### Healthcheck and smoke tests
- Container healthcheck command: `python twitch-archive.py --healthcheck -u vinesauce`

View file

@ -0,0 +1,6 @@
services:
twitch-archive:
gpus: all
environment:
NVIDIA_VISIBLE_DEVICES: ${NVIDIA_VISIBLE_DEVICES:-all}
NVIDIA_DRIVER_CAPABILITIES: ${NVIDIA_DRIVER_CAPABILITIES:-compute,utility,video}

View file

@ -1,8 +1,15 @@
@echo off
setlocal
set NVIDIA_COMPOSE=
if /I "%~1"=="--nvidia" (
set NVIDIA_COMPOSE=-f docker-compose.nvidia.yml
shift
)
if "%~1"=="" (
echo Usage: .\dockerstart.bat streamer [additional args]
echo Usage: .\dockerstart.bat [--nvidia] streamer [additional args]
exit /b 1
)
@ -17,4 +24,4 @@ shift
goto collect_args
:run_compose
docker compose --env-file .env.development -f docker-compose.yml -f docker-compose.override.yml run --rm twitch-archive python twitch-archive.py -u %STREAMER%%EXTRA_ARGS%
docker compose --env-file .env.development -f docker-compose.yml -f docker-compose.override.yml %NVIDIA_COMPOSE% run --rm twitch-archive python twitch-archive.py -u %STREAMER%%EXTRA_ARGS%

View file

@ -4,6 +4,7 @@ Includes fallback support for chat_downloader when VOD-based methods fail.
"""
import os
import sys
import subprocess
import json
import threading
@ -45,6 +46,10 @@ class ContentDownloader:
self.download_live_chat_enabled = config.get('downloadLiveCHAT', True)
self.use_chat_downloader_primary = config.get('useChatDownloaderPrimary', False)
self.use_chat_downloader_fallback = config.get('useChatDownloaderFallback', True)
default_chat_font = 'Arial' if sys.platform.startswith('win') else 'DejaVu Sans'
self.chat_render_font = config.get('chat_render_font', default_chat_font)
self.last_chat_render_attempted = False
self.last_chat_render_succeeded = False
# Initialize chat_downloader if available
self.chat_downloader = None
@ -61,6 +66,11 @@ class ContentDownloader:
self.chat_thread = None
self.chat_thread_success = False
self.chat_thread_error = None
def reset_chat_render_status(self) -> None:
"""Reset chat render tracking before a processing pass."""
self.last_chat_render_attempted = False
self.last_chat_render_succeeded = False
def download_vod(self, vod_info: Dict[str, Any], output_path: str) -> bool:
"""
@ -190,7 +200,7 @@ class ContentDownloader:
'-h', '1080',
'--framerate', '30',
'--outline',
'-f', 'Arial',
'-f', self.chat_render_font,
'--font-size', '22',
'--update-rate', '1.0',
'--offline',
@ -215,6 +225,9 @@ class ContentDownloader:
try:
print(f'{Fore.YELLOW}Rendering chat video...{Style.RESET_ALL}')
print(f'{Fore.CYAN}Using chat font: {self.chat_render_font}{Style.RESET_ALL}')
self.last_chat_render_attempted = True
self.last_chat_render_succeeded = False
# Build complete command
full_cmd = [self.twitch_downloader_path, 'chatrender', '-i', json_path, '-o', video_path] + chat_settings
@ -249,6 +262,7 @@ class ContentDownloader:
print(f'{Fore.RED}✗ Chat video file is too small ({file_size} bytes){Style.RESET_ALL}')
return False
self.last_chat_render_succeeded = True
print(f'{Fore.GREEN}✓ Chat rendered ({file_size:,} bytes){Style.RESET_ALL}')
return True

View file

@ -6,7 +6,7 @@ import os
import subprocess
from colorama import Fore, Style
from .utils import detect_hardware_acceleration, get_hwaccel_encoder
from .utils import detect_hardware_acceleration, get_hwaccel_encoder, resolve_hwaccel_type
class StreamProcessor:
@ -36,6 +36,7 @@ class StreamProcessor:
config.get('ffmpeg_hwaccel', 'auto'),
os_type
)
self.hwaccel_type = resolve_hwaccel_type(self.hwaccel_type, os_type)
def process_raw_stream(self, raw_path: str, output_path: str) -> bool:
"""

View file

@ -269,20 +269,57 @@ def detect_hardware_acceleration(hwaccel_config: str, os_type: str) -> Optional[
if hwaccel_config in ['nvenc', 'qsv', 'amf', 'vaapi']:
return hwaccel_config
# Auto-detect: try to determine available hardware
# Auto-detect: choose only hardware we can reasonably prove is present.
if hwaccel_config == 'auto':
# On Windows, NVIDIA is most common
if os_type == 'windows':
# Could check for nvidia-smi, but just return 'auto' for ffmpeg to decide
return 'auto'
else:
# On Linux, VAAPI is common for Intel/AMD, or NVENC for NVIDIA
# Let ffmpeg auto-detect
return 'auto'
if is_nvidia_runtime_available():
return 'nvenc'
if is_vaapi_runtime_available():
return 'vaapi'
return 'none'
return None
def is_nvidia_runtime_available() -> bool:
"""Return True when the current runtime appears to expose an NVIDIA GPU."""
visible_devices = os.getenv('NVIDIA_VISIBLE_DEVICES', '').strip().lower()
if visible_devices in {'void', 'none'}:
return False
if visible_devices and visible_devices != 'all':
return True
if shutil.which('nvidia-smi'):
return True
return any(
os.path.exists(device_path)
for device_path in ('/dev/nvidiactl', '/dev/nvidia0', '/dev/nvidia-modeset')
)
def is_vaapi_runtime_available() -> bool:
"""Return True when Linux VAAPI render nodes are present."""
return any(
os.path.exists(device_path)
for device_path in ('/dev/dri/renderD128', '/dev/dri/card0')
)
def resolve_hwaccel_type(hwaccel_type: Optional[str], os_type: str) -> Optional[str]:
"""Return a safe hardware acceleration choice for the current runtime."""
if hwaccel_type in (None, 'none'):
return 'none'
if hwaccel_type == 'nvenc':
return 'nvenc' if is_nvidia_runtime_available() else 'none'
if hwaccel_type == 'vaapi':
return 'vaapi' if is_vaapi_runtime_available() else 'none'
# Leave explicit QSV/AMF unchanged for non-container users; container auto-detect no longer picks them blindly.
return hwaccel_type
def get_hwaccel_encoder(hwaccel_type: str) -> str:
"""
Get the appropriate hardware-accelerated encoder for the given acceleration type.

View file

@ -31,7 +31,7 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from modules.constants import DEFAULT_CONFIG
from modules.file_manager import FileManager
from modules.downloader import ContentDownloader
from modules.utils import get_ffmpeg_executable, get_twitch_downloader_executable
from modules.utils import get_ffmpeg_executable, get_twitch_downloader_executable, detect_hardware_acceleration, resolve_hwaccel_type
def load_twitch_archive_module():
@ -541,6 +541,69 @@ class TestLinuxToolResolution(unittest.TestCase):
self.assertEqual(get_twitch_downloader_executable('linux'), '/usr/local/bin/TwitchDownloaderCLI')
@patch('modules.utils.os.path.exists', return_value=False)
@patch('modules.utils.shutil.which', return_value=None)
def test_linux_auto_hwaccel_falls_back_to_software_without_runtime(self, _mock_which, _mock_exists):
detected = detect_hardware_acceleration('auto', 'linux')
self.assertEqual(detected, 'none')
self.assertEqual(resolve_hwaccel_type(detected, 'linux'), 'none')
@patch('modules.utils.shutil.which', return_value='/usr/bin/nvidia-smi')
def test_linux_auto_hwaccel_uses_nvenc_when_nvidia_runtime_visible(self, _mock_which):
detected = detect_hardware_acceleration('auto', 'linux')
self.assertEqual(detected, 'nvenc')
class TestChatRenderBehavior(unittest.TestCase):
"""Regression tests for chat rendering defaults and retry behavior."""
@patch('modules.downloader.sys.platform', 'linux')
@patch('modules.downloader.subprocess.Popen')
def test_linux_chat_render_uses_container_safe_font(self, mock_popen):
with tempfile.TemporaryDirectory() as temp_dir:
json_path = os.path.join(temp_dir, 'chat.json')
video_path = os.path.join(temp_dir, 'chat.mp4')
with open(json_path, 'w', encoding='utf-8') as handle:
json.dump(
{
'comments': [
{
'message': {'body': 'hello world ' * 20},
'commenter': {'display_name': 'tester'}
}
]
},
handle
)
class FakeProcess:
def __init__(self, output_path: str):
self.stdout = iter(['rendering'])
self.output_path = output_path
def wait(self):
with open(self.output_path, 'wb') as handle:
handle.write(b'x' * 2048)
return 0
captured = {}
def build_process(cmd, **_kwargs):
captured['cmd'] = cmd
return FakeProcess(video_path)
mock_popen.side_effect = build_process
downloader = ContentDownloader('TwitchDownloaderCLI', 'ffmpeg', {'downloadCHAT': True})
result = downloader.render_chat(json_path, video_path, '-c:v libx264 "{save_path}"')
self.assertTrue(result)
self.assertIn('DejaVu Sans', captured['cmd'])
class TestMultiStreamerCleanupRegression(unittest.TestCase):
"""Regression tests for multi-streamer conversion and cleanup behavior."""
@ -636,6 +699,59 @@ class TestMultiStreamerCleanupRegression(unittest.TestCase):
upload_filename_base = archiver.file_manager.upload_to_cloud.call_args.args[0]
self.assertFalse(upload_filename_base.startswith('LIVE_'))
def test_process_stream_preserves_files_when_chat_render_fails(self):
manager = self.module.TwitchArchiveManager(specific_streamer='maddoscientist0')
with tempfile.TemporaryDirectory() as temp_dir:
archiver = self._build_archiver(temp_dir)
archiver.downloadCHAT = True
archiver.downloadVOD = True
archiver.vodTimeout = 30
archiver.processor.process_raw_stream.return_value = True
archiver.file_manager.upload_to_cloud.return_value = True
archiver.downloader.last_chat_render_attempted = False
archiver.downloader.last_chat_render_succeeded = False
def write_raw_file(_stream_info, raw_path):
with open(raw_path, 'wb') as handle:
handle.write(b'x' * 4096)
return True
def failed_chat_render(*_args, **_kwargs):
archiver.downloader.last_chat_render_attempted = True
archiver.downloader.last_chat_render_succeeded = False
return False
archiver.recorder.record.side_effect = write_raw_file
archiver.downloader.download_and_render_chat.side_effect = failed_chat_render
archiver.stream_monitor.get_latest_vod.return_value = {
'data': {
'user': {
'videos': {
'edges': [
{
'node': {
'id': '2756589076',
'title': 'Test',
'recordedAt': '2026-04-25T09:14:01Z'
}
}
]
}
}
}
}
stream_info = {
'title': 'Test',
'createdAt': '2026-04-25T09:14:01Z'
}
manager._process_stream(archiver, stream_info, 'stream-id')
archiver.file_manager.clean_raw_file.assert_not_called()
archiver.file_manager.delete_local_files.assert_not_called()
if __name__ == '__main__':
# Run tests with verbose output

View file

@ -425,6 +425,7 @@ class TwitchArchive:
# Process the raw stream file
processing_succeeded = self.processor.process_raw_stream(live_raw_path, live_proc_path)
self.downloader.reset_chat_render_status()
# Wait for live chat download if it was started
live_chat_downloaded = False
@ -585,7 +586,14 @@ class TwitchArchive:
print(f'{Fore.YELLOW}⚠ No matching VOD found for this stream{Style.RESET_ALL}')
# Clean up raw files if configured
if processing_succeeded:
chat_render_retry_needed = (
self.downloader.last_chat_render_attempted and
not self.downloader.last_chat_render_succeeded
)
if chat_render_retry_needed:
print(f'{Fore.YELLOW}⚠ Preserving local files because chat rendering failed and can be retried later{Style.RESET_ALL}')
elif processing_succeeded:
self.file_manager.clean_raw_file(live_raw_path)
elif os.path.exists(live_raw_path):
print(f'{Fore.YELLOW}⚠ Keeping raw file because conversion did not complete successfully{Style.RESET_ALL}')
@ -597,7 +605,7 @@ class TwitchArchive:
)
# Delete local files if configured and upload succeeded
if self.deleteFiles and self.uploadCloud and upload_success:
if self.deleteFiles and self.uploadCloud and upload_success and not chat_render_retry_needed:
self.file_manager.delete_local_files(
filename_base,
live_raw_path,
@ -1204,6 +1212,7 @@ class TwitchArchiveManager:
processing_succeeded = False
if not archiver.onlyRaw:
processing_succeeded = archiver.processor.process_raw_stream(live_raw_path, live_proc_path)
archiver.downloader.reset_chat_render_status()
# Wait for live chat download if it was started
live_chat_downloaded = False
@ -1396,7 +1405,14 @@ class TwitchArchiveManager:
archiver.file_manager.save_metadata(stream_info, filename_base)
# Clean up raw file if configured
if processing_succeeded:
chat_render_retry_needed = (
archiver.downloader.last_chat_render_attempted and
not archiver.downloader.last_chat_render_succeeded
)
if chat_render_retry_needed:
print(f'{Fore.YELLOW}⚠ Preserving local files because chat rendering failed and can be retried later{Style.RESET_ALL}')
elif processing_succeeded:
archiver.file_manager.clean_raw_file(live_raw_path)
elif os.path.exists(live_raw_path):
print(f'{Fore.YELLOW}⚠ Keeping raw file because conversion did not complete successfully{Style.RESET_ALL}')
@ -1408,7 +1424,7 @@ class TwitchArchiveManager:
)
# Delete files if configured
if archiver.deleteFiles and archiver.uploadCloud and upload_success:
if archiver.deleteFiles and archiver.uploadCloud and upload_success and not chat_render_retry_needed:
archiver.file_manager.delete_local_files(
filename_base,
live_raw_path,