diff --git a/README.md b/README.md index 5005181..979e223 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,17 @@ Or use the Windows helper: That batch launcher mirrors the old pattern and expands to a compose `run` command, so you can test any streamer manually. +If the host has the NVIDIA Container Toolkit installed and you want FFmpeg/NVENC inside the container, use the optional NVIDIA override: + +```powershell +.\dockerrebuild.bat +.\dockerstart.bat --nvidia vinesauce --verbose +``` + +The image built by `.\dockerrebuild.bat` already includes the NVIDIA-capable FFmpeg/container toolchain. The optional [docker-compose.nvidia.yml](docker-compose.nvidia.yml) layer is only for runtime GPU passthrough: it requests `gpus: all` and sets `NVIDIA_VISIBLE_DEVICES` plus `NVIDIA_DRIVER_CAPABILITIES=compute,utility,video` for the container. + +On systems without NVIDIA support, keep using the normal command without `--nvidia`; the image still builds the same way, it just runs without GPU passthrough. + ### Healthcheck and smoke tests - Container healthcheck command: `python twitch-archive.py --healthcheck -u vinesauce` diff --git a/docker-compose.nvidia.yml b/docker-compose.nvidia.yml new file mode 100644 index 0000000..19a51b5 --- /dev/null +++ b/docker-compose.nvidia.yml @@ -0,0 +1,6 @@ +services: + twitch-archive: + gpus: all + environment: + NVIDIA_VISIBLE_DEVICES: ${NVIDIA_VISIBLE_DEVICES:-all} + NVIDIA_DRIVER_CAPABILITIES: ${NVIDIA_DRIVER_CAPABILITIES:-compute,utility,video} diff --git a/dockerstart.bat b/dockerstart.bat index 9d1e947..38e7242 100644 --- a/dockerstart.bat +++ b/dockerstart.bat @@ -1,8 +1,15 @@ @echo off setlocal +set NVIDIA_COMPOSE= + +if /I "%~1"=="--nvidia" ( + set NVIDIA_COMPOSE=-f docker-compose.nvidia.yml + shift +) + if "%~1"=="" ( - echo Usage: .\dockerstart.bat streamer [additional args] + echo Usage: .\dockerstart.bat [--nvidia] streamer [additional args] exit /b 1 ) @@ -17,4 +24,4 @@ shift goto collect_args :run_compose -docker compose --env-file .env.development -f docker-compose.yml -f docker-compose.override.yml run --rm twitch-archive python twitch-archive.py -u %STREAMER%%EXTRA_ARGS% \ No newline at end of file +docker compose --env-file .env.development -f docker-compose.yml -f docker-compose.override.yml %NVIDIA_COMPOSE% run --rm twitch-archive python twitch-archive.py -u %STREAMER%%EXTRA_ARGS% \ No newline at end of file diff --git a/modules/downloader.py b/modules/downloader.py index 9f9cc1f..87fe30d 100644 --- a/modules/downloader.py +++ b/modules/downloader.py @@ -4,6 +4,7 @@ Includes fallback support for chat_downloader when VOD-based methods fail. """ import os +import sys import subprocess import json import threading @@ -45,6 +46,10 @@ class ContentDownloader: self.download_live_chat_enabled = config.get('downloadLiveCHAT', True) self.use_chat_downloader_primary = config.get('useChatDownloaderPrimary', False) self.use_chat_downloader_fallback = config.get('useChatDownloaderFallback', True) + default_chat_font = 'Arial' if sys.platform.startswith('win') else 'DejaVu Sans' + self.chat_render_font = config.get('chat_render_font', default_chat_font) + self.last_chat_render_attempted = False + self.last_chat_render_succeeded = False # Initialize chat_downloader if available self.chat_downloader = None @@ -61,6 +66,11 @@ class ContentDownloader: self.chat_thread = None self.chat_thread_success = False self.chat_thread_error = None + + def reset_chat_render_status(self) -> None: + """Reset chat render tracking before a processing pass.""" + self.last_chat_render_attempted = False + self.last_chat_render_succeeded = False def download_vod(self, vod_info: Dict[str, Any], output_path: str) -> bool: """ @@ -190,7 +200,7 @@ class ContentDownloader: '-h', '1080', '--framerate', '30', '--outline', - '-f', 'Arial', + '-f', self.chat_render_font, '--font-size', '22', '--update-rate', '1.0', '--offline', @@ -215,6 +225,9 @@ class ContentDownloader: try: print(f'{Fore.YELLOW}Rendering chat video...{Style.RESET_ALL}') + print(f'{Fore.CYAN}Using chat font: {self.chat_render_font}{Style.RESET_ALL}') + self.last_chat_render_attempted = True + self.last_chat_render_succeeded = False # Build complete command full_cmd = [self.twitch_downloader_path, 'chatrender', '-i', json_path, '-o', video_path] + chat_settings @@ -249,6 +262,7 @@ class ContentDownloader: print(f'{Fore.RED}✗ Chat video file is too small ({file_size} bytes){Style.RESET_ALL}') return False + self.last_chat_render_succeeded = True print(f'{Fore.GREEN}✓ Chat rendered ({file_size:,} bytes){Style.RESET_ALL}') return True diff --git a/modules/processor.py b/modules/processor.py index 9e4518b..58bd47f 100644 --- a/modules/processor.py +++ b/modules/processor.py @@ -6,7 +6,7 @@ import os import subprocess from colorama import Fore, Style -from .utils import detect_hardware_acceleration, get_hwaccel_encoder +from .utils import detect_hardware_acceleration, get_hwaccel_encoder, resolve_hwaccel_type class StreamProcessor: @@ -36,6 +36,7 @@ class StreamProcessor: config.get('ffmpeg_hwaccel', 'auto'), os_type ) + self.hwaccel_type = resolve_hwaccel_type(self.hwaccel_type, os_type) def process_raw_stream(self, raw_path: str, output_path: str) -> bool: """ diff --git a/modules/utils.py b/modules/utils.py index c8bc749..4e3b9b1 100644 --- a/modules/utils.py +++ b/modules/utils.py @@ -269,20 +269,57 @@ def detect_hardware_acceleration(hwaccel_config: str, os_type: str) -> Optional[ if hwaccel_config in ['nvenc', 'qsv', 'amf', 'vaapi']: return hwaccel_config - # Auto-detect: try to determine available hardware + # Auto-detect: choose only hardware we can reasonably prove is present. if hwaccel_config == 'auto': - # On Windows, NVIDIA is most common - if os_type == 'windows': - # Could check for nvidia-smi, but just return 'auto' for ffmpeg to decide - return 'auto' - else: - # On Linux, VAAPI is common for Intel/AMD, or NVENC for NVIDIA - # Let ffmpeg auto-detect - return 'auto' + if is_nvidia_runtime_available(): + return 'nvenc' + if is_vaapi_runtime_available(): + return 'vaapi' + return 'none' return None +def is_nvidia_runtime_available() -> bool: + """Return True when the current runtime appears to expose an NVIDIA GPU.""" + visible_devices = os.getenv('NVIDIA_VISIBLE_DEVICES', '').strip().lower() + if visible_devices in {'void', 'none'}: + return False + if visible_devices and visible_devices != 'all': + return True + + if shutil.which('nvidia-smi'): + return True + + return any( + os.path.exists(device_path) + for device_path in ('/dev/nvidiactl', '/dev/nvidia0', '/dev/nvidia-modeset') + ) + + +def is_vaapi_runtime_available() -> bool: + """Return True when Linux VAAPI render nodes are present.""" + return any( + os.path.exists(device_path) + for device_path in ('/dev/dri/renderD128', '/dev/dri/card0') + ) + + +def resolve_hwaccel_type(hwaccel_type: Optional[str], os_type: str) -> Optional[str]: + """Return a safe hardware acceleration choice for the current runtime.""" + if hwaccel_type in (None, 'none'): + return 'none' + + if hwaccel_type == 'nvenc': + return 'nvenc' if is_nvidia_runtime_available() else 'none' + + if hwaccel_type == 'vaapi': + return 'vaapi' if is_vaapi_runtime_available() else 'none' + + # Leave explicit QSV/AMF unchanged for non-container users; container auto-detect no longer picks them blindly. + return hwaccel_type + + def get_hwaccel_encoder(hwaccel_type: str) -> str: """ Get the appropriate hardware-accelerated encoder for the given acceleration type. diff --git a/test_twitch_archive_simple.py b/test_twitch_archive_simple.py index 96a5f26..445e741 100644 --- a/test_twitch_archive_simple.py +++ b/test_twitch_archive_simple.py @@ -31,7 +31,7 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from modules.constants import DEFAULT_CONFIG from modules.file_manager import FileManager from modules.downloader import ContentDownloader -from modules.utils import get_ffmpeg_executable, get_twitch_downloader_executable +from modules.utils import get_ffmpeg_executable, get_twitch_downloader_executable, detect_hardware_acceleration, resolve_hwaccel_type def load_twitch_archive_module(): @@ -541,6 +541,69 @@ class TestLinuxToolResolution(unittest.TestCase): self.assertEqual(get_twitch_downloader_executable('linux'), '/usr/local/bin/TwitchDownloaderCLI') + @patch('modules.utils.os.path.exists', return_value=False) + @patch('modules.utils.shutil.which', return_value=None) + def test_linux_auto_hwaccel_falls_back_to_software_without_runtime(self, _mock_which, _mock_exists): + detected = detect_hardware_acceleration('auto', 'linux') + + self.assertEqual(detected, 'none') + self.assertEqual(resolve_hwaccel_type(detected, 'linux'), 'none') + + @patch('modules.utils.shutil.which', return_value='/usr/bin/nvidia-smi') + def test_linux_auto_hwaccel_uses_nvenc_when_nvidia_runtime_visible(self, _mock_which): + detected = detect_hardware_acceleration('auto', 'linux') + + self.assertEqual(detected, 'nvenc') + + +class TestChatRenderBehavior(unittest.TestCase): + """Regression tests for chat rendering defaults and retry behavior.""" + + @patch('modules.downloader.sys.platform', 'linux') + @patch('modules.downloader.subprocess.Popen') + def test_linux_chat_render_uses_container_safe_font(self, mock_popen): + with tempfile.TemporaryDirectory() as temp_dir: + json_path = os.path.join(temp_dir, 'chat.json') + video_path = os.path.join(temp_dir, 'chat.mp4') + + with open(json_path, 'w', encoding='utf-8') as handle: + json.dump( + { + 'comments': [ + { + 'message': {'body': 'hello world ' * 20}, + 'commenter': {'display_name': 'tester'} + } + ] + }, + handle + ) + + class FakeProcess: + def __init__(self, output_path: str): + self.stdout = iter(['rendering']) + self.output_path = output_path + + def wait(self): + with open(self.output_path, 'wb') as handle: + handle.write(b'x' * 2048) + return 0 + + captured = {} + + def build_process(cmd, **_kwargs): + captured['cmd'] = cmd + return FakeProcess(video_path) + + mock_popen.side_effect = build_process + + downloader = ContentDownloader('TwitchDownloaderCLI', 'ffmpeg', {'downloadCHAT': True}) + + result = downloader.render_chat(json_path, video_path, '-c:v libx264 "{save_path}"') + + self.assertTrue(result) + self.assertIn('DejaVu Sans', captured['cmd']) + class TestMultiStreamerCleanupRegression(unittest.TestCase): """Regression tests for multi-streamer conversion and cleanup behavior.""" @@ -636,6 +699,59 @@ class TestMultiStreamerCleanupRegression(unittest.TestCase): upload_filename_base = archiver.file_manager.upload_to_cloud.call_args.args[0] self.assertFalse(upload_filename_base.startswith('LIVE_')) + def test_process_stream_preserves_files_when_chat_render_fails(self): + manager = self.module.TwitchArchiveManager(specific_streamer='maddoscientist0') + + with tempfile.TemporaryDirectory() as temp_dir: + archiver = self._build_archiver(temp_dir) + archiver.downloadCHAT = True + archiver.downloadVOD = True + archiver.vodTimeout = 30 + archiver.processor.process_raw_stream.return_value = True + archiver.file_manager.upload_to_cloud.return_value = True + archiver.downloader.last_chat_render_attempted = False + archiver.downloader.last_chat_render_succeeded = False + + def write_raw_file(_stream_info, raw_path): + with open(raw_path, 'wb') as handle: + handle.write(b'x' * 4096) + return True + + def failed_chat_render(*_args, **_kwargs): + archiver.downloader.last_chat_render_attempted = True + archiver.downloader.last_chat_render_succeeded = False + return False + + archiver.recorder.record.side_effect = write_raw_file + archiver.downloader.download_and_render_chat.side_effect = failed_chat_render + archiver.stream_monitor.get_latest_vod.return_value = { + 'data': { + 'user': { + 'videos': { + 'edges': [ + { + 'node': { + 'id': '2756589076', + 'title': 'Test', + 'recordedAt': '2026-04-25T09:14:01Z' + } + } + ] + } + } + } + } + + stream_info = { + 'title': 'Test', + 'createdAt': '2026-04-25T09:14:01Z' + } + + manager._process_stream(archiver, stream_info, 'stream-id') + + archiver.file_manager.clean_raw_file.assert_not_called() + archiver.file_manager.delete_local_files.assert_not_called() + if __name__ == '__main__': # Run tests with verbose output diff --git a/twitch-archive.py b/twitch-archive.py index 81f9526..707bdfe 100644 --- a/twitch-archive.py +++ b/twitch-archive.py @@ -425,6 +425,7 @@ class TwitchArchive: # Process the raw stream file processing_succeeded = self.processor.process_raw_stream(live_raw_path, live_proc_path) + self.downloader.reset_chat_render_status() # Wait for live chat download if it was started live_chat_downloaded = False @@ -585,7 +586,14 @@ class TwitchArchive: print(f'{Fore.YELLOW}⚠ No matching VOD found for this stream{Style.RESET_ALL}') # Clean up raw files if configured - if processing_succeeded: + chat_render_retry_needed = ( + self.downloader.last_chat_render_attempted and + not self.downloader.last_chat_render_succeeded + ) + + if chat_render_retry_needed: + print(f'{Fore.YELLOW}⚠ Preserving local files because chat rendering failed and can be retried later{Style.RESET_ALL}') + elif processing_succeeded: self.file_manager.clean_raw_file(live_raw_path) elif os.path.exists(live_raw_path): print(f'{Fore.YELLOW}⚠ Keeping raw file because conversion did not complete successfully{Style.RESET_ALL}') @@ -597,7 +605,7 @@ class TwitchArchive: ) # Delete local files if configured and upload succeeded - if self.deleteFiles and self.uploadCloud and upload_success: + if self.deleteFiles and self.uploadCloud and upload_success and not chat_render_retry_needed: self.file_manager.delete_local_files( filename_base, live_raw_path, @@ -1204,6 +1212,7 @@ class TwitchArchiveManager: processing_succeeded = False if not archiver.onlyRaw: processing_succeeded = archiver.processor.process_raw_stream(live_raw_path, live_proc_path) + archiver.downloader.reset_chat_render_status() # Wait for live chat download if it was started live_chat_downloaded = False @@ -1396,7 +1405,14 @@ class TwitchArchiveManager: archiver.file_manager.save_metadata(stream_info, filename_base) # Clean up raw file if configured - if processing_succeeded: + chat_render_retry_needed = ( + archiver.downloader.last_chat_render_attempted and + not archiver.downloader.last_chat_render_succeeded + ) + + if chat_render_retry_needed: + print(f'{Fore.YELLOW}⚠ Preserving local files because chat rendering failed and can be retried later{Style.RESET_ALL}') + elif processing_succeeded: archiver.file_manager.clean_raw_file(live_raw_path) elif os.path.exists(live_raw_path): print(f'{Fore.YELLOW}⚠ Keeping raw file because conversion did not complete successfully{Style.RESET_ALL}') @@ -1408,7 +1424,7 @@ class TwitchArchiveManager: ) # Delete files if configured - if archiver.deleteFiles and archiver.uploadCloud and upload_success: + if archiver.deleteFiles and archiver.uploadCloud and upload_success and not chat_render_retry_needed: archiver.file_manager.delete_local_files( filename_base, live_raw_path,