From 38d51636af9b9acade0e13fb64d6cd9102ba4999 Mon Sep 17 00:00:00 2001 From: MaddoScientisto Date: Wed, 11 Feb 2026 13:23:14 +0100 Subject: [PATCH 01/10] Chat monitors stream to end --- modules/downloader.py | 23 ++++++++++++++++++++++- modules/stream_monitor.py | 28 ++++++++++++++++++++++++++++ run_tests.ps1 | 37 +++++++++++++++++++++++++++++++++++++ twitch-archive.py | 2 ++ 4 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 run_tests.ps1 diff --git a/modules/downloader.py b/modules/downloader.py index 2507bd6..a1f166b 100644 --- a/modules/downloader.py +++ b/modules/downloader.py @@ -365,6 +365,7 @@ class ContentDownloader: max_messages: Optional[int] = None, timeout: Optional[float] = None, shutdown_check: Optional[callable] = None, + stream_monitor = None, verbose: bool = False) -> bool: """ Download live chat using chat_downloader library as fallback. @@ -376,6 +377,7 @@ class ContentDownloader: max_messages: Maximum messages to download (None = unlimited) timeout: Stop after this many seconds (None = until stream ends) shutdown_check: Optional callback function that returns True when shutdown requested + stream_monitor: Optional stream monitor to check if stream is still live verbose: Show chat message previews Returns: @@ -414,7 +416,10 @@ class ContentDownloader: # The get_chat with output parameter writes to file automatically # We just need to iterate to trigger the download message_count = 0 - print(f'{Fore.CYAN}Receiving chat messages (press Ctrl+C to stop)...{Style.RESET_ALL}') + last_check_time = time.time() + check_interval = 10.0 # Check if stream is still live every 10 seconds + + print(f'{Fore.CYAN}Receiving chat messages (will stop when stream ends)...{Style.RESET_ALL}') try: for message in chat: # Check for shutdown request @@ -422,6 +427,19 @@ class ContentDownloader: print(f'\n{Fore.YELLOW}⚠ Chat download stopped by shutdown request{Style.RESET_ALL}') break + # Periodically check if stream is still live + current_time = time.time() + if stream_monitor and (current_time - last_check_time) >= check_interval: + last_check_time = current_time + try: + is_live = stream_monitor.is_user_live() + if not is_live: + print(f'\n{Fore.YELLOW}⚠ Stream ended, stopping chat download{Style.RESET_ALL}') + break + except Exception as check_error: + print(f'\n{Fore.YELLOW}⚠ Could not check stream status: {check_error}{Style.RESET_ALL}') + # Continue downloading to avoid false positives from API errors + message_count += 1 # Show progress every 100 messages @@ -467,6 +485,7 @@ class ContentDownloader: def start_chat_downloader_thread(self, username: str, json_path: str, shutdown_check: Optional[callable] = None, + stream_monitor = None, verbose: bool = False) -> threading.Thread: """ Start chat_downloader in a background thread. @@ -475,6 +494,7 @@ class ContentDownloader: username: Twitch username json_path: Path to save chat JSON shutdown_check: Callback to check for shutdown + stream_monitor: Optional stream monitor to check if stream is still live verbose: Show chat previews Returns: @@ -485,6 +505,7 @@ class ContentDownloader: self.chat_thread_success = self.download_live_chat_with_chat_downloader( username, json_path, shutdown_check=shutdown_check, + stream_monitor=stream_monitor, verbose=verbose ) except Exception as e: diff --git a/modules/stream_monitor.py b/modules/stream_monitor.py index 6c1607a..bea3545 100644 --- a/modules/stream_monitor.py +++ b/modules/stream_monitor.py @@ -115,6 +115,34 @@ class StreamMonitor: print(f'{Fore.YELLOW} {str(e)}{Style.RESET_ALL}') sys.exit(1) + def is_user_live(self) -> bool: + """ + Check if the configured user is currently live. + + Returns: + bool: True if user is live, False if offline + + Raises: + Exception: If API request fails (caller should handle) + """ + query = f'query{{user(login: "{self.username}") {{stream{{id title}}}}}}' + + try: + response = requests.post( + TWITCH_GQL_URL, + json={'query': query}, + headers={"Client-ID": TWITCH_GQL_CLIENT_ID}, + timeout=15 + ) + response.raise_for_status() + data = response.json() + stream_data = data.get('data', {}).get('user', {}).get('stream') + return stream_data is not None + + except requests.exceptions.RequestException as e: + # Don't exit, let caller handle this + raise Exception(f"Failed to check if user is live: {str(e)}") + def get_latest_vod(self) -> Optional[Dict[str, Any]]: """ Get the most recent VOD for the configured user. diff --git a/run_tests.ps1 b/run_tests.ps1 new file mode 100644 index 0000000..f547251 --- /dev/null +++ b/run_tests.ps1 @@ -0,0 +1,37 @@ +# PowerShell script to run Twitch Archive unit tests +# Run this script to execute all unit tests + +Write-Host "======================================================================" -ForegroundColor Cyan +Write-Host "TWITCH ARCHIVE - Running Unit Tests" -ForegroundColor Cyan +Write-Host "======================================================================" -ForegroundColor Cyan +Write-Host "" + +# Check if virtual environment exists and activate it +$venvPath = ".\venv314\Scripts\Activate.ps1" +if (Test-Path $venvPath) { + Write-Host "✓ Activating virtual environment..." -ForegroundColor Green + & $venvPath +} else { + Write-Host "⚠ Virtual environment not found at $venvPath" -ForegroundColor Yellow + Write-Host " Continuing with system Python..." -ForegroundColor Yellow +} + +Write-Host "" + +# Run the tests +Write-Host "Running unit tests..." -ForegroundColor Cyan +python test_twitch_archive_simple.py + +# Check exit code +if ($LASTEXITCODE -eq 0) { + Write-Host "" + Write-Host "======================================================================" -ForegroundColor Green + Write-Host "✓ ALL TESTS PASSED" -ForegroundColor Green + Write-Host "======================================================================" -ForegroundColor Green +} else { + Write-Host "" + Write-Host "======================================================================" -ForegroundColor Red + Write-Host "✗ SOME TESTS FAILED" -ForegroundColor Red + Write-Host "======================================================================" -ForegroundColor Red + exit $LASTEXITCODE +} diff --git a/twitch-archive.py b/twitch-archive.py index 5844cde..2fdcb4e 100644 --- a/twitch-archive.py +++ b/twitch-archive.py @@ -1088,6 +1088,7 @@ class TwitchArchiveManager: archiver.downloader.start_chat_downloader_thread( archiver.username, chat_json_path, shutdown_check=lambda: self.shutdown_requested or archiver.shutdown_requested, + stream_monitor=archiver.stream_monitor, verbose=self.verbose ) except Exception as e: @@ -1116,6 +1117,7 @@ class TwitchArchiveManager: archiver.downloader.start_chat_downloader_thread( archiver.username, chat_json_path, shutdown_check=lambda: self.shutdown_requested or archiver.shutdown_requested, + stream_monitor=archiver.stream_monitor, verbose=self.verbose or self.chat_only ) # Wait for completion From 0d3cdfd12c736411ed1d4f33accb5ee3dca488a1 Mon Sep 17 00:00:00 2001 From: MaddoScientisto Date: Wed, 11 Feb 2026 17:44:34 +0100 Subject: [PATCH 02/10] feat: add upload options for pre-merge, merged, and standalone chat videos - Updated global schema to include options for uploading original videos before merging, merged videos, and standalone chat videos. - Modified constants to set default values for new upload options. - Enhanced FileManager to handle new upload options, including conditional file uploads and deletions based on user configuration. - Introduced unit tests for command-line argument parsing, configuration loading, and merging logic, ensuring robust handling of new features. - Added tests for filtering logic, default configurations, and enabled streamer handling. --- config/global.json.example | 3 + config/global.schema.json | 15 + modules/constants.py | 3 + modules/file_manager.py | 90 ++++-- test_twitch_archive.py | 592 ++++++++++++++++++++++++++++++++++ test_twitch_archive_simple.py | 448 +++++++++++++++++++++++++ 6 files changed, 1128 insertions(+), 23 deletions(-) create mode 100644 test_twitch_archive.py create mode 100644 test_twitch_archive_simple.py diff --git a/config/global.json.example b/config/global.json.example index fb37572..f24152c 100644 --- a/config/global.json.example +++ b/config/global.json.example @@ -13,6 +13,9 @@ "mergeChatLayout": "side-by-side", "vodTimeout": 300, "uploadCloud": true, + "uploadPreMergeVideo": true, + "uploadMergedVideo": true, + "uploadChatVideo": false, "deleteFiles": false, "onlyRaw": false, "cleanRaw": true, diff --git a/config/global.schema.json b/config/global.schema.json index 84473ef..393ac33 100644 --- a/config/global.schema.json +++ b/config/global.schema.json @@ -74,6 +74,21 @@ "default": true, "description": "Upload to rclone remote: false = disabled, true = enabled" }, + "uploadPreMergeVideo": { + "type": "boolean", + "default": true, + "description": "Upload original videos before merging with chat (LIVE and VOD files): false = skip, true = upload" + }, + "uploadMergedVideo": { + "type": "boolean", + "default": true, + "description": "Upload merged videos (video + chat combined): false = skip, true = upload" + }, + "uploadChatVideo": { + "type": "boolean", + "default": true, + "description": "Upload standalone chat video files: false = skip, true = upload" + }, "deleteFiles": { "type": "boolean", "default": false, diff --git a/modules/constants.py b/modules/constants.py index 7f09e35..7aa10eb 100644 --- a/modules/constants.py +++ b/modules/constants.py @@ -32,6 +32,9 @@ DEFAULT_CONFIG = { 'mergeChatLayout': 'side-by-side', # Layout: 'side-by-side' or 'overlay' 'vodTimeout': 300, 'uploadCloud': True, + 'uploadPreMergeVideo': True, # Upload original videos before merging + 'uploadMergedVideo': True, # Upload merged videos (video + chat) + 'uploadChatVideo': False, # Upload standalone chat video 'deleteFiles': False, 'onlyRaw': False, 'cleanRaw': True, diff --git a/modules/file_manager.py b/modules/file_manager.py index 8848b8a..97f3909 100644 --- a/modules/file_manager.py +++ b/modules/file_manager.py @@ -9,7 +9,7 @@ import subprocess from typing import List from colorama import Fore, Style -from .constants import PREFIX_LIVE, PREFIX_VOD, PREFIX_CHAT, PREFIX_METADATA +from .constants import PREFIX_LIVE, PREFIX_VOD, PREFIX_CHAT, PREFIX_METADATA, PREFIX_MERGED from .utils import get_bin_path @@ -28,6 +28,9 @@ class FileManager: self.root_path = pathlib.Path(root_path) self.username = username self.upload_cloud = config.get('uploadCloud', True) + self.upload_pre_merge_video = config.get('uploadPreMergeVideo', True) + self.upload_merged_video = config.get('uploadMergedVideo', True) + self.upload_chat_video = config.get('uploadChatVideo', True) self.delete_files = config.get('deleteFiles', False) self.clean_raw = config.get('cleanRaw', True) self.download_vod = config.get('downloadVOD', True) @@ -125,17 +128,35 @@ class FileManager: # Ensure temp directory exists os.makedirs(os.path.dirname(upload_list_path), exist_ok=True) - files_to_upload = [ - f"{PREFIX_LIVE}{filename_base}.ts", - f"{PREFIX_LIVE}{filename_base}.mp4", - f"{PREFIX_LIVE}{filename_base}.mp3", - f"{PREFIX_VOD}{filename_base}.ts", - f"{PREFIX_VOD}{filename_base}.mp4", - f"{PREFIX_VOD}{filename_base}.mp3", - f"{PREFIX_METADATA}{filename_base}.json", - f"{PREFIX_CHAT}{filename_base}.json", - f"{PREFIX_CHAT}{filename_base}.mp4" - ] + files_to_upload = [] + + # Always include metadata and chat JSON + files_to_upload.append(f"{PREFIX_METADATA}{filename_base}.json") + files_to_upload.append(f"{PREFIX_CHAT}{filename_base}.json") + + # Add pre-merge videos (original LIVE and VOD files) + if self.upload_pre_merge_video: + files_to_upload.extend([ + f"{PREFIX_LIVE}{filename_base}.ts", + f"{PREFIX_LIVE}{filename_base}.mp4", + f"{PREFIX_LIVE}{filename_base}.mp3", + f"{PREFIX_VOD}{filename_base}.ts", + f"{PREFIX_VOD}{filename_base}.mp4", + f"{PREFIX_VOD}{filename_base}.mp3" + ]) + + # Add merged videos + if self.upload_merged_video: + files_to_upload.extend([ + f"{PREFIX_MERGED}{filename_base}.mp4", + f"{PREFIX_MERGED}{filename_base}.mp3", + f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}.mp4", + f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}.mp3" + ]) + + # Add standalone chat video + if self.upload_chat_video: + files_to_upload.append(f"{PREFIX_CHAT}{filename_base}.mp4") with open(upload_list_path, 'w') as f: f.write('\n'.join(files_to_upload)) @@ -175,6 +196,8 @@ class FileManager: """ Delete local archive files after successful upload. + Only deletes files that were configured to be uploaded. + Args: filename_base: Base filename (without prefixes/extensions) live_raw_path: Path to live raw file @@ -191,14 +214,15 @@ class FileManager: files_to_delete: List[str] = [] - # Live files - if not self.clean_raw and os.path.exists(live_raw_path): - files_to_delete.append(live_raw_path) - if os.path.exists(live_proc_path): - files_to_delete.append(live_proc_path) + # Live files (only if pre-merge videos are uploaded) + if self.upload_pre_merge_video: + if not self.clean_raw and os.path.exists(live_raw_path): + files_to_delete.append(live_raw_path) + if os.path.exists(live_proc_path): + files_to_delete.append(live_proc_path) - # VOD files - if self.download_vod: + # VOD files (only if pre-merge videos are uploaded) + if self.download_vod and self.upload_pre_merge_video: vod_raw = self.raw_path / f"{PREFIX_VOD}{filename_base}.ts" vod_mp4 = self.video_path / f"{PREFIX_VOD}{filename_base}.mp4" vod_mp3 = self.video_path / f"{PREFIX_VOD}{filename_base}.mp3" @@ -210,17 +234,37 @@ class FileManager: if vod_mp3.exists(): files_to_delete.append(str(vod_mp3)) + # Merged video files (only if merged videos are uploaded) + if self.upload_merged_video: + merged_live_mp4 = self.video_path / f"{PREFIX_MERGED}{filename_base}.mp4" + merged_live_mp3 = self.video_path / f"{PREFIX_MERGED}{filename_base}.mp3" + merged_vod_mp4 = self.video_path / f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}.mp4" + merged_vod_mp3 = self.video_path / f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}.mp3" + + if merged_live_mp4.exists(): + files_to_delete.append(str(merged_live_mp4)) + if merged_live_mp3.exists(): + files_to_delete.append(str(merged_live_mp3)) + if merged_vod_mp4.exists(): + files_to_delete.append(str(merged_vod_mp4)) + if merged_vod_mp3.exists(): + files_to_delete.append(str(merged_vod_mp3)) + # Chat files if self.download_chat: chat_json = self.chat_json_path / f"{PREFIX_CHAT}{filename_base}.json" - chat_mp4 = self.chat_mp4_path / f"{PREFIX_CHAT}{filename_base}.mp4" + # Always delete JSON (it's always uploaded) if chat_json.exists(): files_to_delete.append(str(chat_json)) - if chat_mp4.exists(): - files_to_delete.append(str(chat_mp4)) + + # Only delete chat MP4 if chat videos are uploaded + if self.upload_chat_video: + chat_mp4 = self.chat_mp4_path / f"{PREFIX_CHAT}{filename_base}.mp4" + if chat_mp4.exists(): + files_to_delete.append(str(chat_mp4)) - # Metadata files + # Metadata files (always uploaded) if self.download_metadata: metadata = self.metadata_path / f"{PREFIX_METADATA}{filename_base}.json" if metadata.exists(): diff --git a/test_twitch_archive.py b/test_twitch_archive.py new file mode 100644 index 0000000..4432d81 --- /dev/null +++ b/test_twitch_archive.py @@ -0,0 +1,592 @@ +""" +Unit tests for Twitch Archive command-line options and configuration. + +Tests focus on: +- Command-line argument parsing (via getopt simulation) +- Options and option combinations +- Configuration loading and merging +- Mode selection logic + +Excludes actual download/processing functionality. + +To run these tests: + python test_twitch_archive.py + or + python -m pytest test_twitch_archive.py -v +""" + +import unittest +import sys +import os +import json +import tempfile +import shutil +import getopt +from unittest.mock import patch, MagicMock, Mock +from pathlib import Path + +# Add parent directory to path for imports +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from modules.config import ConfigManager +from modules.constants import DEFAULT_CONFIG + + +class TestCommandLineArgumentParsing(unittest.TestCase): + """Test command-line argument parsing logic using getopt directly.""" + + def setUp(self): + """Set up test fixtures.""" + self.test_dir = tempfile.mkdtemp() + self.original_cwd = os.getcwd() + os.chdir(self.test_dir) + + def tearDown(self): + """Clean up test fixtures.""" + os.chdir(self.original_cwd) + shutil.rmtree(self.test_dir, ignore_errors=True) + + def test_help_short_option(self): + """Test -h option parsing.""" + argv = ['-h'] + opts, args = getopt.getopt( + argv, + "hu:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + # Should parse successfully + self.assertEqual(len(opts), 1) + self.assertEqual(opts[0][0], '-h') + + def test_help_long_option(self): + """Test --help option parsing.""" + argv = ['--help'] + opts, args = getopt.getopt( + argv, + "hu:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 1) + self.assertEqual(opts[0][0], '--help') + + def test_username_short_option(self): + """Test -u username option parsing.""" + argv = ['-u', 'teststreamer'] + opts, args = getopt.getopt( + argv, + "h:u:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 1) + self.assertEqual(opts[0], ('-u', 'teststreamer')) + + def test_username_long_option(self): + """Test --username option parsing.""" + argv = ['--username', 'teststreamer'] + opts, args = getopt.getopt( + argv, + "h:u:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 1) + self.assertEqual(opts[0], ('--username', 'teststreamer')) + + def test_verbose_option(self): + """Test --verbose option parsing.""" + argv = ['--verbose'] + opts, args = getopt.getopt( + argv, + "h:u:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 1) + self.assertEqual(opts[0], ('--verbose', '')) + + def test_chat_only_option(self): + """Test --chat-only option parsing.""" + argv = ['--chat-only'] + opts, args = getopt.getopt( + argv, + "h:u:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 1) + self.assertEqual(opts[0], ('--chat-only', '')) + + def test_legacy_option(self): + """Test --legacy option parsing.""" + argv = ['--legacy'] + opts, args = getopt.getopt( + argv, + "h:u:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 1) + self.assertEqual(opts[0], ('--legacy', '')) + + def test_chat_downloader_options(self): + """Test chat downloader option parsing.""" + argv = ['--use-chat-downloader-primary', '--no-chat-downloader-fallback'] + opts, args = getopt.getopt( + argv, + "h:u:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 2) + self.assertEqual(opts[0], ('--use-chat-downloader-primary', '')) + self.assertEqual(opts[1], ('--no-chat-downloader-fallback', '')) + + def test_legacy_quality_option(self): + """Test -q quality option parsing.""" + argv = ['-q', '720p'] + opts, args = getopt.getopt( + argv, + "h:u:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 1) + self.assertEqual(opts[0], ('-q', '720p')) + + def test_legacy_boolean_options(self): + """Test legacy boolean option parsing.""" + argv = ['-v', '1', '-c', '0', '-m', '1'] + opts, args = getopt.getopt( + argv, + "h:u:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 3) + self.assertEqual(opts[0], ('-v', '1')) + self.assertEqual(opts[1], ('-c', '0')) + self.assertEqual(opts[2], ('-m', '1')) + + def test_invalid_option(self): + """Test that invalid option raises error.""" + argv = ['--invalid-option'] + + with self.assertRaises(getopt.GetoptError): + getopt.getopt( + argv, + "h:u:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + def test_option_combination_username_verbose(self): + """Test combining -u and --verbose options.""" + argv = ['-u', 'testuser', '--verbose'] + opts, args = getopt.getopt( + argv, + "h:u:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 2) + self.assertEqual(opts[0], ('-u', 'testuser')) + self.assertEqual(opts[1], ('--verbose', '')) + + def test_option_combination_all_test_flags(self): + """Test combining all test-related flags.""" + argv = ['-u', 'testuser', '--verbose', '--chat-only', '--use-chat-downloader-primary'] + opts, args = getopt.getopt( + argv, + "h:u:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 4) + opt_dict = dict(opts) + self.assertEqual(opt_dict['-u'], 'testuser') + self.assertIn('--verbose', opt_dict) + self.assertIn('--chat-only', opt_dict) + self.assertIn('--use-chat-downloader-primary', opt_dict) + + def test_option_combination_legacy_mode_with_overrides(self): + """Test legacy mode with multiple overrides.""" + argv = ['--legacy', '-q', '720p', '-v', '1', '-c', '1', '-m', '0'] + opts, args = getopt.getopt( + argv, + "h:u:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 5) + opt_dict = dict(opts) + self.assertIn('--legacy', opt_dict) + self.assertEqual(opt_dict['-q'], '720p') + self.assertEqual(opt_dict['-v'], '1') + self.assertEqual(opt_dict['-c'], '1') + self.assertEqual(opt_dict['-m'], '0') + + +class TestOptionLogicProcessing(unittest.TestCase): + """Test the logic that processes parsed options.""" + + def test_boolean_conversion_true(self): + """Test converting '1' to boolean True.""" + value = '1' + result = bool(int(value)) + self.assertTrue(result) + + def test_boolean_conversion_false(self): + """Test converting '0' to boolean False.""" + value = '0' + result = bool(int(value)) + self.assertFalse(result) + + def test_chat_only_auto_enables_verbose(self): + """Test that chat-only mode should auto-enable verbose.""" + # Simulate the logic from main() + chat_only_mode = True + verbose_mode = False + + if chat_only_mode: + verbose_mode = True + + self.assertTrue(verbose_mode) + + def test_default_chat_downloader_fallback(self): + """Test that chat downloader fallback defaults to enabled.""" + use_chat_downloader_fallback = True # Default + + # Unless explicitly disabled + self.assertTrue(use_chat_downloader_fallback) + + def test_mode_selection_legacy_with_config_json(self): + """Test mode selection logic when config.json exists.""" + # Simulate conditions + use_legacy_mode = False + legacy_config_exists = True + specific_streamer = None + global_config_exists = False + + # Logic from main() + should_use_legacy = use_legacy_mode or ( + legacy_config_exists and not specific_streamer and not global_config_exists + ) + + self.assertTrue(should_use_legacy) + + def test_mode_selection_multi_streamer_with_global_json(self): + """Test mode selection logic when global.json exists.""" + use_legacy_mode = False + legacy_config_exists = True + specific_streamer = None + global_config_exists = True + + should_use_legacy = use_legacy_mode or ( + legacy_config_exists and not specific_streamer and not global_config_exists + ) + + self.assertFalse(should_use_legacy) + + def test_mode_selection_multi_streamer_with_username_flag(self): + """Test mode selection when -u flag is used.""" + use_legacy_mode = False + legacy_config_exists = True + specific_streamer = 'testuser' + global_config_exists = False + + should_use_legacy = use_legacy_mode or ( + legacy_config_exists and not specific_streamer and not global_config_exists + ) + + self.assertFalse(should_use_legacy) + + def test_mode_selection_explicit_legacy_flag(self): + """Test mode selection with explicit --legacy flag.""" + use_legacy_mode = True + legacy_config_exists = False + specific_streamer = None + global_config_exists = True + + should_use_legacy = use_legacy_mode or ( + legacy_config_exists and not specific_streamer and not global_config_exists + ) + + self.assertTrue(should_use_legacy) + + +class TestConfigManager(unittest.TestCase): + """Test ConfigManager functionality.""" + + def setUp(self): + """Set up test fixtures.""" + self.test_dir = tempfile.mkdtemp() + self.original_cwd = os.getcwd() + os.chdir(self.test_dir) + + # Create config directory structure + os.makedirs('config/streamers', exist_ok=True) + + # Patch ConfigManager to use test directory + self.config_dir_patch = patch.object( + ConfigManager, + '__init__', + lambda self: self._init_with_test_dir(self.test_dir) + ) + + def _init_with_test_dir(self, test_dir): + """Initialize ConfigManager with test directory.""" + self.config_dir = Path(test_dir) / "config" + self.streamers_dir = self.config_dir / "streamers" + self.global_config = self._load_global_config() + + def tearDown(self): + """Clean up test fixtures.""" + os.chdir(self.original_cwd) + shutil.rmtree(self.test_dir, ignore_errors=True) + + def test_load_global_config_default(self): + """Test loading default configuration when global.json doesn't exist.""" + # Patch the config_dir to use temp directory + with patch.object(ConfigManager, '_ConfigManager__init__') as mock_init: + manager = ConfigManager.__new__(ConfigManager) + manager.config_dir = Path(self.test_dir) / "config" + manager.streamers_dir = manager.config_dir / "streamers" + manager.global_config = manager._load_global_config() + + # Should have default config values + self.assertIsNotNone(manager.global_config) + self.assertIn('username', manager.global_config) + self.assertIn('quality', manager.global_config) + + def test_load_global_config_from_file(self): + """Test loading global configuration from file.""" + # Create global config + global_config = { + 'quality': '720p', + 'downloadVOD': False, + 'downloadCHAT': True + } + + with open('config/global.json', 'w') as f: + json.dump(global_config, f) + + manager = ConfigManager() + + # Should merge with defaults + self.assertEqual(manager.global_config['quality'], '720p') + self.assertFalse(manager.global_config['downloadVOD']) + self.assertTrue(manager.global_config['downloadCHAT']) + + def test_load_global_config_filters_comments(self): + """Test that global config filters out comment fields.""" + global_config = { + '_comment': 'This is a comment', + 'quality': '720p', + '_note': 'Another comment' + } + + with open('config/global.json', 'w') as f: + json.dump(global_config, f) + + manager = ConfigManager() + + # Comments should be filtered out + self.assertNotIn('_comment', manager.global_config) + self.assertNotIn('_note', manager.global_config) + self.assertEqual(manager.global_config['quality'], '720p') + + def test_load_global_config_filters_schema(self): + """Test that global config filters out $schema field.""" + global_config = { + '$schema': './global.schema.json', + 'quality': '720p' + } + + with open('config/global.json', 'w') as f: + json.dump(global_config, f) + + manager = ConfigManager() + + # $schema should be filtered out + self.assertNotIn('$schema', manager.global_config) + self.assertEqual(manager.global_config['quality'], '720p') + + def test_load_streamer_config_new_streamer(self): + """Test loading config for a new streamer (doesn't exist yet).""" + manager = ConfigManager() + + config = manager.load_streamer_config('newstreamer') + + # Should create default config + self.assertEqual(config['username'], 'newstreamer') + self.assertTrue(config['enabled']) + + # Config file should be created + self.assertTrue(os.path.exists('config/streamers/newstreamer.json')) + + def test_load_streamer_config_existing_streamer(self): + """Test loading config for existing streamer.""" + # Create streamer config + streamer_config = { + 'username': 'existingstreamer', + 'enabled': True, + 'quality': 'source', + 'downloadVOD': True + } + + with open('config/streamers/existingstreamer.json', 'w') as f: + json.dump(streamer_config, f) + + manager = ConfigManager() + config = manager.load_streamer_config('existingstreamer') + + # Should load streamer config + self.assertEqual(config['username'], 'existingstreamer') + self.assertEqual(config['quality'], 'source') + self.assertTrue(config['downloadVOD']) + + def test_load_streamer_config_merges_with_global(self): + """Test that streamer config merges with global config.""" + # Create global config + with open('config/global.json', 'w') as f: + json.dump({ + 'quality': '720p', + 'downloadVOD': True, + 'downloadCHAT': False + }, f) + + # Create streamer config with override + with open('config/streamers/teststreamer.json', 'w') as f: + json.dump({ + 'username': 'teststreamer', + 'enabled': True, + 'quality': 'source' # Override global + }, f) + + manager = ConfigManager() + config = manager.load_streamer_config('teststreamer') + + # Should have streamer's quality (override) + self.assertEqual(config['quality'], 'source') + # Should have global's downloadVOD (inherited) + self.assertTrue(config['downloadVOD']) + # Should have global's downloadCHAT (inherited) + self.assertFalse(config['downloadCHAT']) + + def test_load_streamer_config_filters_comments(self): + """Test that streamer config filters out comments.""" + with open('config/streamers/teststreamer.json', 'w') as f: + json.dump({ + '_comment': 'Test comment', + 'username': 'teststreamer', + 'enabled': True, + '_note': 'Another note' + }, f) + + manager = ConfigManager() + config = manager.load_streamer_config('teststreamer') + + # Comments should be filtered + self.assertNotIn('_comment', config) + self.assertNotIn('_note', config) + self.assertEqual(config['username'], 'teststreamer') + + def test_get_all_enabled_streamers_empty(self): + """Test getting enabled streamers when none exist.""" + manager = ConfigManager() + + streamers = manager.get_all_enabled_streamers() + + self.assertEqual(len(streamers), 0) + + def test_get_all_enabled_streamers_with_enabled(self): + """Test getting enabled streamers.""" + # Create multiple streamer configs + with open('config/streamers/streamer1.json', 'w') as f: + json.dump({'username': 'streamer1', 'enabled': True}, f) + + with open('config/streamers/streamer2.json', 'w') as f: + json.dump({'username': 'streamer2', 'enabled': True}, f) + + with open('config/streamers/streamer3.json', 'w') as f: + json.dump({'username': 'streamer3', 'enabled': False}, f) + + manager = ConfigManager() + streamers = manager.get_all_enabled_streamers() + + # Should return only enabled streamers + self.assertEqual(len(streamers), 2) + self.assertIn('streamer1', streamers) + self.assertIn('streamer2', streamers) + self.assertNotIn('streamer3', streamers) + + def test_get_all_enabled_streamers_default_enabled(self): + """Test that streamers are enabled by default if not specified.""" + # Create config without explicit enabled field + with open('config/streamers/streamer1.json', 'w') as f: + json.dump({'username': 'streamer1'}, f) + + manager = ConfigManager() + streamers = manager.get_all_enabled_streamers() + + # Should be included (defaults to enabled) + self.assertIn('streamer1', streamers) + + def test_create_default_streamer_config(self): + """Test creating default streamer config.""" + manager = ConfigManager() + manager.create_default_streamer_config('newstreamer') + + # File should exist + config_file = Path('config/streamers/newstreamer.json') + self.assertTrue(config_file.exists()) + + # Should have correct structure + with open(config_file) as f: + config = json.load(f) + + self.assertEqual(config['username'], 'newstreamer') + self.assertTrue(config['enabled']) + self.assertIn('$schema', config) + + + + + +if __name__ == '__main__': + # Run tests with verbose output + print("="*70) + print("TWITCH ARCHIVE - Unit Tests for Options and Configuration") + print("="*70) + print() + unittest.main(verbosity=2) diff --git a/test_twitch_archive_simple.py b/test_twitch_archive_simple.py new file mode 100644 index 0000000..0d49da2 --- /dev/null +++ b/test_twitch_archive_simple.py @@ -0,0 +1,448 @@ +""" +Unit tests for Twitch Archive command-line options and configuration. + +Tests focus on: +- Command-line argument parsing (via getopt simulation) +- Options and option combinations +- Configuration logic (filtering, merging, etc.) +- Mode selection logic + +Excludes actual download/processing functionality. + +To run these tests: + python test_twitch_archive_simple.py + or + python -m pytest test_twitch_archive_simple.py -v +""" + +import unittest +import sys +import os +import json +import getopt +from unittest.mock import patch, MagicMock, Mock + +# Add parent directory to path for imports +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from modules.constants import DEFAULT_CONFIG + + +class TestCommandLineArgumentParsing(unittest.TestCase): + """Test command-line argument parsing logic using getopt directly.""" + + def test_help_short_option(self): + """Test -h option parsing.""" + argv = ['-h'] + opts, args = getopt.getopt( + argv, + "hu:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + # Should parse successfully + self.assertEqual(len(opts), 1) + self.assertEqual(opts[0][0], '-h') + + def test_help_long_option(self): + """Test --help option parsing.""" + argv = ['--help'] + opts, args = getopt.getopt( + argv, + "hu:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 1) + self.assertEqual(opts[0][0], '--help') + + def test_username_short_option(self): + """Test -u username option parsing.""" + argv = ['-u', 'teststreamer'] + opts, args = getopt.getopt( + argv, + "hu:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 1) + self.assertEqual(opts[0], ('-u', 'teststreamer')) + + def test_username_long_option(self): + """Test --username option parsing.""" + argv = ['--username', 'teststreamer'] + opts, args = getopt.getopt( + argv, + "hu:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 1) + self.assertEqual(opts[0], ('--username', 'teststreamer')) + + def test_verbose_option(self): + """Test --verbose option parsing.""" + argv = ['--verbose'] + opts, args = getopt.getopt( + argv, + "hu:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 1) + self.assertEqual(opts[0], ('--verbose', '')) + + def test_chat_only_option(self): + """Test --chat-only option parsing.""" + argv = ['--chat-only'] + opts, args = getopt.getopt( + argv, + "hu:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 1) + self.assertEqual(opts[0], ('--chat-only', '')) + + def test_legacy_option(self): + """Test --legacy option parsing.""" + argv = ['--legacy'] + opts, args = getopt.getopt( + argv, + "hu:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 1) + self.assertEqual(opts[0], ('--legacy', '')) + + def test_chat_downloader_options(self): + """Test chat downloader option parsing.""" + argv = ['--use-chat-downloader-primary', '--no-chat-downloader-fallback'] + opts, args = getopt.getopt( + argv, + "hu:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 2) + self.assertEqual(opts[0], ('--use-chat-downloader-primary', '')) + self.assertEqual(opts[1], ('--no-chat-downloader-fallback', '')) + + def test_legacy_quality_option(self): + """Test -q quality option parsing.""" + argv = ['-q', '720p'] + opts, args = getopt.getopt( + argv, + "hu:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 1) + self.assertEqual(opts[0], ('-q', '720p')) + + def test_legacy_boolean_options(self): + """Test legacy boolean option parsing.""" + argv = ['-v', '1', '-c', '0', '-m', '1'] + opts, args = getopt.getopt( + argv, + "hu:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 3) + self.assertEqual(opts[0], ('-v', '1')) + self.assertEqual(opts[1], ('-c', '0')) + self.assertEqual(opts[2], ('-m', '1')) + + def test_invalid_option(self): + """Test that invalid option raises error.""" + argv = ['--invalid-option'] + + with self.assertRaises(getopt.GetoptError): + getopt.getopt( + argv, + "hu:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + def test_option_combination_username_verbose(self): + """Test combining -u and --verbose options.""" + argv = ['-u', 'testuser', '--verbose'] + opts, args = getopt.getopt( + argv, + "hu:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 2) + self.assertEqual(opts[0], ('-u', 'testuser')) + self.assertEqual(opts[1], ('--verbose', '')) + + def test_option_combination_all_test_flags(self): + """Test combining all test-related flags.""" + argv = ['-u', 'testuser', '--verbose', '--chat-only', '--use-chat-downloader-primary'] + opts, args = getopt.getopt( + argv, + "hu:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 4) + opt_dict = dict(opts) + self.assertEqual(opt_dict['-u'], 'testuser') + self.assertIn('--verbose', opt_dict) + self.assertIn('--chat-only', opt_dict) + self.assertIn('--use-chat-downloader-primary', opt_dict) + + def test_option_combination_legacy_mode_with_overrides(self): + """Test legacy mode with multiple overrides.""" + argv = ['--legacy', '-q', '720p', '-v', '1', '-c', '1', '-m', '0'] + opts, args = getopt.getopt( + argv, + "hu:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 5) + opt_dict = dict(opts) + self.assertIn('--legacy', opt_dict) + self.assertEqual(opt_dict['-q'], '720p') + self.assertEqual(opt_dict['-v'], '1') + self.assertEqual(opt_dict['-c'], '1') + self.assertEqual(opt_dict['-m'], '0') + + +class TestOptionLogicProcessing(unittest.TestCase): + """Test the logic that processes parsed options.""" + + def test_boolean_conversion_true(self): + """Test converting '1' to boolean True.""" + value = '1' + result = bool(int(value)) + self.assertTrue(result) + + def test_boolean_conversion_false(self): + """Test converting '0' to boolean False.""" + value = '0' + result = bool(int(value)) + self.assertFalse(result) + + def test_chat_only_auto_enables_verbose(self): + """Test that chat-only mode should auto-enable verbose.""" + # Simulate the logic from main() + chat_only_mode = True + verbose_mode = False + + if chat_only_mode: + verbose_mode = True + + self.assertTrue(verbose_mode) + + def test_default_chat_downloader_fallback(self): + """Test that chat downloader fallback defaults to enabled.""" + use_chat_downloader_fallback = True # Default value + + # Unless explicitly disabled + self.assertTrue(use_chat_downloader_fallback) + + def test_mode_selection_legacy_with_config_json(self): + """Test mode selection logic when config.json exists.""" + # Simulate conditions + use_legacy_mode = False + legacy_config_exists = True + specific_streamer = None + global_config_exists = False + + # Logic from main() function + should_use_legacy = use_legacy_mode or ( + legacy_config_exists and not specific_streamer and not global_config_exists + ) + + self.assertTrue(should_use_legacy) + + def test_mode_selection_multi_streamer_with_global_json(self): + """Test mode selection logic when global.json exists.""" + use_legacy_mode = False + legacy_config_exists = True + specific_streamer = None + global_config_exists = True + + should_use_legacy = use_legacy_mode or ( + legacy_config_exists and not specific_streamer and not global_config_exists + ) + + self.assertFalse(should_use_legacy) + + def test_mode_selection_multi_streamer_with_username_flag(self): + """Test mode selection when -u flag is used.""" + use_legacy_mode = False + legacy_config_exists = True + specific_streamer = 'testuser' + global_config_exists = False + + should_use_legacy = use_legacy_mode or ( + legacy_config_exists and not specific_streamer and not global_config_exists + ) + + self.assertFalse(should_use_legacy) + + def test_mode_selection_explicit_legacy_flag(self): + """Test mode selection with explicit --legacy flag.""" + use_legacy_mode = True + legacy_config_exists = False + specific_streamer = None + global_config_exists = True + + should_use_legacy = use_legacy_mode or ( + legacy_config_exists and not specific_streamer and not global_config_exists + ) + + self.assertTrue(should_use_legacy) + + +class TestConfigLogic(unittest.TestCase): + """Test configuration management logic.""" + + def test_comment_filtering_logic(self): + """Test that comment fields are filtered out.""" + user_config = { + '_comment': 'This is a comment', + 'quality': '720p', + '_note': 'Another comment', + 'username': 'testuser' + } + + # Apply the filtering logic (same as in ConfigManager) + filtered = {k: v for k, v in user_config.items() if not k.startswith('_')} + + self.assertNotIn('_comment', filtered) + self.assertNotIn('_note', filtered) + self.assertIn('quality', filtered) + self.assertIn('username', filtered) + self.assertEqual(filtered['quality'], '720p') + + def test_schema_filtering_logic(self): + """Test that $schema field is filtered out.""" + user_config = { + '$schema': './config.schema.json', + 'quality': '720p', + 'username': 'testuser' + } + + # Apply the filtering logic (same as in ConfigManager) + filtered = {k: v for k, v in user_config.items() + if not k.startswith('_') and k != '$schema'} + + self.assertNotIn('$schema', filtered) + self.assertIn('quality', filtered) + self.assertIn('username', filtered) + + def test_config_merging_logic(self): + """Test config merging logic (streamer overrides global).""" + global_config = { + 'quality': '720p', + 'downloadVOD': True, + 'downloadCHAT': False, + 'username': 'default' + } + + streamer_config = { + 'username': 'specificstreamer', + 'quality': 'source', # Override + # downloadVOD and downloadCHAT inherited from global + } + + # Simulate merging (same as in ConfigManager.load_streamer_config) + merged = global_config.copy() + merged.update(streamer_config) + + # Check overrides + self.assertEqual(merged['quality'], 'source') # Overridden + self.assertEqual(merged['username'], 'specificstreamer') # Overridden + # Check inherited values + self.assertTrue(merged['downloadVOD']) + self.assertFalse(merged['downloadCHAT']) + + def test_default_config_structure(self): + """Test that DEFAULT_CONFIG has expected keys.""" + self.assertIn('username', DEFAULT_CONFIG) + self.assertIn('quality', DEFAULT_CONFIG) + self.assertIn('downloadVOD', DEFAULT_CONFIG) + self.assertIn('downloadCHAT', DEFAULT_CONFIG) + self.assertIn('downloadMETADATA', DEFAULT_CONFIG) + self.assertIn('uploadCloud', DEFAULT_CONFIG) + self.assertIn('deleteFiles', DEFAULT_CONFIG) + self.assertIn('notifications', DEFAULT_CONFIG) + self.assertIn('refresh', DEFAULT_CONFIG) + + def test_enabled_flag_filtering_logic(self): + """Test filtering streamers by enabled flag.""" + streamers = [ + {'username': 'streamer1', 'enabled': True}, + {'username': 'streamer2', 'enabled': True}, + {'username': 'streamer3', 'enabled': False}, + {'username': 'streamer4'}, # No enabled field + ] + + # Simulate filtering logic (same as in ConfigManager.get_all_enabled_streamers) + enabled_streamers = [ + s['username'] for s in streamers + if s.get('enabled', True) # Default to True if not specified + ] + + self.assertIn('streamer1', enabled_streamers) + self.assertIn('streamer2', enabled_streamers) + self.assertNotIn('streamer3', enabled_streamers) + self.assertIn('streamer4', enabled_streamers) # Default enabled + + def test_default_streamer_config_structure(self): + """Test default streamer config structure.""" + # Simulate what create_default_streamer_config creates + username = 'newstreamer' + default_config = { + "$schema": "../streamer.schema.json", + "username": username, + "enabled": True + } + + self.assertEqual(default_config['username'], username) + self.assertTrue(default_config['enabled']) + self.assertIn('$schema', default_config) + + +if __name__ == '__main__': + # Run tests with verbose output + print("="*70) + print("TWITCH ARCHIVE - Unit Tests for Options and Configuration") + print("="*70) + print() + unittest.main(verbosity=2) From 22a1f5b6006d7b652f50722db3e4572d193af544 Mon Sep 17 00:00:00 2001 From: MaddoScientisto Date: Sun, 15 Feb 2026 09:38:58 +0100 Subject: [PATCH 03/10] feat: add standalone chat downloader script and batch file for testing --- modules/downloader.py | 160 +++++++++++++++++++++++++++++++++++++++--- run_chat_only.py | 109 ++++++++++++++++++++++++++++ start_chat_only.bat | 18 +++++ 3 files changed, 279 insertions(+), 8 deletions(-) create mode 100644 run_chat_only.py create mode 100644 start_chat_only.bat diff --git a/modules/downloader.py b/modules/downloader.py index a1f166b..1a7ba00 100644 --- a/modules/downloader.py +++ b/modules/downloader.py @@ -8,6 +8,8 @@ import subprocess import json import threading import time +import socket +import re from typing import Dict, Any, Optional from colorama import Fore, Style @@ -325,6 +327,118 @@ class ContentDownloader: except Exception as e: print(f'{Fore.RED}✗ Failed to start live chat download: {str(e)}{Style.RESET_ALL}') return None + + def _download_live_chat_via_irc(self, username: str, json_path: str, + max_messages: Optional[int] = None, + timeout: Optional[float] = None, + shutdown_check: Optional[callable] = None, + stream_monitor = None, + verbose: bool = False) -> bool: + """ + Simple IRC-based fallback to capture Twitch chat when GraphQL methods fail. + + This writes newline-delimited JSON objects with at least: timestamp (ms), + author (dict with `name`), and `message`. + """ + try: + sock = socket.socket() + sock.connect(('irc.chat.twitch.tv', 6667)) + sock.settimeout(1.0) + + # Request tags & capabilities + sock.sendall(b'CAP REQ :twitch.tv/tags twitch.tv/commands twitch.tv/membership\r\n') + sock.sendall(b'PASS SCHMOOPIIE\r\n') + sock.sendall(b'NICK justinfan67420\r\n') + sock.sendall(f'JOIN #{username}\r\n'.encode('utf-8')) + + messages_written = 0 + start_time = time.time() + + # Open file for streaming newline-delimited JSON + os.makedirs(os.path.dirname(json_path), exist_ok=True) + with open(json_path, 'w', encoding='utf-8') as out_f: + buffer = '' + while True: + # Shutdown/timeouts + if shutdown_check and shutdown_check(): + break + if timeout and (time.time() - start_time) > timeout: + break + if stream_monitor: + try: + if not stream_monitor.is_user_live(): + break + except Exception: + pass + + try: + data = sock.recv(4096).decode('utf-8', 'ignore') + except socket.timeout: + continue + except Exception as e: + print(f'{Fore.YELLOW}⚠ IRC recv error: {e}{Style.RESET_ALL}') + break + + if not data: + continue + + buffer += data + lines = buffer.split('\r\n') + buffer = lines.pop() # remainder + + for line in lines: + if not line: + continue + # Respond to PINGs + if line.startswith('PING'): + try: + sock.sendall(b'PONG :tmi.twitch.tv\r\n') + except Exception: + pass + continue + + # Extract PRIVMSG lines + m = re.match(r'(?:@[^ ]+ )?:([^!]+)!.* PRIVMSG #[^ ]+ :(.+)', line) + if not m: + continue + + author = m.group(1) + msg_text = m.group(2) + timestamp_ms = int(time.time() * 1000) + + item = { + 'timestamp': timestamp_ms, + 'author': {'name': author}, + 'message': msg_text + } + + out_f.write(json.dumps(item, ensure_ascii=False) + '\n') + out_f.flush() + messages_written += 1 + + if verbose and (messages_written % 10 == 0): + print(f'\n{Fore.GREEN}💬 {author}: {Fore.WHITE}{msg_text}{Style.RESET_ALL}') + + if max_messages and messages_written >= max_messages: + break + + if max_messages and messages_written >= max_messages: + break + + sock.close() + + if messages_written > 0: + print(f'\n{Fore.GREEN}✓ IRC fallback captured {messages_written} messages{Style.RESET_ALL}') + return True + else: + print(f'\n{Fore.RED}✗ IRC fallback captured no messages{Style.RESET_ALL}') + return False + + except Exception as e: + print(f'{Fore.RED}✗ IRC fallback failed: {e}{Style.RESET_ALL}') + import traceback + traceback.print_exc() + return False def wait_for_chat_download(self, process: Optional[subprocess.Popen], json_path: str, timeout: int = 300) -> bool: @@ -403,15 +517,45 @@ class ContentDownloader: print(f'{Fore.MAGENTA}[VERBOSE] Timeout: {timeout}s (None = unlimited){Style.RESET_ALL}') print(f'{Fore.MAGENTA}[VERBOSE] Max messages: {max_messages} (None = unlimited){Style.RESET_ALL}') - # Get chat messages + # Get chat messages with a small retry loop to handle transient GQL/network issues print(f'{Fore.CYAN}Connecting to Twitch chat...{Style.RESET_ALL}') - chat = self.chat_downloader.get_chat( - stream_url, - message_types=['text_message'], # Basic text messages - output=json_path, - timeout=timeout, - max_messages=max_messages - ) + chat = None + max_attempts = 3 + for attempt in range(1, max_attempts + 1): + try: + chat = self.chat_downloader.get_chat( + stream_url, + message_types=['text_message'], # Basic text messages + output=json_path, + timeout=timeout, + max_messages=max_messages + ) + break + except Exception as e: + # Provide a clearer, user-facing message for common failures + print(f"{Fore.YELLOW}⚠ chat_downloader attempt {attempt}/{max_attempts} failed: {str(e)}{Style.RESET_ALL}") + # On final attempt, dump traceback to help diagnose library internals + if attempt >= max_attempts: + print(f"{Fore.RED}✗ chat_downloader failed after {max_attempts} attempts. This may be caused by Twitch GraphQL changes or rate-limiting.{Style.RESET_ALL}") + print(f"{Fore.YELLOW} Try upgrading the chat-downloader package: pip install -U chat-downloader{Style.RESET_ALL}") + import traceback + traceback.print_exc() + # Try IRC fallback before giving up + print(f"{Fore.MAGENTA}[VERBOSE] Attempting IRC fallback for chat capture...{Style.RESET_ALL}") + try: + return self._download_live_chat_via_irc(username, json_path, + max_messages=max_messages, + timeout=timeout, + shutdown_check=shutdown_check, + stream_monitor=stream_monitor, + verbose=verbose) + except Exception as fallback_err: + print(f"{Fore.RED}✗ IRC fallback failed: {fallback_err}{Style.RESET_ALL}") + traceback.print_exc() + return False + else: + time.sleep(1) + continue # The get_chat with output parameter writes to file automatically # We just need to iterate to trigger the download diff --git a/run_chat_only.py b/run_chat_only.py new file mode 100644 index 0000000..ff94189 --- /dev/null +++ b/run_chat_only.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +""" +Start chat downloader standalone for testing without recording video. + +Usage: + python run_chat_only.py --username vinesauce [--output path] [--max-messages N] [--timeout S] [--verbose] + +This script uses the project's `ConfigManager` and `FileManager` to create +appropriate directories and then starts the chat downloader in a background +thread. Press Ctrl+C to stop. +""" +import argparse +import time +from datetime import datetime +import os + +from colorama import Fore, Style + +from modules.config import ConfigManager +from modules.file_manager import FileManager +from modules.utils import get_ffmpeg_executable, get_twitch_downloader_executable, detect_operating_system +from modules.downloader import ContentDownloader + + +def main(): + parser = argparse.ArgumentParser(description='Run chat downloader standalone for testing') + parser.add_argument('--username', '-u', required=True, help='Twitch username/channel name') + parser.add_argument('--output', '-o', help='Output JSON path (optional)') + parser.add_argument('--max-messages', type=int, default=None, help='Max messages to capture') + parser.add_argument('--timeout', type=float, default=None, help='Timeout in seconds') + parser.add_argument('--verbose', action='store_true', help='Show verbose/chat previews') + parser.add_argument('--foreground', action='store_true', help='Run downloader in foreground (blocking)') + parser.add_argument('--use-chat-downloader-primary', action='store_true', help='Use chat_downloader as primary method') + parser.add_argument('--use-chat-downloader-fallback', dest='use_chat_downloader_fallback', action='store_true', help='Allow chat_downloader as fallback (default)') + parser.add_argument('--no-chat-downloader-fallback', dest='use_chat_downloader_fallback', action='store_false', help='Disable chat_downloader fallback') + args = parser.parse_args() + + cfg = ConfigManager() + config = cfg.load_streamer_config(args.username) + + # Apply overrides from CLI + if args.use_chat_downloader_primary: + config['useChatDownloaderPrimary'] = True + if args.use_chat_downloader_fallback is not None: + config['useChatDownloaderFallback'] = bool(args.use_chat_downloader_fallback) + + # Ensure directories exist (use configured archive root path) + fm = FileManager(root_path=config.get('root_path', 'archive'), username=args.username, config=config) + fm.initialize_directories() + + # Build default output path if not provided + if args.output: + json_path = args.output + else: + ts = datetime.now().strftime('%Y%m%d_%Hh%Mm%Ss') + json_path = str(fm.chat_json_path / f"CHAT_TEST_{args.username}_{ts}.json") + + # Initialize downloader + os_type = detect_operating_system() + twitch_downloader_path = get_twitch_downloader_executable(os_type) + ffmpeg_path = get_ffmpeg_executable(os_type) + + downloader = ContentDownloader(twitch_downloader_path=twitch_downloader_path, + ffmpeg_path=ffmpeg_path, + config=config) + + print(f"{Fore.CYAN}Starting standalone chat downloader for {args.username}{Style.RESET_ALL}") + print(f"Output path: {json_path}") + + stop_requested = {'stop': False} + + def shutdown_check(): + return stop_requested['stop'] + + # Start thread + thread = downloader.start_chat_downloader_thread( + args.username, + json_path, + shutdown_check=shutdown_check, + stream_monitor=None, + verbose=args.verbose + ) + + try: + if args.foreground: + # Run download directly in foreground + print('Running in foreground; this will block until download completes or interrupted') + success = downloader.download_live_chat_with_chat_downloader( + args.username, + json_path, + max_messages=args.max_messages, + timeout=args.timeout, + shutdown_check=shutdown_check, + stream_monitor=None, + verbose=args.verbose + ) + print('Done, success=' + str(success)) + else: + # Wait for thread to finish or until interrupted + while thread.is_alive(): + time.sleep(0.5) + except KeyboardInterrupt: + print('\nKeyboard interrupt received; stopping downloader...') + stop_requested['stop'] = True + thread.join(timeout=5) + + +if __name__ == '__main__': + main() diff --git a/start_chat_only.bat b/start_chat_only.bat new file mode 100644 index 0000000..e1c4d4a --- /dev/null +++ b/start_chat_only.bat @@ -0,0 +1,18 @@ +rem @echo off +setlocal + +rem Set the path to your virtual environment +set VENV_PATH=.\venv314 + +rem Activate the virtual environment +call "%VENV_PATH%\Scripts\activate.bat" +rem Ensure required packages are installed +pip install -r requirements.txt + +rem Run the standalone chat downloader script +rem Usage: start_chat_only.bat [--output path] [--max-messages N] [--timeout S] [--verbose] [--foreground] [--use-chat-downloader-primary] +rem Pass username as -u and forward additional arguments (mirrors start.bat behavior) +python run_chat_only.py -u %1 %2 %3 %4 %5 %6 %7 %8 %9 + +rem Deactivate the virtual environment +call "%VENV_PATH%\Scripts\deactivate.bat" From b47641feaaa1da648755135d1297e819eff8a8ad Mon Sep 17 00:00:00 2001 From: MaddoScientisto Date: Wed, 18 Feb 2026 18:11:53 +0100 Subject: [PATCH 04/10] feat: enhance chat downloading with stream monitoring and improved file paths --- modules/downloader.py | 10 +++++++ modules/file_manager.py | 65 +++++++++++++++++++++++------------------ run_chat_only.py | 42 ++++++++++++++++++++++++-- 3 files changed, 87 insertions(+), 30 deletions(-) diff --git a/modules/downloader.py b/modules/downloader.py index 1a7ba00..a4ea645 100644 --- a/modules/downloader.py +++ b/modules/downloader.py @@ -505,6 +505,16 @@ class ContentDownloader: if not self.download_live_chat: print(f'{Fore.YELLOW}⚠ downloadLiveCHAT is disabled in config{Style.RESET_ALL}') return False + + # If a stream monitor was provided, check that the user is currently live + if stream_monitor is not None: + try: + if not stream_monitor.is_user_live(): + print(f'{Fore.YELLOW}⚠ Stream is not live; skipping chat download{Style.RESET_ALL}') + return False + except Exception as e: + # If we couldn't determine live status, continue and let chat_downloader handle it + print(f'{Fore.YELLOW}⚠ Could not determine live status: {e} - proceeding with chat download{Style.RESET_ALL}') print(f'\n{Fore.CYAN}Starting live chat download (chat_downloader)...{Style.RESET_ALL}') print(f'{Fore.MAGENTA}[VERBOSE] chat_downloader library version: {ChatDownloader.__module__}{Style.RESET_ALL}') diff --git a/modules/file_manager.py b/modules/file_manager.py index 97f3909..4567cb6 100644 --- a/modules/file_manager.py +++ b/modules/file_manager.py @@ -129,51 +129,60 @@ class FileManager: os.makedirs(os.path.dirname(upload_list_path), exist_ok=True) files_to_upload = [] - - # Always include metadata and chat JSON - files_to_upload.append(f"{PREFIX_METADATA}{filename_base}.json") - files_to_upload.append(f"{PREFIX_CHAT}{filename_base}.json") - - # Add pre-merge videos (original LIVE and VOD files) + + # Build files list relative to root_path so rclone can read them with --files-from + # Metadata and chat JSON + files_to_upload.append(os.path.join(self.username, 'metadata', f"{PREFIX_METADATA}{filename_base}.json")) + files_to_upload.append(os.path.join(self.username, 'chat', 'json', f"{PREFIX_CHAT}{filename_base}.json")) + + # Pre-merge videos (raw .ts in video/raw, mp4/mp3 in video) if self.upload_pre_merge_video: files_to_upload.extend([ - f"{PREFIX_LIVE}{filename_base}.ts", - f"{PREFIX_LIVE}{filename_base}.mp4", - f"{PREFIX_LIVE}{filename_base}.mp3", - f"{PREFIX_VOD}{filename_base}.ts", - f"{PREFIX_VOD}{filename_base}.mp4", - f"{PREFIX_VOD}{filename_base}.mp3" + os.path.join(self.username, 'video', 'raw', f"{PREFIX_LIVE}{filename_base}.ts"), + os.path.join(self.username, 'video', f"{PREFIX_LIVE}{filename_base}.mp4"), + os.path.join(self.username, 'video', f"{PREFIX_LIVE}{filename_base}.mp3"), + os.path.join(self.username, 'video', 'raw', f"{PREFIX_VOD}{filename_base}.ts"), + os.path.join(self.username, 'video', f"{PREFIX_VOD}{filename_base}.mp4"), + os.path.join(self.username, 'video', f"{PREFIX_VOD}{filename_base}.mp3") ]) - - # Add merged videos + + # Merged videos (in video folder) if self.upload_merged_video: files_to_upload.extend([ - f"{PREFIX_MERGED}{filename_base}.mp4", - f"{PREFIX_MERGED}{filename_base}.mp3", - f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}.mp4", - f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}.mp3" + os.path.join(self.username, 'video', f"{PREFIX_MERGED}{filename_base}.mp4"), + os.path.join(self.username, 'video', f"{PREFIX_MERGED}{filename_base}.mp3"), + os.path.join(self.username, 'video', f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}.mp4"), + os.path.join(self.username, 'video', f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}.mp3") ]) - - # Add standalone chat video + + # Standalone chat video (in chat folder) if self.upload_chat_video: - files_to_upload.append(f"{PREFIX_CHAT}{filename_base}.mp4") + files_to_upload.append(os.path.join(self.username, 'chat', f"{PREFIX_CHAT}{filename_base}.mp4")) with open(upload_list_path, 'w') as f: f.write('\n'.join(files_to_upload)) - # Run rclone + # Run rclone using --files-from so the listed paths (relative to root_path) are uploaded. try: - result = subprocess.call([ + cmd = [ 'rclone', 'copy', str(self.root_path.resolve()), self.rclone_path, - '--include-from', upload_list_path - ]) - + '--files-from', upload_list_path + ] + + # Stream rclone output to console so user can see progress/errors + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + if proc.stdout: + for line in proc.stdout: + print(line, end='') + proc.wait() + result = proc.returncode + # Clean up upload list if os.path.exists(upload_list_path): os.remove(upload_list_path) - + if result == 0: print(f'{Fore.GREEN}✓ Upload complete{Style.RESET_ALL}') if notification_callback: @@ -186,7 +195,7 @@ class FileManager: notification_callback(f'✗ Upload Failed - {filename_base}', f'Upload failed with code {result}. Files preserved locally.') return False - + except Exception as e: print(f'{Fore.RED}✗ Upload error: {str(e)}{Style.RESET_ALL}') return False diff --git a/run_chat_only.py b/run_chat_only.py index ff94189..5134f38 100644 --- a/run_chat_only.py +++ b/run_chat_only.py @@ -20,6 +20,7 @@ from modules.config import ConfigManager from modules.file_manager import FileManager from modules.utils import get_ffmpeg_executable, get_twitch_downloader_executable, detect_operating_system from modules.downloader import ContentDownloader +from modules.stream_monitor import StreamMonitor def main(): @@ -72,12 +73,49 @@ def main(): def shutdown_check(): return stop_requested['stop'] - # Start thread + # Prepare stream monitor + stream_monitor = StreamMonitor(args.username) + + # If chat downloads are disabled in config, enter monitoring mode instead + if not downloader.download_live_chat: + print(f"{Fore.YELLOW}⚠ downloadLiveCHAT is disabled in config - entering monitoring mode for {args.username}{Style.RESET_ALL}") + try: + while True: + try: + is_live = stream_monitor.is_user_live() + if is_live: + print(f"{Fore.GREEN}✓ {args.username} is live! Exiting monitor. Run the archiver to record video.{Style.RESET_ALL}") + break + else: + print(f"{Fore.CYAN}{args.username} is offline - checking again in 30s...{Style.RESET_ALL}") + except Exception as e: + print(f"{Fore.YELLOW}⚠ Could not check stream status: {e}{Style.RESET_ALL}") + time.sleep(30) + except KeyboardInterrupt: + print('\nKeyboard interrupt received; stopping monitor...') + return + + # If chat download is enabled, but the stream is currently offline, wait until it goes live + try: + try: + if not stream_monitor.is_user_live(): + print(f"{Fore.CYAN}{args.username} is currently offline - waiting for live stream to start...{Style.RESET_ALL}") + while not stream_monitor.is_user_live(): + time.sleep(10) + except Exception: + # If we cannot determine live status, proceed to start chat downloader anyway + pass + + except KeyboardInterrupt: + print('\nKeyboard interrupt received; exiting...') + return + + # Start thread (stream_monitor passed so downloader can stop when stream ends) thread = downloader.start_chat_downloader_thread( args.username, json_path, shutdown_check=shutdown_check, - stream_monitor=None, + stream_monitor=stream_monitor, verbose=args.verbose ) From 4f488bae45922fa4552f2d644997a7314cd9297e Mon Sep 17 00:00:00 2001 From: MaddoScientisto Date: Sat, 21 Feb 2026 10:40:12 +0100 Subject: [PATCH 05/10] Refactor code structure for improved readability and maintainability --- .github/instructions/blazor.instructions.md | 77 + .gitignore | 46 +- Twitch-Archive-2.sln | 32 + UpgradePlan.md | 214 +++ UpgradePlan2.md | 78 + dotnet/Dockerfile | 18 + dotnet/README.md | 14 + dotnet/TwitchArchive.sln | 33 + dotnet/docker-compose.yml | 11 + .../Api/ITwitchApiClient.cs | 17 + .../TwitchArchive.Core/Api/TwitchApiClient.cs | 111 ++ .../TwitchArchive.Core/Config/AppSettings.cs | 22 + .../Config/ConfigurationService.cs | 88 + .../Config/EffectiveConfig.cs | 36 + .../TwitchArchive.Core/Config/GlobalConfig.cs | 37 + .../Config/IConfigurationService.cs | 18 + .../Config/StreamerConfig.cs | 25 + .../Config/ToolPathResolver.cs | 35 + .../Monitoring/DummyLiveChecker.cs | 17 + .../Monitoring/ILiveChecker.cs | 13 + .../Monitoring/TwitchLiveChecker.cs | 29 + .../Persistence/ArchiveDbContext.cs | 14 + .../Persistence/ISessionRepository.cs | 17 + .../Persistence/Models/ArchiveJob.cs | 16 + .../Persistence/Models/StreamSession.cs | 15 + .../Persistence/Models/StreamerState.cs | 13 + .../Persistence/SessionRepository.cs | 62 + .../Recovery/RecoveryPolicy.cs | 131 ++ .../Services/DownloaderService.cs | 50 + .../Services/FileManagerService.cs | 52 + .../Services/IDownloaderService.cs | 12 + .../Services/IProcessOutputStore.cs | 19 + .../Services/IProcessRunner.cs | 31 + .../Services/IProcessorService.cs | 11 + .../Services/ProcessOutputStore.cs | 59 + .../Services/ProcessRunner.cs | 101 ++ .../Services/ProcessorService.cs | 63 + .../TwitchArchive.Core.csproj | 18 + .../Workers/StreamWorker.cs | 178 ++ .../Workers/StreamWorkerManager.cs | 51 + .../ConfigurationServiceTests.cs | 39 + .../DownloaderServiceTests.cs | 55 + .../EffectiveConfigTests.cs | 27 + .../FileManagerServiceTests.cs | 38 + .../ProcessorServiceTests.cs | 57 + .../RecoveryPolicyTests.cs | 86 + .../SessionRepositoryTests.cs | 49 + .../TwitchApiClientTests.cs | 67 + .../TwitchArchive.Tests.csproj | 25 + dotnet/src/TwitchArchive.Web/App.razor | 14 + .../Hubs/ProcessOutputHub.cs | 20 + .../TwitchArchive.Web/Pages/AddStreamer.razor | 29 + .../TwitchArchive.Web/Pages/AppSettings.razor | 92 + .../Pages/GlobalConfig.razor | 45 + .../src/TwitchArchive.Web/Pages/Index.razor | 64 + .../src/TwitchArchive.Web/Pages/Login.razor | 20 + .../src/TwitchArchive.Web/Pages/Monitor.razor | 37 + .../TwitchArchive.Web/Pages/Sessions.razor | 73 + .../Pages/StreamerConfig.razor | 71 + .../Pages/StreamerDetail.razor | 22 + .../src/TwitchArchive.Web/Pages/_Host.cshtml | 19 + dotnet/src/TwitchArchive.Web/Program.cs | 104 ++ .../Properties/launchSettings.json | 12 + .../TwitchArchive.Web/Services/AuthService.cs | 50 + .../Services/IAuthService.cs | 11 + .../Services/ProcessOutputBroadcaster.cs | 35 + .../Services/SessionCacheService.cs | 38 + .../Services/SessionRefreshHostedService.cs | 50 + .../TwitchArchive.Web/Shared/MainLayout.razor | 21 + .../Shared/ProcessConsole.razor | 93 + .../TwitchArchive.Web.csproj | 16 + dotnet/src/TwitchArchive.Web/_Imports.razor | 10 + dotnet/src/TwitchArchive.Web/archive.db | Bin 0 -> 4096 bytes dotnet/src/TwitchArchive.Web/archive.db-shm | Bin 0 -> 32768 bytes dotnet/src/TwitchArchive.Web/archive.db-wal | Bin 0 -> 20632 bytes .../src/TwitchArchive.Web/wwwroot/css/app.css | 16 + only-vod-chat.py | 25 +- twitch-archive.py | 1565 ----------------- 78 files changed, 3309 insertions(+), 1570 deletions(-) create mode 100644 .github/instructions/blazor.instructions.md create mode 100644 Twitch-Archive-2.sln create mode 100644 UpgradePlan.md create mode 100644 UpgradePlan2.md create mode 100644 dotnet/Dockerfile create mode 100644 dotnet/README.md create mode 100644 dotnet/TwitchArchive.sln create mode 100644 dotnet/docker-compose.yml create mode 100644 dotnet/src/TwitchArchive.Core/Api/ITwitchApiClient.cs create mode 100644 dotnet/src/TwitchArchive.Core/Api/TwitchApiClient.cs create mode 100644 dotnet/src/TwitchArchive.Core/Config/AppSettings.cs create mode 100644 dotnet/src/TwitchArchive.Core/Config/ConfigurationService.cs create mode 100644 dotnet/src/TwitchArchive.Core/Config/EffectiveConfig.cs create mode 100644 dotnet/src/TwitchArchive.Core/Config/GlobalConfig.cs create mode 100644 dotnet/src/TwitchArchive.Core/Config/IConfigurationService.cs create mode 100644 dotnet/src/TwitchArchive.Core/Config/StreamerConfig.cs create mode 100644 dotnet/src/TwitchArchive.Core/Config/ToolPathResolver.cs create mode 100644 dotnet/src/TwitchArchive.Core/Monitoring/DummyLiveChecker.cs create mode 100644 dotnet/src/TwitchArchive.Core/Monitoring/ILiveChecker.cs create mode 100644 dotnet/src/TwitchArchive.Core/Monitoring/TwitchLiveChecker.cs create mode 100644 dotnet/src/TwitchArchive.Core/Persistence/ArchiveDbContext.cs create mode 100644 dotnet/src/TwitchArchive.Core/Persistence/ISessionRepository.cs create mode 100644 dotnet/src/TwitchArchive.Core/Persistence/Models/ArchiveJob.cs create mode 100644 dotnet/src/TwitchArchive.Core/Persistence/Models/StreamSession.cs create mode 100644 dotnet/src/TwitchArchive.Core/Persistence/Models/StreamerState.cs create mode 100644 dotnet/src/TwitchArchive.Core/Persistence/SessionRepository.cs create mode 100644 dotnet/src/TwitchArchive.Core/Recovery/RecoveryPolicy.cs create mode 100644 dotnet/src/TwitchArchive.Core/Services/DownloaderService.cs create mode 100644 dotnet/src/TwitchArchive.Core/Services/FileManagerService.cs create mode 100644 dotnet/src/TwitchArchive.Core/Services/IDownloaderService.cs create mode 100644 dotnet/src/TwitchArchive.Core/Services/IProcessOutputStore.cs create mode 100644 dotnet/src/TwitchArchive.Core/Services/IProcessRunner.cs create mode 100644 dotnet/src/TwitchArchive.Core/Services/IProcessorService.cs create mode 100644 dotnet/src/TwitchArchive.Core/Services/ProcessOutputStore.cs create mode 100644 dotnet/src/TwitchArchive.Core/Services/ProcessRunner.cs create mode 100644 dotnet/src/TwitchArchive.Core/Services/ProcessorService.cs create mode 100644 dotnet/src/TwitchArchive.Core/TwitchArchive.Core.csproj create mode 100644 dotnet/src/TwitchArchive.Core/Workers/StreamWorker.cs create mode 100644 dotnet/src/TwitchArchive.Core/Workers/StreamWorkerManager.cs create mode 100644 dotnet/src/TwitchArchive.Tests/ConfigurationServiceTests.cs create mode 100644 dotnet/src/TwitchArchive.Tests/DownloaderServiceTests.cs create mode 100644 dotnet/src/TwitchArchive.Tests/EffectiveConfigTests.cs create mode 100644 dotnet/src/TwitchArchive.Tests/FileManagerServiceTests.cs create mode 100644 dotnet/src/TwitchArchive.Tests/ProcessorServiceTests.cs create mode 100644 dotnet/src/TwitchArchive.Tests/RecoveryPolicyTests.cs create mode 100644 dotnet/src/TwitchArchive.Tests/SessionRepositoryTests.cs create mode 100644 dotnet/src/TwitchArchive.Tests/TwitchApiClientTests.cs create mode 100644 dotnet/src/TwitchArchive.Tests/TwitchArchive.Tests.csproj create mode 100644 dotnet/src/TwitchArchive.Web/App.razor create mode 100644 dotnet/src/TwitchArchive.Web/Hubs/ProcessOutputHub.cs create mode 100644 dotnet/src/TwitchArchive.Web/Pages/AddStreamer.razor create mode 100644 dotnet/src/TwitchArchive.Web/Pages/AppSettings.razor create mode 100644 dotnet/src/TwitchArchive.Web/Pages/GlobalConfig.razor create mode 100644 dotnet/src/TwitchArchive.Web/Pages/Index.razor create mode 100644 dotnet/src/TwitchArchive.Web/Pages/Login.razor create mode 100644 dotnet/src/TwitchArchive.Web/Pages/Monitor.razor create mode 100644 dotnet/src/TwitchArchive.Web/Pages/Sessions.razor create mode 100644 dotnet/src/TwitchArchive.Web/Pages/StreamerConfig.razor create mode 100644 dotnet/src/TwitchArchive.Web/Pages/StreamerDetail.razor create mode 100644 dotnet/src/TwitchArchive.Web/Pages/_Host.cshtml create mode 100644 dotnet/src/TwitchArchive.Web/Program.cs create mode 100644 dotnet/src/TwitchArchive.Web/Properties/launchSettings.json create mode 100644 dotnet/src/TwitchArchive.Web/Services/AuthService.cs create mode 100644 dotnet/src/TwitchArchive.Web/Services/IAuthService.cs create mode 100644 dotnet/src/TwitchArchive.Web/Services/ProcessOutputBroadcaster.cs create mode 100644 dotnet/src/TwitchArchive.Web/Services/SessionCacheService.cs create mode 100644 dotnet/src/TwitchArchive.Web/Services/SessionRefreshHostedService.cs create mode 100644 dotnet/src/TwitchArchive.Web/Shared/MainLayout.razor create mode 100644 dotnet/src/TwitchArchive.Web/Shared/ProcessConsole.razor create mode 100644 dotnet/src/TwitchArchive.Web/TwitchArchive.Web.csproj create mode 100644 dotnet/src/TwitchArchive.Web/_Imports.razor create mode 100644 dotnet/src/TwitchArchive.Web/archive.db create mode 100644 dotnet/src/TwitchArchive.Web/archive.db-shm create mode 100644 dotnet/src/TwitchArchive.Web/archive.db-wal create mode 100644 dotnet/src/TwitchArchive.Web/wwwroot/css/app.css delete mode 100644 twitch-archive.py diff --git a/.github/instructions/blazor.instructions.md b/.github/instructions/blazor.instructions.md new file mode 100644 index 0000000..4e88cc0 --- /dev/null +++ b/.github/instructions/blazor.instructions.md @@ -0,0 +1,77 @@ +--- +description: 'Blazor component and application patterns' +applyTo: '**/*.razor, **/*.razor.cs, **/*.razor.css' +--- + +## Blazor Code Style and Structure + +- Write idiomatic and efficient Blazor and C# code. +- Follow .NET and Blazor conventions. +- Use Razor Components appropriately for component-based UI development. +- Prefer inline functions for smaller components but separate complex logic into code-behind or service classes. +- Async/await should be used where applicable to ensure non-blocking UI operations. + +## Naming Conventions + +- Follow PascalCase for component names, method names, and public members. +- Use camelCase for private fields and local variables. +- Prefix interface names with "I" (e.g., IUserService). + +## Blazor and .NET Specific Guidelines + +- Utilize Blazor's built-in features for component lifecycle (e.g., OnInitializedAsync, OnParametersSetAsync). +- Use data binding effectively with @bind. +- Leverage Dependency Injection for services in Blazor. +- Structure Blazor components and services following Separation of Concerns. +- Always use the latest version C#, currently C# 13 features like record types, pattern matching, and global usings. + +## Error Handling and Validation + +- Implement proper error handling for Blazor pages and API calls. +- Use logging for error tracking in the backend and consider capturing UI-level errors in Blazor with tools like ErrorBoundary. +- Implement validation using FluentValidation or DataAnnotations in forms. + +## Blazor API and Performance Optimization + +- Utilize Blazor server-side or WebAssembly optimally based on the project requirements. +- Use asynchronous methods (async/await) for API calls or UI actions that could block the main thread. +- Optimize Razor components by reducing unnecessary renders and using StateHasChanged() efficiently. +- Minimize the component render tree by avoiding re-renders unless necessary, using ShouldRender() where appropriate. +- Use EventCallbacks for handling user interactions efficiently, passing only minimal data when triggering events. + +## Caching Strategies + +- Implement in-memory caching for frequently used data, especially for Blazor Server apps. Use IMemoryCache for lightweight caching solutions. +- For Blazor WebAssembly, utilize localStorage or sessionStorage to cache application state between user sessions. +- Consider Distributed Cache strategies (like Redis or SQL Server Cache) for larger applications that need shared state across multiple users or clients. +- Cache API calls by storing responses to avoid redundant calls when data is unlikely to change, thus improving the user experience. + +## State Management Libraries + +- Use Blazor's built-in Cascading Parameters and EventCallbacks for basic state sharing across components. +- Implement advanced state management solutions using libraries like Fluxor or BlazorState when the application grows in complexity. +- For client-side state persistence in Blazor WebAssembly, consider using Blazored.LocalStorage or Blazored.SessionStorage to maintain state between page reloads. +- For server-side Blazor, use Scoped Services and the StateContainer pattern to manage state within user sessions while minimizing re-renders. + +## API Design and Integration + +- Use HttpClient or other appropriate services to communicate with external APIs or your own backend. +- Implement error handling for API calls using try-catch and provide proper user feedback in the UI. + +## Testing and Debugging in Visual Studio + +- All unit testing and integration testing should be done in Visual Studio. +- Test Blazor components and services using xUnit, NUnit, or MSTest. +- Use Moq or NSubstitute for mocking dependencies during tests. +- Debug Blazor UI issues using browser developer tools and Visual Studio's debugging tools for backend and server-side issues. +- For performance profiling and optimization, rely on Visual Studio's diagnostics tools. + +## Security and Authentication + +- Implement Authentication and Authorization in the Blazor app where necessary using ASP.NET Identity or JWT tokens for API authentication. +- Use HTTPS for all web communication and ensure proper CORS policies are implemented. + +## API Documentation and Swagger + +- Use Swagger/OpenAPI for API documentation for your backend API services. +- Ensure XML documentation for models and API methods for enhancing Swagger documentation. diff --git a/.gitignore b/.gitignore index b1a3beb..7433967 100644 --- a/.gitignore +++ b/.gitignore @@ -21,4 +21,48 @@ venv3/** .gitignore bin/** \n+# Ignore any virtual environment directories starting with 'venv' (venv, venv3, venv314, etc.) -venv*/ \ No newline at end of file +venv*/ +.vs/ProjectEvaluation/twitch-archive-2.metadata.v10.bin +.vs/ProjectEvaluation/twitch-archive-2.projects.v10.bin +.vs/ProjectEvaluation/twitch-archive-2.strings.v10.bin +.vs/Twitch-Archive-2/CopilotIndices/18.3.508.13148/CodeChunks.db +.vs/Twitch-Archive-2/CopilotIndices/18.3.508.13148/SemanticSymbols.db +.vs/Twitch-Archive-2/DesignTimeBuild/.dtbcache.v2 +.vs/Twitch-Archive-2/FileContentIndex/843065c8-d80f-4907-b0ae-6d010b3a5699.vsidx +.vs/Twitch-Archive-2/FileContentIndex/ef7e1a3c-80cd-4867-a9a8-2e5099471227.vsidx +.vs/Twitch-Archive-2/v18/.futdcache.v2 +.vs/Twitch-Archive-2/v18/.suo +.vs/Twitch-Archive-2/v18/DocumentLayout.backup.json +.vs/Twitch-Archive-2/v18/DocumentLayout.json +.vscode/settings.json + +# C# / Visual Studio +# Build Folders +bin/ +obj/ + +# Visual Studio files +*.user +*.suo +*.userprefs +*.csproj.user +*.pidb +*.pdb +*.cache +*.ilk +*.log +*.vspscc +*.vssscc + +# Test results and packages +TestResults/ +packages/ +*.nupkg + +# Database and backup +*.dbmdl +*.bak +*.backup +*.orig + +dotnet/.vs/** \ No newline at end of file diff --git a/Twitch-Archive-2.sln b/Twitch-Archive-2.sln new file mode 100644 index 0000000..a41b4f2 --- /dev/null +++ b/Twitch-Archive-2.sln @@ -0,0 +1,32 @@ +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.5.2.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "dotnet", "dotnet", "{71E6E750-85FD-B5BC-4321-E01377EC6231}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{D90AB541-7400-80B1-A0B4-F58D0D439F55}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TwitchArchive.Core", "dotnet\src\TwitchArchive.Core\TwitchArchive.Core.csproj", "{1D11D744-6D0D-BB4D-8B77-30B5CE764821}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {1D11D744-6D0D-BB4D-8B77-30B5CE764821}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1D11D744-6D0D-BB4D-8B77-30B5CE764821}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1D11D744-6D0D-BB4D-8B77-30B5CE764821}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1D11D744-6D0D-BB4D-8B77-30B5CE764821}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {D90AB541-7400-80B1-A0B4-F58D0D439F55} = {71E6E750-85FD-B5BC-4321-E01377EC6231} + {1D11D744-6D0D-BB4D-8B77-30B5CE764821} = {D90AB541-7400-80B1-A0B4-F58D0D439F55} + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {D101688C-0CA3-4CFB-96D4-E1AB9A62EC51} + EndGlobalSection +EndGlobal diff --git a/UpgradePlan.md b/UpgradePlan.md new file mode 100644 index 0000000..6654d3d --- /dev/null +++ b/UpgradePlan.md @@ -0,0 +1,214 @@ +Plan: C# .NET 10 Twitch Archive Rewrite +A complete port of the Python archiver to C# .NET 10 with Blazor Server UI, real-time process output, SQLite state tracking, full DI/service pattern, NLog logging, and a resilient recording engine. Placed under dotnet/ in the existing repo. + +Project Layout +Step 1 — Solution & Project scaffolding +Create the SLN and three projects: + +TwitchArchive.Core — classlib, targets net10.0 +TwitchArchive.Web — Blazor Server (blazorserver), targets net10.0 +TwitchArchive.Tests — xUnit, targets net10.0 +NuGet packages: + +Core: Microsoft.EntityFrameworkCore.Sqlite, Polly, NLog, NLog.Extensions.Logging +Web: all Core packages + Microsoft.AspNetCore.SignalR, NLog.Web.AspNetCore +Tests: xunit, Moq, coverlet.collector, Microsoft.EntityFrameworkCore.InMemory +Step 2 — Configuration models +Mirror the existing JSON schemas as C# POCOs with System.Text.Json attributes: + +GlobalConfig.cs — one property per key in config/global.json.example +StreamerConfig.cs — all fields nullable (override semantics), only Username and Enabled required +EffectiveConfig.cs — computed merge of global + per-streamer; exposes resolved values +AppSettings.cs — app-level settings (password hash, tool paths, .env secrets) +IConfigurationService / ConfigurationService: + +LoadGlobal() / SaveGlobal(GlobalConfig) +LoadStreamer(string username) / SaveStreamer(StreamerConfig) +GetAllStreamers(), GetEffectiveConfig(string username) (merge logic) +Reads/writes global.json and config/streamers/*.json — same files as Python +Step 3 — Infrastructure layer +TwitchApiClient (injectable, mockable): + +GetOAuthTokenAsync() — POST to https://id.twitch.tv/oauth2/token, caches token, refreshes on 401 +CheckStreamStatusAsync(string username) — GQL query for live stream + archiveVideo.id +GetLatestVodAsync(string username) — GQL query for most recent VOD +ValidateUsernameAsync(string username) — Helix /users endpoint +Credentials read from environment (CLIENT-ID, CLIENT-SECRET, OAUTH-PRIVATE-TOKEN) +All methods return typed result objects, never throw on network errors — return Result (or OneOf) +HttpResiliencePolicy (Polly): + +Wraps HttpClient for TwitchApiClient +WaitAndRetryForever with exponential backoff starting at 15 s, doubling, capped at 10 minutes +Only applies to transient errors (5xx, timeout, HttpRequestException) — not 401/404 +Logged via NLog on each retry attempt +ProcessRunner (injectable + mockable for tests): + +RunAsync(ProcessRunOptions options, CancellationToken ct) → int exitCode +StartAsync(ProcessRunOptions options, CancellationToken ct) → IRunningProcess handle (for long-lived processes like streamlink) +Reads stdout and stderr line by line asynchronously +Reports each line to IProcessOutputStore (streamer + job context) +Forwards to NLog +ProcessRunOptions: FileName, Arguments, WorkingDirectory, RedirectOutput + +Step 4 — Core services (all behind interfaces) +IStreamMonitorService / StreamMonitorService + +Wraps TwitchApiClient +CheckIsLiveAsync(string username) → LiveStreamInfo? +GetLatestVodAsync(string username) → VodInfo? +IRecorderService / RecorderService + +StartRecordingAsync(string username, string quality, string outputPath, CancellationToken ct) → Task +Invokes streamlink via ProcessRunner +Passes --hls-live-restart, --stream-segment-threads, optional OAuth header +Returns when streamlink exits (either stream ended or ct cancelled) +IProcessorService / ProcessorService + +ProcessRawStreamAsync(string rawPath, string outputPath, EffectiveConfig cfg, CancellationToken ct) +Builds ffmpeg args: hwaccel, thread count, error recovery flags, faststart, copy codecs +MergeVideoChatAsync(string videoPath, string chatVideoPath, string outputPath, string layout, CancellationToken ct) +IDownloaderService / DownloaderService + +DownloadVodAsync(VodInfo vod, string outputPath, EffectiveConfig cfg, CancellationToken ct) → bool +Invokes TwitchDownloaderCLI videodownload +Chat download methods stubbed with NotImplementedException / commented structure; interface is defined now to keep the architecture clean +IUploadService / UploadService + +UploadAsync(string localRoot, IEnumerable relativeFilePaths, string rcloneDest, CancellationToken ct) → bool +Writes a temp files-from list, invokes rclone copy --files-from +Returns success/failure; preserves local files on failure +IFileManagerService / FileManagerService + +InitializeDirectories(string rootPath, string username) +GetPaths(string rootPath, string username, string filenameBase) → ArchivePaths record (all expected paths) +CleanRawFile(string path, bool cleanRaw) +DeleteLocalFiles(ArchivePaths paths, EffectiveConfig cfg) +GetUniquePath(string path) → adds numeric suffix if file exists +Step 5 — Recording resilience engine +RecoveryPolicy (POCO, unit-testable, no DI deps): + +Encodes a state machine with these states: + +State Meaning +Monitoring Normal polling at refresh interval +Recording streamlink subprocess active +FastReconnect Stream ended; checking every 10 s for up to 2 minutes +SlowReconnect Still not live after 2 min; checking every 60 s concurrently with post-processing +PostProcessing Confirmed ended; ffmpeg / VOD download / upload running +NetworkFault Twitch API unreachable; exponential back-off (30 s → capped at 10 min) +Transitions: + +Recording → streamlink exits → enter FastReconnect, record phase start time +FastReconnect → live confirmed → start new Recording (new filename/segment) +FastReconnect (2 min elapsed) → enter SlowReconnect + kick off PostProcessing concurrently +SlowReconnect → live confirmed → start new Recording +SlowReconnect / Monitoring → API call throws network error → enter NetworkFault +NetworkFault → successful API response → return to previous state (Monitoring or re-enter FastReconnect if we were mid-reconnect) +NetworkFault backoff: 30s * 2^attempt, capped at 600s +RecoveryPolicy is a pure class with a Tick(DateTime now, bool? isLive, bool networkError) method → returns RecoveryDecision (what to do next + sleep duration). Fully unit-testable with no async or DI. + +StreamWorker : BackgroundService + +One instance per enabled streamer +Holds RecoveryPolicy instance +Main loop: evaluate policy decision → execute the corresponding service call → loop +Started/stopped by StreamWorkerManager +Writes job records to SQLite on start/complete/fail +StreamWorkerManager + +StartWorker(string username), StopWorker(string username), RestartWorker(string username) +Called at app startup for all enabled streamers +Called from Web UI on enable/disable/config change +Workers stored in ConcurrentDictionary +Step 6 — Persistence (SQLite + EF Core) +ArchiveDbContext with three tables: + +StreamSessions: Id, StreamerUsername, TwitchStreamId, Title, StartedAt, EndedAt, Status (Recording/Processing/Uploading/Complete/Failed) + +ArchiveJobs: Id, SessionId, JobType (enum: RecordLive, ProcessLive, DownloadVod, ProcessVod, UploadCloud, DeleteLocal), Status, StartedAt, CompletedAt, FilePath, ErrorMessage + +StreamerStates: Username, IsMonitoring, LastCheckedAt, CurrentRecoveryState + +Migrations via EF Core CLI. ISessionRepository / IJobRepository interfaces for testability with in-memory EF provider in tests. + +Step 7 — Process output streaming +IProcessOutputStore: + +AppendLine(string streamerId, Guid jobId, string line, bool isError) +GetRecentLines(string streamerId, Guid jobId, int count = 500) → IReadOnlyList +In-memory circular buffer (1000 lines per job, last 20 jobs per streamer) +ProcessOutputHub : Hub (SignalR): + +Clients call SubscribeToStreamer(string username) → join group streamer:{username} +Clients call SubscribeToJob(Guid jobId) → join group job:{jobId} +Server pushes ReceiveLine(OutputLine line) from ProcessRunner via IHubContext +On subscribe: server immediately sends buffered lines from IProcessOutputStore +Step 8 — Blazor Server Web UI +Authentication: Cookie-based single-password auth via ASP.NET Core minimal auth middleware. Password stored as BCrypt hash in AppSettings. Login.razor page at /login. Protected routes with [Authorize]. + +Pages & Components: + +Dashboard.razor (/) — grid of all configured streamers showing: username, live/offline badge, current recovery state, last recorded session, quick Start/Stop monitoring toggle + +StreamerDetail.razor (/streamer/{username}) — live status, current job pipeline steps (record → process → upload with progress), ProcessOutputConsole.razor showing real-time terminal output via SignalR + +ProcessOutputConsole.razor — reusable Blazor component; subscribes to SignalR on mount, renders an auto-scrolling
 with colored output (stdout = white, stderr = orange/red), handles reconnect
+
+Sessions.razor (/sessions) — paginated list of past archive sessions with job statuses and expandable per-job output
+
+GlobalConfig.razor (/config/global) — EditForm bound to GlobalConfig model with data annotations validation, Save button calls IConfigurationService.SaveGlobal()
+
+StreamerConfig.razor (/config/{username}) — similar form for per-streamer overrides; each field has a nullable toggle (inherit from global vs override)
+
+AddStreamer.razor (/config/new) — minimal form: username + enabled; creates new config/streamers/{username}.json
+
+AppSettings.razor (/settings) — tool paths (streamlink, ffmpeg, TwitchDownloaderCLI, rclone), change password
+
+Step 9 — NLog configuration
+nlog.config (XML): two targets:
+
+Console (colored, with level formatting)
+File rolling (logs/archive-${shortdate}.log, keep 30 days)
+Log structured context: StreamerUsername, JobId, JobType as NLog ScopeContext properties. Service methods open a scope via ILogger.BeginScope(...).
+
+Step 10 — Docker
+Dockerfile (multi-stage):
+
+Build stage: mcr.microsoft.com/dotnet/sdk:10.0
+Runtime stage: mcr.microsoft.com/dotnet/aspnet:10.0 (Linux)
+Install ffmpeg, streamlink (via pip), download TwitchDownloaderCLI binary for linux-x64
+rclone installed via shell script or apt
+Expose port 8080
+ENTRYPOINT ["dotnet", "TwitchArchive.Web.dll"]
+docker-compose.yml:
+
+Volume mounts: ./config:/app/config, ./archive:/app/archive, ./logs:/app/logs
+Environment variables: CLIENT-ID, CLIENT-SECRET, OAUTH-PRIVATE-TOKEN
+Windows dev: run directly with dotnet run; tool paths auto-detected (Windows vs Linux) via RuntimeInformation.IsOSPlatform(OSPlatform.Windows) in ToolPathResolver
+
+Step 11 — Unit tests
+TwitchArchive.Tests covers:
+
+RecoveryPolicyTests — state machine transitions, timing phases, network fault backoff; pure synchronous tests
+ConfigurationServiceTests — JSON load/save/merge with temp files
+TwitchApiClientTests — mocked HttpMessageHandler; OAuth, GQL queries, 401 refresh, network errors
+FileManagerServiceTests — path generation, directory creation with temp directories
+DownloaderServiceTests / RecorderServiceTests — mocked ProcessRunner; verify correct CLI arguments
+UploadServiceTests — mocked ProcessRunner; verify rclone argument construction
+SessionRepositoryTests — EF Core in-memory provider
+EffectiveConfigTests — global + streamer override merge logic
+Verification
+dotnet build dotnet/TwitchArchive.sln — zero warnings, zero errors
+dotnet test dotnet/TwitchArchive.Tests/ — all tests green
+docker compose up --build in the dotnet/ folder → app reachable at http://localhost:8080
+Manual: add a test streamer config, enable monitoring, confirm it polls and records a live stream, runs ffmpeg, uploads via rclone
+Resilience manual test: kill network during recording → verify FastReconnect phase kicks in, resumes after connectivity restored
+Decisions
+
+Blazor Server chosen for real-time terminal output without a separate API; no WASM needed
+streamlink for live + TwitchDownloaderCLI for VOD (same split as Python; streamlink gives better live resilience)
+Simple BCrypt password auth (not full Identity — this is a single-user tool)
+RecoveryPolicy as a pure POCO state machine keeps the resilience logic fully unit-testable without async/mocking
+Polly WaitAndRetryForever on HttpClient handles persistent network failure independently of the application-level state machine; they are complementary — Polly handles individual HTTP call retries, RecoveryPolicy handles the overall workflow state
+Chat download service interface is defined in Step 4 but methods are stubbed — adding implementation later requires only filling in DownloaderService without touching any other layer
+Config files remain in the same format/location so the Python and C# versions can share the same config directory
\ No newline at end of file
diff --git a/UpgradePlan2.md b/UpgradePlan2.md
new file mode 100644
index 0000000..cc4ffbd
--- /dev/null
+++ b/UpgradePlan2.md
@@ -0,0 +1,78 @@
+
+
+
@Body
``` Add a top bar (`
`) with the app title "Twitch Archive" and a hamburger toggle ` + + +@code { + private TwitchArchive.Core.Config.StreamerConfig model = new() { Enabled = true }; + + private void Save() + { + model.Username = model.Username?.Trim().ToLowerInvariant() ?? string.Empty; + if (string.IsNullOrWhiteSpace(model.Username)) return; + ConfigService.SaveStreamer(model); + Nav.NavigateTo($"/config/{model.Username}"); + } +} diff --git a/dotnet/src/TwitchArchive.Web/Pages/AppSettings.razor b/dotnet/src/TwitchArchive.Web/Pages/AppSettings.razor new file mode 100644 index 0000000..9ccb7d3 --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Pages/AppSettings.razor @@ -0,0 +1,92 @@ +@page "/settings" +@using System.Text.Json +@inject TwitchArchive.Web.Services.IAuthService Auth + +

App Settings

+ +@if (saved) +{ +
Saved.
+} + + +
+ + +
+
+ + +
+
+ + +
+
+ + +
+ +
+ +

Change Password

+@if (!string.IsNullOrEmpty(pwError)) +{ +
@pwError
+} +
+ + + + +
+ +@code { + private TwitchArchive.Core.Config.AppSettings model = new(); + private bool saved = false; + private string currentPw = string.Empty; + private string newPw = string.Empty; + private string confirmPw = string.Empty; + private string pwError = string.Empty; + + protected override void OnInitialized() + { + Load(); + } + + private void Load() + { + var file = Path.Combine(AppContext.BaseDirectory, "appsettings.json"); + try + { + if (!File.Exists(file)) { model = new(); return; } + var txt = File.ReadAllText(file); + model = JsonSerializer.Deserialize(txt) ?? new TwitchArchive.Core.Config.AppSettings(); + } + catch { model = new(); } + } + + private void Save() + { + var file = Path.Combine(AppContext.BaseDirectory, "appsettings.json"); + try + { + var txt = JsonSerializer.Serialize(model, new JsonSerializerOptions { WriteIndented = true }); + File.WriteAllText(file, txt); + saved = true; + Auth.Refresh(); + } + catch { } + } + + private void ChangePassword() + { + pwError = string.Empty; + if (!Auth.ValidatePassword(currentPw)) { pwError = "Current password incorrect"; return; } + if (string.IsNullOrWhiteSpace(newPw)) { pwError = "New password required"; return; } + if (newPw != confirmPw) { pwError = "Passwords do not match"; return; } + var hash = BCrypt.Net.BCrypt.HashPassword(newPw); + Auth.SetPasswordHash(hash); + pwError = string.Empty; + } +} diff --git a/dotnet/src/TwitchArchive.Web/Pages/GlobalConfig.razor b/dotnet/src/TwitchArchive.Web/Pages/GlobalConfig.razor new file mode 100644 index 0000000..96ba67d --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Pages/GlobalConfig.razor @@ -0,0 +1,45 @@ +@page "/config/global" +@inject TwitchArchive.Core.Config.IConfigurationService ConfigService + +

Global Configuration

+ +@if (saved) +{ +
Saved.
+} + + +
+ + +
+
+ + +
+
+ + +
+
+ + +
+ +
+ +@code { + private TwitchArchive.Core.Config.GlobalConfig model = new(); + private bool saved = false; + + protected override void OnInitialized() + { + model = ConfigService.LoadGlobal(); + } + + private void Save() + { + ConfigService.SaveGlobal(model); + saved = true; + } +} diff --git a/dotnet/src/TwitchArchive.Web/Pages/Index.razor b/dotnet/src/TwitchArchive.Web/Pages/Index.razor new file mode 100644 index 0000000..9120fe2 --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Pages/Index.razor @@ -0,0 +1,64 @@ +@page "/" +@inject TwitchArchive.Core.Workers.StreamWorkerManager WorkerManager +@inject TwitchArchive.Web.Services.SessionCacheService SessionCache + +

Dashboard

+ +
+ @foreach (var s in streamers) + { +
+
+ @s + @(WorkerManager.IsRunning(s) ? "Live" : "Offline") +
+
+
Last session: @(lastStarts.ContainsKey(s) ? lastStarts[s].ToLocalTime().ToString() : "-")
+
+ + +
+
+
+ } +
+ +@code { + private List streamers = new(); + private Dictionary lastStarts = new(); + + private void OnCacheUpdatedHandler() + { + _ = InvokeAsync(() => { + lastStarts = SessionCache.GetSnapshot(); + StateHasChanged(); + }); + } + + protected override async Task OnInitializedAsync() + { + LoadStreamers(); + lastStarts = SessionCache.GetSnapshot(); + SessionCache.Updated += OnCacheUpdatedHandler; + } + + private void LoadStreamers() + { + var cfgDir = Path.Combine(Environment.CurrentDirectory, "config", "streamers"); + if (Directory.Exists(cfgDir)) + { + streamers = Directory.GetFiles(cfgDir, "*.json").Select(f => Path.GetFileNameWithoutExtension(f)).ToList(); + } + } + + // Index reads from the singleton SessionCacheService; updates are pushed via the Updated event. + + private void Start(string u) { WorkerManager.StartWorker(u); } + private async Task Stop(string u) { await WorkerManager.StopWorkerAsync(u); } + + public async ValueTask DisposeAsync() + { + SessionCache.Updated -= OnCacheUpdatedHandler; + await Task.CompletedTask; + } +} diff --git a/dotnet/src/TwitchArchive.Web/Pages/Login.razor b/dotnet/src/TwitchArchive.Web/Pages/Login.razor new file mode 100644 index 0000000..ff4f1da --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Pages/Login.razor @@ -0,0 +1,20 @@ +@page "/login" +@attribute [AllowAnonymous] +@using Microsoft.AspNetCore.Components + +

Login

+ +@if (error) +{ +
Invalid password
+} + +
+ + +
+ +@code { + [Parameter] + public bool error { get; set; } +} diff --git a/dotnet/src/TwitchArchive.Web/Pages/Monitor.razor b/dotnet/src/TwitchArchive.Web/Pages/Monitor.razor new file mode 100644 index 0000000..4cd79de --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Pages/Monitor.razor @@ -0,0 +1,37 @@ +@page "/monitor" +@inject TwitchArchive.Core.Workers.StreamWorkerManager WorkerManager +@inject TwitchArchive.Core.Services.IProcessOutputStore OutputStore +@using TwitchArchive.Web.Shared + +

Streamer Monitor

+ +
+ + + +
+ +
+ Status: @status +
+ + + +@code { + private string username = "hackerling"; + private string status = "idle"; + + private void Start() + { + if (string.IsNullOrWhiteSpace(username)) return; + WorkerManager.StartWorker(username); + status = WorkerManager.IsRunning(username) ? "running" : "starting"; + } + + private async Task Stop() + { + if (string.IsNullOrWhiteSpace(username)) return; + await WorkerManager.StopWorkerAsync(username); + status = WorkerManager.IsRunning(username) ? "running" : "stopped"; + } +} diff --git a/dotnet/src/TwitchArchive.Web/Pages/Sessions.razor b/dotnet/src/TwitchArchive.Web/Pages/Sessions.razor new file mode 100644 index 0000000..256da43 --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Pages/Sessions.razor @@ -0,0 +1,73 @@ +@page "/sessions" +@using TwitchArchive.Core.Persistence.Models +@inject TwitchArchive.Core.Persistence.ISessionRepository SessionRepo + +

Sessions

+ + + + + + + + + @foreach (var s in sessions) + { + + + + + + + + + @if (expandedSession == s.Id) + { + + } + } + +
IdStreamerStartedEndedStatusJobs
@s.Id@s.StreamerUsername@s.StartedAt.ToLocalTime()@(s.EndedAt?.ToLocalTime().ToString() ?? "-")@s.Status
+
    + @if (jobs?.Any() ?? false) + { + @foreach (var j in jobs) + { +
  • @j.Id - @j.JobType - @j.Status - @j.StartedAt.ToLocalTime() - @(j.FilePath ?? "")
  • + } + } + else + { +
  • No jobs
  • + } +
+
+ +@code { + private List sessions = new(); + private List? jobs; + private long? expandedSession; + + protected override async Task OnInitializedAsync() + { + await Refresh(); + } + + private async Task Refresh() + { + sessions = await SessionRepo.GetRecentSessionsAsync(50); + StateHasChanged(); + } + + private async Task ToggleJobs(long sessionId) + { + if (expandedSession == sessionId) + { + expandedSession = null; + jobs = null; + return; + } + jobs = await SessionRepo.GetJobsForSessionAsync(sessionId); + expandedSession = sessionId; + } +} diff --git a/dotnet/src/TwitchArchive.Web/Pages/StreamerConfig.razor b/dotnet/src/TwitchArchive.Web/Pages/StreamerConfig.razor new file mode 100644 index 0000000..bf0c72a --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Pages/StreamerConfig.razor @@ -0,0 +1,71 @@ +@page "/config/{Username}" +@inject TwitchArchive.Core.Config.IConfigurationService ConfigService +@inject NavigationManager Nav + +

Streamer Config: @Username

+ + +
+ + +
+ +
+ + Override + +
+ +
+ + Override + +
+ +
+ + +
+ +
+ + Override + +
+ + + +
+ +@code { + [Parameter] + public string Username { get; set; } = string.Empty; + + private TwitchArchive.Core.Config.StreamerConfig model = new(); + private TwitchArchive.Core.Config.GlobalConfig? global; + private bool overrideQuality = false; + private bool overrideUpload = false; + private bool overrideStreamlink = false; + + protected override void OnInitialized() + { + global = ConfigService.LoadGlobal(); + var s = ConfigService.LoadStreamer(Username); + if (s != null) model = s; + } + + private void Save() + { + model.Username = Username; + if (!overrideQuality) model.Quality = null; + if (!overrideUpload) model.UploadToCloud = null; + if (!overrideStreamlink) model.StreamlinkPath = null; + ConfigService.SaveStreamer(model); + } + + private void Delete() + { + ConfigService.DeleteStreamer(Username); + Nav.NavigateTo("/"); + } +} diff --git a/dotnet/src/TwitchArchive.Web/Pages/StreamerDetail.razor b/dotnet/src/TwitchArchive.Web/Pages/StreamerDetail.razor new file mode 100644 index 0000000..bafa148 --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Pages/StreamerDetail.razor @@ -0,0 +1,22 @@ +@page "/streamer/{Username}" +@using TwitchArchive.Core.Workers +@inject StreamWorkerManager WorkerManager + +

Streamer: @Username

+ +
+ Status: @(WorkerManager.IsRunning(Username) ? "Live" : "Offline") +
+ +
+
Record
+
Process
+
Upload
+
+ + + +@code { + [Parameter] + public string Username { get; set; } = string.Empty; +} diff --git a/dotnet/src/TwitchArchive.Web/Pages/_Host.cshtml b/dotnet/src/TwitchArchive.Web/Pages/_Host.cshtml new file mode 100644 index 0000000..1463d9e --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Pages/_Host.cshtml @@ -0,0 +1,19 @@ +@page "/" +@namespace TwitchArchive.Web.Pages +@addTagHelper *, Microsoft.AspNetCore.Mvc.TagHelpers + + + + + + Twitch Archive + + + + + + + + + + diff --git a/dotnet/src/TwitchArchive.Web/Program.cs b/dotnet/src/TwitchArchive.Web/Program.cs new file mode 100644 index 0000000..d655b3e --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Program.cs @@ -0,0 +1,104 @@ +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Authentication; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.EntityFrameworkCore; +using TwitchArchive.Core.Services; +using TwitchArchive.Core.Config; +using TwitchArchive.Core.Persistence; +using TwitchArchive.Web.Hubs; +using TwitchArchive.Web.Services; + +var builder = WebApplication.CreateBuilder(args); + +// Add services +builder.Services.AddRazorPages(); +builder.Services.AddServerSideBlazor(); +builder.Services.AddSignalR(); + +// Authentication +builder.Services.AddSingleton(); +builder.Services.AddAuthentication(options => +{ + options.DefaultScheme = Microsoft.AspNetCore.Authentication.Cookies.CookieAuthenticationDefaults.AuthenticationScheme; +}) +.AddCookie(options => { options.LoginPath = "/login"; }); + +// Register Core services +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddHttpClient(client => { /* base config if needed */ }); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); + +// Configuration service for global + per-streamer JSON files in ./config +builder.Services.AddScoped(); + +// Broadcaster forwards output store events to the SignalR hub +builder.Services.AddSingleton(); + +// SQLite DB (file in app folder) +var conn = "Data Source=archive.db"; +// Provide a factory for creating DbContext instances for background work +builder.Services.AddDbContextFactory(opt => opt.UseSqlite(conn)); +// persistence +builder.Services.AddScoped(); + +// Session cache and background refresh +builder.Services.AddSingleton(); +builder.Services.AddHostedService(); + +var app = builder.Build(); + +if (!app.Environment.IsDevelopment()) +{ + app.UseExceptionHandler("/Error"); +} + +// Ensure DB schema exists (creates DB when missing) +using (var scope = app.Services.CreateScope()) +{ + var factory = scope.ServiceProvider.GetRequiredService>(); + try + { + using var db = factory.CreateDbContext(); + db.Database.EnsureCreated(); + } + catch { } +} + +app.UseStaticFiles(); +app.UseRouting(); + +app.UseAuthentication(); +app.UseAuthorization(); + +// login endpoint +app.MapPost("/auth/login", async (Microsoft.AspNetCore.Http.HttpContext http, TwitchArchive.Web.Services.IAuthService auth) => +{ + try + { + var form = await http.Request.ReadFormAsync(); + var pwd = form["password"].ToString(); + if (auth.ValidatePassword(pwd)) + { + var claims = new[] { new System.Security.Claims.Claim(System.Security.Claims.ClaimTypes.Name, "admin") }; + var id = new System.Security.Claims.ClaimsIdentity(claims, Microsoft.AspNetCore.Authentication.Cookies.CookieAuthenticationDefaults.AuthenticationScheme); + var principal = new System.Security.Claims.ClaimsPrincipal(id); + await http.SignInAsync(Microsoft.AspNetCore.Authentication.Cookies.CookieAuthenticationDefaults.AuthenticationScheme, principal); + http.Response.Redirect("/"); + return; + } + } + catch { } + http.Response.Redirect("/login?error=1"); +}); + +app.MapBlazorHub(); +app.MapHub("/processOutputHub"); +app.MapFallbackToPage("/_Host"); + +app.Run(); diff --git a/dotnet/src/TwitchArchive.Web/Properties/launchSettings.json b/dotnet/src/TwitchArchive.Web/Properties/launchSettings.json new file mode 100644 index 0000000..9c88f37 --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Properties/launchSettings.json @@ -0,0 +1,12 @@ +{ + "profiles": { + "TwitchArchive.Web": { + "commandName": "Project", + "launchBrowser": true, + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + }, + "applicationUrl": "https://localhost:64466;http://localhost:64467" + } + } +} \ No newline at end of file diff --git a/dotnet/src/TwitchArchive.Web/Services/AuthService.cs b/dotnet/src/TwitchArchive.Web/Services/AuthService.cs new file mode 100644 index 0000000..12f4e2f --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Services/AuthService.cs @@ -0,0 +1,50 @@ +using System; +using System.IO; +using System.Text.Json; +using TwitchArchive.Core.Config; + +namespace TwitchArchive.Web.Services +{ + public class AuthService : IAuthService + { + private readonly string _file; + private AppSettings _settings = new(); + + public AuthService() + { + _file = Path.Combine(AppContext.BaseDirectory, "appsettings.json"); + Refresh(); + } + + public void Refresh() + { + try + { + if (!File.Exists(_file)) { _settings = new AppSettings(); return; } + var txt = File.ReadAllText(_file); + _settings = JsonSerializer.Deserialize(txt) ?? new AppSettings(); + } + catch { _settings = new AppSettings(); } + } + + public bool ValidatePassword(string plain) + { + if (string.IsNullOrWhiteSpace(_settings.PasswordHash)) return true; + try { return BCrypt.Net.BCrypt.Verify(plain ?? string.Empty, _settings.PasswordHash); } + catch { return false; } + } + + public void SetPasswordHash(string hash) + { + try + { + if (string.IsNullOrWhiteSpace(_file)) return; + _settings.PasswordHash = hash; + var txt = JsonSerializer.Serialize(_settings, new JsonSerializerOptions { WriteIndented = true }); + File.WriteAllText(_file, txt); + Refresh(); + } + catch { } + } + } +} diff --git a/dotnet/src/TwitchArchive.Web/Services/IAuthService.cs b/dotnet/src/TwitchArchive.Web/Services/IAuthService.cs new file mode 100644 index 0000000..cd3b0fd --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Services/IAuthService.cs @@ -0,0 +1,11 @@ +using System.Threading.Tasks; + +namespace TwitchArchive.Web.Services +{ + public interface IAuthService + { + bool ValidatePassword(string plain); + void Refresh(); + void SetPasswordHash(string hash); + } +} diff --git a/dotnet/src/TwitchArchive.Web/Services/ProcessOutputBroadcaster.cs b/dotnet/src/TwitchArchive.Web/Services/ProcessOutputBroadcaster.cs new file mode 100644 index 0000000..641ddf9 --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Services/ProcessOutputBroadcaster.cs @@ -0,0 +1,35 @@ +using Microsoft.AspNetCore.SignalR; +using System; +using TwitchArchive.Core.Services; +using TwitchArchive.Web.Hubs; + +namespace TwitchArchive.Web.Services +{ + public class ProcessOutputBroadcaster : IDisposable + { + private readonly IProcessOutputStore _store; + private readonly IHubContext _hub; + + public ProcessOutputBroadcaster(IProcessOutputStore store, IHubContext hub) + { + _store = store; + _hub = hub; + _store.LineAppended += OnLineAppended; + } + + private void OnLineAppended(string streamer, OutputLine line) + { + try + { + // send to clients; clients should listen on "ReceiveLine" + _hub.Clients.Group(streamer).SendAsync("ReceiveLine", streamer, line); + } + catch { } + } + + public void Dispose() + { + _store.LineAppended -= OnLineAppended; + } + } +} diff --git a/dotnet/src/TwitchArchive.Web/Services/SessionCacheService.cs b/dotnet/src/TwitchArchive.Web/Services/SessionCacheService.cs new file mode 100644 index 0000000..abe7855 --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Services/SessionCacheService.cs @@ -0,0 +1,38 @@ +using System; +using System.Collections.Generic; +using TwitchArchive.Core.Persistence.Models; + +namespace TwitchArchive.Web.Services +{ + public class SessionCacheService + { + private readonly object _lock = new(); + private Dictionary _snapshot = new(); + + public event Action? Updated; + + public void Update(IEnumerable sessions) + { + var next = new Dictionary(); + foreach (var s in sessions) + { + if (!next.ContainsKey(s.StreamerUsername)) next[s.StreamerUsername] = s.StartedAt; + } + + lock (_lock) + { + _snapshot = next; + } + + Updated?.Invoke(); + } + + public Dictionary GetSnapshot() + { + lock (_lock) + { + return new Dictionary(_snapshot); + } + } + } +} diff --git a/dotnet/src/TwitchArchive.Web/Services/SessionRefreshHostedService.cs b/dotnet/src/TwitchArchive.Web/Services/SessionRefreshHostedService.cs new file mode 100644 index 0000000..93523b8 --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Services/SessionRefreshHostedService.cs @@ -0,0 +1,50 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using TwitchArchive.Core.Persistence; + +namespace TwitchArchive.Web.Services +{ + public class SessionRefreshHostedService : BackgroundService + { + private readonly IServiceScopeFactory _scopeFactory; + private readonly SessionCacheService _cache; + private readonly ILogger _logger; + private readonly TimeSpan _interval; + + public SessionRefreshHostedService(IServiceScopeFactory scopeFactory, SessionCacheService cache, ILogger logger) + { + _scopeFactory = scopeFactory ?? throw new ArgumentNullException(nameof(scopeFactory)); + _cache = cache ?? throw new ArgumentNullException(nameof(cache)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _interval = TimeSpan.FromSeconds(10); + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + using var timer = new PeriodicTimer(_interval); + try + { + while (await timer.WaitForNextTickAsync(stoppingToken)) + { + try + { + using var scope = _scopeFactory.CreateScope(); + var repo = scope.ServiceProvider.GetRequiredService(); + var sessions = await repo.GetRecentSessionsAsync(200, stoppingToken).ConfigureAwait(false); + _cache.Update(sessions); + } + catch (OperationCanceledException) { break; } + catch (Exception ex) + { + _logger.LogError(ex, "Error while refreshing sessions"); + } + } + } + catch (OperationCanceledException) { } + } + } +} diff --git a/dotnet/src/TwitchArchive.Web/Shared/MainLayout.razor b/dotnet/src/TwitchArchive.Web/Shared/MainLayout.razor new file mode 100644 index 0000000..dd8a426 --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Shared/MainLayout.razor @@ -0,0 +1,21 @@ +@inherits LayoutComponentBase +
+ +

Twitch Archive

+
+
+ +
+ @Body +
+
+ +@code { + bool sidebarCollapsed; + void ToggleSidebar() => sidebarCollapsed = !sidebarCollapsed; +} diff --git a/dotnet/src/TwitchArchive.Web/Shared/ProcessConsole.razor b/dotnet/src/TwitchArchive.Web/Shared/ProcessConsole.razor new file mode 100644 index 0000000..f5c2c76 --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Shared/ProcessConsole.razor @@ -0,0 +1,93 @@ +@using TwitchArchive.Core.Services +@using Microsoft.AspNetCore.Components +@using Microsoft.AspNetCore.SignalR.Client +@using System.Linq +@inject IProcessOutputStore OutputStore +@inject NavigationManager Navigation + +@inherits ComponentBase +
+ @foreach (var line in lines) + { +
@line.TimestampUtc.ToLocalTime().ToString("HH:mm:ss") - @line.Line
+ } +
+ +@code { + [Parameter] + public string Streamer { get; set; } = string.Empty; + + private List lines = new(); + private HubConnection? hubConnection; + private string? _currentGroup; + + protected override async Task OnInitializedAsync() + { + OutputStore.LineAppended += OnLineAppended; + + hubConnection = new HubConnectionBuilder() + .WithUrl(Navigation.ToAbsoluteUri("/processOutputHub")) + .WithAutomaticReconnect() + .Build(); + + hubConnection.On("ReceiveLine", (streamer, line) => + { + if (!string.Equals(streamer, Streamer, StringComparison.OrdinalIgnoreCase)) return; + // avoid duplicates + if (lines.Any(l => l.TimestampUtc == line.TimestampUtc && l.Line == line.Line)) return; + lines.Add(line); + if (lines.Count > 1000) lines.RemoveAt(0); + InvokeAsync(StateHasChanged); + }); + + await hubConnection.StartAsync(); + if (!string.IsNullOrWhiteSpace(Streamer)) + { + await hubConnection.SendAsync("JoinStreamerGroup", Streamer); + _currentGroup = Streamer; + } + + await base.OnInitializedAsync(); + } + + protected override async Task OnParametersSetAsync() + { + if (hubConnection == null) return; + if (!string.IsNullOrWhiteSpace(_currentGroup) && !string.Equals(_currentGroup, Streamer, StringComparison.OrdinalIgnoreCase)) + { + try { await hubConnection.SendAsync("LeaveStreamerGroup", _currentGroup); } catch { } + _currentGroup = null; + } + + if (!string.IsNullOrWhiteSpace(Streamer) && !string.Equals(_currentGroup, Streamer, StringComparison.OrdinalIgnoreCase)) + { + try { await hubConnection.SendAsync("JoinStreamerGroup", Streamer); _currentGroup = Streamer; } catch { } + } + + await base.OnParametersSetAsync(); + } + + private void OnLineAppended(string streamer, OutputLine line) + { + if (!string.Equals(streamer, Streamer, StringComparison.OrdinalIgnoreCase)) return; + lines.Add(line); + if (lines.Count > 1000) lines.RemoveAt(0); + InvokeAsync(StateHasChanged); + } + + public async ValueTask DisposeAsync() + { + OutputStore.LineAppended -= OnLineAppended; + if (hubConnection != null) + { + try + { + if (!string.IsNullOrWhiteSpace(_currentGroup)) await hubConnection.SendAsync("LeaveStreamerGroup", _currentGroup); + } + catch { } + try { await hubConnection.StopAsync(); } catch { } + try { await hubConnection.DisposeAsync(); } catch { } + hubConnection = null; + } + } +} diff --git a/dotnet/src/TwitchArchive.Web/TwitchArchive.Web.csproj b/dotnet/src/TwitchArchive.Web/TwitchArchive.Web.csproj new file mode 100644 index 0000000..774299b --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/TwitchArchive.Web.csproj @@ -0,0 +1,16 @@ + + + net10.0 + enable + enable + + + + + + + + + + + diff --git a/dotnet/src/TwitchArchive.Web/_Imports.razor b/dotnet/src/TwitchArchive.Web/_Imports.razor new file mode 100644 index 0000000..34b6706 --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/_Imports.razor @@ -0,0 +1,10 @@ +@using System +@using System.Net.Http +@using Microsoft.AspNetCore.Components +@using Microsoft.AspNetCore.Components.Web +@using Microsoft.AspNetCore.Components.Routing +@using Microsoft.AspNetCore.Components.Forms +@using Microsoft.AspNetCore.Authorization +@using TwitchArchive.Core +@using TwitchArchive.Core.Services +@using TwitchArchive.Web.Shared diff --git a/dotnet/src/TwitchArchive.Web/archive.db b/dotnet/src/TwitchArchive.Web/archive.db new file mode 100644 index 0000000000000000000000000000000000000000..4e86411b5803e34b1e4767ce981907694f15c1ed GIT binary patch literal 4096 zcmWFz^vNtqRY=P(%1ta$FlG>7U}9o$P*7lCU|@t|AVoG{WYBBVXt)<@c!Uvb* zxR59PTU}?VO)sU%C_lt3ZkO|6_c(3t%klQuOrP`DzHV0Q>dW)(-Err;lkR@3^Se_& z&mhk*Pn;J51PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB=Di7l_sgAyB43^sNYiG6kY(NC=cE5FJ}W zpiBYna%KK^Urm4j0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF P5FkK+009C7epBEBdYUP} literal 0 HcmV?d00001 diff --git a/dotnet/src/TwitchArchive.Web/archive.db-wal b/dotnet/src/TwitchArchive.Web/archive.db-wal new file mode 100644 index 0000000000000000000000000000000000000000..f7800fa8c36eff9afd343644d3bf18ba0642f320 GIT binary patch literal 20632 zcmeI)!E4h{90%|;?KYWo=&8Hyk)fOsNF4#pBkUE6hShW6sf+P5~?Cfm!~#x4rQ ziHP_Qs0TfWDB@W??CQxsVBkS7A_&8i2p)WCn=Z*Ziyq7OLCJf0@Ar~?-b*2`q!(`w zH>ERQ#wF>56|!L zR?F!#ole^Ulr1@wSw@2@MAxp1zB~PU&Gapkmuv0D?s@cXWb*n^9Td62IIXDsh*-_4 z_-n1Z4Hqz{-fw>j+qd!<+}V6XCjfDdB(0{4MUJ6YQ@xy?+g zdU$v}gN)(uRE?Hz9$}Q)V2oP4K+Et-L&(n4XT5T2{__T|7uGJ9K@6 zT4fq4Mgv-*$|-k}%!tTj(~gWJCMP31r&@igtXwlU=~a8h@nz*eUy85h-`E3zea!=R zL9g4!a}{T+Z?FAj&K{n5UTD7wow6HibsCuRzDa$>tkb-~YraA9I@TDoS(ZAEQSF{s zdkn>wj{BV7T`$=E@HxKvCZ0UQy1-x#Hm(W+5P$##AOHafKmY;|fB*y_5E5v;1K_=1VEw-K_G{(& zBGv^$N+?1A0uX=z1Rwwb2tWV=5P$##22;TOuYe(MUEt-%Z*wp9Z#~Akz+etGt_lJW WfB*y_009U<00Izz00bc57x)908huaz literal 0 HcmV?d00001 diff --git a/dotnet/src/TwitchArchive.Web/wwwroot/css/app.css b/dotnet/src/TwitchArchive.Web/wwwroot/css/app.css new file mode 100644 index 0000000..ad8771f --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/wwwroot/css/app.css @@ -0,0 +1,16 @@ +/* App layout styles for Twitch Archive */ +.page { display:flex; height:100vh; } +.sidebar { width:220px; flex-shrink:0; background:#1e1e2e; color:#cdd6f4; overflow-y:auto; } +.sidebar.collapsed { display:none; } +.main { flex:1; overflow-y:auto; padding:1.5rem; } +.nav-link { display:block; padding:0.6rem 1rem; color:#cdd6f4; text-decoration:none; } +.nav-link.active { background:#313244; border-left:3px solid #89b4fa; } +.topbar { display:none; background:#111; color:#fff; padding:0.6rem 1rem; align-items:center; } +.topbar .hamburger { font-size:1.2rem; margin-right:1rem; background:transparent; border:none; color:inherit; } +@media(max-width:768px) { + .sidebar { display:none; } + .topbar { display:flex; } +} +@media(min-width:769px) { + .topbar { display:none; } +} diff --git a/only-vod-chat.py b/only-vod-chat.py index b3a948d..4d9a92d 100644 --- a/only-vod-chat.py +++ b/only-vod-chat.py @@ -228,7 +228,11 @@ class TwitchArchive: elif self.os == 'linux': subprocess.call([bin_path+"/TwitchDownloaderCLI", "chatupdate", "-i", chat_json_path, "-o", chat_html_path, "-E", "--temp-path", f"{bin_path}/temp"]) if self.username == 'KalathrasLolweapon': print('Uploading html chat to b2 bucket') - subprocess.call(['rclone', 'copy', chat_html_path, 'b2:kala-help/chat_html', '--progress']) + proc = subprocess.Popen(['rclone', 'copy', chat_html_path, 'b2:kala-help/chat_html', '--progress'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + if proc.stdout: + for line in proc.stdout: + print(line, end='') + proc.wait() except Exception as e: print('A ERROR has ocurred and chat will need to be updated to html manually') @@ -236,9 +240,22 @@ class TwitchArchive: print('Uploading files:') if self.os == 'windows': if self.username == 'KalathrasLolweapon': - subprocess.call(['rclone', 'copy', str(pathlib.Path(__file__).parent.resolve())+'/VODS', 'GD:VODS', '--progress']) - subprocess.call(['rclone', 'copy', str(pathlib.Path(__file__).parent.resolve())+'/Chat', 'GD:Chat', '--progress']) - else:subprocess.call(['rclone', 'copy', str(pathlib.Path(__file__).parent.resolve())+'/VODS', 'GD:VODS', '--progress']) + proc = subprocess.Popen(['rclone', 'copy', str(pathlib.Path(__file__).parent.resolve())+'/VODS', 'GD:VODS', '--progress'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + if proc.stdout: + for line in proc.stdout: + print(line, end='') + proc.wait() + proc = subprocess.Popen(['rclone', 'copy', str(pathlib.Path(__file__).parent.resolve())+'/Chat', 'GD:Chat', '--progress'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + if proc.stdout: + for line in proc.stdout: + print(line, end='') + proc.wait() + else: + proc = subprocess.Popen(['rclone', 'copy', str(pathlib.Path(__file__).parent.resolve())+'/VODS', 'GD:VODS', '--progress'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + if proc.stdout: + for line in proc.stdout: + print(line, end='') + proc.wait() elif self.os == 'linux':subprocess.call([bin_path+'/upload.sh', str(pathlib.Path(self.root_path).resolve()),self.username]) if self.deleteFiles == 1: diff --git a/twitch-archive.py b/twitch-archive.py deleted file mode 100644 index 2fdcb4e..0000000 --- a/twitch-archive.py +++ /dev/null @@ -1,1565 +0,0 @@ -""" -Twitch Archive - Automated Twitch Stream Recording & Archiving System - -This script monitors a Twitch channel and automatically: -- Records live streams as they happen -- Downloads VODs (Video on Demand) after the stream ends -- Downloads and renders chat logs -- Saves stream metadata -- Uploads everything to cloud storage (optional) - -Requirements: -- Python 3.7+ -- External tools: streamlink, ffmpeg, TwitchDownloaderCLI, rclone (optional) -- Configuration file: config.json (copy from config.sample.json) -- Environment file: .env (for API credentials) - -Refactored Version 2.0: -This version has been split into multiple modules for better maintainability: -- modules/constants.py: Constants and default configuration -- modules/config.py: Configuration management -- modules/notifications.py: Email notifications -- modules/utils.py: Utility functions -- modules/stream_monitor.py: Stream monitoring and API -- modules/recorder.py: Live stream recording -- modules/processor.py: Video/audio processing -- modules/downloader.py: VOD and chat downloading -- modules/file_manager.py: File and cloud management -""" - -# Standard library imports -import os -import sys -import time -import json -import signal -import getopt -from typing import Dict, Optional, Any -from datetime import datetime, timedelta - -# Third-party imports -from colorama import Fore, Style -from pytz import timezone -from dotenv import load_dotenv, find_dotenv - -# Local module imports -from modules.constants import DEFAULT_CONFIG, PREFIX_LIVE, PREFIX_VOD, PREFIX_CHAT, PREFIX_MERGED, PREFIX_METADATA -from modules.config import ConfigManager -from modules.notifications import NotificationManager -from modules.utils import ( - detect_operating_system, get_ffmpeg_executable, get_twitch_downloader_executable, - get_unique_filename, get_video_duration, verify_streamlink, verify_ffmpeg, verify_twitch_downloader -) -from modules.stream_monitor import StreamMonitor -from modules.recorder import StreamRecorder -from modules.processor import StreamProcessor -from modules.downloader import ContentDownloader -from modules.file_manager import FileManager - - -class TwitchArchive: - """ - Main class for the Twitch Archive system. - - Handles monitoring a Twitch channel, recording live streams, and downloading - VODs, chat logs, and metadata. Can optionally upload to cloud storage. - - Refactored Version 2.0: This class now delegates most functionality to - specialized modules for better code organization. - """ - - def __init__(self, config: Optional[Dict[str, Any]] = None): - """ - Initialize the TwitchArchive with configuration settings. - - Args: - config: Configuration dictionary. If None, loads from legacy config.json - """ - if config is None: - # Legacy mode: load from config.json - self.load_config() - else: - # New mode: use provided config - for key, value in config.items(): - setattr(self, key, value) - - # Initialize system components - self.os_type = detect_operating_system() - self.shutdown_requested = False - self.current_stream_data = {} - - # Initialize component modules (created during run()) - self.stream_monitor = None - self.notification_manager = None - self.file_manager = None - self.recorder = None - self.processor = None - self.downloader = None - - def load_config(self) -> None: - """ - Load configuration from config.json file (legacy support). - - Falls back to default configuration if file is not found or cannot be read. - Filters out comment fields (starting with '_') from the config. - """ - config_file = os.path.join(os.path.dirname(__file__), 'config.json') - - # Start with default configuration - config = DEFAULT_CONFIG.copy() - - # Try to load and merge user configuration - if os.path.exists(config_file): - try: - with open(config_file, 'r', encoding='utf-8') as f: - user_config = json.load(f) - # Filter out comment fields (those starting with '_') - user_config = {k: v for k, v in user_config.items() if not k.startswith('_')} - # Merge user config with defaults (user config takes precedence) - config.update(user_config) - print(f'{Fore.GREEN}✓ Configuration loaded from config.json{Style.RESET_ALL}') - except json.JSONDecodeError as e: - print(f'{Fore.YELLOW}⚠ Warning: Invalid JSON in config.json: {e}{Style.RESET_ALL}') - print(f'{Fore.YELLOW} Using default configuration{Style.RESET_ALL}') - except Exception as e: - print(f'{Fore.YELLOW}⚠ Warning: Could not load config.json: {e}{Style.RESET_ALL}') - print(f'{Fore.YELLOW} Using default configuration{Style.RESET_ALL}') - else: - print(f'{Fore.YELLOW}⚠ Warning: config.json not found{Style.RESET_ALL}') - print(f'{Fore.CYAN} → Copy config.sample.json to config.json and edit it with your settings{Style.RESET_ALL}') - - # Set all configuration values as instance attributes - for key, value in config.items(): - setattr(self, key, value) - - def _load_environment_variables(self) -> None: - """ - Load environment variables from .env file. - - Required variables: - - CLIENT-ID: Twitch API client ID - - CLIENT-SECRET: Twitch API client secret - - OAUTH-PRIVATE-TOKEN: Optional, for accessing subscriber-only streams - - SENDER: Email address for notifications (if enabled) - - RECEIVER: Email address to receive notifications (if enabled) - - PASSWD: Email password for sending notifications (if enabled) - - Raises: - SystemExit: If .env file is not found - """ - if not load_dotenv(find_dotenv()): - print(f'{Fore.RED}✗ ERROR: .env file not found{Style.RESET_ALL}') - print(f'{Fore.CYAN} → Create a .env file with your Twitch API credentials{Style.RESET_ALL}') - print(f'{Fore.CYAN} → Required: CLIENT-ID, CLIENT-SECRET{Style.RESET_ALL}') - sys.exit(1) - - def _initialize_components(self) -> None: - """Initialize all component modules.""" - # Stream monitoring - self.stream_monitor = StreamMonitor(self.username) - - # Notifications - self.notification_manager = NotificationManager( - enabled=self.notifications, - username=self.username - ) - - # File management - self.file_manager = FileManager( - root_path=self.root_path, - username=self.username, - config=vars(self) - ) - self.file_manager.initialize_directories() - - # Recording - self.recorder = StreamRecorder( - username=self.username, - quality=self.quality, - refresh=self.refresh, - hls_segments=self.hls_segments, - streamlink_ttvlol=self.streamlink_ttvlol, - shutdown_callback=lambda: self.shutdown_requested - ) - - # Processing - ffmpeg_path = get_ffmpeg_executable(self.os_type) - self.processor = StreamProcessor( - os_type=self.os_type, - ffmpeg_path=ffmpeg_path, - config=vars(self) - ) - - # Downloading - twitch_downloader_path = get_twitch_downloader_executable(self.os_type) - self.downloader = ContentDownloader( - twitch_downloader_path=twitch_downloader_path, - ffmpeg_path=ffmpeg_path, - config=vars(self) - ) - - def _print_configuration_summary(self) -> None: - """Print a summary of the current configuration to the console.""" - print(f'\n{Fore.CYAN}{"=" * 60}{Style.RESET_ALL}') - print(f'{Fore.CYAN}TWITCH ARCHIVE - Configuration Summary{Style.RESET_ALL}') - print(f'{Fore.CYAN}{"=" * 60}{Style.RESET_ALL}\n') - - # Basic settings - print(f'Streamer: {Fore.GREEN}{self.username}{Style.RESET_ALL}') - print(f'Quality: {Fore.GREEN}{self.quality}{Style.RESET_ALL}') - print(f'Storage: {Fore.GREEN}{os.path.abspath(self.root_path)}{Style.RESET_ALL}') - print(f'Refresh rate: {Fore.GREEN}{self.refresh}s{Style.RESET_ALL}\n') - - # Feature toggles - self._print_toggle('Email notifications', self.notifications) - self._print_toggle('Metadata download', self.downloadMETADATA) - self._print_toggle('VOD download', self.downloadVOD) - self._print_toggle('Chat download & render', self.downloadCHAT) - if self.downloadCHAT: - self._print_toggle(' ↳ Merge video + chat', self.mergeVideoChat) - if self.mergeVideoChat: - print(f' Layout: {Fore.GREEN}{self.mergeChatLayout}{Style.RESET_ALL}') - self._print_toggle('Cloud upload', self.uploadCloud) - - # Warning messages - if self.deleteFiles: - print(f'\n{Fore.RED}⚠ WARNING: Files will be DELETED after processing{Style.RESET_ALL}') - if not self.uploadCloud: - print(f'{Fore.RED}⚠ CRITICAL: Files will be deleted WITHOUT cloud backup!{Style.RESET_ALL}') - print(f'{Fore.YELLOW} Press CTRL+C to stop and change configuration{Style.RESET_ALL}') - else: - print(f'\n{Fore.GREEN}✓ Files will be preserved locally{Style.RESET_ALL}') - - print(f'\n{Fore.CYAN}{"=" * 60}{Style.RESET_ALL}\n') - - def _print_toggle(self, label: str, value: bool) -> None: - """Helper method to print a configuration toggle in a consistent format.""" - status = f'{Fore.GREEN}Enabled{Style.RESET_ALL}' if value else f'{Fore.RED}Disabled{Style.RESET_ALL}' - print(f'{label}: {status}') - - def run(self) -> None: - """ - Main entry point for the application. - - Initializes environment, validates configuration, creates necessary - directories, and starts the monitoring loop. - """ - # Load environment variables - self._load_environment_variables() - - # Initialize all component modules - self._initialize_components() - - # Validate username - self.stream_monitor.validate_username() - - # Verify dependencies - if not verify_streamlink(): - sys.exit(1) - verify_ffmpeg(self.os_type) - if self.downloadVOD or self.downloadCHAT: - verify_twitch_downloader(self.os_type) - - # Print configuration summary - self._print_configuration_summary() - - # Start monitoring - print(f"Monitoring {Fore.GREEN}{self.username}{Style.RESET_ALL} every {Fore.GREEN}{self.refresh}s{Style.RESET_ALL}") - self.notification_manager.send("TWITCH ARCHIVE STARTED", - f"Monitoring {self.username} every {self.refresh} seconds.") - - # Begin the main monitoring loop - self.loopcheck() - - def _interruptible_sleep(self, seconds: float) -> bool: - """ - Sleep for the specified duration, but check for shutdown periodically. - - Args: - seconds: Number of seconds to sleep - - Returns: - bool: True if sleep completed, False if interrupted by shutdown - """ - start_time = time.time() - while time.time() - start_time < seconds: - if self.shutdown_requested: - return False - time.sleep(min(1.0, seconds - (time.time() - start_time))) - return True - - def _signal_handler(self, signum, frame): - """Handle interrupt signals gracefully.""" - if not self.shutdown_requested: - print(f'\n{Fore.YELLOW}{"=" * 60}{Style.RESET_ALL}') - print(f'{Fore.YELLOW}⚠ Shutdown requested. Stopping downloads and finalizing...{Style.RESET_ALL}') - print(f'{Fore.YELLOW}{"=" * 60}{Style.RESET_ALL}\n') - self.shutdown_requested = True - - # Stop current subprocess if running - if self.recorder: - self.recorder.stop() - - def loopcheck(self) -> None: - """ - Main monitoring loop. - - Continuously checks if the streamer is live, and when they are: - 1. Records the live stream - 2. Downloads the VOD - 3. Downloads and renders chat - 4. Uploads everything to cloud storage (if enabled) - 5. Optionally deletes local files after upload - """ - # Set up signal handlers for graceful shutdown - signal.signal(signal.SIGINT, self._signal_handler) - if hasattr(signal, 'SIGTERM'): - signal.signal(signal.SIGTERM, self._signal_handler) - - while not self.shutdown_requested: - try: - # Check stream status using StreamMonitor - response = self.stream_monitor.check_stream_status() - is_live = response['data']['user']['stream'] - - # Stream is offline - if is_live is None: - print(f'{Fore.CYAN}⏳ {self.username} is offline. Checking again in {self.refresh}s...{Style.RESET_ALL}', end='\r') - if self.shutdown_requested: - break - self._interruptible_sleep(self.refresh) - continue - - # Stream is live but not ready yet - if not is_live.get('title'): - print(f'{Fore.YELLOW}⚠ Stream detected but no title yet. Waiting...{Style.RESET_ALL}') - if self.shutdown_requested: - break - self._interruptible_sleep(self.refresh) - continue - - # Stream is live and ready! - print(f'\n{Fore.GREEN}✓ {self.username} is LIVE!{Style.RESET_ALL}') - print(f'{Fore.CYAN}Title: {is_live["title"]}{Style.RESET_ALL}') - - # Create unique stream identifier - stream_id = f"{is_live['createdAt']} - {self.username} - {is_live['title']}" - - # Parse stream start time - live_date = datetime.strptime( - is_live["createdAt"], '%Y-%m-%dT%H:%M:%SZ' - ).replace(tzinfo=timezone('UTC')).astimezone(tz=None).replace(tzinfo=None) - - # Use CURRENT time for filename - current_time = datetime.now() - filename_base = current_time.strftime('%Y%m%d_%Hh%Mm%Ss') - - # Check if stream was already processed - if self.file_manager.is_stream_processed(stream_id): - print(f'{Fore.YELLOW}⚠ Stream was previously recorded, but it\'s still live!{Style.RESET_ALL}') - print(f'{Fore.GREEN}✓ Starting new recording with timestamp: {filename_base}{Style.RESET_ALL}') - else: - self.file_manager.mark_stream_processed(stream_id) - print(f'{Fore.GREEN}✓ New stream detected - starting recording{Style.RESET_ALL}') - - # Determine file paths - live_raw_path = str(self.file_manager.raw_path / f"{PREFIX_LIVE}{filename_base}.ts") - live_proc_ext = '.mp3' if self.quality == 'audio_only' else '.mp4' - live_proc_path = str(self.file_manager.video_path / f"{PREFIX_LIVE}{filename_base}{live_proc_ext}") - - # Ensure unique filenames - live_raw_path = get_unique_filename(live_raw_path) - live_proc_path = get_unique_filename(live_proc_path) - filename_base = os.path.splitext(os.path.basename(live_raw_path))[0].replace(PREFIX_LIVE, "") - - print(f'{Fore.CYAN}Output path: {live_raw_path}{Style.RESET_ALL}') - - # Send notification - self.notification_manager.send(f'🔴 Stream Started - {filename_base}', - f'Title: {is_live["title"]}') - - # Start live chat download if enabled - live_chat_process = None - live_chat_method = None # Track which method was used - chat_json_path = str(self.file_manager.chat_json_path / f"{PREFIX_CHAT}{filename_base}.json") - - if self.downloadLiveCHAT: - vod_id = is_live.get('archiveVideo', {}).get('id') if is_live.get('archiveVideo') else None - stream_url = f"https://twitch.tv/{self.username}" - - live_chat_process, live_chat_method = self.downloader.start_live_chat_download_with_fallback( - vod_id=vod_id, - stream_url=stream_url, - json_path=chat_json_path, - use_chat_downloader_primary=self.use_chat_downloader_primary, - no_chat_downloader_fallback=self.no_chat_downloader_fallback, - verbose=self.verbose - ) - - # Record the live stream - recording_completed = self.recorder.record(is_live, live_raw_path) - - # If shutdown was requested during recording, try to finalize - if self.shutdown_requested: - print(f'{Fore.YELLOW}Attempting to process any recorded content...{Style.RESET_ALL}') - - # Process the raw stream file - self.processor.process_raw_stream(live_raw_path, live_proc_path) - - # Wait for live chat download if it was started - live_chat_downloaded = False - if live_chat_process is not None: - live_chat_downloaded = self.downloader.wait_for_chat_download(live_chat_process, chat_json_path) - - # Render live chat if downloaded successfully - chat_rendered_successfully = False - chat_video_path = None - if live_chat_downloaded: - chat_video_path = str(self.file_manager.chat_mp4_path / f"{PREFIX_CHAT}{filename_base}.mp4") - output_args = self.processor.build_chat_output_args() - - # Wait for chat file to be fully accessible (not locked) - print(f'{Fore.CYAN}Verifying chat file is ready for rendering...{Style.RESET_ALL}') - if not self.downloader.wait_for_file_access(chat_json_path, max_attempts=15, delay=0.5): - print(f'{Fore.RED}✗ Chat file is locked, skipping rendering{Style.RESET_ALL}') - chat_rendered_successfully = False - else: - # Get video duration first (needed for chat conversion and trimming) - ffmpeg_path = get_ffmpeg_executable(self.os_type) - video_duration = get_video_duration(live_proc_path, ffmpeg_path) - print(f'{Fore.CYAN}Video duration for chat rendering: {video_duration}s{Style.RESET_ALL}') - - # Convert chat format if needed (chat_downloader uses different JSON structure) - render_json_path = chat_json_path - if live_chat_method == 'chat_downloader': - print(f'{Fore.CYAN}Converting chat format for rendering...{Style.RESET_ALL}') - converted_path = chat_json_path.replace('.json', '_converted.json') - if self.downloader.convert_chat_downloader_to_twitch_format(chat_json_path, converted_path, video_duration): - render_json_path = converted_path - print(f'{Fore.GREEN}✓ Chat format converted successfully{Style.RESET_ALL}') - else: - print(f'{Fore.RED}✗ Failed to convert chat format{Style.RESET_ALL}') - - chat_rendered_successfully = self.downloader.render_chat( - render_json_path, - chat_video_path, - output_args, - video_duration=video_duration - ) - - # Merge video and chat if configured - merged_video_path = None - if chat_rendered_successfully and self.mergeVideoChat and os.path.exists(live_proc_path) and os.path.exists(chat_video_path): - merged_video_path = str(self.file_manager.video_path / f"{PREFIX_MERGED}{filename_base}{live_proc_ext}") - merge_success = self.processor.merge_video_and_chat( - live_proc_path, - chat_video_path, - merged_video_path, - self.mergeChatLayout - ) - - # Skip VOD/chat download if shutdown was requested or vodTimeout is 0 - vod_response = None - if self.shutdown_requested: - print(f'{Fore.YELLOW}Skipping VOD and chat download due to shutdown request{Style.RESET_ALL}') - elif self.vodTimeout == 0: - print(f'{Fore.CYAN}VOD check disabled (vodTimeout=0). Skipping VOD download.{Style.RESET_ALL}') - else: - # Try to match stream with VOD (with timeout) - print(f'{Fore.CYAN}Waiting for VOD to become available (timeout: {self.vodTimeout}s)...{Style.RESET_ALL}') - vod_found = False - vod_wait_start = time.time() - - while time.time() - vod_wait_start < self.vodTimeout and not self.shutdown_requested: - vod_response = self.stream_monitor.get_latest_vod() - - if vod_response and vod_response['data']['user']['videos']['edges']: - current_vod = vod_response['data']['user']['videos']['edges'][0]['node'] - vod_date = datetime.strptime( - current_vod["recordedAt"], '%Y-%m-%dT%H:%M:%SZ' - ).replace(tzinfo=timezone('UTC')).astimezone(tz=None).replace(tzinfo=None) - - # Check if VOD matches the stream (within 1 minute tolerance) - time_tolerance = timedelta(minutes=1) - if (live_date - time_tolerance) <= vod_date <= (live_date + time_tolerance): - vod_found = True - break - - # Wait before checking again - if not vod_found: - print(f'{Fore.CYAN}VOD not found yet, waiting...{Style.RESET_ALL}', end='\r') - if not self._interruptible_sleep(min(10, self.vodTimeout - (time.time() - vod_wait_start))): - break - - if not vod_found: - if self.shutdown_requested: - print(f'\n{Fore.YELLOW}VOD check interrupted by shutdown{Style.RESET_ALL}') - else: - print(f'\n{Fore.YELLOW}⚠ VOD not found after {self.vodTimeout}s - streamer may have VODs disabled{Style.RESET_ALL}') - print(f'{Fore.CYAN} → Live recording and chat (if enabled) were saved successfully{Style.RESET_ALL}') - vod_response = None - - # Process VOD if found - if not self.shutdown_requested and vod_response and vod_response['data']['user']['videos']['edges']: - current_vod = vod_response['data']['user']['videos']['edges'][0]['node'] - vod_date = datetime.strptime( - current_vod["recordedAt"], '%Y-%m-%dT%H:%M:%SZ' - ).replace(tzinfo=timezone('UTC')).astimezone(tz=None).replace(tzinfo=None) - - # Check if VOD matches the stream (within 1 minute tolerance) - time_tolerance = timedelta(minutes=1) - if (live_date - time_tolerance) <= vod_date <= (live_date + time_tolerance): - print(f'\n{Fore.GREEN}✓ Found matching VOD{Style.RESET_ALL}') - - # Save metadata - self.file_manager.save_metadata(current_vod, filename_base) - - # Download VOD - vod_ext = '.mp3' if self.quality == 'audio_only' else '.mp4' - vod_path = str(self.file_manager.video_path / f"{PREFIX_VOD}{filename_base}{vod_ext}") - self.downloader.download_vod(current_vod, vod_path) - - # Download and render chat from VOD (if not already done via live chat) - if not live_chat_downloaded: - chat_video_path = str(self.file_manager.chat_mp4_path / f"{PREFIX_CHAT}{filename_base}.mp4") - output_args = self.processor.build_chat_output_args() - - # Get VOD duration to trim chat accordingly - ffmpeg_path = get_ffmpeg_executable(self.os_type) - vod_duration = get_video_duration(vod_path, ffmpeg_path) - - chat_rendered_successfully = self.downloader.download_and_render_chat( - current_vod, - chat_json_path, - chat_video_path, - output_args, - video_duration=vod_duration - ) - - # Merge VOD and chat if configured - if chat_rendered_successfully and self.mergeVideoChat and os.path.exists(vod_path) and os.path.exists(chat_video_path): - merged_vod_path = str(self.file_manager.video_path / f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}{vod_ext}") - self.processor.merge_video_and_chat( - vod_path, - chat_video_path, - merged_vod_path, - self.mergeChatLayout - ) - else: - print(f'{Fore.CYAN}Chat already downloaded from live stream, skipping VOD chat download{Style.RESET_ALL}') - - # But still merge VOD with existing chat if configured - if self.mergeVideoChat and os.path.exists(vod_path) and chat_video_path and os.path.exists(chat_video_path): - merged_vod_path = str(self.file_manager.video_path / f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}{vod_ext}") - self.processor.merge_video_and_chat( - vod_path, - chat_video_path, - merged_vod_path, - self.mergeChatLayout - ) - else: - print(f'{Fore.YELLOW}⚠ No matching VOD found for this stream{Style.RESET_ALL}') - - # Clean up raw files if configured - self.file_manager.clean_raw_file(live_raw_path) - - # Upload to cloud if configured - upload_success = self.file_manager.upload_to_cloud( - filename_base, - notification_callback=self.notification_manager.send - ) - - # Delete local files if configured and upload succeeded - if self.deleteFiles and upload_success: - self.file_manager.delete_local_files( - filename_base, - live_raw_path, - live_proc_path, - notification_callback=self.notification_manager.send - ) - - # Done processing this stream - if self.shutdown_requested: - print(f'\n{Fore.YELLOW}{"=" * 60}{Style.RESET_ALL}') - print(f'{Fore.YELLOW}✓ Stream processing stopped by user{Style.RESET_ALL}') - print(f'{Fore.YELLOW}{"=" * 60}{Style.RESET_ALL}\n') - break - else: - print(f'\n{Fore.GREEN}{"=" * 60}{Style.RESET_ALL}') - print(f'{Fore.GREEN}✓ Stream processing complete{Style.RESET_ALL}') - print(f'{Fore.GREEN}{"=" * 60}{Style.RESET_ALL}\n') - self.notification_manager.send(f'✓ Complete - {filename_base}', - 'Stream processing finished. Resuming monitoring.') - self._interruptible_sleep(self.refresh) - - except KeyboardInterrupt: - if not self.shutdown_requested: - self.shutdown_requested = True - print(f'\n{Fore.YELLOW}{"=" * 60}{Style.RESET_ALL}') - print(f'{Fore.YELLOW}⚠ Interrupted. Cleaning up...{Style.RESET_ALL}') - print(f'{Fore.YELLOW}{"=" * 60}{Style.RESET_ALL}\n') - break - - except Exception as e: - print(f'\n{Fore.RED}{"=" * 60}{Style.RESET_ALL}') - print(f'{Fore.RED}✗ ERROR: {str(e)}{Style.RESET_ALL}') - print(f'{Fore.YELLOW}Waiting {self.refresh} seconds before retrying...{Style.RESET_ALL}') - print(f'{Fore.RED}{"=" * 60}{Style.RESET_ALL}\n') - self.notification_manager.send('⚠ Error - Recovery', - f'Error: {str(e)}\nRetrying after {self.refresh} seconds.') - - if self.shutdown_requested: - break - self._interruptible_sleep(self.refresh) - - # Final cleanup message - print(f'{Fore.GREEN}✓ Monitoring stopped cleanly{Style.RESET_ALL}') - - def _upload_to_cloud(self, filename_base: str) -> bool: - """ - Upload archived files to cloud storage using rclone. - - Args: - filename_base: Base filename (without prefixes/extensions) - - Returns: - bool: True if upload succeeded or is disabled, False if failed - """ - if not self.uploadCloud: - return True # Consider upload "successful" if disabled - - print(f'\n{Fore.CYAN}Uploading to cloud storage...{Style.RESET_ALL}') - self.send_notification(f'☁ Uploading - {filename_base}', 'Uploading files to cloud storage') - - # Create list of files to upload - bin_path = self._get_bin_path() - upload_list_path = os.path.join(bin_path, 'temp', 'upload.txt') - - # Ensure temp directory exists - os.makedirs(os.path.dirname(upload_list_path), exist_ok=True) - - files_to_upload = [ - f"{PREFIX_LIVE}{filename_base}.ts", - f"{PREFIX_LIVE}{filename_base}.mp4", - f"{PREFIX_LIVE}{filename_base}.mp3", - f"{PREFIX_VOD}{filename_base}.ts", - f"{PREFIX_VOD}{filename_base}.mp4", - f"{PREFIX_VOD}{filename_base}.mp3", - f"{PREFIX_METADATA}{filename_base}.json", - f"{PREFIX_CHAT}{filename_base}.json", - f"{PREFIX_CHAT}{filename_base}.mp4" - ] - - with open(upload_list_path, 'w') as f: - f.write('\n'.join(files_to_upload)) - - # Run rclone - try: - result = subprocess.call([ - 'rclone', 'copy', - str(pathlib.Path(self.root_path).resolve()), - self.rclone_path, - '--include-from', upload_list_path - ]) - - # Clean up upload list - if os.path.exists(upload_list_path): - os.remove(upload_list_path) - - if result == 0: - print(f'{Fore.GREEN}✓ Upload complete{Style.RESET_ALL}') - self.send_notification(f'✓ Upload Success - {filename_base}', - 'All files uploaded successfully') - return True - else: - print(f'{Fore.RED}✗ Upload failed (exit code: {result}){Style.RESET_ALL}') - print(f'{Fore.YELLOW}Files preserved locally due to upload failure{Style.RESET_ALL}') - self.send_notification(f'✗ Upload Failed - {filename_base}', - f'Upload failed with code {result}. Files preserved locally.') - return False - - except Exception as e: - print(f'{Fore.RED}✗ Upload error: {str(e)}{Style.RESET_ALL}') - return False - - def _delete_local_files(self, filename_base: str, live_raw_path: str, live_proc_path: str) -> None: - """ - Delete local archive files after successful upload. - - Args: - filename_base: Base filename (without prefixes/extensions) - live_raw_path: Path to live raw file - live_proc_path: Path to live processed file - """ - print(f'\n{Fore.RED}{"=" * 60}{Style.RESET_ALL}') - print(f'{Fore.RED}⚠ DELETING LOCAL FILES{Style.RESET_ALL}') - print(f'{Fore.RED}{"=" * 60}{Style.RESET_ALL}\n') - - self.send_notification(f'🗑 Deleting - {filename_base}', - 'Deleting local files after successful upload') - - files_to_delete = [] - - # Live files - if not self.cleanRaw and os.path.exists(live_raw_path): - files_to_delete.append(live_raw_path) - if os.path.exists(live_proc_path): - files_to_delete.append(live_proc_path) - - # VOD files - if self.downloadVOD: - vod_raw = os.path.join(self.raw_path, f"{PREFIX_VOD}{filename_base}.ts") - vod_mp4 = os.path.join(self.video_path, f"{PREFIX_VOD}{filename_base}.mp4") - vod_mp3 = os.path.join(self.video_path, f"{PREFIX_VOD}{filename_base}.mp3") - - if not self.cleanRaw and os.path.exists(vod_raw): - files_to_delete.append(vod_raw) - if os.path.exists(vod_mp4): - files_to_delete.append(vod_mp4) - if os.path.exists(vod_mp3): - files_to_delete.append(vod_mp3) - - # Chat files - if self.downloadCHAT: - chat_json = os.path.join(self.chatJSON_path, f"{PREFIX_CHAT}{filename_base}.json") - chat_mp4 = os.path.join(self.chatMP4_path, f"{PREFIX_CHAT}{filename_base}.mp4") - - if os.path.exists(chat_json): - files_to_delete.append(chat_json) - if os.path.exists(chat_mp4): - files_to_delete.append(chat_mp4) - - # Metadata files - if self.downloadMETADATA: - metadata = os.path.join(self.metadata_path, f"{PREFIX_METADATA}{filename_base}.json") - if os.path.exists(metadata): - files_to_delete.append(metadata) - - # Delete all files - for filepath in files_to_delete: - try: - print(f'{Fore.RED} Deleting: {os.path.basename(filepath)}{Style.RESET_ALL}') - os.remove(filepath) - except Exception as e: - print(f'{Fore.YELLOW} ⚠ Failed to delete {filepath}: {e}{Style.RESET_ALL}') - - print(f'{Fore.RED}\n✓ Cleanup complete{Style.RESET_ALL}') - - -# ============================================================================ -# MULTI-STREAMER MANAGER -# ============================================================================ - -class TwitchArchiveManager: - """ - Manages multiple TwitchArchive instances for monitoring multiple streamers. - """ - - def __init__(self, specific_streamer: Optional[str] = None, verbose: bool = False, - chat_only: bool = False, - use_chat_downloader_primary: bool = False, - use_chat_downloader_fallback: bool = True): - """ - Initialize the manager. - - Args: - specific_streamer: If provided, only monitor this streamer (ignore enabled status) - verbose: Enable verbose debug output - chat_only: Only download chat, skip video recording (test mode) - use_chat_downloader_primary: Use chat_downloader as primary chat source - use_chat_downloader_fallback: Enable chat_downloader fallback - """ - self.config_manager = ConfigManager() - self.specific_streamer = specific_streamer - self.verbose = verbose - self.chat_only = chat_only - self.use_chat_downloader_primary = use_chat_downloader_primary - self.use_chat_downloader_fallback = use_chat_downloader_fallback - self.archivers: Dict[str, TwitchArchive] = {} - self.shutdown_requested = False - self.active_recordings: Dict[str, str] = {} # Track active recordings: {username: stream_id} - - # Setup signal handlers - signal.signal(signal.SIGTERM, self._signal_handler) - signal.signal(signal.SIGINT, self._signal_handler) - - def _signal_handler(self, signum, frame): - """Handle shutdown signals gracefully.""" - print(f'\n{Fore.YELLOW}⚠ Shutdown signal received...{Style.RESET_ALL}') - self.shutdown_requested = True - - # Signal all archivers to shut down - for archiver in self.archivers.values(): - archiver.shutdown_requested = True - - def _get_streamers_to_monitor(self) -> list: - """ - Get list of streamers to monitor. - - Returns: - list: List of streamer usernames to monitor - """ - if self.specific_streamer: - # Monitor only the specified streamer (ignore enabled flag) - return [self.specific_streamer] - else: - # Monitor all enabled streamers - return self.config_manager.get_all_enabled_streamers() - - def _initialize_archiver(self, username: str) -> TwitchArchive: - """ - Initialize a TwitchArchive instance for a streamer. - - Args: - username: Twitch username - - Returns: - TwitchArchive: Initialized archiver instance - """ - config = self.config_manager.load_streamer_config(username) - - # Apply command-line overrides for chat_downloader options - config['useChatDownloaderPrimary'] = self.use_chat_downloader_primary - config['useChatDownloaderFallback'] = self.use_chat_downloader_fallback - - archiver = TwitchArchive(config) - return archiver - - def run(self) -> None: - """ - Main entry point for multi-streamer monitoring. - - Monitors all enabled streamers (or a specific one if provided). - """ - print(f'\n{Fore.CYAN}{"=" * 70}{Style.RESET_ALL}') - print(f'{Fore.CYAN}TWITCH ARCHIVE - Multi-Streamer Mode{Style.RESET_ALL}') - if self.chat_only: - print(f'{Fore.YELLOW}🧪 TEST MODE: Chat-Only (Video Recording Disabled){Style.RESET_ALL}') - print(f'{Fore.CYAN}{"=" * 70}{Style.RESET_ALL}\n') - - # Get streamers to monitor - streamers = self._get_streamers_to_monitor() - - if not streamers: - print(f'{Fore.RED}✗ No streamers configured or enabled{Style.RESET_ALL}') - print(f'{Fore.CYAN}→ Create config files in config/streamers/{Style.RESET_ALL}') - print(f'{Fore.CYAN}→ Or run with -u to create a new config{Style.RESET_ALL}') - sys.exit(1) - - if self.chat_only: - print(f'{Fore.YELLOW}📝 Chat-Only Mode Enabled:{Style.RESET_ALL}') - print(f'{Fore.CYAN} • Verbose logging: ON{Style.RESET_ALL}') - print(f'{Fore.CYAN} • Video recording: DISABLED{Style.RESET_ALL}') - print(f'{Fore.CYAN} • Chat download: ENABLED{Style.RESET_ALL}') - print(f'{Fore.CYAN} • VOD download: DISABLED{Style.RESET_ALL}') - print() - - print(f'{Fore.GREEN}Monitoring {len(streamers)} streamer(s):{Style.RESET_ALL}') - for streamer in streamers: - print(f' • {Fore.CYAN}{streamer}{Style.RESET_ALL}') - print() - - # Initialize archivers for all streamers - for username in streamers: - try: - archiver = self._initialize_archiver(username) - - # Load environment and initialize components - archiver._load_environment_variables() - archiver._initialize_components() - - # Validate username through stream_monitor - archiver.stream_monitor.validate_username() - - self.archivers[username] = archiver - print(f'{Fore.GREEN}✓ Initialized {username}{Style.RESET_ALL}') - except Exception as e: - print(f'{Fore.RED}✗ Failed to initialize {username}: {e}{Style.RESET_ALL}') - import traceback - traceback.print_exc() - - if not self.archivers: - print(f'{Fore.RED}✗ No archivers could be initialized{Style.RESET_ALL}') - sys.exit(1) - - # Verify dependencies once (shared across all streamers) - print(f'\n{Fore.CYAN}Verifying dependencies...{Style.RESET_ALL}') - first_archiver = next(iter(self.archivers.values())) - if not verify_streamlink(): - sys.exit(1) - verify_ffmpeg(first_archiver.os_type) - if first_archiver.downloadVOD or first_archiver.downloadCHAT: - verify_twitch_downloader(first_archiver.os_type) - - # Print configuration summary for each streamer - for username, archiver in self.archivers.items(): - archiver._print_configuration_summary() - - print(f'\n{Fore.GREEN}🚀 Starting monitoring loop...{Style.RESET_ALL}\n') - - # Start monitoring loop - self._monitoring_loop() - - def _monitoring_loop(self) -> None: - """ - Main monitoring loop for all streamers. - - Checks each streamer's status and processes streams as needed. - """ - last_check = {} - last_status_print = time.time() - - while not self.shutdown_requested: - current_time = time.time() - - # Print periodic status every 60 seconds - if current_time - last_status_print >= 60: - status_line = " | ".join([f"{username}: checking" for username in self.archivers.keys()]) - print(f'{Fore.CYAN}[Status] {status_line}{Style.RESET_ALL}') - last_status_print = current_time - - for username, archiver in self.archivers.items(): - # Check if enough time has passed since last check for this streamer - if username not in last_check or (current_time - last_check[username]) >= archiver.refresh: - last_check[username] = current_time - - # Check stream status - try: - response = archiver.stream_monitor.check_stream_status() - - # Debug: Print the full response (if verbose) - if self.verbose: - print(f'\n{Fore.MAGENTA}[DEBUG {username}] API Response: {response}{Style.RESET_ALL}') - - stream_data = response['data']['user']['stream'] if response else None - - if self.verbose: - print(f'{Fore.MAGENTA}[DEBUG {username}] Stream data: {stream_data}{Style.RESET_ALL}') - - if stream_data: - # Stream is live - check if it has required basic data (title and start time) - if stream_data.get('title') and stream_data.get('createdAt'): - # Create composite stream ID like single-streamer mode - # This prevents duplicate recordings in the same session - stream_id = f"{stream_data['createdAt']} - {username} - {stream_data.get('title', 'Untitled')}" - - if self.verbose: - # Check if VOD ID is available (for live chat) - if stream_data.get('archiveVideo') and stream_data['archiveVideo'].get('id'): - print(f'{Fore.MAGENTA}[DEBUG {username}] VOD ID: {stream_data["archiveVideo"]["id"]}{Style.RESET_ALL}') - else: - print(f'{Fore.MAGENTA}[DEBUG {username}] No VOD ID available (VODs may be disabled){Style.RESET_ALL}') - print(f'{Fore.MAGENTA}[DEBUG {username}] Composite Stream ID: {stream_id}{Style.RESET_ALL}') - - # Check if we're currently recording this stream - currently_recording = username in self.active_recordings and self.active_recordings[username] == stream_id - - if self.verbose: - print(f'{Fore.MAGENTA}[DEBUG {username}] Currently recording: {currently_recording}{Style.RESET_ALL}') - print(f'{Fore.MAGENTA}[DEBUG {username}] Active recordings: {self.active_recordings}{Style.RESET_ALL}') - - # Record if not currently recording (ignore .log file - always record if live) - if not currently_recording: - print(f'\n{Fore.GREEN}[{username}] Stream detected!{Style.RESET_ALL}') - print(f'{Fore.CYAN}Title: {stream_data.get("title", "No title")}{Style.RESET_ALL}') - print(f'{Fore.CYAN}Started at: {stream_data["createdAt"]}{Style.RESET_ALL}') - - # Warn if VOD ID not available - if not (stream_data.get('archiveVideo') and stream_data['archiveVideo'].get('id')): - print(f'{Fore.YELLOW}⚠ VOD ID not available - live chat download will be skipped{Style.RESET_ALL}') - print(f'{Fore.YELLOW} Stream recording will proceed normally{Style.RESET_ALL}') - - # Mark as currently recording - self.active_recordings[username] = stream_id - - # Process the stream (this blocks until stream ends) - self._process_stream(archiver, stream_data, stream_id) - - # Mark as processed in log (for record keeping) - archiver.file_manager.mark_stream_processed(stream_id) - - # Remove from active recordings - if username in self.active_recordings: - del self.active_recordings[username] - else: - if self.verbose: - print(f'{Fore.CYAN}[{username}] Currently recording this stream, skipping duplicate...{Style.RESET_ALL}') - else: - # Stream is live but not fully initialized yet - print(f'{Fore.YELLOW}[{username}] Stream starting up, waiting for stream data...{Style.RESET_ALL}') - else: - # Not live - if self.verbose: - print(f'{Fore.CYAN}[{username}] Offline - checking again in {archiver.refresh}s{Style.RESET_ALL}', end='\r') - - except Exception as e: - print(f'{Fore.RED}[{username}] Error checking stream: {e}{Style.RESET_ALL}') - import traceback - traceback.print_exc() - - # Sleep briefly before next iteration - time.sleep(1) - - def _process_stream(self, archiver: TwitchArchive, stream_info: Dict[str, Any], stream_id: str) -> None: - """ - Process a detected stream for a specific archiver. - - Args: - archiver: The TwitchArchive instance - stream_info: Stream information from API - stream_id: Unique stream ID - """ - # Store stream data - archiver.current_stream_data = { - 'stream_id': stream_id, - 'title': stream_info['title'], - 'started_at': stream_info['createdAt'] - } - - # Generate timestamp and filename - timestamp = datetime.now(timezone('UTC')).strftime("%Y%m%d_%Hh%Mm%Ss") - filename_base = f"{PREFIX_LIVE}{archiver.username}_{timestamp}" - - # Parse stream start time - live_date = datetime.strptime( - stream_info["createdAt"], '%Y-%m-%dT%H:%M:%SZ' - ).replace(tzinfo=timezone('UTC')).astimezone(tz=None).replace(tzinfo=None) - - # Define paths - raw_extension = '.ts' - proc_extension = '.mp3' if archiver.quality == 'audio_only' else '.mp4' - - live_raw_path = str(archiver.file_manager.raw_path / f"{filename_base}{raw_extension}") - live_proc_path = str(archiver.file_manager.video_path / f"{filename_base}{proc_extension}") - chat_json_path = str(archiver.file_manager.chat_json_path / f"{PREFIX_CHAT}{filename_base}.json") - - # Send notification - if not self.chat_only: - archiver.notification_manager.send( - f"Stream Started - {archiver.username}", - f"Recording: {stream_info['title']}" - ) - - # Start live chat download if enabled (with fallback support) - live_chat_process = None - live_chat_method = 'failed' - if archiver.downloadLiveCHAT: - if self.verbose or self.chat_only: - print(f'\n{Fore.MAGENTA}[VERBOSE] Starting chat download process...{Style.RESET_ALL}') - print(f'{Fore.MAGENTA}[VERBOSE] downloadLiveCHAT: {archiver.downloadLiveCHAT}{Style.RESET_ALL}') - print(f'{Fore.MAGENTA}[VERBOSE] useChatDownloaderPrimary: {archiver.downloader.use_chat_downloader_primary}{Style.RESET_ALL}') - print(f'{Fore.MAGENTA}[VERBOSE] useChatDownloaderFallback: {archiver.downloader.use_chat_downloader_fallback}{Style.RESET_ALL}') - - # Get VOD ID if available - live_vod_id = None - if stream_info.get('archiveVideo') and stream_info['archiveVideo'].get('id'): - live_vod_id = stream_info['archiveVideo']['id'] - print(f'{Fore.CYAN}Live VOD ID detected: {live_vod_id}{Style.RESET_ALL}') - if self.verbose or self.chat_only: - print(f'{Fore.MAGENTA}[VERBOSE] VOD URL: https://www.twitch.tv/videos/{live_vod_id}{Style.RESET_ALL}') - else: - print(f'{Fore.YELLOW}⚠ No VOD ID available - will use fallback if configured{Style.RESET_ALL}') - if self.verbose or self.chat_only: - print(f'{Fore.MAGENTA}[VERBOSE] This happens when streamer has VODs disabled{Style.RESET_ALL}') - print(f'{Fore.MAGENTA}[VERBOSE] chat_downloader fallback will be used if enabled{Style.RESET_ALL}') - - # Try to start live chat download with fallback - try: - if self.verbose or self.chat_only: - print(f'{Fore.MAGENTA}[VERBOSE] Calling start_live_chat_download_with_fallback(){Style.RESET_ALL}') - print(f'{Fore.MAGENTA}[VERBOSE] Username: {archiver.username}{Style.RESET_ALL}') - print(f'{Fore.MAGENTA}[VERBOSE] VOD ID: {live_vod_id}{Style.RESET_ALL}') - print(f'{Fore.MAGENTA}[VERBOSE] Output path: {chat_json_path}{Style.RESET_ALL}') - - live_chat_process, live_chat_method = archiver.downloader.start_live_chat_download_with_fallback( - archiver.username, live_vod_id, chat_json_path - ) - - if self.verbose or self.chat_only: - print(f'{Fore.MAGENTA}[VERBOSE] Chat download method selected: {live_chat_method}{Style.RESET_ALL}') - print(f'{Fore.MAGENTA}[VERBOSE] Process handle: {live_chat_process}{Style.RESET_ALL}') - - # If chat_downloader is selected, start it in background thread now (before video recording) - if live_chat_method == 'chat_downloader' and not self.chat_only: - if self.verbose: - print(f'{Fore.MAGENTA}[VERBOSE] Starting chat_downloader in background thread...{Style.RESET_ALL}') - try: - print(f'{Fore.CYAN}Starting chat_downloader in background (concurrent with video)...{Style.RESET_ALL}') - archiver.downloader.start_chat_downloader_thread( - archiver.username, chat_json_path, - shutdown_check=lambda: self.shutdown_requested or archiver.shutdown_requested, - stream_monitor=archiver.stream_monitor, - verbose=self.verbose - ) - except Exception as e: - print(f'{Fore.RED}✗ Failed to start chat thread: {e}{Style.RESET_ALL}') - import traceback - traceback.print_exc() - live_chat_method = 'failed' - - except Exception as e: - print(f'{Fore.RED}✗ Failed to start live chat download: {e}{Style.RESET_ALL}') - import traceback - traceback.print_exc() - live_chat_method = 'failed' - - # Record livestream (skip in chat-only mode) - if self.chat_only: - print(f'\n{Fore.YELLOW}🧪 Chat-Only Mode: Skipping video recording{Style.RESET_ALL}') - print(f'{Fore.CYAN}Waiting for chat download to complete...{Style.RESET_ALL}') - - # Start chat download based on method - if live_chat_method == 'chat_downloader': - if self.verbose: - print(f'{Fore.MAGENTA}[VERBOSE] Starting chat_downloader in background thread...{Style.RESET_ALL}') - try: - print(f'{Fore.CYAN}Using chat_downloader for live chat...{Style.RESET_ALL}') - archiver.downloader.start_chat_downloader_thread( - archiver.username, chat_json_path, - shutdown_check=lambda: self.shutdown_requested or archiver.shutdown_requested, - stream_monitor=archiver.stream_monitor, - verbose=self.verbose or self.chat_only - ) - # Wait for completion - live_chat_downloaded = archiver.downloader.wait_for_chat_thread() - except Exception as e: - print(f'{Fore.RED}✗ chat_downloader failed: {e}{Style.RESET_ALL}') - import traceback - traceback.print_exc() - live_chat_downloaded = False - elif live_chat_method == 'twitch_downloader' and live_chat_process is not None: - if self.verbose: - print(f'{Fore.MAGENTA}[VERBOSE] Waiting for TwitchDownloaderCLI process...{Style.RESET_ALL}') - live_chat_downloaded = archiver.downloader.wait_for_chat_download(live_chat_process, chat_json_path) - else: - live_chat_downloaded = False - - # Report results - if live_chat_downloaded: - print(f'\n{Fore.GREEN}✓ Chat-Only Test Complete!{Style.RESET_ALL}') - print(f'{Fore.CYAN}Chat saved to: {chat_json_path}{Style.RESET_ALL}') - if os.path.exists(chat_json_path): - file_size = os.path.getsize(chat_json_path) - print(f'{Fore.CYAN}File size: {file_size / 1024:.2f} KB{Style.RESET_ALL}') - else: - print(f'\n{Fore.RED}✗ Chat download failed{Style.RESET_ALL}') - - return # Exit early, don't process video - - # Normal mode: Record livestream - recording_successful = archiver.recorder.record(stream_info, live_raw_path) - - # Check if raw file exists (may exist even after interrupted recording) - if not os.path.exists(live_raw_path): - print(f'{Fore.RED}✗ No recording file found, skipping processing{Style.RESET_ALL}') - - # Still wait for chat if it's downloading - if live_chat_method == 'chat_downloader' and archiver.downloader.chat_thread is not None: - print(f'{Fore.CYAN}Waiting for chat download to finish...{Style.RESET_ALL}') - archiver.downloader.wait_for_chat_thread(timeout=30) - elif live_chat_method == 'twitch_downloader' and live_chat_process is not None: - print(f'{Fore.CYAN}Waiting for chat download to finish...{Style.RESET_ALL}') - archiver.downloader.wait_for_chat_download(live_chat_process, chat_json_path, timeout=30) - - return - - # Get file size to check if anything was recorded - file_size = os.path.getsize(live_raw_path) - if file_size < 1024: # Less than 1KB means essentially nothing was recorded - print(f'{Fore.RED}✗ Recording file too small ({file_size} bytes), skipping processing{Style.RESET_ALL}') - return - - print(f'{Fore.CYAN}Processing recorded content ({file_size / (1024*1024):.2f} MB)...{Style.RESET_ALL}') - - # Process raw stream - if not archiver.onlyRaw: - archiver.processor.process_raw_stream(live_raw_path, live_proc_path) - - # Wait for live chat download if it was started - live_chat_downloaded = False - chat_rendered_successfully = False - chat_video_path = None - - # Handle different chat download methods - if live_chat_method == 'twitch_downloader' and live_chat_process is not None: - # Wait for TwitchDownloaderCLI process - print(f'{Fore.CYAN}Waiting for live chat download to complete...{Style.RESET_ALL}') - live_chat_downloaded = archiver.downloader.wait_for_chat_download(live_chat_process, chat_json_path) - elif live_chat_method == 'chat_downloader' and archiver.downloader.chat_thread is not None: - # Wait for chat_downloader thread - print(f'{Fore.CYAN}Waiting for live chat download to complete...{Style.RESET_ALL}') - try: - live_chat_downloaded = archiver.downloader.wait_for_chat_thread() - if live_chat_downloaded: - print(f'{Fore.GREEN}✓ Chat download thread completed successfully{Style.RESET_ALL}') - else: - print(f'{Fore.YELLOW}⚠ Chat download thread completed with errors or no messages{Style.RESET_ALL}') - except Exception as e: - print(f'{Fore.RED}✗ Error waiting for chat download thread: {e}{Style.RESET_ALL}') - import traceback - traceback.print_exc() - live_chat_downloaded = False - - # Render live chat if downloaded successfully - if live_chat_downloaded: - chat_video_path = str(archiver.file_manager.chat_mp4_path / f"{PREFIX_CHAT}{filename_base}.mp4") - output_args = archiver.processor.build_chat_output_args() - - # Wait for chat file to be fully accessible (not locked) - print(f'{Fore.CYAN}Verifying chat file is ready for rendering...{Style.RESET_ALL}') - if not archiver.downloader.wait_for_file_access(chat_json_path, max_attempts=15, delay=0.5): - print(f'{Fore.RED}✗ Chat file is locked, skipping rendering{Style.RESET_ALL}') - chat_rendered_successfully = False - else: - # Get video duration first - ffmpeg_path = get_ffmpeg_executable(archiver.os_type) - video_duration = get_video_duration(live_proc_path, ffmpeg_path) - - if video_duration is None: - print(f'{Fore.YELLOW}⚠ Could not detect video duration from {live_proc_path}{Style.RESET_ALL}') - print(f'{Fore.YELLOW} Will use chat message timestamps instead{Style.RESET_ALL}') - else: - print(f'{Fore.CYAN}Video duration for chat rendering: {video_duration}s{Style.RESET_ALL}') - - # Convert chat format if chat_downloader was used - render_json_path = chat_json_path - if live_chat_method == 'chat_downloader': - converted_path = chat_json_path.replace('.json', '_converted.json') - print(f'{Fore.CYAN}Chat downloaded with chat_downloader, converting format...{Style.RESET_ALL}') - if archiver.downloader.convert_chat_downloader_to_twitch_format(chat_json_path, converted_path, video_duration): - render_json_path = converted_path - print(f'{Fore.GREEN}✓ Using converted chat file for rendering{Style.RESET_ALL}') - else: - print(f'{Fore.RED}✗ Format conversion failed, skipping rendering{Style.RESET_ALL}') - chat_rendered_successfully = False - render_json_path = None - - if render_json_path: - chat_rendered_successfully = archiver.downloader.render_chat( - render_json_path, - chat_video_path, - output_args, - video_duration=video_duration - ) - - # Merge video and chat if configured - merged_video_path = None - if chat_rendered_successfully and archiver.mergeVideoChat and os.path.exists(live_proc_path) and os.path.exists(chat_video_path): - merged_video_path = str(archiver.file_manager.video_path / f"{PREFIX_MERGED}{filename_base}{proc_extension}") - archiver.processor.merge_video_and_chat( - live_proc_path, - chat_video_path, - merged_video_path, - archiver.mergeChatLayout - ) - - # Wait for VOD and download it - vod_response = None - if archiver.vodTimeout == 0: - print(f'{Fore.CYAN}VOD check disabled (vodTimeout=0). Skipping VOD download.{Style.RESET_ALL}') - elif archiver.shutdown_requested: - print(f'{Fore.YELLOW}Skipping VOD download due to shutdown request{Style.RESET_ALL}') - else: - # Try to match stream with VOD (with timeout) - print(f'{Fore.CYAN}Waiting for VOD to become available (timeout: {archiver.vodTimeout}s)...{Style.RESET_ALL}') - vod_found = False - vod_wait_start = time.time() - - while time.time() - vod_wait_start < archiver.vodTimeout: - # Check for shutdown request - if archiver.shutdown_requested: - print(f'\n{Fore.YELLOW}VOD check interrupted by shutdown{Style.RESET_ALL}') - break - - vod_response = archiver.stream_monitor.get_latest_vod() - - if vod_response and vod_response['data']['user']['videos']['edges']: - current_vod = vod_response['data']['user']['videos']['edges'][0]['node'] - vod_date = datetime.strptime( - current_vod["recordedAt"], '%Y-%m-%dT%H:%M:%SZ' - ).replace(tzinfo=timezone('UTC')).astimezone(tz=None).replace(tzinfo=None) - - # Check if VOD matches the stream (within 1 minute tolerance) - time_tolerance = timedelta(minutes=1) - if (live_date - time_tolerance) <= vod_date <= (live_date + time_tolerance): - vod_found = True - break - - # Wait before checking again - if not vod_found: - print(f'{Fore.CYAN}VOD not found yet, waiting...{Style.RESET_ALL}', end='\r') - time.sleep(min(10, archiver.vodTimeout - (time.time() - vod_wait_start))) - - if not vod_found: - print(f'\n{Fore.YELLOW}⚠ VOD not found after {archiver.vodTimeout}s - streamer may have VODs disabled{Style.RESET_ALL}') - print(f'{Fore.CYAN} → Live recording and chat (if enabled) were saved successfully{Style.RESET_ALL}') - vod_response = None - - # Process VOD if found - if vod_response and vod_response['data']['user']['videos']['edges']: - current_vod = vod_response['data']['user']['videos']['edges'][0]['node'] - vod_date = datetime.strptime( - current_vod["recordedAt"], '%Y-%m-%dT%H:%M:%SZ' - ).replace(tzinfo=timezone('UTC')).astimezone(tz=None).replace(tzinfo=None) - - # Check if VOD matches the stream (within 1 minute tolerance) - time_tolerance = timedelta(minutes=1) - if (live_date - time_tolerance) <= vod_date <= (live_date + time_tolerance): - print(f'\n{Fore.GREEN}✓ Found matching VOD{Style.RESET_ALL}') - - # Save metadata - if archiver.downloadMETADATA: - archiver.file_manager.save_metadata(current_vod, filename_base) - - # Download VOD - if archiver.downloadVOD: - vod_ext = '.mp3' if archiver.quality == 'audio_only' else '.mp4' - vod_path = str(archiver.file_manager.video_path / f"{PREFIX_VOD}{filename_base}{vod_ext}") - archiver.downloader.download_vod(current_vod, vod_path) - - # Download and render chat from VOD (if not already done via live chat) - if archiver.downloadCHAT and not live_chat_downloaded: - chat_video_path = str(archiver.file_manager.chat_mp4_path / f"{PREFIX_CHAT}{filename_base}.mp4") - output_args = archiver.processor.build_chat_output_args() - - # Get VOD duration to trim chat accordingly - ffmpeg_path = get_ffmpeg_executable(archiver.os_type) - vod_duration = get_video_duration(vod_path, ffmpeg_path) - - chat_rendered_successfully = archiver.downloader.download_and_render_chat( - current_vod, - chat_json_path, - chat_video_path, - output_args, - video_duration=vod_duration - ) - - # Merge VOD and chat if configured - if chat_rendered_successfully and archiver.mergeVideoChat and os.path.exists(vod_path) and os.path.exists(chat_video_path): - merged_vod_path = str(archiver.file_manager.video_path / f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}{vod_ext}") - archiver.processor.merge_video_and_chat( - vod_path, - chat_video_path, - merged_vod_path, - archiver.mergeChatLayout - ) - elif live_chat_downloaded: - print(f'{Fore.CYAN}Chat already downloaded from live stream, skipping VOD chat download{Style.RESET_ALL}') - - # But still merge VOD with existing chat if configured - if archiver.mergeVideoChat and archiver.downloadVOD and os.path.exists(vod_path) and chat_video_path and os.path.exists(chat_video_path): - merged_vod_path = str(archiver.file_manager.video_path / f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}{vod_ext}") - archiver.processor.merge_video_and_chat( - vod_path, - chat_video_path, - merged_vod_path, - archiver.mergeChatLayout - ) - else: - print(f'{Fore.YELLOW}⚠ No matching VOD found for this stream{Style.RESET_ALL}') - elif archiver.downloadMETADATA: - # Save what metadata we have from the live stream - archiver.file_manager.save_metadata(stream_info, filename_base) - - # Clean up raw file if configured - archiver.file_manager.clean_raw_file(live_raw_path) - - # Upload to cloud if configured - upload_success = archiver.file_manager.upload_to_cloud( - filename_base, - notification_callback=archiver.notification_manager.send - ) - - # Delete files if configured - if archiver.deleteFiles and upload_success: - archiver.file_manager.delete_local_files( - filename_base, - live_raw_path, - live_proc_path, - notification_callback=archiver.notification_manager.send - ) - - # Send completion notification - archiver.notification_manager.send( - f"Stream Archived - {archiver.username}", - f"Completed: {stream_info['title']}" - ) - - -# ============================================================================ -# COMMAND-LINE INTERFACE -# ============================================================================ - -def main(argv: list) -> None: - """ - Main entry point for command-line execution. - - Parses command-line arguments and starts the archive system. - - Args: - argv: Command-line arguments - """ - specific_streamer = None - use_legacy_mode = False - - help_msg = f''' -{Fore.CYAN}{"=" * 70} -TWITCH ARCHIVE - Automated Stream Recording & Archiving -{"=" * 70}{Style.RESET_ALL} - -{Fore.GREEN}USAGE:{Style.RESET_ALL} - python twitch-archive.py [OPTIONS] - -{Fore.GREEN}MODES:{Style.RESET_ALL} - • Multi-Streamer Mode (default): - Monitor all enabled streamers from config/streamers/*.json - - • Single-Streamer Mode: - Use -u to monitor only one streamer - - • Legacy Mode: - Uses config.json if it exists (deprecated) - -{Fore.GREEN}OPTIONS:{Style.RESET_ALL} - -h, --help Display this help information - -u, --username Monitor only this Twitch channel - --verbose Enable verbose debug output - --legacy Force legacy mode (use config.json) - --chat-only Test mode: Only download chat (skip video recording) - Automatically enables verbose logging - --use-chat-downloader-primary Use chat_downloader as primary chat source (for testing) - --no-chat-downloader-fallback Disable chat_downloader fallback - -{Fore.GREEN}LEGACY OPTIONS (when using --legacy):{Style.RESET_ALL} - -q, --quality Stream quality: best/source, high/720p, - medium/480p, low/360p, audio_only - -a, --ttv-lol <0|1> Enable ad-blocking (1) or disable (0) - -v, --vod <0|1> Download VODs after stream ends - -c, --chat <0|1> Download and render chat - -m, --metadata <0|1> Download stream metadata - -r, --upload <0|1> Upload to cloud storage via rclone - -d, --delete <0|1> Delete local files after upload (CAREFUL!) - -n, --notifications <0|1> Send email notifications - -{Fore.YELLOW}TIPS:{Style.RESET_ALL} - • Create config/global.json for default settings - • Create config/streamers/.json for each streamer - • Set enabled: true/false in each streamer config - • Set up API credentials in .env file - -{Fore.CYAN}EXAMPLES:{Style.RESET_ALL} - python twitch-archive.py # Monitor all enabled streamers - python twitch-archive.py -u vinesauce # Monitor only vinesauce - python twitch-archive.py -u hackerling --verbose # Monitor with debug output - python twitch-archive.py -u streamername --chat-only # Test chat download only (no video) - python twitch-archive.py --use-chat-downloader-primary # Test chat_downloader library - python twitch-archive.py --legacy # Use old config.json mode - -{Fore.CYAN}{"=" * 70}{Style.RESET_ALL} - ''' - - try: - opts, args = getopt.getopt( - argv, - "h:u:q:a:v:c:m:r:d:n:", - ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", - "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", - "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] - ) - except getopt.GetoptError as e: - print(f'{Fore.RED}Error: {e}{Style.RESET_ALL}\n') - print(help_msg) - sys.exit(2) - - # Check if legacy mode is requested or if config.json exists (fallback) - legacy_config_exists = os.path.exists(os.path.join(os.path.dirname(__file__), 'config.json')) - - # Parse command line args - legacy_overrides = {} - verbose_mode = False - chat_only_mode = False - use_chat_downloader_primary = False - use_chat_downloader_fallback = True # Default to enabled - for opt, arg in opts: - if opt in ('-h', '--help'): - print(help_msg) - sys.exit(0) - elif opt in ("-u", "--username"): - specific_streamer = arg - elif opt == "--verbose": - verbose_mode = True - elif opt == "--chat-only": - chat_only_mode = True - verbose_mode = True # Auto-enable verbose for chat-only mode - elif opt == "--legacy": - use_legacy_mode = True - elif opt == "--use-chat-downloader-primary": - use_chat_downloader_primary = True - elif opt == "--no-chat-downloader-fallback": - use_chat_downloader_fallback = False - if opt in ('-h', '--help'): - print(help_msg) - sys.exit(0) - elif opt in ("-u", "--username"): - specific_streamer = arg - elif opt == "--verbose": - verbose_mode = True - elif opt == "--legacy": - use_legacy_mode = True - # Legacy options (only used in legacy mode) - elif opt in ("-q", "--quality"): - legacy_overrides['quality'] = arg - elif opt in ("-a", "--ttv-lol"): - legacy_overrides['streamlink_ttvlol'] = bool(int(arg)) - elif opt in ("-v", "--vod"): - legacy_overrides['downloadVOD'] = bool(int(arg)) - elif opt in ("-c", "--chat"): - legacy_overrides['downloadCHAT'] = bool(int(arg)) - elif opt in ("-m", "--metadata"): - legacy_overrides['downloadMETADATA'] = bool(int(arg)) - elif opt in ("-r", "--upload"): - legacy_overrides['uploadCloud'] = bool(int(arg)) - elif opt in ("-d", "--delete"): - legacy_overrides['deleteFiles'] = bool(int(arg)) - elif opt in ("-n", "--notifications"): - legacy_overrides['notifications'] = bool(int(arg)) - - # Determine which mode to use - if use_legacy_mode or (legacy_config_exists and not specific_streamer and not os.path.exists('config/global.json')): - # Legacy mode: single streamer using config.json - print(f'{Fore.YELLOW}⚠ Using legacy mode (config.json){Style.RESET_ALL}') - print(f'{Fore.CYAN}→ Consider migrating to new config structure (config/global.json + config/streamers/*.json){Style.RESET_ALL}\n') - - twitch_archive = TwitchArchive() # Loads from config.json - - # Apply command-line overrides - for key, value in legacy_overrides.items(): - setattr(twitch_archive, key, value) - - # Apply chat_downloader options - if hasattr(twitch_archive.downloader, 'use_chat_downloader_primary'): - twitch_archive.downloader.use_chat_downloader_primary = use_chat_downloader_primary - if hasattr(twitch_archive.downloader, 'use_chat_downloader_fallback'): - twitch_archive.downloader.use_chat_downloader_fallback = use_chat_downloader_fallback - - # Start the archive system - twitch_archive.run() - else: - # New multi-streamer mode - manager = TwitchArchiveManager( - specific_streamer=specific_streamer, - verbose=verbose_mode, - chat_only=chat_only_mode, - use_chat_downloader_primary=use_chat_downloader_primary, - use_chat_downloader_fallback=use_chat_downloader_fallback - ) - manager.run() - - -if __name__ == "__main__": - try: - main(sys.argv[1:]) - except KeyboardInterrupt: - # Suppress stack trace for clean exit - print(f'\n{Fore.GREEN}✓ Graceful shutdown complete{Style.RESET_ALL}') - sys.exit(0) \ No newline at end of file From 1ecf7501f43d7078fa15dd098cf26928568b6212 Mon Sep 17 00:00:00 2001 From: MaddoScientisto Date: Sun, 22 Feb 2026 23:06:18 +0100 Subject: [PATCH 06/10] Implemented features --- twitch-archive.py | 1565 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1565 insertions(+) create mode 100644 twitch-archive.py diff --git a/twitch-archive.py b/twitch-archive.py new file mode 100644 index 0000000..2fdcb4e --- /dev/null +++ b/twitch-archive.py @@ -0,0 +1,1565 @@ +""" +Twitch Archive - Automated Twitch Stream Recording & Archiving System + +This script monitors a Twitch channel and automatically: +- Records live streams as they happen +- Downloads VODs (Video on Demand) after the stream ends +- Downloads and renders chat logs +- Saves stream metadata +- Uploads everything to cloud storage (optional) + +Requirements: +- Python 3.7+ +- External tools: streamlink, ffmpeg, TwitchDownloaderCLI, rclone (optional) +- Configuration file: config.json (copy from config.sample.json) +- Environment file: .env (for API credentials) + +Refactored Version 2.0: +This version has been split into multiple modules for better maintainability: +- modules/constants.py: Constants and default configuration +- modules/config.py: Configuration management +- modules/notifications.py: Email notifications +- modules/utils.py: Utility functions +- modules/stream_monitor.py: Stream monitoring and API +- modules/recorder.py: Live stream recording +- modules/processor.py: Video/audio processing +- modules/downloader.py: VOD and chat downloading +- modules/file_manager.py: File and cloud management +""" + +# Standard library imports +import os +import sys +import time +import json +import signal +import getopt +from typing import Dict, Optional, Any +from datetime import datetime, timedelta + +# Third-party imports +from colorama import Fore, Style +from pytz import timezone +from dotenv import load_dotenv, find_dotenv + +# Local module imports +from modules.constants import DEFAULT_CONFIG, PREFIX_LIVE, PREFIX_VOD, PREFIX_CHAT, PREFIX_MERGED, PREFIX_METADATA +from modules.config import ConfigManager +from modules.notifications import NotificationManager +from modules.utils import ( + detect_operating_system, get_ffmpeg_executable, get_twitch_downloader_executable, + get_unique_filename, get_video_duration, verify_streamlink, verify_ffmpeg, verify_twitch_downloader +) +from modules.stream_monitor import StreamMonitor +from modules.recorder import StreamRecorder +from modules.processor import StreamProcessor +from modules.downloader import ContentDownloader +from modules.file_manager import FileManager + + +class TwitchArchive: + """ + Main class for the Twitch Archive system. + + Handles monitoring a Twitch channel, recording live streams, and downloading + VODs, chat logs, and metadata. Can optionally upload to cloud storage. + + Refactored Version 2.0: This class now delegates most functionality to + specialized modules for better code organization. + """ + + def __init__(self, config: Optional[Dict[str, Any]] = None): + """ + Initialize the TwitchArchive with configuration settings. + + Args: + config: Configuration dictionary. If None, loads from legacy config.json + """ + if config is None: + # Legacy mode: load from config.json + self.load_config() + else: + # New mode: use provided config + for key, value in config.items(): + setattr(self, key, value) + + # Initialize system components + self.os_type = detect_operating_system() + self.shutdown_requested = False + self.current_stream_data = {} + + # Initialize component modules (created during run()) + self.stream_monitor = None + self.notification_manager = None + self.file_manager = None + self.recorder = None + self.processor = None + self.downloader = None + + def load_config(self) -> None: + """ + Load configuration from config.json file (legacy support). + + Falls back to default configuration if file is not found or cannot be read. + Filters out comment fields (starting with '_') from the config. + """ + config_file = os.path.join(os.path.dirname(__file__), 'config.json') + + # Start with default configuration + config = DEFAULT_CONFIG.copy() + + # Try to load and merge user configuration + if os.path.exists(config_file): + try: + with open(config_file, 'r', encoding='utf-8') as f: + user_config = json.load(f) + # Filter out comment fields (those starting with '_') + user_config = {k: v for k, v in user_config.items() if not k.startswith('_')} + # Merge user config with defaults (user config takes precedence) + config.update(user_config) + print(f'{Fore.GREEN}✓ Configuration loaded from config.json{Style.RESET_ALL}') + except json.JSONDecodeError as e: + print(f'{Fore.YELLOW}⚠ Warning: Invalid JSON in config.json: {e}{Style.RESET_ALL}') + print(f'{Fore.YELLOW} Using default configuration{Style.RESET_ALL}') + except Exception as e: + print(f'{Fore.YELLOW}⚠ Warning: Could not load config.json: {e}{Style.RESET_ALL}') + print(f'{Fore.YELLOW} Using default configuration{Style.RESET_ALL}') + else: + print(f'{Fore.YELLOW}⚠ Warning: config.json not found{Style.RESET_ALL}') + print(f'{Fore.CYAN} → Copy config.sample.json to config.json and edit it with your settings{Style.RESET_ALL}') + + # Set all configuration values as instance attributes + for key, value in config.items(): + setattr(self, key, value) + + def _load_environment_variables(self) -> None: + """ + Load environment variables from .env file. + + Required variables: + - CLIENT-ID: Twitch API client ID + - CLIENT-SECRET: Twitch API client secret + - OAUTH-PRIVATE-TOKEN: Optional, for accessing subscriber-only streams + - SENDER: Email address for notifications (if enabled) + - RECEIVER: Email address to receive notifications (if enabled) + - PASSWD: Email password for sending notifications (if enabled) + + Raises: + SystemExit: If .env file is not found + """ + if not load_dotenv(find_dotenv()): + print(f'{Fore.RED}✗ ERROR: .env file not found{Style.RESET_ALL}') + print(f'{Fore.CYAN} → Create a .env file with your Twitch API credentials{Style.RESET_ALL}') + print(f'{Fore.CYAN} → Required: CLIENT-ID, CLIENT-SECRET{Style.RESET_ALL}') + sys.exit(1) + + def _initialize_components(self) -> None: + """Initialize all component modules.""" + # Stream monitoring + self.stream_monitor = StreamMonitor(self.username) + + # Notifications + self.notification_manager = NotificationManager( + enabled=self.notifications, + username=self.username + ) + + # File management + self.file_manager = FileManager( + root_path=self.root_path, + username=self.username, + config=vars(self) + ) + self.file_manager.initialize_directories() + + # Recording + self.recorder = StreamRecorder( + username=self.username, + quality=self.quality, + refresh=self.refresh, + hls_segments=self.hls_segments, + streamlink_ttvlol=self.streamlink_ttvlol, + shutdown_callback=lambda: self.shutdown_requested + ) + + # Processing + ffmpeg_path = get_ffmpeg_executable(self.os_type) + self.processor = StreamProcessor( + os_type=self.os_type, + ffmpeg_path=ffmpeg_path, + config=vars(self) + ) + + # Downloading + twitch_downloader_path = get_twitch_downloader_executable(self.os_type) + self.downloader = ContentDownloader( + twitch_downloader_path=twitch_downloader_path, + ffmpeg_path=ffmpeg_path, + config=vars(self) + ) + + def _print_configuration_summary(self) -> None: + """Print a summary of the current configuration to the console.""" + print(f'\n{Fore.CYAN}{"=" * 60}{Style.RESET_ALL}') + print(f'{Fore.CYAN}TWITCH ARCHIVE - Configuration Summary{Style.RESET_ALL}') + print(f'{Fore.CYAN}{"=" * 60}{Style.RESET_ALL}\n') + + # Basic settings + print(f'Streamer: {Fore.GREEN}{self.username}{Style.RESET_ALL}') + print(f'Quality: {Fore.GREEN}{self.quality}{Style.RESET_ALL}') + print(f'Storage: {Fore.GREEN}{os.path.abspath(self.root_path)}{Style.RESET_ALL}') + print(f'Refresh rate: {Fore.GREEN}{self.refresh}s{Style.RESET_ALL}\n') + + # Feature toggles + self._print_toggle('Email notifications', self.notifications) + self._print_toggle('Metadata download', self.downloadMETADATA) + self._print_toggle('VOD download', self.downloadVOD) + self._print_toggle('Chat download & render', self.downloadCHAT) + if self.downloadCHAT: + self._print_toggle(' ↳ Merge video + chat', self.mergeVideoChat) + if self.mergeVideoChat: + print(f' Layout: {Fore.GREEN}{self.mergeChatLayout}{Style.RESET_ALL}') + self._print_toggle('Cloud upload', self.uploadCloud) + + # Warning messages + if self.deleteFiles: + print(f'\n{Fore.RED}⚠ WARNING: Files will be DELETED after processing{Style.RESET_ALL}') + if not self.uploadCloud: + print(f'{Fore.RED}⚠ CRITICAL: Files will be deleted WITHOUT cloud backup!{Style.RESET_ALL}') + print(f'{Fore.YELLOW} Press CTRL+C to stop and change configuration{Style.RESET_ALL}') + else: + print(f'\n{Fore.GREEN}✓ Files will be preserved locally{Style.RESET_ALL}') + + print(f'\n{Fore.CYAN}{"=" * 60}{Style.RESET_ALL}\n') + + def _print_toggle(self, label: str, value: bool) -> None: + """Helper method to print a configuration toggle in a consistent format.""" + status = f'{Fore.GREEN}Enabled{Style.RESET_ALL}' if value else f'{Fore.RED}Disabled{Style.RESET_ALL}' + print(f'{label}: {status}') + + def run(self) -> None: + """ + Main entry point for the application. + + Initializes environment, validates configuration, creates necessary + directories, and starts the monitoring loop. + """ + # Load environment variables + self._load_environment_variables() + + # Initialize all component modules + self._initialize_components() + + # Validate username + self.stream_monitor.validate_username() + + # Verify dependencies + if not verify_streamlink(): + sys.exit(1) + verify_ffmpeg(self.os_type) + if self.downloadVOD or self.downloadCHAT: + verify_twitch_downloader(self.os_type) + + # Print configuration summary + self._print_configuration_summary() + + # Start monitoring + print(f"Monitoring {Fore.GREEN}{self.username}{Style.RESET_ALL} every {Fore.GREEN}{self.refresh}s{Style.RESET_ALL}") + self.notification_manager.send("TWITCH ARCHIVE STARTED", + f"Monitoring {self.username} every {self.refresh} seconds.") + + # Begin the main monitoring loop + self.loopcheck() + + def _interruptible_sleep(self, seconds: float) -> bool: + """ + Sleep for the specified duration, but check for shutdown periodically. + + Args: + seconds: Number of seconds to sleep + + Returns: + bool: True if sleep completed, False if interrupted by shutdown + """ + start_time = time.time() + while time.time() - start_time < seconds: + if self.shutdown_requested: + return False + time.sleep(min(1.0, seconds - (time.time() - start_time))) + return True + + def _signal_handler(self, signum, frame): + """Handle interrupt signals gracefully.""" + if not self.shutdown_requested: + print(f'\n{Fore.YELLOW}{"=" * 60}{Style.RESET_ALL}') + print(f'{Fore.YELLOW}⚠ Shutdown requested. Stopping downloads and finalizing...{Style.RESET_ALL}') + print(f'{Fore.YELLOW}{"=" * 60}{Style.RESET_ALL}\n') + self.shutdown_requested = True + + # Stop current subprocess if running + if self.recorder: + self.recorder.stop() + + def loopcheck(self) -> None: + """ + Main monitoring loop. + + Continuously checks if the streamer is live, and when they are: + 1. Records the live stream + 2. Downloads the VOD + 3. Downloads and renders chat + 4. Uploads everything to cloud storage (if enabled) + 5. Optionally deletes local files after upload + """ + # Set up signal handlers for graceful shutdown + signal.signal(signal.SIGINT, self._signal_handler) + if hasattr(signal, 'SIGTERM'): + signal.signal(signal.SIGTERM, self._signal_handler) + + while not self.shutdown_requested: + try: + # Check stream status using StreamMonitor + response = self.stream_monitor.check_stream_status() + is_live = response['data']['user']['stream'] + + # Stream is offline + if is_live is None: + print(f'{Fore.CYAN}⏳ {self.username} is offline. Checking again in {self.refresh}s...{Style.RESET_ALL}', end='\r') + if self.shutdown_requested: + break + self._interruptible_sleep(self.refresh) + continue + + # Stream is live but not ready yet + if not is_live.get('title'): + print(f'{Fore.YELLOW}⚠ Stream detected but no title yet. Waiting...{Style.RESET_ALL}') + if self.shutdown_requested: + break + self._interruptible_sleep(self.refresh) + continue + + # Stream is live and ready! + print(f'\n{Fore.GREEN}✓ {self.username} is LIVE!{Style.RESET_ALL}') + print(f'{Fore.CYAN}Title: {is_live["title"]}{Style.RESET_ALL}') + + # Create unique stream identifier + stream_id = f"{is_live['createdAt']} - {self.username} - {is_live['title']}" + + # Parse stream start time + live_date = datetime.strptime( + is_live["createdAt"], '%Y-%m-%dT%H:%M:%SZ' + ).replace(tzinfo=timezone('UTC')).astimezone(tz=None).replace(tzinfo=None) + + # Use CURRENT time for filename + current_time = datetime.now() + filename_base = current_time.strftime('%Y%m%d_%Hh%Mm%Ss') + + # Check if stream was already processed + if self.file_manager.is_stream_processed(stream_id): + print(f'{Fore.YELLOW}⚠ Stream was previously recorded, but it\'s still live!{Style.RESET_ALL}') + print(f'{Fore.GREEN}✓ Starting new recording with timestamp: {filename_base}{Style.RESET_ALL}') + else: + self.file_manager.mark_stream_processed(stream_id) + print(f'{Fore.GREEN}✓ New stream detected - starting recording{Style.RESET_ALL}') + + # Determine file paths + live_raw_path = str(self.file_manager.raw_path / f"{PREFIX_LIVE}{filename_base}.ts") + live_proc_ext = '.mp3' if self.quality == 'audio_only' else '.mp4' + live_proc_path = str(self.file_manager.video_path / f"{PREFIX_LIVE}{filename_base}{live_proc_ext}") + + # Ensure unique filenames + live_raw_path = get_unique_filename(live_raw_path) + live_proc_path = get_unique_filename(live_proc_path) + filename_base = os.path.splitext(os.path.basename(live_raw_path))[0].replace(PREFIX_LIVE, "") + + print(f'{Fore.CYAN}Output path: {live_raw_path}{Style.RESET_ALL}') + + # Send notification + self.notification_manager.send(f'🔴 Stream Started - {filename_base}', + f'Title: {is_live["title"]}') + + # Start live chat download if enabled + live_chat_process = None + live_chat_method = None # Track which method was used + chat_json_path = str(self.file_manager.chat_json_path / f"{PREFIX_CHAT}{filename_base}.json") + + if self.downloadLiveCHAT: + vod_id = is_live.get('archiveVideo', {}).get('id') if is_live.get('archiveVideo') else None + stream_url = f"https://twitch.tv/{self.username}" + + live_chat_process, live_chat_method = self.downloader.start_live_chat_download_with_fallback( + vod_id=vod_id, + stream_url=stream_url, + json_path=chat_json_path, + use_chat_downloader_primary=self.use_chat_downloader_primary, + no_chat_downloader_fallback=self.no_chat_downloader_fallback, + verbose=self.verbose + ) + + # Record the live stream + recording_completed = self.recorder.record(is_live, live_raw_path) + + # If shutdown was requested during recording, try to finalize + if self.shutdown_requested: + print(f'{Fore.YELLOW}Attempting to process any recorded content...{Style.RESET_ALL}') + + # Process the raw stream file + self.processor.process_raw_stream(live_raw_path, live_proc_path) + + # Wait for live chat download if it was started + live_chat_downloaded = False + if live_chat_process is not None: + live_chat_downloaded = self.downloader.wait_for_chat_download(live_chat_process, chat_json_path) + + # Render live chat if downloaded successfully + chat_rendered_successfully = False + chat_video_path = None + if live_chat_downloaded: + chat_video_path = str(self.file_manager.chat_mp4_path / f"{PREFIX_CHAT}{filename_base}.mp4") + output_args = self.processor.build_chat_output_args() + + # Wait for chat file to be fully accessible (not locked) + print(f'{Fore.CYAN}Verifying chat file is ready for rendering...{Style.RESET_ALL}') + if not self.downloader.wait_for_file_access(chat_json_path, max_attempts=15, delay=0.5): + print(f'{Fore.RED}✗ Chat file is locked, skipping rendering{Style.RESET_ALL}') + chat_rendered_successfully = False + else: + # Get video duration first (needed for chat conversion and trimming) + ffmpeg_path = get_ffmpeg_executable(self.os_type) + video_duration = get_video_duration(live_proc_path, ffmpeg_path) + print(f'{Fore.CYAN}Video duration for chat rendering: {video_duration}s{Style.RESET_ALL}') + + # Convert chat format if needed (chat_downloader uses different JSON structure) + render_json_path = chat_json_path + if live_chat_method == 'chat_downloader': + print(f'{Fore.CYAN}Converting chat format for rendering...{Style.RESET_ALL}') + converted_path = chat_json_path.replace('.json', '_converted.json') + if self.downloader.convert_chat_downloader_to_twitch_format(chat_json_path, converted_path, video_duration): + render_json_path = converted_path + print(f'{Fore.GREEN}✓ Chat format converted successfully{Style.RESET_ALL}') + else: + print(f'{Fore.RED}✗ Failed to convert chat format{Style.RESET_ALL}') + + chat_rendered_successfully = self.downloader.render_chat( + render_json_path, + chat_video_path, + output_args, + video_duration=video_duration + ) + + # Merge video and chat if configured + merged_video_path = None + if chat_rendered_successfully and self.mergeVideoChat and os.path.exists(live_proc_path) and os.path.exists(chat_video_path): + merged_video_path = str(self.file_manager.video_path / f"{PREFIX_MERGED}{filename_base}{live_proc_ext}") + merge_success = self.processor.merge_video_and_chat( + live_proc_path, + chat_video_path, + merged_video_path, + self.mergeChatLayout + ) + + # Skip VOD/chat download if shutdown was requested or vodTimeout is 0 + vod_response = None + if self.shutdown_requested: + print(f'{Fore.YELLOW}Skipping VOD and chat download due to shutdown request{Style.RESET_ALL}') + elif self.vodTimeout == 0: + print(f'{Fore.CYAN}VOD check disabled (vodTimeout=0). Skipping VOD download.{Style.RESET_ALL}') + else: + # Try to match stream with VOD (with timeout) + print(f'{Fore.CYAN}Waiting for VOD to become available (timeout: {self.vodTimeout}s)...{Style.RESET_ALL}') + vod_found = False + vod_wait_start = time.time() + + while time.time() - vod_wait_start < self.vodTimeout and not self.shutdown_requested: + vod_response = self.stream_monitor.get_latest_vod() + + if vod_response and vod_response['data']['user']['videos']['edges']: + current_vod = vod_response['data']['user']['videos']['edges'][0]['node'] + vod_date = datetime.strptime( + current_vod["recordedAt"], '%Y-%m-%dT%H:%M:%SZ' + ).replace(tzinfo=timezone('UTC')).astimezone(tz=None).replace(tzinfo=None) + + # Check if VOD matches the stream (within 1 minute tolerance) + time_tolerance = timedelta(minutes=1) + if (live_date - time_tolerance) <= vod_date <= (live_date + time_tolerance): + vod_found = True + break + + # Wait before checking again + if not vod_found: + print(f'{Fore.CYAN}VOD not found yet, waiting...{Style.RESET_ALL}', end='\r') + if not self._interruptible_sleep(min(10, self.vodTimeout - (time.time() - vod_wait_start))): + break + + if not vod_found: + if self.shutdown_requested: + print(f'\n{Fore.YELLOW}VOD check interrupted by shutdown{Style.RESET_ALL}') + else: + print(f'\n{Fore.YELLOW}⚠ VOD not found after {self.vodTimeout}s - streamer may have VODs disabled{Style.RESET_ALL}') + print(f'{Fore.CYAN} → Live recording and chat (if enabled) were saved successfully{Style.RESET_ALL}') + vod_response = None + + # Process VOD if found + if not self.shutdown_requested and vod_response and vod_response['data']['user']['videos']['edges']: + current_vod = vod_response['data']['user']['videos']['edges'][0]['node'] + vod_date = datetime.strptime( + current_vod["recordedAt"], '%Y-%m-%dT%H:%M:%SZ' + ).replace(tzinfo=timezone('UTC')).astimezone(tz=None).replace(tzinfo=None) + + # Check if VOD matches the stream (within 1 minute tolerance) + time_tolerance = timedelta(minutes=1) + if (live_date - time_tolerance) <= vod_date <= (live_date + time_tolerance): + print(f'\n{Fore.GREEN}✓ Found matching VOD{Style.RESET_ALL}') + + # Save metadata + self.file_manager.save_metadata(current_vod, filename_base) + + # Download VOD + vod_ext = '.mp3' if self.quality == 'audio_only' else '.mp4' + vod_path = str(self.file_manager.video_path / f"{PREFIX_VOD}{filename_base}{vod_ext}") + self.downloader.download_vod(current_vod, vod_path) + + # Download and render chat from VOD (if not already done via live chat) + if not live_chat_downloaded: + chat_video_path = str(self.file_manager.chat_mp4_path / f"{PREFIX_CHAT}{filename_base}.mp4") + output_args = self.processor.build_chat_output_args() + + # Get VOD duration to trim chat accordingly + ffmpeg_path = get_ffmpeg_executable(self.os_type) + vod_duration = get_video_duration(vod_path, ffmpeg_path) + + chat_rendered_successfully = self.downloader.download_and_render_chat( + current_vod, + chat_json_path, + chat_video_path, + output_args, + video_duration=vod_duration + ) + + # Merge VOD and chat if configured + if chat_rendered_successfully and self.mergeVideoChat and os.path.exists(vod_path) and os.path.exists(chat_video_path): + merged_vod_path = str(self.file_manager.video_path / f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}{vod_ext}") + self.processor.merge_video_and_chat( + vod_path, + chat_video_path, + merged_vod_path, + self.mergeChatLayout + ) + else: + print(f'{Fore.CYAN}Chat already downloaded from live stream, skipping VOD chat download{Style.RESET_ALL}') + + # But still merge VOD with existing chat if configured + if self.mergeVideoChat and os.path.exists(vod_path) and chat_video_path and os.path.exists(chat_video_path): + merged_vod_path = str(self.file_manager.video_path / f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}{vod_ext}") + self.processor.merge_video_and_chat( + vod_path, + chat_video_path, + merged_vod_path, + self.mergeChatLayout + ) + else: + print(f'{Fore.YELLOW}⚠ No matching VOD found for this stream{Style.RESET_ALL}') + + # Clean up raw files if configured + self.file_manager.clean_raw_file(live_raw_path) + + # Upload to cloud if configured + upload_success = self.file_manager.upload_to_cloud( + filename_base, + notification_callback=self.notification_manager.send + ) + + # Delete local files if configured and upload succeeded + if self.deleteFiles and upload_success: + self.file_manager.delete_local_files( + filename_base, + live_raw_path, + live_proc_path, + notification_callback=self.notification_manager.send + ) + + # Done processing this stream + if self.shutdown_requested: + print(f'\n{Fore.YELLOW}{"=" * 60}{Style.RESET_ALL}') + print(f'{Fore.YELLOW}✓ Stream processing stopped by user{Style.RESET_ALL}') + print(f'{Fore.YELLOW}{"=" * 60}{Style.RESET_ALL}\n') + break + else: + print(f'\n{Fore.GREEN}{"=" * 60}{Style.RESET_ALL}') + print(f'{Fore.GREEN}✓ Stream processing complete{Style.RESET_ALL}') + print(f'{Fore.GREEN}{"=" * 60}{Style.RESET_ALL}\n') + self.notification_manager.send(f'✓ Complete - {filename_base}', + 'Stream processing finished. Resuming monitoring.') + self._interruptible_sleep(self.refresh) + + except KeyboardInterrupt: + if not self.shutdown_requested: + self.shutdown_requested = True + print(f'\n{Fore.YELLOW}{"=" * 60}{Style.RESET_ALL}') + print(f'{Fore.YELLOW}⚠ Interrupted. Cleaning up...{Style.RESET_ALL}') + print(f'{Fore.YELLOW}{"=" * 60}{Style.RESET_ALL}\n') + break + + except Exception as e: + print(f'\n{Fore.RED}{"=" * 60}{Style.RESET_ALL}') + print(f'{Fore.RED}✗ ERROR: {str(e)}{Style.RESET_ALL}') + print(f'{Fore.YELLOW}Waiting {self.refresh} seconds before retrying...{Style.RESET_ALL}') + print(f'{Fore.RED}{"=" * 60}{Style.RESET_ALL}\n') + self.notification_manager.send('⚠ Error - Recovery', + f'Error: {str(e)}\nRetrying after {self.refresh} seconds.') + + if self.shutdown_requested: + break + self._interruptible_sleep(self.refresh) + + # Final cleanup message + print(f'{Fore.GREEN}✓ Monitoring stopped cleanly{Style.RESET_ALL}') + + def _upload_to_cloud(self, filename_base: str) -> bool: + """ + Upload archived files to cloud storage using rclone. + + Args: + filename_base: Base filename (without prefixes/extensions) + + Returns: + bool: True if upload succeeded or is disabled, False if failed + """ + if not self.uploadCloud: + return True # Consider upload "successful" if disabled + + print(f'\n{Fore.CYAN}Uploading to cloud storage...{Style.RESET_ALL}') + self.send_notification(f'☁ Uploading - {filename_base}', 'Uploading files to cloud storage') + + # Create list of files to upload + bin_path = self._get_bin_path() + upload_list_path = os.path.join(bin_path, 'temp', 'upload.txt') + + # Ensure temp directory exists + os.makedirs(os.path.dirname(upload_list_path), exist_ok=True) + + files_to_upload = [ + f"{PREFIX_LIVE}{filename_base}.ts", + f"{PREFIX_LIVE}{filename_base}.mp4", + f"{PREFIX_LIVE}{filename_base}.mp3", + f"{PREFIX_VOD}{filename_base}.ts", + f"{PREFIX_VOD}{filename_base}.mp4", + f"{PREFIX_VOD}{filename_base}.mp3", + f"{PREFIX_METADATA}{filename_base}.json", + f"{PREFIX_CHAT}{filename_base}.json", + f"{PREFIX_CHAT}{filename_base}.mp4" + ] + + with open(upload_list_path, 'w') as f: + f.write('\n'.join(files_to_upload)) + + # Run rclone + try: + result = subprocess.call([ + 'rclone', 'copy', + str(pathlib.Path(self.root_path).resolve()), + self.rclone_path, + '--include-from', upload_list_path + ]) + + # Clean up upload list + if os.path.exists(upload_list_path): + os.remove(upload_list_path) + + if result == 0: + print(f'{Fore.GREEN}✓ Upload complete{Style.RESET_ALL}') + self.send_notification(f'✓ Upload Success - {filename_base}', + 'All files uploaded successfully') + return True + else: + print(f'{Fore.RED}✗ Upload failed (exit code: {result}){Style.RESET_ALL}') + print(f'{Fore.YELLOW}Files preserved locally due to upload failure{Style.RESET_ALL}') + self.send_notification(f'✗ Upload Failed - {filename_base}', + f'Upload failed with code {result}. Files preserved locally.') + return False + + except Exception as e: + print(f'{Fore.RED}✗ Upload error: {str(e)}{Style.RESET_ALL}') + return False + + def _delete_local_files(self, filename_base: str, live_raw_path: str, live_proc_path: str) -> None: + """ + Delete local archive files after successful upload. + + Args: + filename_base: Base filename (without prefixes/extensions) + live_raw_path: Path to live raw file + live_proc_path: Path to live processed file + """ + print(f'\n{Fore.RED}{"=" * 60}{Style.RESET_ALL}') + print(f'{Fore.RED}⚠ DELETING LOCAL FILES{Style.RESET_ALL}') + print(f'{Fore.RED}{"=" * 60}{Style.RESET_ALL}\n') + + self.send_notification(f'🗑 Deleting - {filename_base}', + 'Deleting local files after successful upload') + + files_to_delete = [] + + # Live files + if not self.cleanRaw and os.path.exists(live_raw_path): + files_to_delete.append(live_raw_path) + if os.path.exists(live_proc_path): + files_to_delete.append(live_proc_path) + + # VOD files + if self.downloadVOD: + vod_raw = os.path.join(self.raw_path, f"{PREFIX_VOD}{filename_base}.ts") + vod_mp4 = os.path.join(self.video_path, f"{PREFIX_VOD}{filename_base}.mp4") + vod_mp3 = os.path.join(self.video_path, f"{PREFIX_VOD}{filename_base}.mp3") + + if not self.cleanRaw and os.path.exists(vod_raw): + files_to_delete.append(vod_raw) + if os.path.exists(vod_mp4): + files_to_delete.append(vod_mp4) + if os.path.exists(vod_mp3): + files_to_delete.append(vod_mp3) + + # Chat files + if self.downloadCHAT: + chat_json = os.path.join(self.chatJSON_path, f"{PREFIX_CHAT}{filename_base}.json") + chat_mp4 = os.path.join(self.chatMP4_path, f"{PREFIX_CHAT}{filename_base}.mp4") + + if os.path.exists(chat_json): + files_to_delete.append(chat_json) + if os.path.exists(chat_mp4): + files_to_delete.append(chat_mp4) + + # Metadata files + if self.downloadMETADATA: + metadata = os.path.join(self.metadata_path, f"{PREFIX_METADATA}{filename_base}.json") + if os.path.exists(metadata): + files_to_delete.append(metadata) + + # Delete all files + for filepath in files_to_delete: + try: + print(f'{Fore.RED} Deleting: {os.path.basename(filepath)}{Style.RESET_ALL}') + os.remove(filepath) + except Exception as e: + print(f'{Fore.YELLOW} ⚠ Failed to delete {filepath}: {e}{Style.RESET_ALL}') + + print(f'{Fore.RED}\n✓ Cleanup complete{Style.RESET_ALL}') + + +# ============================================================================ +# MULTI-STREAMER MANAGER +# ============================================================================ + +class TwitchArchiveManager: + """ + Manages multiple TwitchArchive instances for monitoring multiple streamers. + """ + + def __init__(self, specific_streamer: Optional[str] = None, verbose: bool = False, + chat_only: bool = False, + use_chat_downloader_primary: bool = False, + use_chat_downloader_fallback: bool = True): + """ + Initialize the manager. + + Args: + specific_streamer: If provided, only monitor this streamer (ignore enabled status) + verbose: Enable verbose debug output + chat_only: Only download chat, skip video recording (test mode) + use_chat_downloader_primary: Use chat_downloader as primary chat source + use_chat_downloader_fallback: Enable chat_downloader fallback + """ + self.config_manager = ConfigManager() + self.specific_streamer = specific_streamer + self.verbose = verbose + self.chat_only = chat_only + self.use_chat_downloader_primary = use_chat_downloader_primary + self.use_chat_downloader_fallback = use_chat_downloader_fallback + self.archivers: Dict[str, TwitchArchive] = {} + self.shutdown_requested = False + self.active_recordings: Dict[str, str] = {} # Track active recordings: {username: stream_id} + + # Setup signal handlers + signal.signal(signal.SIGTERM, self._signal_handler) + signal.signal(signal.SIGINT, self._signal_handler) + + def _signal_handler(self, signum, frame): + """Handle shutdown signals gracefully.""" + print(f'\n{Fore.YELLOW}⚠ Shutdown signal received...{Style.RESET_ALL}') + self.shutdown_requested = True + + # Signal all archivers to shut down + for archiver in self.archivers.values(): + archiver.shutdown_requested = True + + def _get_streamers_to_monitor(self) -> list: + """ + Get list of streamers to monitor. + + Returns: + list: List of streamer usernames to monitor + """ + if self.specific_streamer: + # Monitor only the specified streamer (ignore enabled flag) + return [self.specific_streamer] + else: + # Monitor all enabled streamers + return self.config_manager.get_all_enabled_streamers() + + def _initialize_archiver(self, username: str) -> TwitchArchive: + """ + Initialize a TwitchArchive instance for a streamer. + + Args: + username: Twitch username + + Returns: + TwitchArchive: Initialized archiver instance + """ + config = self.config_manager.load_streamer_config(username) + + # Apply command-line overrides for chat_downloader options + config['useChatDownloaderPrimary'] = self.use_chat_downloader_primary + config['useChatDownloaderFallback'] = self.use_chat_downloader_fallback + + archiver = TwitchArchive(config) + return archiver + + def run(self) -> None: + """ + Main entry point for multi-streamer monitoring. + + Monitors all enabled streamers (or a specific one if provided). + """ + print(f'\n{Fore.CYAN}{"=" * 70}{Style.RESET_ALL}') + print(f'{Fore.CYAN}TWITCH ARCHIVE - Multi-Streamer Mode{Style.RESET_ALL}') + if self.chat_only: + print(f'{Fore.YELLOW}🧪 TEST MODE: Chat-Only (Video Recording Disabled){Style.RESET_ALL}') + print(f'{Fore.CYAN}{"=" * 70}{Style.RESET_ALL}\n') + + # Get streamers to monitor + streamers = self._get_streamers_to_monitor() + + if not streamers: + print(f'{Fore.RED}✗ No streamers configured or enabled{Style.RESET_ALL}') + print(f'{Fore.CYAN}→ Create config files in config/streamers/{Style.RESET_ALL}') + print(f'{Fore.CYAN}→ Or run with -u to create a new config{Style.RESET_ALL}') + sys.exit(1) + + if self.chat_only: + print(f'{Fore.YELLOW}📝 Chat-Only Mode Enabled:{Style.RESET_ALL}') + print(f'{Fore.CYAN} • Verbose logging: ON{Style.RESET_ALL}') + print(f'{Fore.CYAN} • Video recording: DISABLED{Style.RESET_ALL}') + print(f'{Fore.CYAN} • Chat download: ENABLED{Style.RESET_ALL}') + print(f'{Fore.CYAN} • VOD download: DISABLED{Style.RESET_ALL}') + print() + + print(f'{Fore.GREEN}Monitoring {len(streamers)} streamer(s):{Style.RESET_ALL}') + for streamer in streamers: + print(f' • {Fore.CYAN}{streamer}{Style.RESET_ALL}') + print() + + # Initialize archivers for all streamers + for username in streamers: + try: + archiver = self._initialize_archiver(username) + + # Load environment and initialize components + archiver._load_environment_variables() + archiver._initialize_components() + + # Validate username through stream_monitor + archiver.stream_monitor.validate_username() + + self.archivers[username] = archiver + print(f'{Fore.GREEN}✓ Initialized {username}{Style.RESET_ALL}') + except Exception as e: + print(f'{Fore.RED}✗ Failed to initialize {username}: {e}{Style.RESET_ALL}') + import traceback + traceback.print_exc() + + if not self.archivers: + print(f'{Fore.RED}✗ No archivers could be initialized{Style.RESET_ALL}') + sys.exit(1) + + # Verify dependencies once (shared across all streamers) + print(f'\n{Fore.CYAN}Verifying dependencies...{Style.RESET_ALL}') + first_archiver = next(iter(self.archivers.values())) + if not verify_streamlink(): + sys.exit(1) + verify_ffmpeg(first_archiver.os_type) + if first_archiver.downloadVOD or first_archiver.downloadCHAT: + verify_twitch_downloader(first_archiver.os_type) + + # Print configuration summary for each streamer + for username, archiver in self.archivers.items(): + archiver._print_configuration_summary() + + print(f'\n{Fore.GREEN}🚀 Starting monitoring loop...{Style.RESET_ALL}\n') + + # Start monitoring loop + self._monitoring_loop() + + def _monitoring_loop(self) -> None: + """ + Main monitoring loop for all streamers. + + Checks each streamer's status and processes streams as needed. + """ + last_check = {} + last_status_print = time.time() + + while not self.shutdown_requested: + current_time = time.time() + + # Print periodic status every 60 seconds + if current_time - last_status_print >= 60: + status_line = " | ".join([f"{username}: checking" for username in self.archivers.keys()]) + print(f'{Fore.CYAN}[Status] {status_line}{Style.RESET_ALL}') + last_status_print = current_time + + for username, archiver in self.archivers.items(): + # Check if enough time has passed since last check for this streamer + if username not in last_check or (current_time - last_check[username]) >= archiver.refresh: + last_check[username] = current_time + + # Check stream status + try: + response = archiver.stream_monitor.check_stream_status() + + # Debug: Print the full response (if verbose) + if self.verbose: + print(f'\n{Fore.MAGENTA}[DEBUG {username}] API Response: {response}{Style.RESET_ALL}') + + stream_data = response['data']['user']['stream'] if response else None + + if self.verbose: + print(f'{Fore.MAGENTA}[DEBUG {username}] Stream data: {stream_data}{Style.RESET_ALL}') + + if stream_data: + # Stream is live - check if it has required basic data (title and start time) + if stream_data.get('title') and stream_data.get('createdAt'): + # Create composite stream ID like single-streamer mode + # This prevents duplicate recordings in the same session + stream_id = f"{stream_data['createdAt']} - {username} - {stream_data.get('title', 'Untitled')}" + + if self.verbose: + # Check if VOD ID is available (for live chat) + if stream_data.get('archiveVideo') and stream_data['archiveVideo'].get('id'): + print(f'{Fore.MAGENTA}[DEBUG {username}] VOD ID: {stream_data["archiveVideo"]["id"]}{Style.RESET_ALL}') + else: + print(f'{Fore.MAGENTA}[DEBUG {username}] No VOD ID available (VODs may be disabled){Style.RESET_ALL}') + print(f'{Fore.MAGENTA}[DEBUG {username}] Composite Stream ID: {stream_id}{Style.RESET_ALL}') + + # Check if we're currently recording this stream + currently_recording = username in self.active_recordings and self.active_recordings[username] == stream_id + + if self.verbose: + print(f'{Fore.MAGENTA}[DEBUG {username}] Currently recording: {currently_recording}{Style.RESET_ALL}') + print(f'{Fore.MAGENTA}[DEBUG {username}] Active recordings: {self.active_recordings}{Style.RESET_ALL}') + + # Record if not currently recording (ignore .log file - always record if live) + if not currently_recording: + print(f'\n{Fore.GREEN}[{username}] Stream detected!{Style.RESET_ALL}') + print(f'{Fore.CYAN}Title: {stream_data.get("title", "No title")}{Style.RESET_ALL}') + print(f'{Fore.CYAN}Started at: {stream_data["createdAt"]}{Style.RESET_ALL}') + + # Warn if VOD ID not available + if not (stream_data.get('archiveVideo') and stream_data['archiveVideo'].get('id')): + print(f'{Fore.YELLOW}⚠ VOD ID not available - live chat download will be skipped{Style.RESET_ALL}') + print(f'{Fore.YELLOW} Stream recording will proceed normally{Style.RESET_ALL}') + + # Mark as currently recording + self.active_recordings[username] = stream_id + + # Process the stream (this blocks until stream ends) + self._process_stream(archiver, stream_data, stream_id) + + # Mark as processed in log (for record keeping) + archiver.file_manager.mark_stream_processed(stream_id) + + # Remove from active recordings + if username in self.active_recordings: + del self.active_recordings[username] + else: + if self.verbose: + print(f'{Fore.CYAN}[{username}] Currently recording this stream, skipping duplicate...{Style.RESET_ALL}') + else: + # Stream is live but not fully initialized yet + print(f'{Fore.YELLOW}[{username}] Stream starting up, waiting for stream data...{Style.RESET_ALL}') + else: + # Not live + if self.verbose: + print(f'{Fore.CYAN}[{username}] Offline - checking again in {archiver.refresh}s{Style.RESET_ALL}', end='\r') + + except Exception as e: + print(f'{Fore.RED}[{username}] Error checking stream: {e}{Style.RESET_ALL}') + import traceback + traceback.print_exc() + + # Sleep briefly before next iteration + time.sleep(1) + + def _process_stream(self, archiver: TwitchArchive, stream_info: Dict[str, Any], stream_id: str) -> None: + """ + Process a detected stream for a specific archiver. + + Args: + archiver: The TwitchArchive instance + stream_info: Stream information from API + stream_id: Unique stream ID + """ + # Store stream data + archiver.current_stream_data = { + 'stream_id': stream_id, + 'title': stream_info['title'], + 'started_at': stream_info['createdAt'] + } + + # Generate timestamp and filename + timestamp = datetime.now(timezone('UTC')).strftime("%Y%m%d_%Hh%Mm%Ss") + filename_base = f"{PREFIX_LIVE}{archiver.username}_{timestamp}" + + # Parse stream start time + live_date = datetime.strptime( + stream_info["createdAt"], '%Y-%m-%dT%H:%M:%SZ' + ).replace(tzinfo=timezone('UTC')).astimezone(tz=None).replace(tzinfo=None) + + # Define paths + raw_extension = '.ts' + proc_extension = '.mp3' if archiver.quality == 'audio_only' else '.mp4' + + live_raw_path = str(archiver.file_manager.raw_path / f"{filename_base}{raw_extension}") + live_proc_path = str(archiver.file_manager.video_path / f"{filename_base}{proc_extension}") + chat_json_path = str(archiver.file_manager.chat_json_path / f"{PREFIX_CHAT}{filename_base}.json") + + # Send notification + if not self.chat_only: + archiver.notification_manager.send( + f"Stream Started - {archiver.username}", + f"Recording: {stream_info['title']}" + ) + + # Start live chat download if enabled (with fallback support) + live_chat_process = None + live_chat_method = 'failed' + if archiver.downloadLiveCHAT: + if self.verbose or self.chat_only: + print(f'\n{Fore.MAGENTA}[VERBOSE] Starting chat download process...{Style.RESET_ALL}') + print(f'{Fore.MAGENTA}[VERBOSE] downloadLiveCHAT: {archiver.downloadLiveCHAT}{Style.RESET_ALL}') + print(f'{Fore.MAGENTA}[VERBOSE] useChatDownloaderPrimary: {archiver.downloader.use_chat_downloader_primary}{Style.RESET_ALL}') + print(f'{Fore.MAGENTA}[VERBOSE] useChatDownloaderFallback: {archiver.downloader.use_chat_downloader_fallback}{Style.RESET_ALL}') + + # Get VOD ID if available + live_vod_id = None + if stream_info.get('archiveVideo') and stream_info['archiveVideo'].get('id'): + live_vod_id = stream_info['archiveVideo']['id'] + print(f'{Fore.CYAN}Live VOD ID detected: {live_vod_id}{Style.RESET_ALL}') + if self.verbose or self.chat_only: + print(f'{Fore.MAGENTA}[VERBOSE] VOD URL: https://www.twitch.tv/videos/{live_vod_id}{Style.RESET_ALL}') + else: + print(f'{Fore.YELLOW}⚠ No VOD ID available - will use fallback if configured{Style.RESET_ALL}') + if self.verbose or self.chat_only: + print(f'{Fore.MAGENTA}[VERBOSE] This happens when streamer has VODs disabled{Style.RESET_ALL}') + print(f'{Fore.MAGENTA}[VERBOSE] chat_downloader fallback will be used if enabled{Style.RESET_ALL}') + + # Try to start live chat download with fallback + try: + if self.verbose or self.chat_only: + print(f'{Fore.MAGENTA}[VERBOSE] Calling start_live_chat_download_with_fallback(){Style.RESET_ALL}') + print(f'{Fore.MAGENTA}[VERBOSE] Username: {archiver.username}{Style.RESET_ALL}') + print(f'{Fore.MAGENTA}[VERBOSE] VOD ID: {live_vod_id}{Style.RESET_ALL}') + print(f'{Fore.MAGENTA}[VERBOSE] Output path: {chat_json_path}{Style.RESET_ALL}') + + live_chat_process, live_chat_method = archiver.downloader.start_live_chat_download_with_fallback( + archiver.username, live_vod_id, chat_json_path + ) + + if self.verbose or self.chat_only: + print(f'{Fore.MAGENTA}[VERBOSE] Chat download method selected: {live_chat_method}{Style.RESET_ALL}') + print(f'{Fore.MAGENTA}[VERBOSE] Process handle: {live_chat_process}{Style.RESET_ALL}') + + # If chat_downloader is selected, start it in background thread now (before video recording) + if live_chat_method == 'chat_downloader' and not self.chat_only: + if self.verbose: + print(f'{Fore.MAGENTA}[VERBOSE] Starting chat_downloader in background thread...{Style.RESET_ALL}') + try: + print(f'{Fore.CYAN}Starting chat_downloader in background (concurrent with video)...{Style.RESET_ALL}') + archiver.downloader.start_chat_downloader_thread( + archiver.username, chat_json_path, + shutdown_check=lambda: self.shutdown_requested or archiver.shutdown_requested, + stream_monitor=archiver.stream_monitor, + verbose=self.verbose + ) + except Exception as e: + print(f'{Fore.RED}✗ Failed to start chat thread: {e}{Style.RESET_ALL}') + import traceback + traceback.print_exc() + live_chat_method = 'failed' + + except Exception as e: + print(f'{Fore.RED}✗ Failed to start live chat download: {e}{Style.RESET_ALL}') + import traceback + traceback.print_exc() + live_chat_method = 'failed' + + # Record livestream (skip in chat-only mode) + if self.chat_only: + print(f'\n{Fore.YELLOW}🧪 Chat-Only Mode: Skipping video recording{Style.RESET_ALL}') + print(f'{Fore.CYAN}Waiting for chat download to complete...{Style.RESET_ALL}') + + # Start chat download based on method + if live_chat_method == 'chat_downloader': + if self.verbose: + print(f'{Fore.MAGENTA}[VERBOSE] Starting chat_downloader in background thread...{Style.RESET_ALL}') + try: + print(f'{Fore.CYAN}Using chat_downloader for live chat...{Style.RESET_ALL}') + archiver.downloader.start_chat_downloader_thread( + archiver.username, chat_json_path, + shutdown_check=lambda: self.shutdown_requested or archiver.shutdown_requested, + stream_monitor=archiver.stream_monitor, + verbose=self.verbose or self.chat_only + ) + # Wait for completion + live_chat_downloaded = archiver.downloader.wait_for_chat_thread() + except Exception as e: + print(f'{Fore.RED}✗ chat_downloader failed: {e}{Style.RESET_ALL}') + import traceback + traceback.print_exc() + live_chat_downloaded = False + elif live_chat_method == 'twitch_downloader' and live_chat_process is not None: + if self.verbose: + print(f'{Fore.MAGENTA}[VERBOSE] Waiting for TwitchDownloaderCLI process...{Style.RESET_ALL}') + live_chat_downloaded = archiver.downloader.wait_for_chat_download(live_chat_process, chat_json_path) + else: + live_chat_downloaded = False + + # Report results + if live_chat_downloaded: + print(f'\n{Fore.GREEN}✓ Chat-Only Test Complete!{Style.RESET_ALL}') + print(f'{Fore.CYAN}Chat saved to: {chat_json_path}{Style.RESET_ALL}') + if os.path.exists(chat_json_path): + file_size = os.path.getsize(chat_json_path) + print(f'{Fore.CYAN}File size: {file_size / 1024:.2f} KB{Style.RESET_ALL}') + else: + print(f'\n{Fore.RED}✗ Chat download failed{Style.RESET_ALL}') + + return # Exit early, don't process video + + # Normal mode: Record livestream + recording_successful = archiver.recorder.record(stream_info, live_raw_path) + + # Check if raw file exists (may exist even after interrupted recording) + if not os.path.exists(live_raw_path): + print(f'{Fore.RED}✗ No recording file found, skipping processing{Style.RESET_ALL}') + + # Still wait for chat if it's downloading + if live_chat_method == 'chat_downloader' and archiver.downloader.chat_thread is not None: + print(f'{Fore.CYAN}Waiting for chat download to finish...{Style.RESET_ALL}') + archiver.downloader.wait_for_chat_thread(timeout=30) + elif live_chat_method == 'twitch_downloader' and live_chat_process is not None: + print(f'{Fore.CYAN}Waiting for chat download to finish...{Style.RESET_ALL}') + archiver.downloader.wait_for_chat_download(live_chat_process, chat_json_path, timeout=30) + + return + + # Get file size to check if anything was recorded + file_size = os.path.getsize(live_raw_path) + if file_size < 1024: # Less than 1KB means essentially nothing was recorded + print(f'{Fore.RED}✗ Recording file too small ({file_size} bytes), skipping processing{Style.RESET_ALL}') + return + + print(f'{Fore.CYAN}Processing recorded content ({file_size / (1024*1024):.2f} MB)...{Style.RESET_ALL}') + + # Process raw stream + if not archiver.onlyRaw: + archiver.processor.process_raw_stream(live_raw_path, live_proc_path) + + # Wait for live chat download if it was started + live_chat_downloaded = False + chat_rendered_successfully = False + chat_video_path = None + + # Handle different chat download methods + if live_chat_method == 'twitch_downloader' and live_chat_process is not None: + # Wait for TwitchDownloaderCLI process + print(f'{Fore.CYAN}Waiting for live chat download to complete...{Style.RESET_ALL}') + live_chat_downloaded = archiver.downloader.wait_for_chat_download(live_chat_process, chat_json_path) + elif live_chat_method == 'chat_downloader' and archiver.downloader.chat_thread is not None: + # Wait for chat_downloader thread + print(f'{Fore.CYAN}Waiting for live chat download to complete...{Style.RESET_ALL}') + try: + live_chat_downloaded = archiver.downloader.wait_for_chat_thread() + if live_chat_downloaded: + print(f'{Fore.GREEN}✓ Chat download thread completed successfully{Style.RESET_ALL}') + else: + print(f'{Fore.YELLOW}⚠ Chat download thread completed with errors or no messages{Style.RESET_ALL}') + except Exception as e: + print(f'{Fore.RED}✗ Error waiting for chat download thread: {e}{Style.RESET_ALL}') + import traceback + traceback.print_exc() + live_chat_downloaded = False + + # Render live chat if downloaded successfully + if live_chat_downloaded: + chat_video_path = str(archiver.file_manager.chat_mp4_path / f"{PREFIX_CHAT}{filename_base}.mp4") + output_args = archiver.processor.build_chat_output_args() + + # Wait for chat file to be fully accessible (not locked) + print(f'{Fore.CYAN}Verifying chat file is ready for rendering...{Style.RESET_ALL}') + if not archiver.downloader.wait_for_file_access(chat_json_path, max_attempts=15, delay=0.5): + print(f'{Fore.RED}✗ Chat file is locked, skipping rendering{Style.RESET_ALL}') + chat_rendered_successfully = False + else: + # Get video duration first + ffmpeg_path = get_ffmpeg_executable(archiver.os_type) + video_duration = get_video_duration(live_proc_path, ffmpeg_path) + + if video_duration is None: + print(f'{Fore.YELLOW}⚠ Could not detect video duration from {live_proc_path}{Style.RESET_ALL}') + print(f'{Fore.YELLOW} Will use chat message timestamps instead{Style.RESET_ALL}') + else: + print(f'{Fore.CYAN}Video duration for chat rendering: {video_duration}s{Style.RESET_ALL}') + + # Convert chat format if chat_downloader was used + render_json_path = chat_json_path + if live_chat_method == 'chat_downloader': + converted_path = chat_json_path.replace('.json', '_converted.json') + print(f'{Fore.CYAN}Chat downloaded with chat_downloader, converting format...{Style.RESET_ALL}') + if archiver.downloader.convert_chat_downloader_to_twitch_format(chat_json_path, converted_path, video_duration): + render_json_path = converted_path + print(f'{Fore.GREEN}✓ Using converted chat file for rendering{Style.RESET_ALL}') + else: + print(f'{Fore.RED}✗ Format conversion failed, skipping rendering{Style.RESET_ALL}') + chat_rendered_successfully = False + render_json_path = None + + if render_json_path: + chat_rendered_successfully = archiver.downloader.render_chat( + render_json_path, + chat_video_path, + output_args, + video_duration=video_duration + ) + + # Merge video and chat if configured + merged_video_path = None + if chat_rendered_successfully and archiver.mergeVideoChat and os.path.exists(live_proc_path) and os.path.exists(chat_video_path): + merged_video_path = str(archiver.file_manager.video_path / f"{PREFIX_MERGED}{filename_base}{proc_extension}") + archiver.processor.merge_video_and_chat( + live_proc_path, + chat_video_path, + merged_video_path, + archiver.mergeChatLayout + ) + + # Wait for VOD and download it + vod_response = None + if archiver.vodTimeout == 0: + print(f'{Fore.CYAN}VOD check disabled (vodTimeout=0). Skipping VOD download.{Style.RESET_ALL}') + elif archiver.shutdown_requested: + print(f'{Fore.YELLOW}Skipping VOD download due to shutdown request{Style.RESET_ALL}') + else: + # Try to match stream with VOD (with timeout) + print(f'{Fore.CYAN}Waiting for VOD to become available (timeout: {archiver.vodTimeout}s)...{Style.RESET_ALL}') + vod_found = False + vod_wait_start = time.time() + + while time.time() - vod_wait_start < archiver.vodTimeout: + # Check for shutdown request + if archiver.shutdown_requested: + print(f'\n{Fore.YELLOW}VOD check interrupted by shutdown{Style.RESET_ALL}') + break + + vod_response = archiver.stream_monitor.get_latest_vod() + + if vod_response and vod_response['data']['user']['videos']['edges']: + current_vod = vod_response['data']['user']['videos']['edges'][0]['node'] + vod_date = datetime.strptime( + current_vod["recordedAt"], '%Y-%m-%dT%H:%M:%SZ' + ).replace(tzinfo=timezone('UTC')).astimezone(tz=None).replace(tzinfo=None) + + # Check if VOD matches the stream (within 1 minute tolerance) + time_tolerance = timedelta(minutes=1) + if (live_date - time_tolerance) <= vod_date <= (live_date + time_tolerance): + vod_found = True + break + + # Wait before checking again + if not vod_found: + print(f'{Fore.CYAN}VOD not found yet, waiting...{Style.RESET_ALL}', end='\r') + time.sleep(min(10, archiver.vodTimeout - (time.time() - vod_wait_start))) + + if not vod_found: + print(f'\n{Fore.YELLOW}⚠ VOD not found after {archiver.vodTimeout}s - streamer may have VODs disabled{Style.RESET_ALL}') + print(f'{Fore.CYAN} → Live recording and chat (if enabled) were saved successfully{Style.RESET_ALL}') + vod_response = None + + # Process VOD if found + if vod_response and vod_response['data']['user']['videos']['edges']: + current_vod = vod_response['data']['user']['videos']['edges'][0]['node'] + vod_date = datetime.strptime( + current_vod["recordedAt"], '%Y-%m-%dT%H:%M:%SZ' + ).replace(tzinfo=timezone('UTC')).astimezone(tz=None).replace(tzinfo=None) + + # Check if VOD matches the stream (within 1 minute tolerance) + time_tolerance = timedelta(minutes=1) + if (live_date - time_tolerance) <= vod_date <= (live_date + time_tolerance): + print(f'\n{Fore.GREEN}✓ Found matching VOD{Style.RESET_ALL}') + + # Save metadata + if archiver.downloadMETADATA: + archiver.file_manager.save_metadata(current_vod, filename_base) + + # Download VOD + if archiver.downloadVOD: + vod_ext = '.mp3' if archiver.quality == 'audio_only' else '.mp4' + vod_path = str(archiver.file_manager.video_path / f"{PREFIX_VOD}{filename_base}{vod_ext}") + archiver.downloader.download_vod(current_vod, vod_path) + + # Download and render chat from VOD (if not already done via live chat) + if archiver.downloadCHAT and not live_chat_downloaded: + chat_video_path = str(archiver.file_manager.chat_mp4_path / f"{PREFIX_CHAT}{filename_base}.mp4") + output_args = archiver.processor.build_chat_output_args() + + # Get VOD duration to trim chat accordingly + ffmpeg_path = get_ffmpeg_executable(archiver.os_type) + vod_duration = get_video_duration(vod_path, ffmpeg_path) + + chat_rendered_successfully = archiver.downloader.download_and_render_chat( + current_vod, + chat_json_path, + chat_video_path, + output_args, + video_duration=vod_duration + ) + + # Merge VOD and chat if configured + if chat_rendered_successfully and archiver.mergeVideoChat and os.path.exists(vod_path) and os.path.exists(chat_video_path): + merged_vod_path = str(archiver.file_manager.video_path / f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}{vod_ext}") + archiver.processor.merge_video_and_chat( + vod_path, + chat_video_path, + merged_vod_path, + archiver.mergeChatLayout + ) + elif live_chat_downloaded: + print(f'{Fore.CYAN}Chat already downloaded from live stream, skipping VOD chat download{Style.RESET_ALL}') + + # But still merge VOD with existing chat if configured + if archiver.mergeVideoChat and archiver.downloadVOD and os.path.exists(vod_path) and chat_video_path and os.path.exists(chat_video_path): + merged_vod_path = str(archiver.file_manager.video_path / f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}{vod_ext}") + archiver.processor.merge_video_and_chat( + vod_path, + chat_video_path, + merged_vod_path, + archiver.mergeChatLayout + ) + else: + print(f'{Fore.YELLOW}⚠ No matching VOD found for this stream{Style.RESET_ALL}') + elif archiver.downloadMETADATA: + # Save what metadata we have from the live stream + archiver.file_manager.save_metadata(stream_info, filename_base) + + # Clean up raw file if configured + archiver.file_manager.clean_raw_file(live_raw_path) + + # Upload to cloud if configured + upload_success = archiver.file_manager.upload_to_cloud( + filename_base, + notification_callback=archiver.notification_manager.send + ) + + # Delete files if configured + if archiver.deleteFiles and upload_success: + archiver.file_manager.delete_local_files( + filename_base, + live_raw_path, + live_proc_path, + notification_callback=archiver.notification_manager.send + ) + + # Send completion notification + archiver.notification_manager.send( + f"Stream Archived - {archiver.username}", + f"Completed: {stream_info['title']}" + ) + + +# ============================================================================ +# COMMAND-LINE INTERFACE +# ============================================================================ + +def main(argv: list) -> None: + """ + Main entry point for command-line execution. + + Parses command-line arguments and starts the archive system. + + Args: + argv: Command-line arguments + """ + specific_streamer = None + use_legacy_mode = False + + help_msg = f''' +{Fore.CYAN}{"=" * 70} +TWITCH ARCHIVE - Automated Stream Recording & Archiving +{"=" * 70}{Style.RESET_ALL} + +{Fore.GREEN}USAGE:{Style.RESET_ALL} + python twitch-archive.py [OPTIONS] + +{Fore.GREEN}MODES:{Style.RESET_ALL} + • Multi-Streamer Mode (default): + Monitor all enabled streamers from config/streamers/*.json + + • Single-Streamer Mode: + Use -u to monitor only one streamer + + • Legacy Mode: + Uses config.json if it exists (deprecated) + +{Fore.GREEN}OPTIONS:{Style.RESET_ALL} + -h, --help Display this help information + -u, --username Monitor only this Twitch channel + --verbose Enable verbose debug output + --legacy Force legacy mode (use config.json) + --chat-only Test mode: Only download chat (skip video recording) + Automatically enables verbose logging + --use-chat-downloader-primary Use chat_downloader as primary chat source (for testing) + --no-chat-downloader-fallback Disable chat_downloader fallback + +{Fore.GREEN}LEGACY OPTIONS (when using --legacy):{Style.RESET_ALL} + -q, --quality Stream quality: best/source, high/720p, + medium/480p, low/360p, audio_only + -a, --ttv-lol <0|1> Enable ad-blocking (1) or disable (0) + -v, --vod <0|1> Download VODs after stream ends + -c, --chat <0|1> Download and render chat + -m, --metadata <0|1> Download stream metadata + -r, --upload <0|1> Upload to cloud storage via rclone + -d, --delete <0|1> Delete local files after upload (CAREFUL!) + -n, --notifications <0|1> Send email notifications + +{Fore.YELLOW}TIPS:{Style.RESET_ALL} + • Create config/global.json for default settings + • Create config/streamers/.json for each streamer + • Set enabled: true/false in each streamer config + • Set up API credentials in .env file + +{Fore.CYAN}EXAMPLES:{Style.RESET_ALL} + python twitch-archive.py # Monitor all enabled streamers + python twitch-archive.py -u vinesauce # Monitor only vinesauce + python twitch-archive.py -u hackerling --verbose # Monitor with debug output + python twitch-archive.py -u streamername --chat-only # Test chat download only (no video) + python twitch-archive.py --use-chat-downloader-primary # Test chat_downloader library + python twitch-archive.py --legacy # Use old config.json mode + +{Fore.CYAN}{"=" * 70}{Style.RESET_ALL} + ''' + + try: + opts, args = getopt.getopt( + argv, + "h:u:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + except getopt.GetoptError as e: + print(f'{Fore.RED}Error: {e}{Style.RESET_ALL}\n') + print(help_msg) + sys.exit(2) + + # Check if legacy mode is requested or if config.json exists (fallback) + legacy_config_exists = os.path.exists(os.path.join(os.path.dirname(__file__), 'config.json')) + + # Parse command line args + legacy_overrides = {} + verbose_mode = False + chat_only_mode = False + use_chat_downloader_primary = False + use_chat_downloader_fallback = True # Default to enabled + for opt, arg in opts: + if opt in ('-h', '--help'): + print(help_msg) + sys.exit(0) + elif opt in ("-u", "--username"): + specific_streamer = arg + elif opt == "--verbose": + verbose_mode = True + elif opt == "--chat-only": + chat_only_mode = True + verbose_mode = True # Auto-enable verbose for chat-only mode + elif opt == "--legacy": + use_legacy_mode = True + elif opt == "--use-chat-downloader-primary": + use_chat_downloader_primary = True + elif opt == "--no-chat-downloader-fallback": + use_chat_downloader_fallback = False + if opt in ('-h', '--help'): + print(help_msg) + sys.exit(0) + elif opt in ("-u", "--username"): + specific_streamer = arg + elif opt == "--verbose": + verbose_mode = True + elif opt == "--legacy": + use_legacy_mode = True + # Legacy options (only used in legacy mode) + elif opt in ("-q", "--quality"): + legacy_overrides['quality'] = arg + elif opt in ("-a", "--ttv-lol"): + legacy_overrides['streamlink_ttvlol'] = bool(int(arg)) + elif opt in ("-v", "--vod"): + legacy_overrides['downloadVOD'] = bool(int(arg)) + elif opt in ("-c", "--chat"): + legacy_overrides['downloadCHAT'] = bool(int(arg)) + elif opt in ("-m", "--metadata"): + legacy_overrides['downloadMETADATA'] = bool(int(arg)) + elif opt in ("-r", "--upload"): + legacy_overrides['uploadCloud'] = bool(int(arg)) + elif opt in ("-d", "--delete"): + legacy_overrides['deleteFiles'] = bool(int(arg)) + elif opt in ("-n", "--notifications"): + legacy_overrides['notifications'] = bool(int(arg)) + + # Determine which mode to use + if use_legacy_mode or (legacy_config_exists and not specific_streamer and not os.path.exists('config/global.json')): + # Legacy mode: single streamer using config.json + print(f'{Fore.YELLOW}⚠ Using legacy mode (config.json){Style.RESET_ALL}') + print(f'{Fore.CYAN}→ Consider migrating to new config structure (config/global.json + config/streamers/*.json){Style.RESET_ALL}\n') + + twitch_archive = TwitchArchive() # Loads from config.json + + # Apply command-line overrides + for key, value in legacy_overrides.items(): + setattr(twitch_archive, key, value) + + # Apply chat_downloader options + if hasattr(twitch_archive.downloader, 'use_chat_downloader_primary'): + twitch_archive.downloader.use_chat_downloader_primary = use_chat_downloader_primary + if hasattr(twitch_archive.downloader, 'use_chat_downloader_fallback'): + twitch_archive.downloader.use_chat_downloader_fallback = use_chat_downloader_fallback + + # Start the archive system + twitch_archive.run() + else: + # New multi-streamer mode + manager = TwitchArchiveManager( + specific_streamer=specific_streamer, + verbose=verbose_mode, + chat_only=chat_only_mode, + use_chat_downloader_primary=use_chat_downloader_primary, + use_chat_downloader_fallback=use_chat_downloader_fallback + ) + manager.run() + + +if __name__ == "__main__": + try: + main(sys.argv[1:]) + except KeyboardInterrupt: + # Suppress stack trace for clean exit + print(f'\n{Fore.GREEN}✓ Graceful shutdown complete{Style.RESET_ALL}') + sys.exit(0) \ No newline at end of file From e5e60999bf223287f63cf3bfd6e4103c14e6366f Mon Sep 17 00:00:00 2001 From: MaddoScientisto Date: Sun, 22 Feb 2026 23:06:40 +0100 Subject: [PATCH 07/10] Refactor global configuration page and navigation; add media library page; enhance streamer configuration with detailed options - Removed the global configuration form and redirected to the consolidated settings page. - Updated the dashboard to provide feedback when no streamers are configured and added edit links for each streamer. - Introduced a new media library page to display media files from the configured archive root. - Enhanced the streamer configuration page with additional options for overrides and settings, including a confirmation modal for deletion. - Updated the layout and styles for improved user experience and navigation. - Switched from file-based password storage to database-backed user credentials management in AuthService. - Applied EF migrations on application startup to ensure database schema is up-to-date. --- .gitignore | 3 +- Twitch-Archive-2.sln | 32 -- .../ArchiveDbContextFactory.cs | 17 + .../Config/ConfigurationService.cs | 33 +- .../Config/EffectiveConfig.cs | 37 ++- .../TwitchArchive.Core/Config/GlobalConfig.cs | 87 +++++ .../Config/StreamerConfig.cs | 80 +++++ .../Persistence/ArchiveDbContext.cs | 1 + .../Persistence/ArchiveDbInitializer.cs | 30 ++ .../Persistence/Models/UserCredential.cs | 8 + .../EffectiveConfigDefaultsTests.cs | 38 +++ .../TwitchArchive.Web/Pages/AddStreamer.razor | 195 ++++++++++- .../TwitchArchive.Web/Pages/AppSettings.razor | 178 ++++++++-- .../Pages/GlobalConfig.razor | 41 +-- .../src/TwitchArchive.Web/Pages/Index.razor | 36 +- .../src/TwitchArchive.Web/Pages/Media.razor | 59 ++++ .../Pages/StreamerConfig.razor | 310 ++++++++++++++++-- .../Pages/StreamerDetail.razor | 1 + dotnet/src/TwitchArchive.Web/Program.cs | 3 +- .../TwitchArchive.Web/Services/AuthService.cs | 65 ++-- .../TwitchArchive.Web/Shared/MainLayout.razor | 24 +- dotnet/src/TwitchArchive.Web/archive.db-shm | Bin 32768 -> 32768 bytes dotnet/src/TwitchArchive.Web/archive.db-wal | Bin 20632 -> 65952 bytes .../src/TwitchArchive.Web/wwwroot/css/app.css | 38 ++- 24 files changed, 1152 insertions(+), 164 deletions(-) delete mode 100644 Twitch-Archive-2.sln create mode 100644 dotnet/src/TwitchArchive.Core/ArchiveDbContextFactory.cs create mode 100644 dotnet/src/TwitchArchive.Core/Persistence/ArchiveDbInitializer.cs create mode 100644 dotnet/src/TwitchArchive.Core/Persistence/Models/UserCredential.cs create mode 100644 dotnet/src/TwitchArchive.Tests/EffectiveConfigDefaultsTests.cs create mode 100644 dotnet/src/TwitchArchive.Web/Pages/Media.razor diff --git a/.gitignore b/.gitignore index 7433967..ff0421f 100644 --- a/.gitignore +++ b/.gitignore @@ -65,4 +65,5 @@ packages/ *.backup *.orig -dotnet/.vs/** \ No newline at end of file +dotnet/.vs/** +.vs/** \ No newline at end of file diff --git a/Twitch-Archive-2.sln b/Twitch-Archive-2.sln deleted file mode 100644 index a41b4f2..0000000 --- a/Twitch-Archive-2.sln +++ /dev/null @@ -1,32 +0,0 @@ -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.5.2.0 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "dotnet", "dotnet", "{71E6E750-85FD-B5BC-4321-E01377EC6231}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{D90AB541-7400-80B1-A0B4-F58D0D439F55}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TwitchArchive.Core", "dotnet\src\TwitchArchive.Core\TwitchArchive.Core.csproj", "{1D11D744-6D0D-BB4D-8B77-30B5CE764821}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Any CPU = Debug|Any CPU - Release|Any CPU = Release|Any CPU - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {1D11D744-6D0D-BB4D-8B77-30B5CE764821}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {1D11D744-6D0D-BB4D-8B77-30B5CE764821}.Debug|Any CPU.Build.0 = Debug|Any CPU - {1D11D744-6D0D-BB4D-8B77-30B5CE764821}.Release|Any CPU.ActiveCfg = Release|Any CPU - {1D11D744-6D0D-BB4D-8B77-30B5CE764821}.Release|Any CPU.Build.0 = Release|Any CPU - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection - GlobalSection(NestedProjects) = preSolution - {D90AB541-7400-80B1-A0B4-F58D0D439F55} = {71E6E750-85FD-B5BC-4321-E01377EC6231} - {1D11D744-6D0D-BB4D-8B77-30B5CE764821} = {D90AB541-7400-80B1-A0B4-F58D0D439F55} - EndGlobalSection - GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {D101688C-0CA3-4CFB-96D4-E1AB9A62EC51} - EndGlobalSection -EndGlobal diff --git a/dotnet/src/TwitchArchive.Core/ArchiveDbContextFactory.cs b/dotnet/src/TwitchArchive.Core/ArchiveDbContextFactory.cs new file mode 100644 index 0000000..984f7c5 --- /dev/null +++ b/dotnet/src/TwitchArchive.Core/ArchiveDbContextFactory.cs @@ -0,0 +1,17 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Design; +using TwitchArchive.Core.Persistence; + +namespace TwitchArchive.Core +{ + public class ArchiveDbContextFactory : IDesignTimeDbContextFactory + { + public ArchiveDbContext CreateDbContext(string[] args) + { + var builder = new DbContextOptionsBuilder(); + var conn = "Data Source=archive.db"; + builder.UseSqlite(conn); + return new ArchiveDbContext(builder.Options); + } + } +} diff --git a/dotnet/src/TwitchArchive.Core/Config/ConfigurationService.cs b/dotnet/src/TwitchArchive.Core/Config/ConfigurationService.cs index dc76fde..9dc3ba6 100644 --- a/dotnet/src/TwitchArchive.Core/Config/ConfigurationService.cs +++ b/dotnet/src/TwitchArchive.Core/Config/ConfigurationService.cs @@ -14,12 +14,43 @@ namespace TwitchArchive.Core.Config public ConfigurationService(string? basePath = null) { - _basePath = basePath ?? Path.Combine(AppContext.BaseDirectory, "config"); + // If a basePath was explicitly provided, use it. Otherwise try to find a + // repository-level `config` folder by walking parent directories from the + // application base. This ensures the web app uses the same config/ files + // as the repository (global.json, config/streamers/*.json) when available. + if (!string.IsNullOrWhiteSpace(basePath)) + { + _basePath = basePath; + } + else + { + var found = FindExistingConfigFolder(); + _basePath = found ?? Path.Combine(AppContext.BaseDirectory, "config"); + } + _streamersPath = Path.Combine(_basePath, "streamers"); Directory.CreateDirectory(_basePath); Directory.CreateDirectory(_streamersPath); } + private string? FindExistingConfigFolder() + { + var start = AppContext.BaseDirectory ?? Environment.CurrentDirectory; + var dir = new DirectoryInfo(start); + for (int i = 0; i < 8 && dir != null; i++) + { + var candidate = Path.Combine(dir.FullName, "config"); + if (Directory.Exists(candidate)) + { + // prefer candidate if it contains global.json or streamers + if (File.Exists(Path.Combine(candidate, "global.json")) || Directory.Exists(Path.Combine(candidate, "streamers"))) + return candidate; + } + dir = dir.Parent; + } + return null; + } + public GlobalConfig LoadGlobal() { var file = Path.Combine(_basePath, "global.json"); diff --git a/dotnet/src/TwitchArchive.Core/Config/EffectiveConfig.cs b/dotnet/src/TwitchArchive.Core/Config/EffectiveConfig.cs index 82c8c4a..2fecf71 100644 --- a/dotnet/src/TwitchArchive.Core/Config/EffectiveConfig.cs +++ b/dotnet/src/TwitchArchive.Core/Config/EffectiveConfig.cs @@ -14,10 +14,44 @@ namespace TwitchArchive.Core.Config public int RefreshIntervalSeconds { get; init; } public int StreamSegmentThreads { get; init; } public string? DefaultQuality { get; init; } + // Defaults that can be overridden per-streamer + public DefaultsSection Defaults { get; init; } = new DefaultsSection(); public static EffectiveConfig Merge(GlobalConfig global, StreamerConfig? streamer) { streamer ??= new StreamerConfig(); + var d = new TwitchArchive.Core.Config.DefaultsSection(); + // start with global defaults + if (global?.Defaults != null) d = global.Defaults; + // apply per-streamer overrides when present + var mergedDefaults = new TwitchArchive.Core.Config.DefaultsSection + { + DownloadVOD = streamer.DownloadVOD ?? d.DownloadVOD, + DownloadCHAT = streamer.DownloadCHAT ?? d.DownloadCHAT, + DownloadLiveCHAT = streamer.DownloadLiveCHAT ?? d.DownloadLiveCHAT, + MergeVideoChat = streamer.MergeVideoChat ?? d.MergeVideoChat, + MergeChatLayout = streamer.MergeChatLayout ?? d.MergeChatLayout, + VodTimeout = streamer.VodTimeout ?? d.VodTimeout, + UploadCloud = streamer.UploadCloud ?? d.UploadCloud, + UploadPreMergeVideo = streamer.UploadPreMergeVideo ?? d.UploadPreMergeVideo, + UploadMergedVideo = streamer.UploadMergedVideo ?? d.UploadMergedVideo, + UploadChatVideo = streamer.UploadChatVideo ?? d.UploadChatVideo, + DeleteFiles = streamer.DeleteFiles ?? d.DeleteFiles, + OnlyRaw = streamer.OnlyRaw ?? d.OnlyRaw, + CleanRaw = streamer.CleanRaw ?? d.CleanRaw, + HlsSegments = streamer.HlsSegments ?? d.HlsSegments, + HlsSegmentsVOD = streamer.HlsSegmentsVOD ?? d.HlsSegmentsVOD, + StreamlinkTtvlol = streamer.StreamlinkTtvlol ?? d.StreamlinkTtvlol, + FfmpegHwaccel = streamer.FfmpegHwaccel ?? d.FfmpegHwaccel, + FfmpegThreads = streamer.FfmpegThreads ?? d.FfmpegThreads, + FfmpegAudioCodec = streamer.FfmpegAudioCodec ?? d.FfmpegAudioCodec, + FfmpegAudioSamplerate = streamer.FfmpegAudioSamplerate ?? d.FfmpegAudioSamplerate, + FfmpegAudioBitrate = streamer.FfmpegAudioBitrate ?? d.FfmpegAudioBitrate, + FfmpegErrorRecovery = streamer.FfmpegErrorRecovery ?? d.FfmpegErrorRecovery, + FfmpegFaststart = streamer.FfmpegFaststart ?? d.FfmpegFaststart, + FfmpegProgress = streamer.FfmpegProgress ?? d.FfmpegProgress + }; + return new EffectiveConfig { ArchiveRoot = streamer.Username != null ? (global.ArchiveRoot ?? string.Empty) : (global.ArchiveRoot ?? string.Empty), @@ -29,7 +63,8 @@ namespace TwitchArchive.Core.Config UploadDestination = streamer.UploadDestination ?? global.UploadDestination, RefreshIntervalSeconds = global.RefreshIntervalSeconds, StreamSegmentThreads = global.StreamSegmentThreads, - DefaultQuality = streamer.Quality ?? global.DefaultQuality + DefaultQuality = streamer.Quality ?? global.DefaultQuality, + Defaults = mergedDefaults }; } } diff --git a/dotnet/src/TwitchArchive.Core/Config/GlobalConfig.cs b/dotnet/src/TwitchArchive.Core/Config/GlobalConfig.cs index cb7fecf..840ba8e 100644 --- a/dotnet/src/TwitchArchive.Core/Config/GlobalConfig.cs +++ b/dotnet/src/TwitchArchive.Core/Config/GlobalConfig.cs @@ -1,3 +1,4 @@ +using System.ComponentModel.DataAnnotations; using System.Text.Json.Serialization; namespace TwitchArchive.Core.Config @@ -26,12 +27,98 @@ namespace TwitchArchive.Core.Config public string? UploadDestination { get; set; } [JsonPropertyName("refresh_interval_seconds")] + [Range(5, 86400, ErrorMessage = "Refresh interval must be between 5 and 86400 seconds.")] public int RefreshIntervalSeconds { get; set; } = 60; [JsonPropertyName("stream_segment_threads")] + [Range(1, 64, ErrorMessage = "Stream segment threads must be between 1 and 64.")] public int StreamSegmentThreads { get; set; } = 4; [JsonPropertyName("default_quality")] public string? DefaultQuality { get; set; } = "best"; + + // Defaults section for per-streamer fallbacks + [JsonPropertyName("defaults")] + public DefaultsSection Defaults { get; set; } = new DefaultsSection(); + } + + public class DefaultsSection + { + [JsonPropertyName("downloadVOD")] + public bool DownloadVOD { get; set; } = true; + + [JsonPropertyName("downloadCHAT")] + public bool DownloadCHAT { get; set; } = true; + + [JsonPropertyName("downloadLiveCHAT")] + public bool DownloadLiveCHAT { get; set; } = true; + + [JsonPropertyName("mergeVideoChat")] + public bool MergeVideoChat { get; set; } = false; + + [JsonPropertyName("mergeChatLayout")] + public string MergeChatLayout { get; set; } = "side-by-side"; + + [JsonPropertyName("vodTimeout")] + [Range(0, 86400, ErrorMessage = "VOD timeout must be between 0 and 86400 seconds.")] + public int VodTimeout { get; set; } = 300; + + [JsonPropertyName("uploadCloud")] + public bool UploadCloud { get; set; } = false; + + [JsonPropertyName("uploadPreMergeVideo")] + public bool UploadPreMergeVideo { get; set; } = true; + + [JsonPropertyName("uploadMergedVideo")] + public bool UploadMergedVideo { get; set; } = true; + + [JsonPropertyName("uploadChatVideo")] + public bool UploadChatVideo { get; set; } = false; + + [JsonPropertyName("deleteFiles")] + public bool DeleteFiles { get; set; } = false; + + [JsonPropertyName("onlyRaw")] + public bool OnlyRaw { get; set; } = false; + + [JsonPropertyName("cleanRaw")] + public bool CleanRaw { get; set; } = true; + + [JsonPropertyName("hls_segments")] + [Range(1, 50, ErrorMessage = "HLS segments must be between 1 and 50.")] + public int HlsSegments { get; set; } = 3; + + [JsonPropertyName("hls_segmentsVOD")] + [Range(1, 200, ErrorMessage = "HLS segments (VOD) must be between 1 and 200.")] + public int HlsSegmentsVOD { get; set; } = 10; + + [JsonPropertyName("streamlink_ttvlol")] + public bool StreamlinkTtvlol { get; set; } = false; + + [JsonPropertyName("ffmpeg_hwaccel")] + public string FfmpegHwaccel { get; set; } = "auto"; + + [JsonPropertyName("ffmpeg_threads")] + [Range(0, 128, ErrorMessage = "FFmpeg threads must be between 0 and 128.")] + public int FfmpegThreads { get; set; } = 0; + + [JsonPropertyName("ffmpeg_audio_codec")] + public string FfmpegAudioCodec { get; set; } = "aac"; + + [JsonPropertyName("ffmpeg_audio_samplerate")] + [Range(8000, 192000, ErrorMessage = "Audio sample rate must be between 8000 and 192000.")] + public int FfmpegAudioSamplerate { get; set; } = 48000; + + [JsonPropertyName("ffmpeg_audio_bitrate")] + public string FfmpegAudioBitrate { get; set; } = "192k"; + + [JsonPropertyName("ffmpeg_error_recovery")] + public bool FfmpegErrorRecovery { get; set; } = true; + + [JsonPropertyName("ffmpeg_faststart")] + public bool FfmpegFaststart { get; set; } = true; + + [JsonPropertyName("ffmpeg_progress")] + public bool FfmpegProgress { get; set; } = false; } } diff --git a/dotnet/src/TwitchArchive.Core/Config/StreamerConfig.cs b/dotnet/src/TwitchArchive.Core/Config/StreamerConfig.cs index 1c3a334..e490785 100644 --- a/dotnet/src/TwitchArchive.Core/Config/StreamerConfig.cs +++ b/dotnet/src/TwitchArchive.Core/Config/StreamerConfig.cs @@ -1,3 +1,4 @@ +using System.ComponentModel.DataAnnotations; using System.Text.Json.Serialization; namespace TwitchArchive.Core.Config @@ -5,6 +6,7 @@ namespace TwitchArchive.Core.Config public class StreamerConfig { [JsonPropertyName("username")] + [Required] public string Username { get; set; } = string.Empty; [JsonPropertyName("enabled")] @@ -21,5 +23,83 @@ namespace TwitchArchive.Core.Config [JsonPropertyName("streamlink_path")] public string? StreamlinkPath { get; set; } + + // Per-streamer override options matching GlobalConfig.Defaults + [JsonPropertyName("downloadVOD")] + public bool? DownloadVOD { get; set; } + + [JsonPropertyName("downloadCHAT")] + public bool? DownloadCHAT { get; set; } + + [JsonPropertyName("downloadLiveCHAT")] + public bool? DownloadLiveCHAT { get; set; } + + [JsonPropertyName("mergeVideoChat")] + public bool? MergeVideoChat { get; set; } + + [JsonPropertyName("mergeChatLayout")] + public string? MergeChatLayout { get; set; } + + [JsonPropertyName("vodTimeout")] + [Range(0, 86400, ErrorMessage = "VOD timeout must be between 0 and 86400 seconds.")] + public int? VodTimeout { get; set; } + + [JsonPropertyName("uploadCloud")] + public bool? UploadCloud { get; set; } + + [JsonPropertyName("uploadPreMergeVideo")] + public bool? UploadPreMergeVideo { get; set; } + + [JsonPropertyName("uploadMergedVideo")] + public bool? UploadMergedVideo { get; set; } + + [JsonPropertyName("uploadChatVideo")] + public bool? UploadChatVideo { get; set; } + + [JsonPropertyName("deleteFiles")] + public bool? DeleteFiles { get; set; } + + [JsonPropertyName("onlyRaw")] + public bool? OnlyRaw { get; set; } + + [JsonPropertyName("cleanRaw")] + public bool? CleanRaw { get; set; } + + [JsonPropertyName("hls_segments")] + [Range(1, 50, ErrorMessage = "HLS segments must be between 1 and 50.")] + public int? HlsSegments { get; set; } + + [JsonPropertyName("hls_segmentsVOD")] + [Range(1, 200, ErrorMessage = "HLS segments (VOD) must be between 1 and 200.")] + public int? HlsSegmentsVOD { get; set; } + + [JsonPropertyName("streamlink_ttvlol")] + public bool? StreamlinkTtvlol { get; set; } + + [JsonPropertyName("ffmpeg_hwaccel")] + public string? FfmpegHwaccel { get; set; } + + [JsonPropertyName("ffmpeg_threads")] + [Range(0, 128, ErrorMessage = "FFmpeg threads must be between 0 and 128.")] + public int? FfmpegThreads { get; set; } + + [JsonPropertyName("ffmpeg_audio_codec")] + public string? FfmpegAudioCodec { get; set; } + + [JsonPropertyName("ffmpeg_audio_samplerate")] + [Range(8000, 192000, ErrorMessage = "Audio sample rate must be between 8000 and 192000.")] + public int? FfmpegAudioSamplerate { get; set; } + + [JsonPropertyName("ffmpeg_audio_bitrate")] + public string? FfmpegAudioBitrate { get; set; } + + [JsonPropertyName("ffmpeg_error_recovery")] + public bool? FfmpegErrorRecovery { get; set; } + + [JsonPropertyName("ffmpeg_faststart")] + public bool? FfmpegFaststart { get; set; } + + [JsonPropertyName("ffmpeg_progress")] + public bool? FfmpegProgress { get; set; } } } diff --git a/dotnet/src/TwitchArchive.Core/Persistence/ArchiveDbContext.cs b/dotnet/src/TwitchArchive.Core/Persistence/ArchiveDbContext.cs index c2b3b1d..94bcf4d 100644 --- a/dotnet/src/TwitchArchive.Core/Persistence/ArchiveDbContext.cs +++ b/dotnet/src/TwitchArchive.Core/Persistence/ArchiveDbContext.cs @@ -10,5 +10,6 @@ namespace TwitchArchive.Core.Persistence public DbSet StreamSessions { get; set; } = null!; public DbSet ArchiveJobs { get; set; } = null!; public DbSet StreamerStates { get; set; } = null!; + public DbSet UserCredentials { get; set; } = null!; } } diff --git a/dotnet/src/TwitchArchive.Core/Persistence/ArchiveDbInitializer.cs b/dotnet/src/TwitchArchive.Core/Persistence/ArchiveDbInitializer.cs new file mode 100644 index 0000000..3ef78a8 --- /dev/null +++ b/dotnet/src/TwitchArchive.Core/Persistence/ArchiveDbInitializer.cs @@ -0,0 +1,30 @@ +using System; +using Microsoft.EntityFrameworkCore; + +namespace TwitchArchive.Core.Persistence +{ + public static class ArchiveDbInitializer + { + public static void EnsureUserCredentialsTable(IDbContextFactory factory) + { + if (factory == null) throw new ArgumentNullException(nameof(factory)); + try + { + using var ctx = factory.CreateDbContext(); + var conn = ctx.Database.GetDbConnection(); + try { conn.Open(); } catch { /* ignore open errors */ } + using var cmd = conn.CreateCommand(); + // Create table if it doesn't exist (SQLite syntax) + cmd.CommandText = @"CREATE TABLE IF NOT EXISTS UserCredentials ( + Id INTEGER PRIMARY KEY AUTOINCREMENT, + PasswordHash TEXT NOT NULL + );"; + cmd.ExecuteNonQuery(); + } + catch + { + // Initialization should not crash the app; log if needed + } + } + } +} diff --git a/dotnet/src/TwitchArchive.Core/Persistence/Models/UserCredential.cs b/dotnet/src/TwitchArchive.Core/Persistence/Models/UserCredential.cs new file mode 100644 index 0000000..8cb56de --- /dev/null +++ b/dotnet/src/TwitchArchive.Core/Persistence/Models/UserCredential.cs @@ -0,0 +1,8 @@ +namespace TwitchArchive.Core.Persistence.Models +{ + public class UserCredential + { + public int Id { get; set; } + public string PasswordHash { get; set; } = string.Empty; + } +} diff --git a/dotnet/src/TwitchArchive.Tests/EffectiveConfigDefaultsTests.cs b/dotnet/src/TwitchArchive.Tests/EffectiveConfigDefaultsTests.cs new file mode 100644 index 0000000..ac54d76 --- /dev/null +++ b/dotnet/src/TwitchArchive.Tests/EffectiveConfigDefaultsTests.cs @@ -0,0 +1,38 @@ +using Xunit; +using TwitchArchive.Core.Config; + +namespace TwitchArchive.Tests +{ + public class EffectiveConfigDefaultsTests + { + [Fact] + public void Merge_AppliesStreamerOverridesOverGlobalDefaults() + { + var global = new GlobalConfig + { + DefaultQuality = "best", + Defaults = new DefaultsSection + { + DownloadVOD = true, + MergeVideoChat = false, + VodTimeout = 300 + } + }; + + var streamer = new StreamerConfig + { + Username = "test", + DownloadVOD = false, + MergeVideoChat = true, + VodTimeout = 10 + }; + + var eff = EffectiveConfig.Merge(global, streamer); + + Assert.False(eff.Defaults.DownloadVOD); + Assert.True(eff.Defaults.MergeVideoChat); + Assert.Equal(10, eff.Defaults.VodTimeout); + Assert.Equal("best", eff.DefaultQuality); + } + } +} diff --git a/dotnet/src/TwitchArchive.Web/Pages/AddStreamer.razor b/dotnet/src/TwitchArchive.Web/Pages/AddStreamer.razor index e345a2b..e02e055 100644 --- a/dotnet/src/TwitchArchive.Web/Pages/AddStreamer.razor +++ b/dotnet/src/TwitchArchive.Web/Pages/AddStreamer.razor @@ -1,28 +1,207 @@ -@page "/config/new" +@page "/addstreamer" @inject TwitchArchive.Core.Config.IConfigurationService ConfigService @inject NavigationManager Nav

Add Streamer

-
- - + + +
+ +
-
- - +
+ + +
+
+ + +
+
+ + +
+ +
+ +
+ + @if (showAdvanced) + { +
+

Advanced per-streamer defaults

+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + + + + + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + + + + + + + + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ } + +
+
- @code { private TwitchArchive.Core.Config.StreamerConfig model = new() { Enabled = true }; + private bool showAdvanced = false; + + // local fields for binding nullable/global override values + private bool downloadVOD = true; + private bool downloadCHAT = true; + private bool downloadLiveCHAT = true; + private bool mergeVideoChat = false; + private int? vodTimeout; + private bool uploadPreMergeVideo = true; + private bool uploadMergedVideo = true; + private bool uploadChatVideo = false; + private bool deleteFiles = false; + private bool onlyRaw = false; + private bool cleanRaw = true; + private int? hlsSegments; + private int? hlsSegmentsVOD; + private bool streamlinkTtvlol = false; + private string? ffmpegHwaccel; + private int? ffmpegThreads; + private string? ffmpegAudioCodec; + private int? ffmpegAudioSamplerate; + private string? ffmpegAudioBitrate; + private bool ffmpegErrorRecovery = true; + private bool ffmpegFaststart = true; + private bool ffmpegProgress = false; + + private void ToggleAdvanced() => showAdvanced = !showAdvanced; private void Save() { model.Username = model.Username?.Trim().ToLowerInvariant() ?? string.Empty; if (string.IsNullOrWhiteSpace(model.Username)) return; + + // map local fields into nullable model properties + model.DownloadVOD = downloadVOD; + model.DownloadCHAT = downloadCHAT; + model.DownloadLiveCHAT = downloadLiveCHAT; + model.MergeVideoChat = mergeVideoChat; + model.VodTimeout = vodTimeout; + model.UploadPreMergeVideo = uploadPreMergeVideo; + model.UploadMergedVideo = uploadMergedVideo; + model.UploadChatVideo = uploadChatVideo; + model.DeleteFiles = deleteFiles; + model.OnlyRaw = onlyRaw; + model.CleanRaw = cleanRaw; + model.HlsSegments = hlsSegments; + model.HlsSegmentsVOD = hlsSegmentsVOD; + model.StreamlinkTtvlol = streamlinkTtvlol; + model.FfmpegHwaccel = ffmpegHwaccel; + model.FfmpegThreads = ffmpegThreads; + model.FfmpegAudioCodec = ffmpegAudioCodec; + model.FfmpegAudioSamplerate = ffmpegAudioSamplerate; + model.FfmpegAudioBitrate = ffmpegAudioBitrate; + model.FfmpegErrorRecovery = ffmpegErrorRecovery; + model.FfmpegFaststart = ffmpegFaststart; + model.FfmpegProgress = ffmpegProgress; + ConfigService.SaveStreamer(model); Nav.NavigateTo($"/config/{model.Username}"); } diff --git a/dotnet/src/TwitchArchive.Web/Pages/AppSettings.razor b/dotnet/src/TwitchArchive.Web/Pages/AppSettings.razor index 9ccb7d3..a666c0b 100644 --- a/dotnet/src/TwitchArchive.Web/Pages/AppSettings.razor +++ b/dotnet/src/TwitchArchive.Web/Pages/AppSettings.razor @@ -1,35 +1,174 @@ @page "/settings" @using System.Text.Json +@inject TwitchArchive.Core.Config.IConfigurationService ConfigService @inject TwitchArchive.Web.Services.IAuthService Auth -

App Settings

+

Settings

@if (saved) {
Saved.
} - + +
+ + +
- +
- +
- +
- + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ +
-
-

Change Password

+
+ + @if (showDefaults) + { + + +
+

Defaults

+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + + + + + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + + + + + + + + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ +
+
+
+ } +
+ +
+ +
+ +

Change Password

@if (!string.IsNullOrEmpty(pwError)) {
@pwError
@@ -42,8 +181,9 @@
@code { - private TwitchArchive.Core.Config.AppSettings model = new(); + private TwitchArchive.Core.Config.GlobalConfig globalModel = new(); private bool saved = false; + private bool showDefaults = false; private string currentPw = string.Empty; private string newPw = string.Empty; private string confirmPw = string.Empty; @@ -56,29 +196,26 @@ private void Load() { - var file = Path.Combine(AppContext.BaseDirectory, "appsettings.json"); try { - if (!File.Exists(file)) { model = new(); return; } - var txt = File.ReadAllText(file); - model = JsonSerializer.Deserialize(txt) ?? new TwitchArchive.Core.Config.AppSettings(); + globalModel = ConfigService.LoadGlobal() ?? new TwitchArchive.Core.Config.GlobalConfig(); } - catch { model = new(); } + catch { globalModel = new TwitchArchive.Core.Config.GlobalConfig(); } } - private void Save() + private void SaveGlobal() { - var file = Path.Combine(AppContext.BaseDirectory, "appsettings.json"); try { - var txt = JsonSerializer.Serialize(model, new JsonSerializerOptions { WriteIndented = true }); - File.WriteAllText(file, txt); + ConfigService.SaveGlobal(globalModel); saved = true; - Auth.Refresh(); + // optionally notify auth or other services } catch { } } + private void ToggleDefaults() => showDefaults = !showDefaults; + private void ChangePassword() { pwError = string.Empty; @@ -87,6 +224,7 @@ if (newPw != confirmPw) { pwError = "Passwords do not match"; return; } var hash = BCrypt.Net.BCrypt.HashPassword(newPw); Auth.SetPasswordHash(hash); - pwError = string.Empty; + Auth.Refresh(); + saved = true; } } diff --git a/dotnet/src/TwitchArchive.Web/Pages/GlobalConfig.razor b/dotnet/src/TwitchArchive.Web/Pages/GlobalConfig.razor index 96ba67d..1ec6d41 100644 --- a/dotnet/src/TwitchArchive.Web/Pages/GlobalConfig.razor +++ b/dotnet/src/TwitchArchive.Web/Pages/GlobalConfig.razor @@ -1,45 +1,10 @@ @page "/config/global" -@inject TwitchArchive.Core.Config.IConfigurationService ConfigService - -

Global Configuration

- -@if (saved) -{ -
Saved.
-} - - -
- - -
-
- - -
-
- - -
-
- - -
- -
+@inject NavigationManager Nav @code { - private TwitchArchive.Core.Config.GlobalConfig model = new(); - private bool saved = false; - protected override void OnInitialized() { - model = ConfigService.LoadGlobal(); - } - - private void Save() - { - ConfigService.SaveGlobal(model); - saved = true; + // Consolidated settings are now at /settings + Nav.NavigateTo("/settings", true); } } diff --git a/dotnet/src/TwitchArchive.Web/Pages/Index.razor b/dotnet/src/TwitchArchive.Web/Pages/Index.razor index 9120fe2..e25b8cd 100644 --- a/dotnet/src/TwitchArchive.Web/Pages/Index.razor +++ b/dotnet/src/TwitchArchive.Web/Pages/Index.razor @@ -4,12 +4,18 @@

Dashboard

+@if (streamers.Count == 0) +{ +
No streamers configured. Add one on the Add Streamer page.
+} +
@foreach (var s in streamers) {
@s + Edit @(WorkerManager.IsRunning(s) ? "Live" : "Offline")
@@ -23,6 +29,12 @@ }
+@* Show global feedback when there are streamers but no recent sessions *@ +@if (streamers.Count > 0 && (lastStarts == null || lastStarts.Count == 0)) +{ +
No recent sessions found for configured streamers.
+} + @code { private List streamers = new(); private Dictionary lastStarts = new(); @@ -44,13 +56,33 @@ private void LoadStreamers() { - var cfgDir = Path.Combine(Environment.CurrentDirectory, "config", "streamers"); - if (Directory.Exists(cfgDir)) + // Try to find the config/streamers folder from the app content root and parent folders. + string? cfgDir = FindConfigStreamersFolder(); + if (!string.IsNullOrEmpty(cfgDir) && Directory.Exists(cfgDir)) { streamers = Directory.GetFiles(cfgDir, "*.json").Select(f => Path.GetFileNameWithoutExtension(f)).ToList(); } } + private string? FindConfigStreamersFolder() + { + // Prefer ContentRoot if available, fall back to Environment.CurrentDirectory. + var start = AppContext.BaseDirectory ?? Environment.CurrentDirectory; + var dir = new DirectoryInfo(start); + for (int i = 0; i < 6 && dir != null; i++) + { + var candidate = Path.Combine(dir.FullName, "config", "streamers"); + if (Directory.Exists(candidate)) return candidate; + dir = dir.Parent; + } + + // final attempt: repo-root relative (use project parent heuristics) + var alt = Path.Combine(Environment.CurrentDirectory, "..", "..", "..", "config", "streamers"); + try { alt = Path.GetFullPath(alt); } catch { } + if (Directory.Exists(alt)) return alt; + return null; + } + // Index reads from the singleton SessionCacheService; updates are pushed via the Updated event. private void Start(string u) { WorkerManager.StartWorker(u); } diff --git a/dotnet/src/TwitchArchive.Web/Pages/Media.razor b/dotnet/src/TwitchArchive.Web/Pages/Media.razor new file mode 100644 index 0000000..a42c78c --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Pages/Media.razor @@ -0,0 +1,59 @@ +@page "/media" +@inject TwitchArchive.Core.Config.IConfigurationService ConfigService + +

Media Library

+ +@if (string.IsNullOrWhiteSpace(archiveRoot)) +{ +
Archive root is not configured. Set it on the Settings page.
+} +else if (!Directory.Exists(archiveRoot)) +{ +
Archive root '@archiveRoot' does not exist on disk.
+} +else +{ + @if (entries.Count == 0) + { +
No media files found in '@archiveRoot'.
+ } + else + { + @foreach (var kv in entries) + { +
+

@kv.Key

+
    + @foreach (var f in kv.Value) + { +
  • @f.Name - @((f.Length/1024.0/1024.0).ToString("0.00")) MB - @f.LastWriteTime.ToLocalTime()
  • + } +
+
+ } + } +} + +@code { + private string? archiveRoot; + private Dictionary> entries = new(); + + protected override void OnInitialized() + { + var g = ConfigService.LoadGlobal(); + archiveRoot = g?.ArchiveRoot; + if (!string.IsNullOrWhiteSpace(archiveRoot) && Directory.Exists(archiveRoot)) + { + var di = new DirectoryInfo(archiveRoot); + foreach (var dir in di.GetDirectories()) + { + try + { + var files = dir.GetFiles("*.mp4").Concat(dir.GetFiles("*.mkv")).Concat(dir.GetFiles("*.ts")).Concat(dir.GetFiles("*.flv")).OrderByDescending(f => f.LastWriteTime).ToList(); + if (files.Count > 0) entries[dir.Name] = files; + } + catch { } + } + } + } +} diff --git a/dotnet/src/TwitchArchive.Web/Pages/StreamerConfig.razor b/dotnet/src/TwitchArchive.Web/Pages/StreamerConfig.razor index bf0c72a..1b58f8a 100644 --- a/dotnet/src/TwitchArchive.Web/Pages/StreamerConfig.razor +++ b/dotnet/src/TwitchArchive.Web/Pages/StreamerConfig.razor @@ -5,67 +5,331 @@

Streamer Config: @Username

+ + + @if (saved) + { +
Saved.
+ }
- - + + +
+ +
+ + Override +
- - Override - + + Override +
-
- - Override - +
+ +
-
- - +
+ + Override +
-
- - Override - +
+

Per-streamer overrides

+
+ + Override + +
+
+ + Override + +
+
+ + Override + +
+
+ + Override + + + + + +
+
+ + Override + +
+
+ + Override + +
+
+ + Override + +
+
+ + Override + + + + + + + + +
+
+ + Override + +
+
+ + Override + +
+
+ + Override + +
+
+ + Override + +
+
+ + Override + +
+
+ + Override + +
+
+ + Override + +
+
+ + Override + +
+
+ + Override + +
+
+ + Override + +
+
+ + Override + +
+
+ + Override + +
+
+ + Override + +
+
+ + Override + +
+
+ + Override + +
- - + + +@* Confirmation modal *@ +@if (showConfirm) +{ + +} + @code { [Parameter] public string Username { get; set; } = string.Empty; private TwitchArchive.Core.Config.StreamerConfig model = new(); private TwitchArchive.Core.Config.GlobalConfig? global; + private bool saved = false; + private bool showConfirm = false; private bool overrideQuality = false; private bool overrideUpload = false; private bool overrideStreamlink = false; + private bool overrideDownloadVOD = false; + private bool overrideDownloadCHAT = false; + private bool overrideMergeVideoChat = false; + private bool overrideMergeChatLayout = false; + private bool overrideVodTimeout = false; + private bool overrideDeleteFiles = false; + private bool overrideHlsSegments = false; + private bool overrideFfmpegHwaccel = false; + private bool overrideFfmpegThreads = false; + private bool overrideFfmpegAudioBitrate = false; + private bool overrideDownloadLiveCHAT = false; + private bool overrideUploadPreMergeVideo = false; + private bool overrideUploadMergedVideo = false; + private bool overrideUploadChatVideo = false; + private bool overrideOnlyRaw = false; + private bool overrideCleanRaw = false; + private bool overrideHlsSegmentsVOD = false; + private bool overrideStreamlinkTtvlol = false; + private bool overrideFfmpegAudioCodec = false; + private bool overrideFfmpegAudioSamplerate = false; + private bool overrideFfmpegErrorRecovery = false; + private bool overrideFfmpegFaststart = false; + private bool overrideFfmpegProgress = false; + + // local values for nullable per-streamer settings (bind safely) + private bool downloadVODVal; + private bool downloadCHATVal; + private bool downloadLiveCHATVal; + private bool mergeVideoChatVal; + private string mergeChatLayoutVal = "side-by-side"; + private int? vodTimeoutVal; + private bool deleteFilesVal; + private int? hlsSegmentsVal; + private string ffmpegHwaccelVal = "auto"; + private int? ffmpegThreadsVal; + private string? ffmpegAudioBitrateVal; + private bool uploadPreMergeVideoVal; + private bool uploadMergedVideoVal; + private bool uploadChatVideoVal; + private bool onlyRawVal; + private bool cleanRawVal; + private int? hlsSegmentsVODVal; + private bool streamlinkTtvlolVal; + private string? ffmpegAudioCodecVal; + private int? ffmpegAudioSamplerateVal; + private bool ffmpegErrorRecoveryVal; + private bool ffmpegFaststartVal; + private bool ffmpegProgressVal; + private bool uploadToCloudVal; protected override void OnInitialized() { global = ConfigService.LoadGlobal(); var s = ConfigService.LoadStreamer(Username); if (s != null) model = s; + // initialize local values from model or global defaults + downloadVODVal = model.DownloadVOD ?? global?.Defaults.DownloadVOD ?? true; + downloadCHATVal = model.DownloadCHAT ?? global?.Defaults.DownloadCHAT ?? true; + downloadLiveCHATVal = model.DownloadLiveCHAT ?? global?.Defaults.DownloadLiveCHAT ?? true; + mergeVideoChatVal = model.MergeVideoChat ?? global?.Defaults.MergeVideoChat ?? false; + mergeChatLayoutVal = model.MergeChatLayout ?? global?.Defaults.MergeChatLayout ?? "side-by-side"; + vodTimeoutVal = model.VodTimeout ?? global?.Defaults.VodTimeout ?? 300; + deleteFilesVal = model.DeleteFiles ?? global?.Defaults.DeleteFiles ?? false; + hlsSegmentsVal = model.HlsSegments ?? global?.Defaults.HlsSegments ?? 3; + ffmpegHwaccelVal = model.FfmpegHwaccel ?? global?.Defaults.FfmpegHwaccel ?? "auto"; + ffmpegThreadsVal = model.FfmpegThreads ?? global?.Defaults.FfmpegThreads ?? 0; + ffmpegAudioBitrateVal = model.FfmpegAudioBitrate ?? global?.Defaults.FfmpegAudioBitrate ?? "192k"; + uploadPreMergeVideoVal = model.UploadPreMergeVideo ?? global?.Defaults.UploadPreMergeVideo ?? true; + uploadMergedVideoVal = model.UploadMergedVideo ?? global?.Defaults.UploadMergedVideo ?? true; + uploadChatVideoVal = model.UploadChatVideo ?? global?.Defaults.UploadChatVideo ?? false; + onlyRawVal = model.OnlyRaw ?? global?.Defaults.OnlyRaw ?? false; + cleanRawVal = model.CleanRaw ?? global?.Defaults.CleanRaw ?? true; + hlsSegmentsVODVal = model.HlsSegmentsVOD ?? global?.Defaults.HlsSegmentsVOD ?? 10; + streamlinkTtvlolVal = model.StreamlinkTtvlol ?? global?.Defaults.StreamlinkTtvlol ?? false; + ffmpegAudioCodecVal = model.FfmpegAudioCodec ?? global?.Defaults.FfmpegAudioCodec ?? "aac"; + ffmpegAudioSamplerateVal = model.FfmpegAudioSamplerate ?? global?.Defaults.FfmpegAudioSamplerate ?? 48000; + ffmpegErrorRecoveryVal = model.FfmpegErrorRecovery ?? global?.Defaults.FfmpegErrorRecovery ?? true; + ffmpegFaststartVal = model.FfmpegFaststart ?? global?.Defaults.FfmpegFaststart ?? true; + ffmpegProgressVal = model.FfmpegProgress ?? global?.Defaults.FfmpegProgress ?? false; + uploadToCloudVal = model.UploadToCloud ?? global?.UploadToCloud ?? false; } private void Save() { model.Username = Username; if (!overrideQuality) model.Quality = null; - if (!overrideUpload) model.UploadToCloud = null; - if (!overrideStreamlink) model.StreamlinkPath = null; + // Upload to cloud + model.UploadToCloud = overrideUpload ? uploadToCloudVal : (bool?)null; + // Streamlink path + model.StreamlinkPath = overrideStreamlink ? model.StreamlinkPath : null; + // Per-streamer values: map local values when overridden, otherwise clear + model.DownloadVOD = overrideDownloadVOD ? downloadVODVal : (bool?)null; + model.DownloadCHAT = overrideDownloadCHAT ? downloadCHATVal : (bool?)null; + model.DownloadLiveCHAT = overrideDownloadLiveCHAT ? downloadLiveCHATVal : (bool?)null; + model.MergeVideoChat = overrideMergeVideoChat ? mergeVideoChatVal : (bool?)null; + model.MergeChatLayout = overrideMergeChatLayout ? mergeChatLayoutVal : null; + model.VodTimeout = overrideVodTimeout ? vodTimeoutVal : (int?)null; + model.DeleteFiles = overrideDeleteFiles ? deleteFilesVal : (bool?)null; + model.HlsSegments = overrideHlsSegments ? hlsSegmentsVal : (int?)null; + model.FfmpegHwaccel = overrideFfmpegHwaccel ? ffmpegHwaccelVal : null; + model.FfmpegThreads = overrideFfmpegThreads ? ffmpegThreadsVal : (int?)null; + model.FfmpegAudioBitrate = overrideFfmpegAudioBitrate ? ffmpegAudioBitrateVal : null; + model.UploadPreMergeVideo = overrideUploadPreMergeVideo ? uploadPreMergeVideoVal : (bool?)null; + model.UploadMergedVideo = overrideUploadMergedVideo ? uploadMergedVideoVal : (bool?)null; + model.UploadChatVideo = overrideUploadChatVideo ? uploadChatVideoVal : (bool?)null; + model.OnlyRaw = overrideOnlyRaw ? onlyRawVal : (bool?)null; + model.CleanRaw = overrideCleanRaw ? cleanRawVal : (bool?)null; + model.HlsSegmentsVOD = overrideHlsSegmentsVOD ? hlsSegmentsVODVal : (int?)null; + model.StreamlinkTtvlol = overrideStreamlinkTtvlol ? streamlinkTtvlolVal : (bool?)null; + model.FfmpegAudioCodec = overrideFfmpegAudioCodec ? ffmpegAudioCodecVal : null; + model.FfmpegAudioSamplerate = overrideFfmpegAudioSamplerate ? ffmpegAudioSamplerateVal : (int?)null; + model.FfmpegErrorRecovery = overrideFfmpegErrorRecovery ? ffmpegErrorRecoveryVal : (bool?)null; + model.FfmpegFaststart = overrideFfmpegFaststart ? ffmpegFaststartVal : (bool?)null; + model.FfmpegProgress = overrideFfmpegProgress ? ffmpegProgressVal : (bool?)null; ConfigService.SaveStreamer(model); + saved = true; } - private void Delete() + private void ConfirmDelete() { - ConfigService.DeleteStreamer(Username); - Nav.NavigateTo("/"); + try + { + ConfigService.DeleteStreamer(Username); + showConfirm = false; + Nav.NavigateTo("/"); + } + catch + { + // ignore + } } } diff --git a/dotnet/src/TwitchArchive.Web/Pages/StreamerDetail.razor b/dotnet/src/TwitchArchive.Web/Pages/StreamerDetail.razor index bafa148..c9ac1e6 100644 --- a/dotnet/src/TwitchArchive.Web/Pages/StreamerDetail.razor +++ b/dotnet/src/TwitchArchive.Web/Pages/StreamerDetail.razor @@ -6,6 +6,7 @@
Status: @(WorkerManager.IsRunning(Username) ? "Live" : "Offline") + Edit Settings
diff --git a/dotnet/src/TwitchArchive.Web/Program.cs b/dotnet/src/TwitchArchive.Web/Program.cs index d655b3e..12ddccb 100644 --- a/dotnet/src/TwitchArchive.Web/Program.cs +++ b/dotnet/src/TwitchArchive.Web/Program.cs @@ -65,7 +65,8 @@ using (var scope = app.Services.CreateScope()) try { using var db = factory.CreateDbContext(); - db.Database.EnsureCreated(); + // Apply any pending EF migrations (creates/updates schema as needed) + db.Database.Migrate(); } catch { } } diff --git a/dotnet/src/TwitchArchive.Web/Services/AuthService.cs b/dotnet/src/TwitchArchive.Web/Services/AuthService.cs index 12f4e2f..be83fc0 100644 --- a/dotnet/src/TwitchArchive.Web/Services/AuthService.cs +++ b/dotnet/src/TwitchArchive.Web/Services/AuthService.cs @@ -1,18 +1,22 @@ using System; -using System.IO; -using System.Text.Json; -using TwitchArchive.Core.Config; +using System.Linq; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using TwitchArchive.Core.Persistence; +using TwitchArchive.Core.Persistence.Models; namespace TwitchArchive.Web.Services { public class AuthService : IAuthService { - private readonly string _file; - private AppSettings _settings = new(); + private readonly IDbContextFactory _dbFactory; + private readonly ILogger _log; + private string? _cachedHash; - public AuthService() + public AuthService(IDbContextFactory dbFactory, ILogger log) { - _file = Path.Combine(AppContext.BaseDirectory, "appsettings.json"); + _dbFactory = dbFactory ?? throw new ArgumentNullException(nameof(dbFactory)); + _log = log; Refresh(); } @@ -20,31 +24,52 @@ namespace TwitchArchive.Web.Services { try { - if (!File.Exists(_file)) { _settings = new AppSettings(); return; } - var txt = File.ReadAllText(_file); - _settings = JsonSerializer.Deserialize(txt) ?? new AppSettings(); + using var ctx = _dbFactory.CreateDbContext(); + var u = ctx.UserCredentials.AsNoTracking().FirstOrDefault(); + _cachedHash = u?.PasswordHash; + } + catch (Exception ex) + { + _log?.LogWarning(ex, "Failed to read password from database"); + _cachedHash = null; } - catch { _settings = new AppSettings(); } } public bool ValidatePassword(string plain) { - if (string.IsNullOrWhiteSpace(_settings.PasswordHash)) return true; - try { return BCrypt.Net.BCrypt.Verify(plain ?? string.Empty, _settings.PasswordHash); } - catch { return false; } + // If no password configured, allow access (default open) + if (string.IsNullOrWhiteSpace(_cachedHash)) return true; + try { return BCrypt.Net.BCrypt.Verify(plain ?? string.Empty, _cachedHash); } + catch (Exception ex) + { + _log?.LogWarning(ex, "Password verification failed"); + return false; + } } public void SetPasswordHash(string hash) { try { - if (string.IsNullOrWhiteSpace(_file)) return; - _settings.PasswordHash = hash; - var txt = JsonSerializer.Serialize(_settings, new JsonSerializerOptions { WriteIndented = true }); - File.WriteAllText(_file, txt); - Refresh(); + using var ctx = _dbFactory.CreateDbContext(); + var existing = ctx.UserCredentials.FirstOrDefault(); + if (existing == null) + { + existing = new UserCredential { PasswordHash = hash }; + ctx.UserCredentials.Add(existing); + } + else + { + existing.PasswordHash = hash; + ctx.UserCredentials.Update(existing); + } + ctx.SaveChanges(); + _cachedHash = hash; + } + catch (Exception ex) + { + _log?.LogError(ex, "Failed to save password to database"); } - catch { } } } } diff --git a/dotnet/src/TwitchArchive.Web/Shared/MainLayout.razor b/dotnet/src/TwitchArchive.Web/Shared/MainLayout.razor index dd8a426..cf6d00c 100644 --- a/dotnet/src/TwitchArchive.Web/Shared/MainLayout.razor +++ b/dotnet/src/TwitchArchive.Web/Shared/MainLayout.razor @@ -1,17 +1,27 @@ @inherits LayoutComponentBase
- -

Twitch Archive

+
+ +

Twitch Archive

+ +
- @Body +
@Body
diff --git a/dotnet/src/TwitchArchive.Web/archive.db-shm b/dotnet/src/TwitchArchive.Web/archive.db-shm index 08698a3d3a5a3a3747bb810dfeefe4a32fd9fe22..b4164f6d0f635525509dd2609b3027c8deb39c3a 100644 GIT binary patch delta 235 zcmZo@U}|V!s+V}A%K!pQK+MR%ARqvwIf2;KviE0t1k_|B(Pxn2CXJVxzPs&}IfUAO_LwPz+*o05QllFfsOx3oV`48G&j!7`Qhs q{LZxbkLwR^MxgLtCQeQUE(SgZeg=V!3%@ck@_@t`c{eT;R0RMt5j%DO delta 162 zcmZo@U}|V!s+V}A%K!t63=9ISKn@7IzdW~UPTl(zX>WWv(ih%R`{44qC*YQUG^y%g qW`oT9j|8CNObo0O8>Kg1(Q=yjz=Dx+h)RJELJ2#?XyEiJZPfawwTFzk;X62`?tf`3Aa1|#&f!eT_)R<5*EfM^ps(eiSRcmzp2Nb4@w z&v%pN6eE=>?abas?VS`OyIchcL)HwE_8ZHT74vEHR(fdsrD;iHR(3{_L|Q%g@THwekP5y+6#^fE?)SxqB( zBTiE4F`~^E4Y(Mkq*}2SXLnzp{ZP?LKboaCl1ZH$ohA7csakYcw5m4tutu_tmh3I4 zc31b7E%t?ni*@?7R7N>Qo>GtKrr8!tT^`ex)v;R&HZ-?f1+^|4?n>XSG(S_P%`RAOrbfL-$CWK1WCS1f9ps)dQJ%Ttv%Ryv3Dmkztnuk;sOy8L73 zE1OW&@00Izz00bZa0SG_<0uX>ea|*EU`z!qdJ3oE>?N>IRdLR7) z&3UqMR1kmw1Rwwb2tWV=5P$##AOL|Xf#MhdKE5FT?}1N#PJg%`{Q^}=ScCusAOHaf zKmY;|fB*y_009UzrvS?p(8l`(&VKdF-ZQt3zk_~(<~-RrDhNOT0uX=z1Rwwb2tWV= z5P(3HfFPQYY^q8+YI|)xqKoR-1Rw!SfEbb$nFGJ4k0SK>z{}fB*y_ z009UP4l8Da#pWebXpRVRc+)c_gLD!MpH0dNy^tK zFIl6i*!GyL>vO7>-Y4rBYnHuI!oEgz^dj$IF)<>~rq(D;8}be!?_iyLn#emyb4Su& zy6hg`$>RmOcAt7RblZ0l@dEWej<5{^5P$##AOHafKmY;|fB*y_&@=*;c!3Rv<9}b> zb^4Dw@d7M1z-2mmHV$F6@dB+pUclM@lPO++E?|NH1Rwwb2tWV=5V%_cC;U!V_u!zt z&|mznJ((2ugq0~xHrUhcLTYMy=^B^i8+Xsr9ai^5uv+C!U%>P4ehh2?7v+ z00bZa0SG_<0uX=z1Rwx`|DC{AyTcasd3_`OUSGd&P#E-$4ULWs`3AP{7#{HsZ}0PZ z$Gl#8Se9`F+L?hb7Os4GvB5Zkb`uB7PjF}B2-?55x!o5m;|Sar+TSwOjadjl00Izz z00baVU*N<+m#cep)Lw|zh>YK-=!UAzTVvPDE8X?tC Date: Sun, 22 Feb 2026 23:12:11 +0100 Subject: [PATCH 08/10] Added git lfs --- .gitattributes | 18 +++++++++++++++++- README.md | 23 +++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/.gitattributes b/.gitattributes index 8e459b5..f887d97 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,6 +1,22 @@ +# Git LFS tracking for large media and binaries + +# Binaries in bin/ +bin/* filter=lfs diff=lfs merge=lfs -text +bin/TwitchDownloaderCLI* filter=lfs diff=lfs merge=lfs -text +bin/ffmpeg filter=lfs diff=lfs merge=lfs -text +# Video files under archive (raw/ and common video extensions) +archive/**/video/** filter=lfs diff=lfs merge=lfs -text +archive/**/video/raw/** filter=lfs diff=lfs merge=lfs -text +*.mp4 filter=lfs diff=lfs merge=lfs -text +*.mkv filter=lfs diff=lfs merge=lfs -text +*.mov filter=lfs diff=lfs merge=lfs -text +# Large archives and executables +*.zip filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.exe filter=lfs diff=lfs merge=lfs -text +*.dll filter=lfs diff=lfs merge=lfs -text # Auto detect text files and perform LF normalization * text=auto -bin/ffmpeg filter=lfs diff=lfs merge=lfs -text bin/ffmpeg.exe filter=lfs diff=lfs merge=lfs -text bin/TwitchDownloaderCLI filter=lfs diff=lfs merge=lfs -text bin/TwitchDownloaderCLI.exe filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md index 40d7471..e2e2c1f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,29 @@ # Twitch Archive Inspired by https://github.com/EnterGin/Auto-Stream-Recording-Twitch +## Git LFS + +This repository stores large media files (recorded video and some binaries). Use Git LFS to manage large objects. + +Quick setup (Windows): + +1. Install Git LFS: `git lfs install` +2. Ensure `.gitattributes` is committed (this repo includes one). +3. If you already have large files tracked by normal Git, migrate them: + +```powershell +git lfs install +git lfs track "*.mp4" "*.mkv" "bin/*" +git add .gitattributes +git add -A +git commit -m "Migrate large files to LFS" +git push origin main +``` + +Notes: +- Git LFS needs server-side support. If using GitHub, enable Git LFS on the remote and ensure you have sufficient bandwidth/storage quota. +- You can customize tracked patterns in `.gitattributes`. + Python script to check, download live stream, VOD, chat and upload them to any cloud storage supported by rclone. ## ⚡ FFmpeg 8.0 Enhanced From f97e0200d65bfd5ccd138f43d4c2d16cb966d8cb Mon Sep 17 00:00:00 2001 From: MaddoScientisto Date: Sat, 25 Apr 2026 11:54:03 +0200 Subject: [PATCH 09/10] Refactor downloader and file manager for improved rclone integration and add healthcheck and smoke test options - Renamed download flags in ContentDownloader for clarity. - Enhanced FileManager with methods to build upload paths and verify existing files for rclone uploads. - Updated StreamProcessor to return success status for stream processing. - Added rclone smoke test and healthcheck functions to validate configuration and tool availability. - Improved environment variable handling with a utility function. - Updated TwitchArchive to incorporate new rclone verification and processing logic. - Added unit tests for new functionality and refactored existing tests for clarity and coverage. Co-authored-by: Copilot --- .dockerignore | 18 ++ .env.development | 16 ++ .env.production | 16 ++ .../workflows/publish-python-container.yml | 127 +++++++++ .gitignore | 2 + README.md | 67 +++++ docker-compose.override.yml | 17 ++ docker-compose.yml | 28 ++ docker/entrypoint.sh | 6 + docker/python.Dockerfile | 49 ++++ dockerrebuild.bat | 4 + dockerstart.bat | 20 ++ .../Pages/StreamerConfig.razor | 243 +++++------------- dotnet/src/TwitchArchive.Web/archive.db-shm | Bin 32768 -> 32768 bytes dotnet/src/TwitchArchive.Web/archive.db-wal | Bin 65952 -> 90672 bytes modules/downloader.py | 14 +- modules/file_manager.py | 199 +++++++++----- modules/processor.py | 71 +++-- modules/recorder.py | 4 +- modules/stream_monitor.py | 7 +- modules/utils.py | 38 +++ test_twitch_archive_simple.py | 198 ++++++++++++++ twitch-archive.py | 158 ++++++++++-- 23 files changed, 1013 insertions(+), 289 deletions(-) create mode 100644 .dockerignore create mode 100644 .env.development create mode 100644 .env.production create mode 100644 .forgejo/workflows/publish-python-container.yml create mode 100644 docker-compose.override.yml create mode 100644 docker-compose.yml create mode 100644 docker/entrypoint.sh create mode 100644 docker/python.Dockerfile create mode 100644 dockerrebuild.bat create mode 100644 dockerstart.bat diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..49463af --- /dev/null +++ b/.dockerignore @@ -0,0 +1,18 @@ +.git +.github +.forgejo +.vs +.vscode +venv*/ +__pycache__/ +*.pyc +archive/ +dotnet/ +tests/ +.pytest_cache/ +bin/temp/ +bin/ffmpeg +bin/ffmpeg.exe +bin/ffprobe +bin/TwitchDownloaderCLI +bin/TwitchDownloaderCLI.exe \ No newline at end of file diff --git a/.env.development b/.env.development new file mode 100644 index 0000000..4ccc47f --- /dev/null +++ b/.env.development @@ -0,0 +1,16 @@ +TWITCH_ARCHIVE_DEV_IMAGE=twitch-archive-local +TWITCH_ARCHIVE_CONTAINER_NAME=twitch-archive-dev +TWITCH_ARCHIVE_APP_ENV_FILE=./.env.development +TWITCH_ARCHIVE_ARCHIVE_BIND=./archive +TWITCH_ARCHIVE_CONFIG_BIND=./config +TWITCH_ARCHIVE_ARGS=-u vinesauce --verbose +TWITCH_ARCHIVE_HEALTHCHECK_STREAMER=vinesauce +TWITCH_ARCHIVE_RCLONE_CONFIG=/app/config/rclone.conf +PYTHONUNBUFFERED=1 +TZ=UTC +CLIENT-ID=vdyevjvllziylzwsm3y925p79pwtua +CLIENT-SECRET=y906xadsmf22q54suuzsmfnfav3jc7 +OAUTH-PRIVATE-TOKEN=ll4kvlmxuajfgi9lgi5d8mkeglsyvm +SENDER= +RECEIVER= +PASSWD= \ No newline at end of file diff --git a/.env.production b/.env.production new file mode 100644 index 0000000..b1c4197 --- /dev/null +++ b/.env.production @@ -0,0 +1,16 @@ +TWITCH_ARCHIVE_IMAGE=forgejo.maddoscientisto.net/maddo/twitch-archive:latest +TWITCH_ARCHIVE_CONTAINER_NAME=twitch-archive +TWITCH_ARCHIVE_APP_ENV_FILE=./.env.production +TWITCH_ARCHIVE_ARCHIVE_BIND=./archive +TWITCH_ARCHIVE_CONFIG_BIND=./config +TWITCH_ARCHIVE_ARGS=-u vinesauce +TWITCH_ARCHIVE_HEALTHCHECK_STREAMER=vinesauce +TWITCH_ARCHIVE_RCLONE_CONFIG=/app/config/rclone.conf +PYTHONUNBUFFERED=1 +TZ=UTC +CLIENT-ID=vdyevjvllziylzwsm3y925p79pwtua +CLIENT-SECRET=y906xadsmf22q54suuzsmfnfav3jc7 +OAUTH-PRIVATE-TOKEN=ll4kvlmxuajfgi9lgi5d8mkeglsyvm +SENDER= +RECEIVER= +PASSWD= \ No newline at end of file diff --git a/.forgejo/workflows/publish-python-container.yml b/.forgejo/workflows/publish-python-container.yml new file mode 100644 index 0000000..831fa51 --- /dev/null +++ b/.forgejo/workflows/publish-python-container.yml @@ -0,0 +1,127 @@ +name: Publish Twitch Archive Container + +on: + push: + branches: + - master + - main + paths: + - docker/python.Dockerfile + - docker/entrypoint.sh + - docker-compose.yml + - docker-compose.override.yml + - requirements.txt + - twitch-archive.py + - run_chat_only.py + - modules/** + - .forgejo/workflows/publish-python-container.yml + workflow_dispatch: + +env: + REGISTRY: ${{ vars.FORGEJO_REGISTRY }} + IMAGE_NAMESPACE: ${{ vars.IMAGE_NAMESPACE }} + IMAGE_NAME: ${{ vars.IMAGE_NAME != '' && vars.IMAGE_NAME || 'twitch-archive' }} + BUILD_CONTEXT: . + DOCKERFILE_PATH: docker/python.Dockerfile + +jobs: + publish: + runs-on: docker + env: + DOCKER_HOST: ${{ vars.DOCKER_HOST != '' && vars.DOCKER_HOST || 'tcp://172.17.0.1:2375' }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Validate workflow variables + run: | + set -eu + if [ -z "${REGISTRY}" ]; then echo "vars.FORGEJO_REGISTRY is required"; exit 1; fi + if [ -z "${IMAGE_NAMESPACE}" ]; then echo "vars.IMAGE_NAMESPACE is required"; exit 1; fi + if [ ! -f "${DOCKERFILE_PATH}" ]; then echo "Dockerfile not found at ${DOCKERFILE_PATH}"; exit 1; fi + if [ ! -f "requirements.txt" ]; then echo "requirements.txt is missing"; exit 1; fi + + - name: Ensure Docker CLI exists + run: | + set -eu + if command -v docker >/dev/null 2>&1; then + docker --version + exit 0 + fi + + ARCH="$(uname -m)" + case "${ARCH}" in + x86_64) DOCKER_ARCH="x86_64" ;; + aarch64|arm64) DOCKER_ARCH="aarch64" ;; + *) echo "Unsupported architecture for Docker CLI bootstrap: ${ARCH}"; exit 1 ;; + esac + + DOCKER_CLI_VERSION="27.5.1" + curl -fsSL "https://download.docker.com/linux/static/stable/${DOCKER_ARCH}/docker-${DOCKER_CLI_VERSION}.tgz" -o docker.tgz + tar -xzf docker.tgz + mkdir -p "${HOME}/.local/bin" + mv docker/docker "${HOME}/.local/bin/docker" + chmod +x "${HOME}/.local/bin/docker" + echo "${HOME}/.local/bin" >> "${FORGEJO_PATH}" + "${HOME}/.local/bin/docker" --version + + - name: Ensure Docker Buildx exists + run: | + set -eu + if docker buildx version >/dev/null 2>&1; then + docker buildx version + exit 0 + fi + + ARCH="$(uname -m)" + case "${ARCH}" in + x86_64) BUILDX_ARCH="amd64" ;; + aarch64|arm64) BUILDX_ARCH="arm64" ;; + *) echo "Unsupported architecture for Docker Buildx bootstrap: ${ARCH}"; exit 1 ;; + esac + + BUILDX_VERSION="v0.21.1" + mkdir -p "${HOME}/.docker/cli-plugins" + curl -fsSL "https://github.com/docker/buildx/releases/download/${BUILDX_VERSION}/buildx-${BUILDX_VERSION}.linux-${BUILDX_ARCH}" -o "${HOME}/.docker/cli-plugins/docker-buildx" + chmod +x "${HOME}/.docker/cli-plugins/docker-buildx" + docker buildx version + + - name: Check Docker daemon connectivity + run: | + set -eu + echo "Using DOCKER_HOST=${DOCKER_HOST}" + docker version + docker info >/dev/null + + - name: Create Buildx builder + run: | + set -eu + docker buildx rm forgejo-builder >/dev/null 2>&1 || true + docker buildx create --name forgejo-builder --driver docker-container --use + docker buildx inspect --bootstrap + + - name: Validate registry secrets + run: | + set -eu + if [ -z "${{ secrets.FORGEJO_REGISTRY_USERNAME }}" ]; then echo "secrets.FORGEJO_REGISTRY_USERNAME is required"; exit 1; fi + if [ -z "${{ secrets.FORGEJO_REGISTRY_TOKEN }}" ]; then echo "secrets.FORGEJO_REGISTRY_TOKEN is required"; exit 1; fi + + - name: Login to registry + run: | + set -eu + echo "${{ secrets.FORGEJO_REGISTRY_TOKEN }}" | docker login "${REGISTRY}" -u "${{ secrets.FORGEJO_REGISTRY_USERNAME }}" --password-stdin + + - name: Build and push image + run: | + set -eu + IMAGE_REF="${REGISTRY}/${IMAGE_NAMESPACE}/${IMAGE_NAME}" + SHA_TAG="${IMAGE_REF}:sha-${FORGEJO_SHA}" + BRANCH_TAG="${IMAGE_REF}:${FORGEJO_REF_NAME}" + docker buildx build \ + --file "${DOCKERFILE_PATH}" \ + --tag "${SHA_TAG}" \ + --tag "${BRANCH_TAG}" \ + --tag "${IMAGE_REF}:latest" \ + --push \ + "${BUILD_CONTEXT}" \ No newline at end of file diff --git a/.gitignore b/.gitignore index ff0421f..49c4387 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ config/global.json # Streamer-specific configurations (personal settings) config/streamers/*.json +config/rclone.conf +config/rclone.conf.* # Python cache __pycache__/ diff --git a/README.md b/README.md index e2e2c1f..5005181 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,73 @@ Notes: Python script to check, download live stream, VOD, chat and upload them to any cloud storage supported by rclone. +## Docker + +This repository now includes a Python-only container setup for the archiver. The dotnet subapp is not part of this container flow. + +Files: + +- `docker/python.Dockerfile`: production image for the Python archiver +- `docker-compose.yml`: deployment-oriented compose file +- `docker-compose.override.yml`: local development and testing override +- `.env.production`: production container and app environment template +- `.env.development`: development container and app environment template +- `dockerstart.bat`: Windows helper to run the container like the old batch launcher + +### Container layout + +- Mount your external archive folder to `/app/archive` +- Mount your external config folder to `/app/config` +- Put your `rclone.conf` file at `/app/config/rclone.conf` on the mounted host path +- The container exports `RCLONE_CONFIG=/app/config/rclone.conf`, so rclone will use that file automatically + +### Production deployment + +1. Edit `.env.production` with your image name, Twitch credentials, bind paths, and default arguments. +2. Place your streamer JSON files and `rclone.conf` in the mounted config folder. +3. Start the container: + +```powershell +docker compose --env-file .env.production up -d +``` + +4. Follow logs: + +```powershell +docker compose --env-file .env.production logs -f twitch-archive +``` + +### Development and local testing + +The override compose file builds the image locally and mounts the repository for faster iteration. + +Start it with: + +```powershell +docker compose --env-file .env.development -f docker-compose.yml -f docker-compose.override.yml up --build +``` + +Run a one-off manual test for another streamer: + +```powershell +docker compose --env-file .env.development -f docker-compose.yml -f docker-compose.override.yml run --rm twitch-archive python twitch-archive.py -u hackerling --verbose +``` + +Or use the Windows helper: + +```powershell +.\dockerstart.bat vinesauce --verbose +``` + +That batch launcher mirrors the old pattern and expands to a compose `run` command, so you can test any streamer manually. + +### Healthcheck and smoke tests + +- Container healthcheck command: `python twitch-archive.py --healthcheck -u vinesauce` +- Rclone smoke test command: `python twitch-archive.py -u vinesauce --rclone-smoke-test` + +The healthcheck verifies config loading plus `streamlink`, `ffmpeg`, `TwitchDownloaderCLI`, and `rclone` availability. The smoke test writes a tiny file, uploads it with the configured rclone remote, and prints the live rclone output into the container logs. + ## ⚡ FFmpeg 8.0 Enhanced Now with FFmpeg 8.0+ support featuring hardware acceleration and performance improvements! - **5-10x faster encoding** with NVIDIA, Intel, or AMD GPUs diff --git a/docker-compose.override.yml b/docker-compose.override.yml new file mode 100644 index 0000000..c084df9 --- /dev/null +++ b/docker-compose.override.yml @@ -0,0 +1,17 @@ +services: + twitch-archive: + build: + context: . + dockerfile: docker/python.Dockerfile + image: ${TWITCH_ARCHIVE_DEV_IMAGE:-twitch-archive-local} + restart: "no" + env_file: + - ${TWITCH_ARCHIVE_APP_ENV_FILE:-./.env.development} + command: + - sh + - -lc + - python twitch-archive.py ${TWITCH_ARCHIVE_ARGS:--u vinesauce --verbose} + volumes: + - .:/app + - ${TWITCH_ARCHIVE_ARCHIVE_BIND:-./archive}:/app/archive + - ${TWITCH_ARCHIVE_CONFIG_BIND:-./config}:/app/config \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..68bc572 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,28 @@ +services: + twitch-archive: + image: ${TWITCH_ARCHIVE_IMAGE:-forgejo.maddoscientisto.net/maddo/twitch-archive:latest} + container_name: ${TWITCH_ARCHIVE_CONTAINER_NAME:-twitch-archive} + restart: unless-stopped + init: true + env_file: + - ${TWITCH_ARCHIVE_APP_ENV_FILE:-./.env.production} + environment: + PYTHONUNBUFFERED: ${PYTHONUNBUFFERED:-1} + TZ: ${TZ:-UTC} + RCLONE_CONFIG: ${TWITCH_ARCHIVE_RCLONE_CONFIG:-/app/config/rclone.conf} + TWITCH_ARCHIVE_HEALTHCHECK_STREAMER: ${TWITCH_ARCHIVE_HEALTHCHECK_STREAMER:-vinesauce} + command: + - sh + - -lc + - python twitch-archive.py ${TWITCH_ARCHIVE_ARGS:--u vinesauce} + volumes: + - ${TWITCH_ARCHIVE_ARCHIVE_BIND:-./archive}:/app/archive + - ${TWITCH_ARCHIVE_CONFIG_BIND:-./config}:/app/config + healthcheck: + test: + - CMD-SHELL + - python twitch-archive.py --healthcheck -u ${TWITCH_ARCHIVE_HEALTHCHECK_STREAMER:-vinesauce} + interval: 30s + timeout: 10s + retries: 3 + start_period: 30s \ No newline at end of file diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100644 index 0000000..d327347 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,6 @@ +#!/bin/sh +set -eu + +mkdir -p /app/archive /app/config /app/bin/temp + +exec "$@" \ No newline at end of file diff --git a/docker/python.Dockerfile b/docker/python.Dockerfile new file mode 100644 index 0000000..789f738 --- /dev/null +++ b/docker/python.Dockerfile @@ -0,0 +1,49 @@ +FROM python:3.12-slim + +ARG TWITCH_DOWNLOADER_VERSION=1.56.4 +ARG TARGETARCH + +ENV PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + RCLONE_CONFIG=/app/config/rclone.conf + +WORKDIR /app + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + ffmpeg \ + libicu-dev \ + rclone \ + unzip \ + && rm -rf /var/lib/apt/lists/* + +RUN set -eux; \ + case "${TARGETARCH}" in \ + amd64) twitch_downloader_arch='Linux-x64' ;; \ + arm64) twitch_downloader_arch='LinuxArm64' ;; \ + arm) twitch_downloader_arch='LinuxArm' ;; \ + *) echo "Unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; \ + esac; \ + curl -fsSL -o /tmp/TwitchDownloaderCLI.zip "https://github.com/lay295/TwitchDownloader/releases/download/${TWITCH_DOWNLOADER_VERSION}/TwitchDownloaderCLI-${TWITCH_DOWNLOADER_VERSION}-${twitch_downloader_arch}.zip"; \ + mkdir -p /tmp/twitchdownloader; \ + unzip -j /tmp/TwitchDownloaderCLI.zip TwitchDownloaderCLI -d /tmp/twitchdownloader; \ + install -m 0755 /tmp/twitchdownloader/TwitchDownloaderCLI /usr/local/bin/TwitchDownloaderCLI; \ + rm -rf /tmp/TwitchDownloaderCLI.zip /tmp/twitchdownloader + +COPY requirements.txt ./requirements.txt + +RUN python -m pip install --upgrade pip \ + && python -m pip install -r requirements.txt + +COPY . /app + +RUN mkdir -p /app/archive /app/config /app/bin/temp + +COPY docker/entrypoint.sh /usr/local/bin/twitch-archive-entrypoint + +RUN chmod +x /usr/local/bin/twitch-archive-entrypoint + +ENTRYPOINT ["twitch-archive-entrypoint"] +CMD ["python", "twitch-archive.py", "-u", "vinesauce"] \ No newline at end of file diff --git a/dockerrebuild.bat b/dockerrebuild.bat new file mode 100644 index 0000000..f333c1c --- /dev/null +++ b/dockerrebuild.bat @@ -0,0 +1,4 @@ +@echo off +setlocal + +docker compose --env-file .env.development -f docker-compose.yml -f docker-compose.override.yml build --no-cache --pull twitch-archive diff --git a/dockerstart.bat b/dockerstart.bat new file mode 100644 index 0000000..9d1e947 --- /dev/null +++ b/dockerstart.bat @@ -0,0 +1,20 @@ +@echo off +setlocal + +if "%~1"=="" ( + echo Usage: .\dockerstart.bat streamer [additional args] + exit /b 1 +) + +set STREAMER=%~1 +shift + +set EXTRA_ARGS= +:collect_args +if "%~1"=="" goto run_compose +set EXTRA_ARGS=%EXTRA_ARGS% %~1 +shift +goto collect_args + +:run_compose +docker compose --env-file .env.development -f docker-compose.yml -f docker-compose.override.yml run --rm twitch-archive python twitch-archive.py -u %STREAMER%%EXTRA_ARGS% \ No newline at end of file diff --git a/dotnet/src/TwitchArchive.Web/Pages/StreamerConfig.razor b/dotnet/src/TwitchArchive.Web/Pages/StreamerConfig.razor index 1b58f8a..7c9b082 100644 --- a/dotnet/src/TwitchArchive.Web/Pages/StreamerConfig.razor +++ b/dotnet/src/TwitchArchive.Web/Pages/StreamerConfig.razor @@ -17,49 +17,39 @@
- - Override - + +
- Override - +
- +
-
- - Override - -
+ @* Streamlink path is global-only; not configurable per-streamer *@
-

Per-streamer overrides

+

Per-streamer settings

- Override - +
- Override - +
- Override - +
- Override - + @@ -67,105 +57,35 @@
- Override - +
- Override - -
-
- - Override - -
-
- - Override - - - - - - - - -
-
- - Override - -
-
- - Override - +
- Override - +
- Override - +
- Override - +
- Override - +
- Override - +
- Override - -
-
- - Override - -
-
- - Override - -
-
- - Override - -
-
- - Override - -
-
- - Override - -
-
- - Override - -
-
- - Override - +
@@ -193,37 +113,10 @@ @code { [Parameter] public string Username { get; set; } = string.Empty; - private TwitchArchive.Core.Config.StreamerConfig model = new(); private TwitchArchive.Core.Config.GlobalConfig? global; private bool saved = false; private bool showConfirm = false; - private bool overrideQuality = false; - private bool overrideUpload = false; - private bool overrideStreamlink = false; - private bool overrideDownloadVOD = false; - private bool overrideDownloadCHAT = false; - private bool overrideMergeVideoChat = false; - private bool overrideMergeChatLayout = false; - private bool overrideVodTimeout = false; - private bool overrideDeleteFiles = false; - private bool overrideHlsSegments = false; - private bool overrideFfmpegHwaccel = false; - private bool overrideFfmpegThreads = false; - private bool overrideFfmpegAudioBitrate = false; - private bool overrideDownloadLiveCHAT = false; - private bool overrideUploadPreMergeVideo = false; - private bool overrideUploadMergedVideo = false; - private bool overrideUploadChatVideo = false; - private bool overrideOnlyRaw = false; - private bool overrideCleanRaw = false; - private bool overrideHlsSegmentsVOD = false; - private bool overrideStreamlinkTtvlol = false; - private bool overrideFfmpegAudioCodec = false; - private bool overrideFfmpegAudioSamplerate = false; - private bool overrideFfmpegErrorRecovery = false; - private bool overrideFfmpegFaststart = false; - private bool overrideFfmpegProgress = false; // local values for nullable per-streamer settings (bind safely) private bool downloadVODVal; @@ -233,28 +126,18 @@ private string mergeChatLayoutVal = "side-by-side"; private int? vodTimeoutVal; private bool deleteFilesVal; - private int? hlsSegmentsVal; - private string ffmpegHwaccelVal = "auto"; - private int? ffmpegThreadsVal; - private string? ffmpegAudioBitrateVal; private bool uploadPreMergeVideoVal; private bool uploadMergedVideoVal; private bool uploadChatVideoVal; private bool onlyRawVal; private bool cleanRawVal; - private int? hlsSegmentsVODVal; - private bool streamlinkTtvlolVal; - private string? ffmpegAudioCodecVal; - private int? ffmpegAudioSamplerateVal; - private bool ffmpegErrorRecoveryVal; - private bool ffmpegFaststartVal; - private bool ffmpegProgressVal; private bool uploadToCloudVal; protected override void OnInitialized() { global = ConfigService.LoadGlobal(); var s = ConfigService.LoadStreamer(Username); + var isNew = s == null; if (s != null) model = s; // initialize local values from model or global defaults downloadVODVal = model.DownloadVOD ?? global?.Defaults.DownloadVOD ?? true; @@ -264,57 +147,67 @@ mergeChatLayoutVal = model.MergeChatLayout ?? global?.Defaults.MergeChatLayout ?? "side-by-side"; vodTimeoutVal = model.VodTimeout ?? global?.Defaults.VodTimeout ?? 300; deleteFilesVal = model.DeleteFiles ?? global?.Defaults.DeleteFiles ?? false; - hlsSegmentsVal = model.HlsSegments ?? global?.Defaults.HlsSegments ?? 3; - ffmpegHwaccelVal = model.FfmpegHwaccel ?? global?.Defaults.FfmpegHwaccel ?? "auto"; - ffmpegThreadsVal = model.FfmpegThreads ?? global?.Defaults.FfmpegThreads ?? 0; - ffmpegAudioBitrateVal = model.FfmpegAudioBitrate ?? global?.Defaults.FfmpegAudioBitrate ?? "192k"; uploadPreMergeVideoVal = model.UploadPreMergeVideo ?? global?.Defaults.UploadPreMergeVideo ?? true; uploadMergedVideoVal = model.UploadMergedVideo ?? global?.Defaults.UploadMergedVideo ?? true; uploadChatVideoVal = model.UploadChatVideo ?? global?.Defaults.UploadChatVideo ?? false; onlyRawVal = model.OnlyRaw ?? global?.Defaults.OnlyRaw ?? false; cleanRawVal = model.CleanRaw ?? global?.Defaults.CleanRaw ?? true; - hlsSegmentsVODVal = model.HlsSegmentsVOD ?? global?.Defaults.HlsSegmentsVOD ?? 10; - streamlinkTtvlolVal = model.StreamlinkTtvlol ?? global?.Defaults.StreamlinkTtvlol ?? false; - ffmpegAudioCodecVal = model.FfmpegAudioCodec ?? global?.Defaults.FfmpegAudioCodec ?? "aac"; - ffmpegAudioSamplerateVal = model.FfmpegAudioSamplerate ?? global?.Defaults.FfmpegAudioSamplerate ?? 48000; - ffmpegErrorRecoveryVal = model.FfmpegErrorRecovery ?? global?.Defaults.FfmpegErrorRecovery ?? true; - ffmpegFaststartVal = model.FfmpegFaststart ?? global?.Defaults.FfmpegFaststart ?? true; - ffmpegProgressVal = model.FfmpegProgress ?? global?.Defaults.FfmpegProgress ?? false; uploadToCloudVal = model.UploadToCloud ?? global?.UploadToCloud ?? false; + + // when creating a new streamer config, populate model with global defaults so + // the streamer config stores initial values and subsequent edits use streamer values + if (isNew) + { + model.Quality = model.Quality ?? global?.DefaultQuality; + model.UploadToCloud = uploadToCloudVal; + model.UploadDestination = model.UploadDestination ?? global?.UploadDestination; + model.DownloadVOD = downloadVODVal; + model.DownloadCHAT = downloadCHATVal; + model.DownloadLiveCHAT = downloadLiveCHATVal; + model.MergeVideoChat = mergeVideoChatVal; + model.MergeChatLayout = mergeChatLayoutVal; + model.VodTimeout = vodTimeoutVal; + model.DeleteFiles = deleteFilesVal; + model.UploadPreMergeVideo = uploadPreMergeVideoVal; + model.UploadMergedVideo = uploadMergedVideoVal; + model.UploadChatVideo = uploadChatVideoVal; + model.OnlyRaw = onlyRawVal; + model.CleanRaw = cleanRawVal; + } } private void Save() { model.Username = Username; - if (!overrideQuality) model.Quality = null; + if (string.IsNullOrWhiteSpace(model.Quality)) model.Quality = null; // Upload to cloud - model.UploadToCloud = overrideUpload ? uploadToCloudVal : (bool?)null; - // Streamlink path - model.StreamlinkPath = overrideStreamlink ? model.StreamlinkPath : null; - // Per-streamer values: map local values when overridden, otherwise clear - model.DownloadVOD = overrideDownloadVOD ? downloadVODVal : (bool?)null; - model.DownloadCHAT = overrideDownloadCHAT ? downloadCHATVal : (bool?)null; - model.DownloadLiveCHAT = overrideDownloadLiveCHAT ? downloadLiveCHATVal : (bool?)null; - model.MergeVideoChat = overrideMergeVideoChat ? mergeVideoChatVal : (bool?)null; - model.MergeChatLayout = overrideMergeChatLayout ? mergeChatLayoutVal : null; - model.VodTimeout = overrideVodTimeout ? vodTimeoutVal : (int?)null; - model.DeleteFiles = overrideDeleteFiles ? deleteFilesVal : (bool?)null; - model.HlsSegments = overrideHlsSegments ? hlsSegmentsVal : (int?)null; - model.FfmpegHwaccel = overrideFfmpegHwaccel ? ffmpegHwaccelVal : null; - model.FfmpegThreads = overrideFfmpegThreads ? ffmpegThreadsVal : (int?)null; - model.FfmpegAudioBitrate = overrideFfmpegAudioBitrate ? ffmpegAudioBitrateVal : null; - model.UploadPreMergeVideo = overrideUploadPreMergeVideo ? uploadPreMergeVideoVal : (bool?)null; - model.UploadMergedVideo = overrideUploadMergedVideo ? uploadMergedVideoVal : (bool?)null; - model.UploadChatVideo = overrideUploadChatVideo ? uploadChatVideoVal : (bool?)null; - model.OnlyRaw = overrideOnlyRaw ? onlyRawVal : (bool?)null; - model.CleanRaw = overrideCleanRaw ? cleanRawVal : (bool?)null; - model.HlsSegmentsVOD = overrideHlsSegmentsVOD ? hlsSegmentsVODVal : (int?)null; - model.StreamlinkTtvlol = overrideStreamlinkTtvlol ? streamlinkTtvlolVal : (bool?)null; - model.FfmpegAudioCodec = overrideFfmpegAudioCodec ? ffmpegAudioCodecVal : null; - model.FfmpegAudioSamplerate = overrideFfmpegAudioSamplerate ? ffmpegAudioSamplerateVal : (int?)null; - model.FfmpegErrorRecovery = overrideFfmpegErrorRecovery ? ffmpegErrorRecoveryVal : (bool?)null; - model.FfmpegFaststart = overrideFfmpegFaststart ? ffmpegFaststartVal : (bool?)null; - model.FfmpegProgress = overrideFfmpegProgress ? ffmpegProgressVal : (bool?)null; + model.UploadToCloud = uploadToCloudVal; + // Ensure global-only settings are not stored per-streamer + model.StreamlinkPath = null; + model.HlsSegments = null; + model.HlsSegmentsVOD = null; + model.StreamlinkTtvlol = null; + model.FfmpegHwaccel = null; + model.FfmpegThreads = null; + model.FfmpegAudioBitrate = null; + model.FfmpegAudioCodec = null; + model.FfmpegAudioSamplerate = null; + model.FfmpegErrorRecovery = null; + model.FfmpegFaststart = null; + model.FfmpegProgress = null; + // Per-streamer values: always map local values into the model + model.DownloadVOD = downloadVODVal; + model.DownloadCHAT = downloadCHATVal; + model.DownloadLiveCHAT = downloadLiveCHATVal; + model.MergeVideoChat = mergeVideoChatVal; + model.MergeChatLayout = mergeChatLayoutVal; + model.VodTimeout = vodTimeoutVal; + model.DeleteFiles = deleteFilesVal; + model.UploadPreMergeVideo = uploadPreMergeVideoVal; + model.UploadMergedVideo = uploadMergedVideoVal; + model.UploadChatVideo = uploadChatVideoVal; + model.OnlyRaw = onlyRawVal; + model.CleanRaw = cleanRawVal; ConfigService.SaveStreamer(model); saved = true; } diff --git a/dotnet/src/TwitchArchive.Web/archive.db-shm b/dotnet/src/TwitchArchive.Web/archive.db-shm index b4164f6d0f635525509dd2609b3027c8deb39c3a..fd7d358d6bf188b95258115ddec0890a64a0e1c8 100644 GIT binary patch delta 180 zcmZo@U}|V!s+V}A%K!pQK+MR%ARq>$If1y!ev54G^*pzE0t1k_|B(Pxn2CXJVx#-U2XammKS*r;=k!l-2CcDthst*1=FgqTFU~Wn`phZf0s>io^Mj diff --git a/modules/downloader.py b/modules/downloader.py index a4ea645..9f9cc1f 100644 --- a/modules/downloader.py +++ b/modules/downloader.py @@ -40,9 +40,9 @@ class ContentDownloader: self.ffmpeg_path = ffmpeg_path self.quality = config.get('quality', 'best') self.hls_segments_vod = config.get('hls_segmentsVOD', 10) - self.download_vod = config.get('downloadVOD', True) - self.download_chat = config.get('downloadCHAT', True) - self.download_live_chat = config.get('downloadLiveCHAT', True) + self.download_vod_enabled = config.get('downloadVOD', True) + self.download_chat_enabled = config.get('downloadCHAT', True) + self.download_live_chat_enabled = config.get('downloadLiveCHAT', True) self.use_chat_downloader_primary = config.get('useChatDownloaderPrimary', False) self.use_chat_downloader_fallback = config.get('useChatDownloaderFallback', True) @@ -73,7 +73,7 @@ class ContentDownloader: Returns: bool: True if download succeeded, False otherwise """ - if not self.download_vod: + if not self.download_vod_enabled: return False print(f'\n{Fore.CYAN}Downloading VOD: {vod_info["title"]}{Style.RESET_ALL}') @@ -272,7 +272,7 @@ class ContentDownloader: Returns: bool: True if succeeded, False otherwise """ - if not self.download_chat: + if not self.download_chat_enabled: return False print(f'\n{Fore.CYAN}Downloading chat: {vod_info["title"]}{Style.RESET_ALL}') @@ -298,7 +298,7 @@ class ContentDownloader: Returns: subprocess.Popen: The process handle, or None if failed to start """ - if not self.download_live_chat: + if not self.download_live_chat_enabled: return None print(f'\n{Fore.CYAN}Starting live chat download...{Style.RESET_ALL}') @@ -502,7 +502,7 @@ class ContentDownloader: print(f'{Fore.YELLOW} Install with: pip install chat-downloader{Style.RESET_ALL}') return False - if not self.download_live_chat: + if not self.download_live_chat_enabled: print(f'{Fore.YELLOW}⚠ downloadLiveCHAT is disabled in config{Style.RESET_ALL}') return False diff --git a/modules/file_manager.py b/modules/file_manager.py index 4567cb6..b024e8f 100644 --- a/modules/file_manager.py +++ b/modules/file_manager.py @@ -45,6 +45,128 @@ class FileManager: self.chat_mp4_path = self.root_path / username / "chat" self.metadata_path = self.root_path / username / "metadata" self.log_file = self.root_path / ".log" + + def _to_rclone_relative_path(self, *parts: str) -> str: + """Build a POSIX-style relative path for rclone --files-from.""" + return pathlib.PurePosixPath(*parts).as_posix() + + def _build_upload_relative_paths(self, filename_base: str) -> List[str]: + """Build the candidate upload list relative to root_path for rclone.""" + files_to_upload: List[str] = [ + self._to_rclone_relative_path(self.username, 'metadata', f"{PREFIX_METADATA}{filename_base}.json"), + self._to_rclone_relative_path(self.username, 'chat', 'json', f"{PREFIX_CHAT}{filename_base}.json") + ] + + if self.upload_pre_merge_video: + files_to_upload.extend([ + self._to_rclone_relative_path(self.username, 'video', 'raw', f"{PREFIX_LIVE}{filename_base}.ts"), + self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_LIVE}{filename_base}.mp4"), + self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_LIVE}{filename_base}.mp3"), + self._to_rclone_relative_path(self.username, 'video', 'raw', f"{PREFIX_VOD}{filename_base}.ts"), + self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_VOD}{filename_base}.mp4"), + self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_VOD}{filename_base}.mp3") + ]) + + if self.upload_merged_video: + files_to_upload.extend([ + self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_MERGED}{filename_base}.mp4"), + self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_MERGED}{filename_base}.mp3"), + self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}.mp4"), + self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}.mp3") + ]) + + if self.upload_chat_video: + files_to_upload.append(self._to_rclone_relative_path(self.username, 'chat', f"{PREFIX_CHAT}{filename_base}.mp4")) + + return files_to_upload + + def _get_existing_upload_relative_paths(self, relative_paths: List[str]) -> List[str]: + """Filter candidate upload paths to the files that actually exist.""" + existing_paths: List[str] = [] + for relative_path in relative_paths: + if (self.root_path / pathlib.PurePosixPath(relative_path)).exists(): + existing_paths.append(relative_path) + return existing_paths + + def _run_rclone_copy(self, relative_paths: List[str], description: str) -> bool: + """Run rclone copy for a set of paths relative to root_path.""" + existing_paths = self._get_existing_upload_relative_paths(relative_paths) + missing_paths = [path for path in relative_paths if path not in existing_paths] + + if not existing_paths: + print(f'{Fore.RED}✗ Upload skipped: no matching files found for {description}{Style.RESET_ALL}') + for missing_path in missing_paths: + print(f'{Fore.YELLOW} Missing: {missing_path}{Style.RESET_ALL}') + return False + + if missing_paths: + print(f'{Fore.YELLOW}⚠ Some configured upload files were not found and will be skipped{Style.RESET_ALL}') + for missing_path in missing_paths: + print(f'{Fore.YELLOW} Missing: {missing_path}{Style.RESET_ALL}') + + print(f'{Fore.CYAN}rclone source: {self.root_path.resolve()}{Style.RESET_ALL}') + print(f'{Fore.CYAN}rclone destination: {self.rclone_path}{Style.RESET_ALL}') + print(f'{Fore.CYAN}Files queued for upload: {len(existing_paths)}{Style.RESET_ALL}') + + bin_path = get_bin_path() + upload_list_path = os.path.join(bin_path, 'temp', 'upload.txt') + os.makedirs(os.path.dirname(upload_list_path), exist_ok=True) + + with open(upload_list_path, 'w', encoding='utf-8', newline='\n') as f: + f.write('\n'.join(existing_paths)) + f.write('\n') + + try: + cmd = [ + 'rclone', 'copy', + str(self.root_path.resolve()), + self.rclone_path, + '--files-from', upload_list_path, + '--progress' + ] + + print(f'{Fore.CYAN}Running: {' '.join(cmd)}{Style.RESET_ALL}') + + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + if proc.stdout: + for line in proc.stdout: + print(line, end='') + proc.wait() + return proc.returncode == 0 + finally: + if os.path.exists(upload_list_path): + os.remove(upload_list_path) + + def run_rclone_smoke_test(self) -> bool: + """Create and upload a tiny metadata file to verify rclone output and configuration.""" + smoke_name = 'RCLONE_SMOKE_TEST' + smoke_relative_path = self._to_rclone_relative_path( + self.username, + 'metadata', + f"{PREFIX_METADATA}{smoke_name}.json" + ) + smoke_file_path = self.root_path / pathlib.PurePosixPath(smoke_relative_path) + + smoke_payload = { + 'type': 'rclone_smoke_test', + 'username': self.username + } + + smoke_file_path.parent.mkdir(parents=True, exist_ok=True) + with open(smoke_file_path, 'w', encoding='utf-8') as f: + json.dump(smoke_payload, f, indent=2) + + print(f'{Fore.CYAN}Created smoke-test file: {smoke_file_path}{Style.RESET_ALL}') + try: + result = self._run_rclone_copy([smoke_relative_path], 'rclone smoke test') + if result: + print(f'{Fore.GREEN}✓ Rclone smoke test completed{Style.RESET_ALL}') + else: + print(f'{Fore.RED}✗ Rclone smoke test failed{Style.RESET_ALL}') + return result + finally: + if smoke_file_path.exists(): + smoke_file_path.unlink() def initialize_directories(self) -> None: """Create all necessary directory structures.""" @@ -120,81 +242,24 @@ class FileManager: print(f'\n{Fore.CYAN}Uploading to cloud storage...{Style.RESET_ALL}') if notification_callback: notification_callback(f'☁ Uploading - {filename_base}', 'Uploading files to cloud storage') - - # Create list of files to upload - bin_path = get_bin_path() - upload_list_path = os.path.join(bin_path, 'temp', 'upload.txt') - - # Ensure temp directory exists - os.makedirs(os.path.dirname(upload_list_path), exist_ok=True) - - files_to_upload = [] - # Build files list relative to root_path so rclone can read them with --files-from - # Metadata and chat JSON - files_to_upload.append(os.path.join(self.username, 'metadata', f"{PREFIX_METADATA}{filename_base}.json")) - files_to_upload.append(os.path.join(self.username, 'chat', 'json', f"{PREFIX_CHAT}{filename_base}.json")) + files_to_upload = self._build_upload_relative_paths(filename_base) - # Pre-merge videos (raw .ts in video/raw, mp4/mp3 in video) - if self.upload_pre_merge_video: - files_to_upload.extend([ - os.path.join(self.username, 'video', 'raw', f"{PREFIX_LIVE}{filename_base}.ts"), - os.path.join(self.username, 'video', f"{PREFIX_LIVE}{filename_base}.mp4"), - os.path.join(self.username, 'video', f"{PREFIX_LIVE}{filename_base}.mp3"), - os.path.join(self.username, 'video', 'raw', f"{PREFIX_VOD}{filename_base}.ts"), - os.path.join(self.username, 'video', f"{PREFIX_VOD}{filename_base}.mp4"), - os.path.join(self.username, 'video', f"{PREFIX_VOD}{filename_base}.mp3") - ]) - - # Merged videos (in video folder) - if self.upload_merged_video: - files_to_upload.extend([ - os.path.join(self.username, 'video', f"{PREFIX_MERGED}{filename_base}.mp4"), - os.path.join(self.username, 'video', f"{PREFIX_MERGED}{filename_base}.mp3"), - os.path.join(self.username, 'video', f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}.mp4"), - os.path.join(self.username, 'video', f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}.mp3") - ]) - - # Standalone chat video (in chat folder) - if self.upload_chat_video: - files_to_upload.append(os.path.join(self.username, 'chat', f"{PREFIX_CHAT}{filename_base}.mp4")) - - with open(upload_list_path, 'w') as f: - f.write('\n'.join(files_to_upload)) - - # Run rclone using --files-from so the listed paths (relative to root_path) are uploaded. try: - cmd = [ - 'rclone', 'copy', - str(self.root_path.resolve()), - self.rclone_path, - '--files-from', upload_list_path - ] + result = self._run_rclone_copy(files_to_upload, f'archive batch {filename_base}') - # Stream rclone output to console so user can see progress/errors - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) - if proc.stdout: - for line in proc.stdout: - print(line, end='') - proc.wait() - result = proc.returncode - - # Clean up upload list - if os.path.exists(upload_list_path): - os.remove(upload_list_path) - - if result == 0: + if result: print(f'{Fore.GREEN}✓ Upload complete{Style.RESET_ALL}') if notification_callback: notification_callback(f'✓ Upload Success - {filename_base}', 'All files uploaded successfully') return True - else: - print(f'{Fore.RED}✗ Upload failed (exit code: {result}){Style.RESET_ALL}') - print(f'{Fore.YELLOW}Files preserved locally due to upload failure{Style.RESET_ALL}') - if notification_callback: - notification_callback(f'✗ Upload Failed - {filename_base}', - f'Upload failed with code {result}. Files preserved locally.') - return False + + print(f'{Fore.RED}✗ Upload failed{Style.RESET_ALL}') + print(f'{Fore.YELLOW}Files preserved locally due to upload failure{Style.RESET_ALL}') + if notification_callback: + notification_callback(f'✗ Upload Failed - {filename_base}', + 'Upload failed. Files preserved locally. Check rclone output above.') + return False except Exception as e: print(f'{Fore.RED}✗ Upload error: {str(e)}{Style.RESET_ALL}') diff --git a/modules/processor.py b/modules/processor.py index b1cae71..9e4518b 100644 --- a/modules/processor.py +++ b/modules/processor.py @@ -37,37 +37,78 @@ class StreamProcessor: os_type ) - def process_raw_stream(self, raw_path: str, output_path: str) -> None: + def process_raw_stream(self, raw_path: str, output_path: str) -> bool: """ Process raw .ts file into mp4/mp3 using ffmpeg. Args: raw_path: Path to the raw .ts file output_path: Path for the processed output file + + Returns: + bool: True when conversion succeeded, False otherwise """ if not os.path.exists(raw_path): print(f'{Fore.YELLOW}⚠ Raw file not found, skipping processing{Style.RESET_ALL}') - return + return False if self.only_raw: print(f'{Fore.CYAN}Keeping raw .ts file (onlyRaw mode){Style.RESET_ALL}') - return + return False print(f'{Fore.YELLOW}Processing raw stream file...{Style.RESET_ALL}') # Build ffmpeg command based on quality if self.quality == 'audio_only': - self._process_audio(raw_path, output_path) + result = self._process_audio(raw_path, output_path) else: - self._process_video(raw_path, output_path) + result = self._process_video(raw_path, output_path) - print(f'{Fore.GREEN}✓ Stream processed successfully{Style.RESET_ALL}') + if result: + print(f'{Fore.GREEN}✓ Stream processed successfully{Style.RESET_ALL}') + else: + print(f'{Fore.RED}✗ Stream processing failed{Style.RESET_ALL}') + + return result - def _process_audio(self, raw_path: str, output_path: str) -> None: + def _run_ffmpeg_command(self, cmd: list, output_path: str) -> bool: + """Run FFmpeg while streaming its output to the terminal.""" + print(f'{Fore.CYAN}Running FFmpeg: {' '.join(cmd)}{Style.RESET_ALL}') + + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + encoding='utf-8', + errors='replace' + ) + + if process.stdout: + for line in process.stdout: + print(line, end='') + + result = process.wait() + if result != 0: + print(f'{Fore.RED}✗ FFmpeg exited with code: {result}{Style.RESET_ALL}') + return False + + if not os.path.exists(output_path): + print(f'{Fore.RED}✗ FFmpeg did not create output: {output_path}{Style.RESET_ALL}') + return False + + if os.path.getsize(output_path) == 0: + print(f'{Fore.RED}✗ FFmpeg created an empty output file: {output_path}{Style.RESET_ALL}') + return False + + return True + + def _process_audio(self, raw_path: str, output_path: str) -> bool: """Process audio-only stream.""" # Audio-only conversion with modern AAC encoding cmd = [ self.ffmpeg_path, + '-y', '-i', raw_path, '-vn', # No video '-c:a', self.ffmpeg_audio_codec, @@ -85,14 +126,9 @@ class StreamProcessor: cmd.extend(['-movflags', '+faststart']) cmd.append(output_path) - - # Run FFmpeg - if self.ffmpeg_progress: - subprocess.call(cmd) - else: - subprocess.call(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) + return self._run_ffmpeg_command(cmd, output_path) - def _process_video(self, raw_path: str, output_path: str) -> None: + def _process_video(self, raw_path: str, output_path: str) -> bool: """Process video stream.""" cmd = [ self.ffmpeg_path, @@ -135,12 +171,7 @@ class StreamProcessor: cmd.extend(['-movflags', '+faststart']) cmd.append(output_path) - - # Run FFmpeg - if self.ffmpeg_progress: - subprocess.call(cmd) - else: - subprocess.call(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) + return self._run_ffmpeg_command(cmd, output_path) def build_chat_output_args(self) -> str: """ diff --git a/modules/recorder.py b/modules/recorder.py index fa9c9d1..3518591 100644 --- a/modules/recorder.py +++ b/modules/recorder.py @@ -7,6 +7,8 @@ import subprocess from typing import Dict, Any, Optional from colorama import Fore, Style +from .utils import get_env_value + class StreamRecorder: """Handles live stream recording using streamlink.""" @@ -68,7 +70,7 @@ class StreamRecorder: print(f'{Fore.YELLOW} Consider disabling streamlink_ttvlol in config or using alternative methods{Style.RESET_ALL}') # Add authentication if available - oauth_token = os.getenv("OAUTH-PRIVATE-TOKEN", "") + oauth_token = get_env_value("OAUTH-PRIVATE-TOKEN", "OAUTH_PRIVATE_TOKEN", default="") if oauth_token and oauth_token != "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx": cmd.extend(['--twitch-api-header', f'Authorization=OAuth {oauth_token}']) diff --git a/modules/stream_monitor.py b/modules/stream_monitor.py index bea3545..a6b89be 100644 --- a/modules/stream_monitor.py +++ b/modules/stream_monitor.py @@ -9,6 +9,7 @@ import requests from colorama import Fore, Style from .constants import TWITCH_OAUTH_URL, TWITCH_API_URL, TWITCH_GQL_URL, TWITCH_GQL_CLIENT_ID +from .utils import get_env_value class StreamMonitor: @@ -40,7 +41,9 @@ class StreamMonitor: return self._oauth_token try: - url = f"{TWITCH_OAUTH_URL}?client_id={os.getenv('CLIENT-ID')}&client_secret={os.getenv('CLIENT-SECRET')}&grant_type=client_credentials" + client_id = get_env_value('CLIENT-ID', 'CLIENT_ID') + client_secret = get_env_value('CLIENT-SECRET', 'CLIENT_SECRET') + url = f"{TWITCH_OAUTH_URL}?client_id={client_id}&client_secret={client_secret}&grant_type=client_credentials" response = requests.post(url, timeout=15) response.raise_for_status() self._oauth_token = response.json()['access_token'] @@ -69,7 +72,7 @@ class StreamMonitor: url = f'{TWITCH_API_URL}/users?login={self.username}' headers = { "Authorization": f"Bearer {self.get_oauth_token()}", - "Client-ID": os.getenv('CLIENT-ID') + "Client-ID": get_env_value('CLIENT-ID', 'CLIENT_ID') } response = requests.get(url, headers=headers, timeout=15) response.raise_for_status() diff --git a/modules/utils.py b/modules/utils.py index 280afa8..c8bc749 100644 --- a/modules/utils.py +++ b/modules/utils.py @@ -4,6 +4,7 @@ Utility functions and helpers for Twitch Archive. import os import sys +import shutil import pathlib import subprocess from typing import Optional @@ -35,6 +36,15 @@ def get_bin_path() -> str: return str(pathlib.Path(__file__).parent.parent.resolve() / "bin") +def get_env_value(*names: str, default: Optional[str] = None) -> Optional[str]: + """Return the first non-empty environment variable from the provided names.""" + for name in names: + value = os.getenv(name) + if value not in (None, ""): + return value + return default + + def get_ffmpeg_executable(os_type: str) -> str: """ Get the platform-specific ffmpeg executable path. @@ -48,6 +58,11 @@ def get_ffmpeg_executable(os_type: str) -> str: bin_path = get_bin_path() if os_type == 'windows': return os.path.join(bin_path, 'ffmpeg.exe') + + system_ffmpeg = shutil.which('ffmpeg') + if system_ffmpeg: + return system_ffmpeg + return os.path.join(bin_path, 'ffmpeg') @@ -64,6 +79,11 @@ def get_twitch_downloader_executable(os_type: str) -> str: bin_path = get_bin_path() if os_type == 'windows': return os.path.join(bin_path, 'TwitchDownloaderCLI.exe') + + system_twitch_downloader = shutil.which('TwitchDownloaderCLI') + if system_twitch_downloader: + return system_twitch_downloader + return os.path.join(bin_path, 'TwitchDownloaderCLI') @@ -164,6 +184,24 @@ def verify_streamlink() -> bool: return False +def verify_rclone() -> bool: + """Verify that rclone is available on PATH.""" + try: + result = subprocess.run(['rclone', 'version'], + capture_output=True, + text=True, + timeout=5) + if result.returncode == 0: + version_line = result.stdout.strip().splitlines()[0] if result.stdout.strip() else 'unknown' + print(f'{Fore.GREEN}✓ Rclone found ({version_line}){Style.RESET_ALL}') + return True + raise FileNotFoundError() + except (FileNotFoundError, subprocess.TimeoutExpired, IndexError): + print(f'{Fore.RED}✗ ERROR: rclone not found{Style.RESET_ALL}') + print(f'{Fore.CYAN} → Install rclone and ensure it is on PATH{Style.RESET_ALL}') + return False + + def verify_ffmpeg(os_type: str) -> bool: """ Verify that ffmpeg is available. diff --git a/test_twitch_archive_simple.py b/test_twitch_archive_simple.py index 0d49da2..96a5f26 100644 --- a/test_twitch_archive_simple.py +++ b/test_twitch_archive_simple.py @@ -20,12 +20,27 @@ import sys import os import json import getopt +import tempfile +import importlib.util +from pathlib import Path from unittest.mock import patch, MagicMock, Mock # Add parent directory to path for imports sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from modules.constants import DEFAULT_CONFIG +from modules.file_manager import FileManager +from modules.downloader import ContentDownloader +from modules.utils import get_ffmpeg_executable, get_twitch_downloader_executable + + +def load_twitch_archive_module(): + """Load the main script module for targeted regression tests.""" + module_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'twitch-archive.py') + spec = importlib.util.spec_from_file_location('twitch_archive_main', module_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module class TestCommandLineArgumentParsing(unittest.TestCase): @@ -115,6 +130,34 @@ class TestCommandLineArgumentParsing(unittest.TestCase): self.assertEqual(len(opts), 1) self.assertEqual(opts[0], ('--chat-only', '')) + + def test_rclone_smoke_test_option(self): + """Test --rclone-smoke-test option parsing.""" + argv = ['--rclone-smoke-test'] + opts, args = getopt.getopt( + argv, + "hu:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "healthcheck", "rclone-smoke-test", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 1) + self.assertEqual(opts[0], ('--rclone-smoke-test', '')) + + def test_healthcheck_option(self): + """Test --healthcheck option parsing.""" + argv = ['--healthcheck'] + opts, args = getopt.getopt( + argv, + "hu:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "healthcheck", "rclone-smoke-test", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 1) + self.assertEqual(opts[0], ('--healthcheck', '')) def test_legacy_option(self): """Test --legacy option parsing.""" @@ -439,6 +482,161 @@ class TestConfigLogic(unittest.TestCase): self.assertIn('$schema', default_config) +class TestFileManagerUploadPaths(unittest.TestCase): + """Test rclone upload path preparation.""" + + def test_build_upload_relative_paths_uses_forward_slashes(self): + """Rclone --files-from entries must use POSIX separators on Windows.""" + with tempfile.TemporaryDirectory() as temp_dir: + manager = FileManager( + root_path=temp_dir, + username='testuser', + config={ + 'uploadCloud': True, + 'uploadPreMergeVideo': True, + 'uploadMergedVideo': True, + 'uploadChatVideo': True + } + ) + + relative_paths = manager._build_upload_relative_paths('20260424_12h00m00s') + + self.assertTrue(relative_paths) + self.assertTrue(all('\\' not in path for path in relative_paths)) + self.assertIn('testuser/metadata/METADA_20260424_12h00m00s.json', relative_paths) + self.assertIn('testuser/chat/json/CHAT_20260424_12h00m00s.json', relative_paths) + + +class TestDownloaderConfiguration(unittest.TestCase): + """Regression tests for downloader config wiring.""" + + def test_download_vod_method_not_shadowed_by_boolean_flag(self): + """Config booleans must not overwrite callable downloader methods.""" + downloader = ContentDownloader( + twitch_downloader_path='TwitchDownloaderCLI', + ffmpeg_path='ffmpeg', + config={ + 'downloadVOD': True, + 'downloadCHAT': True, + 'downloadLiveCHAT': True + } + ) + + self.assertTrue(callable(downloader.download_vod)) + self.assertTrue(downloader.download_vod_enabled) + + +class TestLinuxToolResolution(unittest.TestCase): + """Ensure Linux containers prefer their own installed toolchain.""" + + @patch('modules.utils.shutil.which') + def test_linux_prefers_system_ffmpeg(self, mock_which): + mock_which.return_value = '/usr/bin/ffmpeg' + + self.assertEqual(get_ffmpeg_executable('linux'), '/usr/bin/ffmpeg') + + @patch('modules.utils.shutil.which') + def test_linux_prefers_system_twitch_downloader(self, mock_which): + mock_which.return_value = '/usr/local/bin/TwitchDownloaderCLI' + + self.assertEqual(get_twitch_downloader_executable('linux'), '/usr/local/bin/TwitchDownloaderCLI') + + +class TestMultiStreamerCleanupRegression(unittest.TestCase): + """Regression tests for multi-streamer conversion and cleanup behavior.""" + + def setUp(self): + self.module = load_twitch_archive_module() + + def _build_archiver(self, temp_dir: str, upload_cloud: bool = True): + archiver = MagicMock() + archiver.username = 'maddoscientist0' + archiver.os_type = 'linux' + archiver.quality = 'best' + archiver.downloadLiveCHAT = False + archiver.downloadCHAT = False + archiver.downloadVOD = False + archiver.downloadMETADATA = False + archiver.mergeVideoChat = False + archiver.mergeChatLayout = 'side-by-side' + archiver.onlyRaw = False + archiver.vodTimeout = 0 + archiver.shutdown_requested = False + archiver.deleteFiles = True + archiver.uploadCloud = upload_cloud + + archiver.notification_manager = MagicMock() + archiver.recorder = MagicMock() + archiver.processor = MagicMock() + archiver.downloader = MagicMock() + archiver.stream_monitor = MagicMock() + archiver.file_manager = MagicMock() + archiver.file_manager.raw_path = Path(temp_dir) / 'raw' + archiver.file_manager.video_path = Path(temp_dir) / 'video' + archiver.file_manager.chat_json_path = Path(temp_dir) / 'chat_json' + archiver.file_manager.chat_mp4_path = Path(temp_dir) / 'chat' + + os.makedirs(archiver.file_manager.raw_path, exist_ok=True) + os.makedirs(archiver.file_manager.video_path, exist_ok=True) + os.makedirs(archiver.file_manager.chat_json_path, exist_ok=True) + os.makedirs(archiver.file_manager.chat_mp4_path, exist_ok=True) + + return archiver + + def test_process_stream_keeps_raw_when_conversion_fails(self): + manager = self.module.TwitchArchiveManager(specific_streamer='maddoscientist0') + + with tempfile.TemporaryDirectory() as temp_dir: + archiver = self._build_archiver(temp_dir) + archiver.processor.process_raw_stream.return_value = False + archiver.file_manager.upload_to_cloud.return_value = True + + def write_raw_file(_stream_info, raw_path): + with open(raw_path, 'wb') as handle: + handle.write(b'x' * 4096) + return True + + archiver.recorder.record.side_effect = write_raw_file + + stream_info = { + 'title': 'Test', + 'createdAt': '2026-04-25T09:14:01Z' + } + + manager._process_stream(archiver, stream_info, 'stream-id') + + archiver.file_manager.clean_raw_file.assert_not_called() + archiver.file_manager.delete_local_files.assert_called_once() + + def test_process_stream_only_deletes_rendered_files_after_real_upload(self): + manager = self.module.TwitchArchiveManager(specific_streamer='maddoscientist0') + + with tempfile.TemporaryDirectory() as temp_dir: + archiver = self._build_archiver(temp_dir, upload_cloud=False) + archiver.processor.process_raw_stream.return_value = True + archiver.file_manager.upload_to_cloud.return_value = True + + def write_raw_file(_stream_info, raw_path): + with open(raw_path, 'wb') as handle: + handle.write(b'x' * 4096) + return True + + archiver.recorder.record.side_effect = write_raw_file + + stream_info = { + 'title': 'Test', + 'createdAt': '2026-04-25T09:14:01Z' + } + + manager._process_stream(archiver, stream_info, 'stream-id') + + archiver.file_manager.clean_raw_file.assert_called_once() + archiver.file_manager.delete_local_files.assert_not_called() + + upload_filename_base = archiver.file_manager.upload_to_cloud.call_args.args[0] + self.assertFalse(upload_filename_base.startswith('LIVE_')) + + if __name__ == '__main__': # Run tests with verbose output print("="*70) diff --git a/twitch-archive.py b/twitch-archive.py index 2fdcb4e..81f9526 100644 --- a/twitch-archive.py +++ b/twitch-archive.py @@ -34,6 +34,8 @@ import time import json import signal import getopt +import pathlib +import subprocess from typing import Dict, Optional, Any from datetime import datetime, timedelta @@ -48,7 +50,8 @@ from modules.config import ConfigManager from modules.notifications import NotificationManager from modules.utils import ( detect_operating_system, get_ffmpeg_executable, get_twitch_downloader_executable, - get_unique_filename, get_video_duration, verify_streamlink, verify_ffmpeg, verify_twitch_downloader + get_unique_filename, get_video_duration, verify_streamlink, verify_ffmpeg, verify_twitch_downloader, + verify_rclone, get_env_value ) from modules.stream_monitor import StreamMonitor from modules.recorder import StreamRecorder @@ -147,10 +150,25 @@ class TwitchArchive: Raises: SystemExit: If .env file is not found """ - if not load_dotenv(find_dotenv()): + dotenv_loaded = load_dotenv(find_dotenv()) + has_required_env = bool( + get_env_value('CLIENT-ID', 'CLIENT_ID') and + get_env_value('CLIENT-SECRET', 'CLIENT_SECRET') + ) + + if not dotenv_loaded and has_required_env: + print(f'{Fore.GREEN}✓ Twitch API credentials loaded from process environment{Style.RESET_ALL}') + return + + if not dotenv_loaded and not has_required_env: print(f'{Fore.RED}✗ ERROR: .env file not found{Style.RESET_ALL}') - print(f'{Fore.CYAN} → Create a .env file with your Twitch API credentials{Style.RESET_ALL}') - print(f'{Fore.CYAN} → Required: CLIENT-ID, CLIENT-SECRET{Style.RESET_ALL}') + print(f'{Fore.CYAN} → Create a .env file with your Twitch API credentials or pass them via environment variables{Style.RESET_ALL}') + print(f'{Fore.CYAN} → Required: CLIENT-ID/CLIENT_ID and CLIENT-SECRET/CLIENT_SECRET{Style.RESET_ALL}') + sys.exit(1) + + if not has_required_env: + print(f'{Fore.RED}✗ ERROR: Twitch API credentials are missing{Style.RESET_ALL}') + print(f'{Fore.CYAN} → Required: CLIENT-ID/CLIENT_ID and CLIENT-SECRET/CLIENT_SECRET{Style.RESET_ALL}') sys.exit(1) def _initialize_components(self) -> None: @@ -259,6 +277,8 @@ class TwitchArchive: verify_ffmpeg(self.os_type) if self.downloadVOD or self.downloadCHAT: verify_twitch_downloader(self.os_type) + if self.uploadCloud and not verify_rclone(): + sys.exit(1) # Print configuration summary self._print_configuration_summary() @@ -404,7 +424,7 @@ class TwitchArchive: print(f'{Fore.YELLOW}Attempting to process any recorded content...{Style.RESET_ALL}') # Process the raw stream file - self.processor.process_raw_stream(live_raw_path, live_proc_path) + processing_succeeded = self.processor.process_raw_stream(live_raw_path, live_proc_path) # Wait for live chat download if it was started live_chat_downloaded = False @@ -426,7 +446,11 @@ class TwitchArchive: else: # Get video duration first (needed for chat conversion and trimming) ffmpeg_path = get_ffmpeg_executable(self.os_type) - video_duration = get_video_duration(live_proc_path, ffmpeg_path) + if not processing_succeeded or not os.path.exists(live_proc_path): + print(f'{Fore.YELLOW}⚠ Processed video file is unavailable, skipping chat render{Style.RESET_ALL}') + video_duration = None + else: + video_duration = get_video_duration(live_proc_path, ffmpeg_path) print(f'{Fore.CYAN}Video duration for chat rendering: {video_duration}s{Style.RESET_ALL}') # Convert chat format if needed (chat_downloader uses different JSON structure) @@ -561,7 +585,10 @@ class TwitchArchive: print(f'{Fore.YELLOW}⚠ No matching VOD found for this stream{Style.RESET_ALL}') # Clean up raw files if configured - self.file_manager.clean_raw_file(live_raw_path) + if processing_succeeded: + self.file_manager.clean_raw_file(live_raw_path) + elif os.path.exists(live_raw_path): + print(f'{Fore.YELLOW}⚠ Keeping raw file because conversion did not complete successfully{Style.RESET_ALL}') # Upload to cloud if configured upload_success = self.file_manager.upload_to_cloud( @@ -570,7 +597,7 @@ class TwitchArchive: ) # Delete local files if configured and upload succeeded - if self.deleteFiles and upload_success: + if self.deleteFiles and self.uploadCloud and upload_success: self.file_manager.delete_local_files( filename_base, live_raw_path, @@ -890,6 +917,8 @@ class TwitchArchiveManager: verify_ffmpeg(first_archiver.os_type) if first_archiver.downloadVOD or first_archiver.downloadCHAT: verify_twitch_downloader(first_archiver.os_type) + if any(archiver.uploadCloud for archiver in self.archivers.values()) and not verify_rclone(): + sys.exit(1) # Print configuration summary for each streamer for username, archiver in self.archivers.items(): @@ -1018,7 +1047,7 @@ class TwitchArchiveManager: # Generate timestamp and filename timestamp = datetime.now(timezone('UTC')).strftime("%Y%m%d_%Hh%Mm%Ss") - filename_base = f"{PREFIX_LIVE}{archiver.username}_{timestamp}" + filename_base = f"{archiver.username}_{timestamp}" # Parse stream start time live_date = datetime.strptime( @@ -1029,8 +1058,8 @@ class TwitchArchiveManager: raw_extension = '.ts' proc_extension = '.mp3' if archiver.quality == 'audio_only' else '.mp4' - live_raw_path = str(archiver.file_manager.raw_path / f"{filename_base}{raw_extension}") - live_proc_path = str(archiver.file_manager.video_path / f"{filename_base}{proc_extension}") + live_raw_path = str(archiver.file_manager.raw_path / f"{PREFIX_LIVE}{filename_base}{raw_extension}") + live_proc_path = str(archiver.file_manager.video_path / f"{PREFIX_LIVE}{filename_base}{proc_extension}") chat_json_path = str(archiver.file_manager.chat_json_path / f"{PREFIX_CHAT}{filename_base}.json") # Send notification @@ -1172,8 +1201,9 @@ class TwitchArchiveManager: print(f'{Fore.CYAN}Processing recorded content ({file_size / (1024*1024):.2f} MB)...{Style.RESET_ALL}') # Process raw stream + processing_succeeded = False if not archiver.onlyRaw: - archiver.processor.process_raw_stream(live_raw_path, live_proc_path) + processing_succeeded = archiver.processor.process_raw_stream(live_raw_path, live_proc_path) # Wait for live chat download if it was started live_chat_downloaded = False @@ -1212,8 +1242,12 @@ class TwitchArchiveManager: chat_rendered_successfully = False else: # Get video duration first - ffmpeg_path = get_ffmpeg_executable(archiver.os_type) - video_duration = get_video_duration(live_proc_path, ffmpeg_path) + if not processing_succeeded or not os.path.exists(live_proc_path): + print(f'{Fore.YELLOW}⚠ Processed video file is unavailable, skipping chat render{Style.RESET_ALL}') + video_duration = None + else: + ffmpeg_path = get_ffmpeg_executable(archiver.os_type) + video_duration = get_video_duration(live_proc_path, ffmpeg_path) if video_duration is None: print(f'{Fore.YELLOW}⚠ Could not detect video duration from {live_proc_path}{Style.RESET_ALL}') @@ -1362,7 +1396,10 @@ class TwitchArchiveManager: archiver.file_manager.save_metadata(stream_info, filename_base) # Clean up raw file if configured - archiver.file_manager.clean_raw_file(live_raw_path) + if processing_succeeded: + archiver.file_manager.clean_raw_file(live_raw_path) + elif os.path.exists(live_raw_path): + print(f'{Fore.YELLOW}⚠ Keeping raw file because conversion did not complete successfully{Style.RESET_ALL}') # Upload to cloud if configured upload_success = archiver.file_manager.upload_to_cloud( @@ -1371,7 +1408,7 @@ class TwitchArchiveManager: ) # Delete files if configured - if archiver.deleteFiles and upload_success: + if archiver.deleteFiles and archiver.uploadCloud and upload_success: archiver.file_manager.delete_local_files( filename_base, live_raw_path, @@ -1386,6 +1423,79 @@ class TwitchArchiveManager: ) +def run_rclone_smoke_test(specific_streamer: Optional[str] = None) -> int: + """Run a one-off rclone smoke test using the configured upload destination.""" + config_manager = ConfigManager() + + if specific_streamer: + username = specific_streamer + else: + enabled_streamers = config_manager.get_all_enabled_streamers() + if not enabled_streamers: + print(f'{Fore.RED}✗ No enabled streamers available for smoke test{Style.RESET_ALL}') + print(f'{Fore.CYAN}→ Use -u or enable a streamer config{Style.RESET_ALL}') + return 1 + username = enabled_streamers[0] + + config = config_manager.load_streamer_config(username) + file_manager = FileManager( + root_path=config.get('root_path', 'archive'), + username=username, + config=config + ) + file_manager.initialize_directories() + + print(f'\n{Fore.CYAN}{"=" * 70}{Style.RESET_ALL}') + print(f'{Fore.CYAN}TWITCH ARCHIVE - Rclone Smoke Test{Style.RESET_ALL}') + print(f'{Fore.CYAN}{"=" * 70}{Style.RESET_ALL}') + print(f'{Fore.GREEN}Streamer: {username}{Style.RESET_ALL}') + print(f'{Fore.GREEN}Remote: {config.get("rclone_path", "")}{Style.RESET_ALL}\n') + + return 0 if file_manager.run_rclone_smoke_test() else 1 + + +def run_healthcheck(specific_streamer: Optional[str] = None) -> int: + """Run a local readiness check suitable for Docker health checks.""" + config_manager = ConfigManager() + + if specific_streamer: + username = specific_streamer + else: + enabled_streamers = config_manager.get_all_enabled_streamers() + username = enabled_streamers[0] if enabled_streamers else 'vinesauce' + + config = config_manager.load_streamer_config(username) + archive = TwitchArchive(config) + + try: + archive._load_environment_variables() + except SystemExit: + return 1 + + archive._initialize_components() + + checks_ok = True + if not verify_streamlink(): + checks_ok = False + if not verify_ffmpeg(archive.os_type): + checks_ok = False + if (archive.downloadVOD or archive.downloadCHAT) and not verify_twitch_downloader(archive.os_type): + checks_ok = False + if archive.uploadCloud: + if not verify_rclone(): + checks_ok = False + rclone_config_path = os.getenv('RCLONE_CONFIG') + if rclone_config_path and not os.path.exists(rclone_config_path): + print(f'{Fore.RED}✗ ERROR: RCLONE_CONFIG points to a missing file: {rclone_config_path}{Style.RESET_ALL}') + checks_ok = False + + if not checks_ok: + return 1 + + print(f'{Fore.GREEN}✓ Healthcheck OK for {username}{Style.RESET_ALL}') + return 0 + + # ============================================================================ # COMMAND-LINE INTERFACE # ============================================================================ @@ -1401,6 +1511,8 @@ def main(argv: list) -> None: """ specific_streamer = None use_legacy_mode = False + rclone_smoke_test_mode = False + healthcheck_mode = False help_msg = f''' {Fore.CYAN}{"=" * 70} @@ -1427,6 +1539,8 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving --legacy Force legacy mode (use config.json) --chat-only Test mode: Only download chat (skip video recording) Automatically enables verbose logging + --healthcheck Validate config and tool availability, then exit + --rclone-smoke-test Create a small test file and upload it with rclone --use-chat-downloader-primary Use chat_downloader as primary chat source (for testing) --no-chat-downloader-fallback Disable chat_downloader fallback @@ -1464,7 +1578,7 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving "h:u:q:a:v:c:m:r:d:n:", ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", - "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + "chat-only", "healthcheck", "rclone-smoke-test", "use-chat-downloader-primary", "no-chat-downloader-fallback"] ) except getopt.GetoptError as e: print(f'{Fore.RED}Error: {e}{Style.RESET_ALL}\n') @@ -1491,6 +1605,10 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving elif opt == "--chat-only": chat_only_mode = True verbose_mode = True # Auto-enable verbose for chat-only mode + elif opt == "--healthcheck": + healthcheck_mode = True + elif opt == "--rclone-smoke-test": + rclone_smoke_test_mode = True elif opt == "--legacy": use_legacy_mode = True elif opt == "--use-chat-downloader-primary": @@ -1523,6 +1641,12 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving legacy_overrides['deleteFiles'] = bool(int(arg)) elif opt in ("-n", "--notifications"): legacy_overrides['notifications'] = bool(int(arg)) + + if rclone_smoke_test_mode: + sys.exit(run_rclone_smoke_test(specific_streamer)) + + if healthcheck_mode: + sys.exit(run_healthcheck(specific_streamer)) # Determine which mode to use if use_legacy_mode or (legacy_config_exists and not specific_streamer and not os.path.exists('config/global.json')): From ec44981a9d460df8610e112c1f83191cdc0fdcb8 Mon Sep 17 00:00:00 2001 From: MaddoScientisto Date: Sat, 25 Apr 2026 12:28:59 +0200 Subject: [PATCH 10/10] Add NVIDIA support for FFmpeg in Docker and enhance chat rendering functionality - Introduced a new docker-compose.nvidia.yml for NVIDIA GPU support. - Updated dockerstart.bat to allow optional NVIDIA runtime. - Enhanced ContentDownloader to manage chat rendering status and font settings. - Improved hardware acceleration detection in utils.py. - Added tests for hardware acceleration and chat rendering behavior. Co-authored-by: Copilot --- README.md | 11 ++++ docker-compose.nvidia.yml | 6 ++ dockerstart.bat | 11 +++- modules/downloader.py | 16 ++++- modules/processor.py | 3 +- modules/utils.py | 55 +++++++++++++--- test_twitch_archive_simple.py | 118 +++++++++++++++++++++++++++++++++- twitch-archive.py | 24 +++++-- 8 files changed, 226 insertions(+), 18 deletions(-) create mode 100644 docker-compose.nvidia.yml diff --git a/README.md b/README.md index 5005181..979e223 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,17 @@ Or use the Windows helper: That batch launcher mirrors the old pattern and expands to a compose `run` command, so you can test any streamer manually. +If the host has the NVIDIA Container Toolkit installed and you want FFmpeg/NVENC inside the container, use the optional NVIDIA override: + +```powershell +.\dockerrebuild.bat +.\dockerstart.bat --nvidia vinesauce --verbose +``` + +The image built by `.\dockerrebuild.bat` already includes the NVIDIA-capable FFmpeg/container toolchain. The optional [docker-compose.nvidia.yml](docker-compose.nvidia.yml) layer is only for runtime GPU passthrough: it requests `gpus: all` and sets `NVIDIA_VISIBLE_DEVICES` plus `NVIDIA_DRIVER_CAPABILITIES=compute,utility,video` for the container. + +On systems without NVIDIA support, keep using the normal command without `--nvidia`; the image still builds the same way, it just runs without GPU passthrough. + ### Healthcheck and smoke tests - Container healthcheck command: `python twitch-archive.py --healthcheck -u vinesauce` diff --git a/docker-compose.nvidia.yml b/docker-compose.nvidia.yml new file mode 100644 index 0000000..19a51b5 --- /dev/null +++ b/docker-compose.nvidia.yml @@ -0,0 +1,6 @@ +services: + twitch-archive: + gpus: all + environment: + NVIDIA_VISIBLE_DEVICES: ${NVIDIA_VISIBLE_DEVICES:-all} + NVIDIA_DRIVER_CAPABILITIES: ${NVIDIA_DRIVER_CAPABILITIES:-compute,utility,video} diff --git a/dockerstart.bat b/dockerstart.bat index 9d1e947..38e7242 100644 --- a/dockerstart.bat +++ b/dockerstart.bat @@ -1,8 +1,15 @@ @echo off setlocal +set NVIDIA_COMPOSE= + +if /I "%~1"=="--nvidia" ( + set NVIDIA_COMPOSE=-f docker-compose.nvidia.yml + shift +) + if "%~1"=="" ( - echo Usage: .\dockerstart.bat streamer [additional args] + echo Usage: .\dockerstart.bat [--nvidia] streamer [additional args] exit /b 1 ) @@ -17,4 +24,4 @@ shift goto collect_args :run_compose -docker compose --env-file .env.development -f docker-compose.yml -f docker-compose.override.yml run --rm twitch-archive python twitch-archive.py -u %STREAMER%%EXTRA_ARGS% \ No newline at end of file +docker compose --env-file .env.development -f docker-compose.yml -f docker-compose.override.yml %NVIDIA_COMPOSE% run --rm twitch-archive python twitch-archive.py -u %STREAMER%%EXTRA_ARGS% \ No newline at end of file diff --git a/modules/downloader.py b/modules/downloader.py index 9f9cc1f..87fe30d 100644 --- a/modules/downloader.py +++ b/modules/downloader.py @@ -4,6 +4,7 @@ Includes fallback support for chat_downloader when VOD-based methods fail. """ import os +import sys import subprocess import json import threading @@ -45,6 +46,10 @@ class ContentDownloader: self.download_live_chat_enabled = config.get('downloadLiveCHAT', True) self.use_chat_downloader_primary = config.get('useChatDownloaderPrimary', False) self.use_chat_downloader_fallback = config.get('useChatDownloaderFallback', True) + default_chat_font = 'Arial' if sys.platform.startswith('win') else 'DejaVu Sans' + self.chat_render_font = config.get('chat_render_font', default_chat_font) + self.last_chat_render_attempted = False + self.last_chat_render_succeeded = False # Initialize chat_downloader if available self.chat_downloader = None @@ -61,6 +66,11 @@ class ContentDownloader: self.chat_thread = None self.chat_thread_success = False self.chat_thread_error = None + + def reset_chat_render_status(self) -> None: + """Reset chat render tracking before a processing pass.""" + self.last_chat_render_attempted = False + self.last_chat_render_succeeded = False def download_vod(self, vod_info: Dict[str, Any], output_path: str) -> bool: """ @@ -190,7 +200,7 @@ class ContentDownloader: '-h', '1080', '--framerate', '30', '--outline', - '-f', 'Arial', + '-f', self.chat_render_font, '--font-size', '22', '--update-rate', '1.0', '--offline', @@ -215,6 +225,9 @@ class ContentDownloader: try: print(f'{Fore.YELLOW}Rendering chat video...{Style.RESET_ALL}') + print(f'{Fore.CYAN}Using chat font: {self.chat_render_font}{Style.RESET_ALL}') + self.last_chat_render_attempted = True + self.last_chat_render_succeeded = False # Build complete command full_cmd = [self.twitch_downloader_path, 'chatrender', '-i', json_path, '-o', video_path] + chat_settings @@ -249,6 +262,7 @@ class ContentDownloader: print(f'{Fore.RED}✗ Chat video file is too small ({file_size} bytes){Style.RESET_ALL}') return False + self.last_chat_render_succeeded = True print(f'{Fore.GREEN}✓ Chat rendered ({file_size:,} bytes){Style.RESET_ALL}') return True diff --git a/modules/processor.py b/modules/processor.py index 9e4518b..58bd47f 100644 --- a/modules/processor.py +++ b/modules/processor.py @@ -6,7 +6,7 @@ import os import subprocess from colorama import Fore, Style -from .utils import detect_hardware_acceleration, get_hwaccel_encoder +from .utils import detect_hardware_acceleration, get_hwaccel_encoder, resolve_hwaccel_type class StreamProcessor: @@ -36,6 +36,7 @@ class StreamProcessor: config.get('ffmpeg_hwaccel', 'auto'), os_type ) + self.hwaccel_type = resolve_hwaccel_type(self.hwaccel_type, os_type) def process_raw_stream(self, raw_path: str, output_path: str) -> bool: """ diff --git a/modules/utils.py b/modules/utils.py index c8bc749..4e3b9b1 100644 --- a/modules/utils.py +++ b/modules/utils.py @@ -269,20 +269,57 @@ def detect_hardware_acceleration(hwaccel_config: str, os_type: str) -> Optional[ if hwaccel_config in ['nvenc', 'qsv', 'amf', 'vaapi']: return hwaccel_config - # Auto-detect: try to determine available hardware + # Auto-detect: choose only hardware we can reasonably prove is present. if hwaccel_config == 'auto': - # On Windows, NVIDIA is most common - if os_type == 'windows': - # Could check for nvidia-smi, but just return 'auto' for ffmpeg to decide - return 'auto' - else: - # On Linux, VAAPI is common for Intel/AMD, or NVENC for NVIDIA - # Let ffmpeg auto-detect - return 'auto' + if is_nvidia_runtime_available(): + return 'nvenc' + if is_vaapi_runtime_available(): + return 'vaapi' + return 'none' return None +def is_nvidia_runtime_available() -> bool: + """Return True when the current runtime appears to expose an NVIDIA GPU.""" + visible_devices = os.getenv('NVIDIA_VISIBLE_DEVICES', '').strip().lower() + if visible_devices in {'void', 'none'}: + return False + if visible_devices and visible_devices != 'all': + return True + + if shutil.which('nvidia-smi'): + return True + + return any( + os.path.exists(device_path) + for device_path in ('/dev/nvidiactl', '/dev/nvidia0', '/dev/nvidia-modeset') + ) + + +def is_vaapi_runtime_available() -> bool: + """Return True when Linux VAAPI render nodes are present.""" + return any( + os.path.exists(device_path) + for device_path in ('/dev/dri/renderD128', '/dev/dri/card0') + ) + + +def resolve_hwaccel_type(hwaccel_type: Optional[str], os_type: str) -> Optional[str]: + """Return a safe hardware acceleration choice for the current runtime.""" + if hwaccel_type in (None, 'none'): + return 'none' + + if hwaccel_type == 'nvenc': + return 'nvenc' if is_nvidia_runtime_available() else 'none' + + if hwaccel_type == 'vaapi': + return 'vaapi' if is_vaapi_runtime_available() else 'none' + + # Leave explicit QSV/AMF unchanged for non-container users; container auto-detect no longer picks them blindly. + return hwaccel_type + + def get_hwaccel_encoder(hwaccel_type: str) -> str: """ Get the appropriate hardware-accelerated encoder for the given acceleration type. diff --git a/test_twitch_archive_simple.py b/test_twitch_archive_simple.py index 96a5f26..445e741 100644 --- a/test_twitch_archive_simple.py +++ b/test_twitch_archive_simple.py @@ -31,7 +31,7 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from modules.constants import DEFAULT_CONFIG from modules.file_manager import FileManager from modules.downloader import ContentDownloader -from modules.utils import get_ffmpeg_executable, get_twitch_downloader_executable +from modules.utils import get_ffmpeg_executable, get_twitch_downloader_executable, detect_hardware_acceleration, resolve_hwaccel_type def load_twitch_archive_module(): @@ -541,6 +541,69 @@ class TestLinuxToolResolution(unittest.TestCase): self.assertEqual(get_twitch_downloader_executable('linux'), '/usr/local/bin/TwitchDownloaderCLI') + @patch('modules.utils.os.path.exists', return_value=False) + @patch('modules.utils.shutil.which', return_value=None) + def test_linux_auto_hwaccel_falls_back_to_software_without_runtime(self, _mock_which, _mock_exists): + detected = detect_hardware_acceleration('auto', 'linux') + + self.assertEqual(detected, 'none') + self.assertEqual(resolve_hwaccel_type(detected, 'linux'), 'none') + + @patch('modules.utils.shutil.which', return_value='/usr/bin/nvidia-smi') + def test_linux_auto_hwaccel_uses_nvenc_when_nvidia_runtime_visible(self, _mock_which): + detected = detect_hardware_acceleration('auto', 'linux') + + self.assertEqual(detected, 'nvenc') + + +class TestChatRenderBehavior(unittest.TestCase): + """Regression tests for chat rendering defaults and retry behavior.""" + + @patch('modules.downloader.sys.platform', 'linux') + @patch('modules.downloader.subprocess.Popen') + def test_linux_chat_render_uses_container_safe_font(self, mock_popen): + with tempfile.TemporaryDirectory() as temp_dir: + json_path = os.path.join(temp_dir, 'chat.json') + video_path = os.path.join(temp_dir, 'chat.mp4') + + with open(json_path, 'w', encoding='utf-8') as handle: + json.dump( + { + 'comments': [ + { + 'message': {'body': 'hello world ' * 20}, + 'commenter': {'display_name': 'tester'} + } + ] + }, + handle + ) + + class FakeProcess: + def __init__(self, output_path: str): + self.stdout = iter(['rendering']) + self.output_path = output_path + + def wait(self): + with open(self.output_path, 'wb') as handle: + handle.write(b'x' * 2048) + return 0 + + captured = {} + + def build_process(cmd, **_kwargs): + captured['cmd'] = cmd + return FakeProcess(video_path) + + mock_popen.side_effect = build_process + + downloader = ContentDownloader('TwitchDownloaderCLI', 'ffmpeg', {'downloadCHAT': True}) + + result = downloader.render_chat(json_path, video_path, '-c:v libx264 "{save_path}"') + + self.assertTrue(result) + self.assertIn('DejaVu Sans', captured['cmd']) + class TestMultiStreamerCleanupRegression(unittest.TestCase): """Regression tests for multi-streamer conversion and cleanup behavior.""" @@ -636,6 +699,59 @@ class TestMultiStreamerCleanupRegression(unittest.TestCase): upload_filename_base = archiver.file_manager.upload_to_cloud.call_args.args[0] self.assertFalse(upload_filename_base.startswith('LIVE_')) + def test_process_stream_preserves_files_when_chat_render_fails(self): + manager = self.module.TwitchArchiveManager(specific_streamer='maddoscientist0') + + with tempfile.TemporaryDirectory() as temp_dir: + archiver = self._build_archiver(temp_dir) + archiver.downloadCHAT = True + archiver.downloadVOD = True + archiver.vodTimeout = 30 + archiver.processor.process_raw_stream.return_value = True + archiver.file_manager.upload_to_cloud.return_value = True + archiver.downloader.last_chat_render_attempted = False + archiver.downloader.last_chat_render_succeeded = False + + def write_raw_file(_stream_info, raw_path): + with open(raw_path, 'wb') as handle: + handle.write(b'x' * 4096) + return True + + def failed_chat_render(*_args, **_kwargs): + archiver.downloader.last_chat_render_attempted = True + archiver.downloader.last_chat_render_succeeded = False + return False + + archiver.recorder.record.side_effect = write_raw_file + archiver.downloader.download_and_render_chat.side_effect = failed_chat_render + archiver.stream_monitor.get_latest_vod.return_value = { + 'data': { + 'user': { + 'videos': { + 'edges': [ + { + 'node': { + 'id': '2756589076', + 'title': 'Test', + 'recordedAt': '2026-04-25T09:14:01Z' + } + } + ] + } + } + } + } + + stream_info = { + 'title': 'Test', + 'createdAt': '2026-04-25T09:14:01Z' + } + + manager._process_stream(archiver, stream_info, 'stream-id') + + archiver.file_manager.clean_raw_file.assert_not_called() + archiver.file_manager.delete_local_files.assert_not_called() + if __name__ == '__main__': # Run tests with verbose output diff --git a/twitch-archive.py b/twitch-archive.py index 81f9526..707bdfe 100644 --- a/twitch-archive.py +++ b/twitch-archive.py @@ -425,6 +425,7 @@ class TwitchArchive: # Process the raw stream file processing_succeeded = self.processor.process_raw_stream(live_raw_path, live_proc_path) + self.downloader.reset_chat_render_status() # Wait for live chat download if it was started live_chat_downloaded = False @@ -585,7 +586,14 @@ class TwitchArchive: print(f'{Fore.YELLOW}⚠ No matching VOD found for this stream{Style.RESET_ALL}') # Clean up raw files if configured - if processing_succeeded: + chat_render_retry_needed = ( + self.downloader.last_chat_render_attempted and + not self.downloader.last_chat_render_succeeded + ) + + if chat_render_retry_needed: + print(f'{Fore.YELLOW}⚠ Preserving local files because chat rendering failed and can be retried later{Style.RESET_ALL}') + elif processing_succeeded: self.file_manager.clean_raw_file(live_raw_path) elif os.path.exists(live_raw_path): print(f'{Fore.YELLOW}⚠ Keeping raw file because conversion did not complete successfully{Style.RESET_ALL}') @@ -597,7 +605,7 @@ class TwitchArchive: ) # Delete local files if configured and upload succeeded - if self.deleteFiles and self.uploadCloud and upload_success: + if self.deleteFiles and self.uploadCloud and upload_success and not chat_render_retry_needed: self.file_manager.delete_local_files( filename_base, live_raw_path, @@ -1204,6 +1212,7 @@ class TwitchArchiveManager: processing_succeeded = False if not archiver.onlyRaw: processing_succeeded = archiver.processor.process_raw_stream(live_raw_path, live_proc_path) + archiver.downloader.reset_chat_render_status() # Wait for live chat download if it was started live_chat_downloaded = False @@ -1396,7 +1405,14 @@ class TwitchArchiveManager: archiver.file_manager.save_metadata(stream_info, filename_base) # Clean up raw file if configured - if processing_succeeded: + chat_render_retry_needed = ( + archiver.downloader.last_chat_render_attempted and + not archiver.downloader.last_chat_render_succeeded + ) + + if chat_render_retry_needed: + print(f'{Fore.YELLOW}⚠ Preserving local files because chat rendering failed and can be retried later{Style.RESET_ALL}') + elif processing_succeeded: archiver.file_manager.clean_raw_file(live_raw_path) elif os.path.exists(live_raw_path): print(f'{Fore.YELLOW}⚠ Keeping raw file because conversion did not complete successfully{Style.RESET_ALL}') @@ -1408,7 +1424,7 @@ class TwitchArchiveManager: ) # Delete files if configured - if archiver.deleteFiles and archiver.uploadCloud and upload_success: + if archiver.deleteFiles and archiver.uploadCloud and upload_success and not chat_render_retry_needed: archiver.file_manager.delete_local_files( filename_base, live_raw_path,