From b50a4bad02228aa0125b1d5f304fd81d2b5a24bd Mon Sep 17 00:00:00 2001 From: MaddoScientisto Date: Tue, 10 Feb 2026 23:42:22 +0100 Subject: [PATCH] Add chat_downloader support with fallback for live chat downloads - Integrated chat_downloader as an optional dependency for downloading live chat. - Implemented fallback logic to use chat_downloader when TwitchDownloaderCLI fails or is not available. - Enhanced chat rendering process to handle different JSON formats from chat_downloader. - Updated requirements.txt to include chat-downloader package. - Modified start.bat to allow passing additional arguments for flexibility. - Improved error handling and logging for chat download processes. - Added command-line options for testing chat-only mode and configuring chat_downloader behavior. --- modules/downloader.py | 520 +++++++++++++++++++++++++++++++++++++++++- requirements.txt | 3 +- start.bat | 3 +- twitch-archive.py | 320 ++++++++++++++++++++++---- 4 files changed, 800 insertions(+), 46 deletions(-) diff --git a/modules/downloader.py b/modules/downloader.py index fcd4921..2507bd6 100644 --- a/modules/downloader.py +++ b/modules/downloader.py @@ -1,14 +1,26 @@ """ VOD and chat downloading functionality using TwitchDownloaderCLI. +Includes fallback support for chat_downloader when VOD-based methods fail. """ import os import subprocess +import json +import threading +import time from typing import Dict, Any, Optional from colorama import Fore, Style from .utils import get_bin_path +# Try to import chat_downloader (optional dependency) +try: + from chat_downloader import ChatDownloader + CHAT_DOWNLOADER_AVAILABLE = True +except ImportError: + CHAT_DOWNLOADER_AVAILABLE = False + ChatDownloader = None + class ContentDownloader: """Handles VOD and chat downloading using TwitchDownloaderCLI.""" @@ -29,6 +41,24 @@ class ContentDownloader: self.download_vod = config.get('downloadVOD', True) self.download_chat = config.get('downloadCHAT', True) self.download_live_chat = config.get('downloadLiveCHAT', True) + self.use_chat_downloader_primary = config.get('useChatDownloaderPrimary', False) + self.use_chat_downloader_fallback = config.get('useChatDownloaderFallback', True) + + # Initialize chat_downloader if available + self.chat_downloader = None + if CHAT_DOWNLOADER_AVAILABLE: + try: + self.chat_downloader = ChatDownloader() + except Exception as e: + print(f'{Fore.YELLOW}⚠ Failed to initialize chat_downloader: {e}{Style.RESET_ALL}') + elif self.use_chat_downloader_primary or self.use_chat_downloader_fallback: + print(f'{Fore.YELLOW}⚠ chat_downloader not available but requested in config{Style.RESET_ALL}') + print(f'{Fore.YELLOW} Install with: pip install chat-downloader{Style.RESET_ALL}') + + # Thread management for chat_downloader + self.chat_thread = None + self.chat_thread_success = False + self.chat_thread_error = None def download_vod(self, vod_info: Dict[str, Any], output_path: str) -> bool: """ @@ -156,6 +186,7 @@ class ContentDownloader: '--background-color', '#FF111111', '-w', '500', '-h', '1080', + '--framerate', '30', '--outline', '-f', 'Arial', '--font-size', '22', @@ -166,6 +197,9 @@ class ContentDownloader: '--collision', 'Rename' ] + # Always start from beginning + chat_settings.extend(['-b', '0']) + # Trim chat to match video duration if provided if video_duration is not None and video_duration > 0: # Format duration as seconds with 1 decimal place @@ -183,13 +217,37 @@ class ContentDownloader: # Build complete command full_cmd = [self.twitch_downloader_path, 'chatrender', '-i', json_path, '-o', video_path] + chat_settings - result = subprocess.call(full_cmd) + # Capture output to see what TwitchDownloaderCLI says + process = subprocess.Popen( + full_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + encoding='utf-8', + errors='replace' + ) + + # Stream output in real-time + for line in process.stdout: + print(line.rstrip()) + + result = process.wait() if result != 0: print(f'{Fore.RED}✗ Chat render failed with exit code: {result}{Style.RESET_ALL}') return False + + # Verify the output file was created and has content + if not os.path.exists(video_path): + print(f'{Fore.RED}✗ Chat video file was not created{Style.RESET_ALL}') + return False + + file_size = os.path.getsize(video_path) + if file_size < 1024: # Less than 1KB is suspicious + print(f'{Fore.RED}✗ Chat video file is too small ({file_size} bytes){Style.RESET_ALL}') + return False - print(f'{Fore.GREEN}✓ Chat rendered{Style.RESET_ALL}') + print(f'{Fore.GREEN}✓ Chat rendered ({file_size:,} bytes){Style.RESET_ALL}') return True except Exception as e: @@ -302,3 +360,461 @@ class ContentDownloader: except Exception as e: print(f'{Fore.RED}✗ Error waiting for chat download: {str(e)}{Style.RESET_ALL}') return False + + def download_live_chat_with_chat_downloader(self, username: str, json_path: str, + max_messages: Optional[int] = None, + timeout: Optional[float] = None, + shutdown_check: Optional[callable] = None, + verbose: bool = False) -> bool: + """ + Download live chat using chat_downloader library as fallback. + This works even when VOD is disabled on the channel. + + Args: + username: Twitch username/channel name + json_path: Path to save chat JSON + max_messages: Maximum messages to download (None = unlimited) + timeout: Stop after this many seconds (None = until stream ends) + shutdown_check: Optional callback function that returns True when shutdown requested + verbose: Show chat message previews + + Returns: + bool: True if chat download succeeded, False otherwise + """ + if not CHAT_DOWNLOADER_AVAILABLE or self.chat_downloader is None: + print(f'{Fore.RED}✗ chat_downloader not available{Style.RESET_ALL}') + print(f'{Fore.YELLOW} Install with: pip install chat-downloader{Style.RESET_ALL}') + return False + + if not self.download_live_chat: + print(f'{Fore.YELLOW}⚠ downloadLiveCHAT is disabled in config{Style.RESET_ALL}') + return False + + print(f'\n{Fore.CYAN}Starting live chat download (chat_downloader)...{Style.RESET_ALL}') + print(f'{Fore.MAGENTA}[VERBOSE] chat_downloader library version: {ChatDownloader.__module__}{Style.RESET_ALL}') + + try: + # Construct Twitch stream URL + stream_url = f'https://www.twitch.tv/{username}' + print(f'{Fore.YELLOW}Downloading chat from: {stream_url}{Style.RESET_ALL}') + print(f'{Fore.MAGENTA}[VERBOSE] Output path: {json_path}{Style.RESET_ALL}') + print(f'{Fore.MAGENTA}[VERBOSE] Timeout: {timeout}s (None = unlimited){Style.RESET_ALL}') + print(f'{Fore.MAGENTA}[VERBOSE] Max messages: {max_messages} (None = unlimited){Style.RESET_ALL}') + + # Get chat messages + print(f'{Fore.CYAN}Connecting to Twitch chat...{Style.RESET_ALL}') + chat = self.chat_downloader.get_chat( + stream_url, + message_types=['text_message'], # Basic text messages + output=json_path, + timeout=timeout, + max_messages=max_messages + ) + + # The get_chat with output parameter writes to file automatically + # We just need to iterate to trigger the download + message_count = 0 + print(f'{Fore.CYAN}Receiving chat messages (press Ctrl+C to stop)...{Style.RESET_ALL}') + try: + for message in chat: + # Check for shutdown request + if shutdown_check and shutdown_check(): + print(f'\n{Fore.YELLOW}⚠ Chat download stopped by shutdown request{Style.RESET_ALL}') + break + + message_count += 1 + + # Show progress every 100 messages + if message_count % 100 == 0: + print(f'{Fore.CYAN} Downloaded {message_count} messages...{Style.RESET_ALL}', end='\r') + + # Show chat previews in verbose mode (every 10 messages) + if verbose and message_count % 10 == 0: + author = message.get('author', {}).get('name', 'Unknown') + msg_text = message.get('message', 'N/A') + # Truncate long messages + if len(msg_text) > 60: + msg_text = msg_text[:60] + '...' + print(f'\n{Fore.GREEN}💬 {author}: {Fore.WHITE}{msg_text}{Style.RESET_ALL}') + + # Print sample message every 500 for extra debugging + if message_count % 500 == 0: + print(f'\n{Fore.MAGENTA}[VERBOSE] Sample message #{message_count}: {message.get("message", "N/A")[:50]}...{Style.RESET_ALL}') + except KeyboardInterrupt: + print(f'\n{Fore.YELLOW}⚠ Chat download interrupted by user{Style.RESET_ALL}') + except Exception as e: + # Stream might have ended + print(f'\n{Fore.YELLOW}Chat download stopped: {str(e)}{Style.RESET_ALL}') + + # Check if file was created + if os.path.exists(json_path): + file_size = os.path.getsize(json_path) + if file_size > 100: + print(f'\n{Fore.GREEN}✓ Live chat downloaded ({message_count} messages, {file_size} bytes){Style.RESET_ALL}') + return True + else: + print(f'\n{Fore.RED}✗ Chat file too small ({file_size} bytes){Style.RESET_ALL}') + return False + else: + print(f'\n{Fore.RED}✗ Chat file was not created{Style.RESET_ALL}') + return False + + except Exception as e: + print(f'{Fore.RED}✗ chat_downloader failed: {str(e)}{Style.RESET_ALL}') + import traceback + traceback.print_exc() + return False + + def start_chat_downloader_thread(self, username: str, json_path: str, + shutdown_check: Optional[callable] = None, + verbose: bool = False) -> threading.Thread: + """ + Start chat_downloader in a background thread. + + Args: + username: Twitch username + json_path: Path to save chat JSON + shutdown_check: Callback to check for shutdown + verbose: Show chat previews + + Returns: + threading.Thread: The thread running the download + """ + def download_thread(): + try: + self.chat_thread_success = self.download_live_chat_with_chat_downloader( + username, json_path, + shutdown_check=shutdown_check, + verbose=verbose + ) + except Exception as e: + self.chat_thread_error = e + self.chat_thread_success = False + print(f'\n{Fore.RED}✗ Chat thread error: {e}{Style.RESET_ALL}') + import traceback + traceback.print_exc() + + thread = threading.Thread(target=download_thread, daemon=True) + thread.start() + self.chat_thread = thread + return thread + + def wait_for_chat_thread(self, timeout: Optional[float] = None) -> bool: + """ + Wait for chat download thread to complete. + + Args: + timeout: Maximum time to wait in seconds + + Returns: + bool: True if chat download succeeded + """ + if self.chat_thread is None: + return False + + try: + self.chat_thread.join(timeout=timeout) + + # Give a moment for file to be fully written and closed + if self.chat_thread_success: + print(f'{Fore.CYAN}Ensuring chat file is fully written and closed...{Style.RESET_ALL}') + time.sleep(1.0) # Increased from 0.5s + + # Force garbage collection to help close file handles + import gc + gc.collect() + time.sleep(0.5) + + return self.chat_thread_success + except Exception as e: + print(f'{Fore.RED}✗ Error waiting for chat thread: {e}{Style.RESET_ALL}') + return False + finally: + self.chat_thread = None + + def wait_for_file_access(self, file_path: str, max_attempts: int = 10, delay: float = 0.5) -> bool: + """ + Wait for a file to be accessible (not locked by another process). + + Args: + file_path: Path to the file to check + max_attempts: Maximum number of attempts + delay: Delay between attempts in seconds + + Returns: + bool: True if file is accessible, False otherwise + """ + for attempt in range(max_attempts): + try: + # Try to open the file for reading with shared access + with open(file_path, 'r', encoding='utf-8') as f: + # Try to read a bit to ensure it's really accessible + f.read(1) + print(f'{Fore.GREEN}✓ File is accessible and ready to use{Style.RESET_ALL}') + return True + except PermissionError: + if attempt < max_attempts - 1: + print(f'{Fore.YELLOW}⚠ File still locked, waiting... (attempt {attempt + 1}/{max_attempts}){Style.RESET_ALL}') + time.sleep(delay) + else: + print(f'{Fore.RED}✗ File still locked after {max_attempts} attempts{Style.RESET_ALL}') + return False + except Exception as e: + print(f'{Fore.RED}✗ Error checking file access: {e}{Style.RESET_ALL}') + return False + + return False + + def convert_chat_downloader_to_twitch_format(self, input_path: str, output_path: str, + video_duration: Optional[float] = None) -> bool: + """ + Convert chat_downloader JSON format to TwitchDownloaderCLI format. + + Args: + input_path: Path to chat_downloader JSON file + output_path: Path to save converted TwitchDownloaderCLI format + video_duration: Optional video duration in seconds (overrides calculated duration) + + Returns: + bool: True if conversion succeeded, False otherwise + """ + try: + from datetime import datetime, timezone as dt_timezone + print(f'{Fore.CYAN}Converting chat format for rendering...{Style.RESET_ALL}') + + # Read chat_downloader format (JSON array) + with open(input_path, 'r', encoding='utf-8') as f: + messages = json.load(f) + + # Ensure we have a list + if not isinstance(messages, list): + print(f'{Fore.RED}✗ Expected JSON array, got {type(messages)}{Style.RESET_ALL}') + return False + + print(f'{Fore.CYAN}Converting {len(messages)} messages...{Style.RESET_ALL}') + + # Track first message timestamp for offset calculation + first_timestamp = None + + # Convert to TwitchDownloaderCLI format + comments = [] + for msg in messages: + # Skip non-dictionary items + if not isinstance(msg, dict): + continue + + # Extract data from chat_downloader format + # Timestamp is in microseconds, convert to seconds + timestamp_us = msg.get('timestamp', 0) + timestamp_sec = timestamp_us / 1000000.0 if timestamp_us else 0.0 + + # Track first timestamp for offset calculation + if first_timestamp is None: + first_timestamp = timestamp_sec + + # Calculate offset from start of stream + content_offset_seconds = timestamp_sec - first_timestamp + + # Extract author info + author = msg.get('author', {}) + author_name = author.get('display_name', author.get('name', 'Unknown')) + author_login = author.get('name', author_name.lower()) + author_id = author.get('id', '') + + # Extract message text + message_text = msg.get('message', '') + + # Format timestamp as ISO 8601 + dt = datetime.fromtimestamp(timestamp_sec, tz=dt_timezone.utc) + created_at = dt.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z' + + # Get user color (chat_downloader uses "colour" - British spelling) + user_color = msg.get('colour', msg.get('color', '#FFFFFF')) + if not user_color: + user_color = '#FFFFFF' + + comment = { + "_id": msg.get('message_id', f"msg_{len(comments)}"), + "created_at": created_at, + "updated_at": created_at, + "channel_id": msg.get('channel_id', ''), + "content_type": "video", + "content_id": "", + "content_offset_seconds": content_offset_seconds, + "commenter": { + "display_name": author_name, + "name": author_login, + "_id": str(author_id), + "type": "user", + "bio": None, + "created_at": created_at, + "updated_at": created_at, + "logo": None + }, + "source": "chat", + "state": "published", + "message": { + "body": message_text, + "bits_spent": 0, + "fragments": [ + { + "text": message_text, + "emoticon": None + } + ], + "is_action": False, + "user_badges": [], + "user_color": user_color, + "user_notice_params": {} + }, + "more_replies": False + } + + comments.append(comment) + + if not comments: + print(f'{Fore.RED}✗ No valid comments to convert{Style.RESET_ALL}') + return False + + # Use provided video duration, or calculate from last message + print(f'{Fore.MAGENTA}[DEBUG] Input video_duration parameter: {video_duration}{Style.RESET_ALL}') + if video_duration is None: + video_duration = int(comments[-1]["content_offset_seconds"]) if comments else 0 + print(f'{Fore.MAGENTA}[DEBUG] No video duration provided, calculated from chat: {video_duration}s{Style.RESET_ALL}') + else: + video_duration = int(video_duration) + print(f'{Fore.MAGENTA}[DEBUG] Using provided video duration: {video_duration}s{Style.RESET_ALL}') + + # Debug output + print(f'{Fore.MAGENTA}[DEBUG] First message offset: {comments[0]["content_offset_seconds"]:.2f}s{Style.RESET_ALL}') + print(f'{Fore.MAGENTA}[DEBUG] Last message offset: {comments[-1]["content_offset_seconds"]:.2f}s{Style.RESET_ALL}') + print(f'{Fore.MAGENTA}[DEBUG] Calculated duration: {video_duration}s{Style.RESET_ALL}') + print(f'{Fore.MAGENTA}[DEBUG] Sample comment structure:{Style.RESET_ALL}') + print(f'{Fore.MAGENTA} ID: {comments[0]["_id"]}{Style.RESET_ALL}') + print(f'{Fore.MAGENTA} Offset: {comments[0]["content_offset_seconds"]}{Style.RESET_ALL}') + print(f'{Fore.MAGENTA} Message: {comments[0]["message"]["body"][:50]}{Style.RESET_ALL}') + print(f'{Fore.MAGENTA}[DEBUG] Final JSON duration will be: {video_duration}s{Style.RESET_ALL}') + print(f'{Fore.MAGENTA}[DEBUG] Comment count: {len(comments)}{Style.RESET_ALL}') + + # Create final structure matching TwitchDownloaderCLI format + from datetime import datetime as dt + created_at_iso = dt.now().isoformat() + 'Z' + + output_data = { + "FileInfo": { + "Version": { + "Major": 1, + "Minor": 4, + "Patch": 0 + }, + "CreatedAt": created_at_iso, + "UpdatedAt": "0001-01-01T00:00:00" + }, + "streamer": { + "name": comments[0]["channel_id"] if comments else "", + "login": comments[0]["channel_id"] if comments else "", + "id": int(comments[0]["channel_id"]) if comments and comments[0]["channel_id"].isdigit() else 0 + }, + "video": { + "title": "Live Chat", + "description": None, + "id": None, + "created_at": comments[0]["created_at"] if comments else created_at_iso, + "start": 0, + "end": video_duration, + "length": video_duration, + "viewCount": 0, + "game": None, + "chapters": [] + }, + "comments": comments, + "embeddedData": None + } + + # Write to output file + with open(output_path, 'w', encoding='utf-8') as f: + json.dump(output_data, f, indent=2, ensure_ascii=False) + + # Verify output + file_size = os.path.getsize(output_path) + print(f'{Fore.MAGENTA}[DEBUG] Converted JSON file: {file_size:,} bytes{Style.RESET_ALL}') + + print(f'{Fore.GREEN}✓ Chat format converted successfully{Style.RESET_ALL}') + return True + + except Exception as e: + print(f'{Fore.RED}✗ Chat format conversion failed: {e}{Style.RESET_ALL}') + import traceback + traceback.print_exc() + return False + + def start_live_chat_download_with_fallback(self, username: str, vod_id: Optional[str], + json_path: str) -> tuple[Optional[subprocess.Popen], str]: + """ + Start live chat download with automatic fallback. + + Tries TwitchDownloaderCLI first (if VOD ID available and not using chat_downloader as primary), + falls back to chat_downloader if that fails or if VOD ID is not available. + + Args: + username: Twitch username + vod_id: Optional VOD ID (may be None if VODs disabled) + json_path: Path to save chat JSON + + Returns: + tuple: (process_handle or None, method_used) + method_used is one of: 'twitch_downloader', 'chat_downloader', 'failed' + """ + print(f'\n{Fore.MAGENTA}[VERBOSE] === CHAT DOWNLOAD FALLBACK LOGIC ==={Style.RESET_ALL}') + print(f'{Fore.MAGENTA}[VERBOSE] Username: {username}{Style.RESET_ALL}') + print(f'{Fore.MAGENTA}[VERBOSE] VOD ID: {vod_id}{Style.RESET_ALL}') + print(f'{Fore.MAGENTA}[VERBOSE] Primary method: {"chat_downloader" if self.use_chat_downloader_primary else "TwitchDownloaderCLI"}{Style.RESET_ALL}') + print(f'{Fore.MAGENTA}[VERBOSE] Fallback enabled: {self.use_chat_downloader_fallback}{Style.RESET_ALL}') + + # Determine primary method + use_twitch_downloader_first = ( + vod_id is not None and + not self.use_chat_downloader_primary and + self.download_live_chat + ) + + print(f'{Fore.MAGENTA}[VERBOSE] Will try TwitchDownloaderCLI first: {use_twitch_downloader_first}{Style.RESET_ALL}') + + # Try TwitchDownloaderCLI first if conditions met + if use_twitch_downloader_first: + print(f'{Fore.CYAN}Attempting live chat download with TwitchDownloaderCLI...{Style.RESET_ALL}') + print(f'{Fore.MAGENTA}[VERBOSE] Starting TwitchDownloaderCLI process...{Style.RESET_ALL}') + process = self.start_live_chat_download(vod_id, json_path) + if process is not None: + print(f'{Fore.GREEN}✓ TwitchDownloaderCLI started successfully{Style.RESET_ALL}') + print(f'{Fore.MAGENTA}[VERBOSE] Process PID: {process.pid}{Style.RESET_ALL}') + return (process, 'twitch_downloader') + else: + print(f'{Fore.YELLOW}⚠ TwitchDownloaderCLI failed to start{Style.RESET_ALL}') + print(f'{Fore.MAGENTA}[VERBOSE] Will try fallback method...{Style.RESET_ALL}') + + # Try chat_downloader as fallback (or as primary) + if self.use_chat_downloader_fallback or self.use_chat_downloader_primary: + if vod_id is None: + print(f'{Fore.YELLOW}⚠ No VOD ID available - using chat_downloader fallback{Style.RESET_ALL}') + print(f'{Fore.MAGENTA}[VERBOSE] This typically means VODs are disabled on this channel{Style.RESET_ALL}') + + if CHAT_DOWNLOADER_AVAILABLE and self.chat_downloader is not None: + print(f'{Fore.CYAN}Using chat_downloader as {"primary method" if self.use_chat_downloader_primary else "fallback"}{Style.RESET_ALL}') + print(f'{Fore.MAGENTA}[VERBOSE] chat_downloader is available and initialized{Style.RESET_ALL}') + # Return special marker for chat_downloader + return (None, 'chat_downloader') + else: + print(f'{Fore.RED}✗ chat_downloader not available for fallback{Style.RESET_ALL}') + print(f'{Fore.MAGENTA}[VERBOSE] CHAT_DOWNLOADER_AVAILABLE: {CHAT_DOWNLOADER_AVAILABLE}{Style.RESET_ALL}') + print(f'{Fore.MAGENTA}[VERBOSE] chat_downloader instance: {self.chat_downloader}{Style.RESET_ALL}') + print(f'{Fore.YELLOW} Install with: pip install chat-downloader{Style.RESET_ALL}') + else: + print(f'{Fore.YELLOW}⚠ Fallback disabled in configuration{Style.RESET_ALL}') + + # Both methods failed or unavailable + if vod_id is None: + print(f'{Fore.RED}✗ No VOD ID and no fallback available - cannot download live chat{Style.RESET_ALL}') + + print(f'{Fore.MAGENTA}[VERBOSE] All chat download methods exhausted{Style.RESET_ALL}') + return (None, 'failed') diff --git a/requirements.txt b/requirements.txt index 1aa42ed..6b41073 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,5 @@ colorama==0.4.6 python-dotenv==1.0.1 pytz==2024.2 requests==2.32.3 -streamlink==8.2.0 \ No newline at end of file +streamlink==8.2.0 +chat-downloader>=0.2.8 \ No newline at end of file diff --git a/start.bat b/start.bat index 79b9174..746bffb 100644 --- a/start.bat +++ b/start.bat @@ -10,7 +10,8 @@ rem Activate the virtual environment pip install -r requirements.txt rem Run the desired command in the virtual environment - python twitch-archive.py -u %1 + rem Pass username as -u and forward all additional arguments + python twitch-archive.py -u %1 %2 %3 %4 %5 %6 %7 %8 %9 rem Deactivate the virtual environment call "%VENV_PATH%\Scripts\deactivate.bat" diff --git a/twitch-archive.py b/twitch-archive.py index 58b0158..5844cde 100644 --- a/twitch-archive.py +++ b/twitch-archive.py @@ -380,14 +380,21 @@ class TwitchArchive: # Start live chat download if enabled live_chat_process = None + live_chat_method = None # Track which method was used chat_json_path = str(self.file_manager.chat_json_path / f"{PREFIX_CHAT}{filename_base}.json") - if self.downloadLiveCHAT and is_live.get('archiveVideo') and is_live['archiveVideo'].get('id'): - live_vod_id = is_live['archiveVideo']['id'] - print(f'{Fore.CYAN}Live VOD ID detected: {live_vod_id}{Style.RESET_ALL}') - live_chat_process = self.downloader.start_live_chat_download(live_vod_id, chat_json_path) - elif self.downloadLiveCHAT: - print(f'{Fore.YELLOW}⚠ No VOD ID available yet for live chat download{Style.RESET_ALL}') + if self.downloadLiveCHAT: + vod_id = is_live.get('archiveVideo', {}).get('id') if is_live.get('archiveVideo') else None + stream_url = f"https://twitch.tv/{self.username}" + + live_chat_process, live_chat_method = self.downloader.start_live_chat_download_with_fallback( + vod_id=vod_id, + stream_url=stream_url, + json_path=chat_json_path, + use_chat_downloader_primary=self.use_chat_downloader_primary, + no_chat_downloader_fallback=self.no_chat_downloader_fallback, + verbose=self.verbose + ) # Record the live stream recording_completed = self.recorder.record(is_live, live_raw_path) @@ -411,16 +418,34 @@ class TwitchArchive: chat_video_path = str(self.file_manager.chat_mp4_path / f"{PREFIX_CHAT}{filename_base}.mp4") output_args = self.processor.build_chat_output_args() - # Get video duration to trim chat accordingly - ffmpeg_path = get_ffmpeg_executable(self.os_type) - video_duration = get_video_duration(live_proc_path, ffmpeg_path) - - chat_rendered_successfully = self.downloader.render_chat( - chat_json_path, - chat_video_path, - output_args, - video_duration=video_duration - ) + # Wait for chat file to be fully accessible (not locked) + print(f'{Fore.CYAN}Verifying chat file is ready for rendering...{Style.RESET_ALL}') + if not self.downloader.wait_for_file_access(chat_json_path, max_attempts=15, delay=0.5): + print(f'{Fore.RED}✗ Chat file is locked, skipping rendering{Style.RESET_ALL}') + chat_rendered_successfully = False + else: + # Get video duration first (needed for chat conversion and trimming) + ffmpeg_path = get_ffmpeg_executable(self.os_type) + video_duration = get_video_duration(live_proc_path, ffmpeg_path) + print(f'{Fore.CYAN}Video duration for chat rendering: {video_duration}s{Style.RESET_ALL}') + + # Convert chat format if needed (chat_downloader uses different JSON structure) + render_json_path = chat_json_path + if live_chat_method == 'chat_downloader': + print(f'{Fore.CYAN}Converting chat format for rendering...{Style.RESET_ALL}') + converted_path = chat_json_path.replace('.json', '_converted.json') + if self.downloader.convert_chat_downloader_to_twitch_format(chat_json_path, converted_path, video_duration): + render_json_path = converted_path + print(f'{Fore.GREEN}✓ Chat format converted successfully{Style.RESET_ALL}') + else: + print(f'{Fore.RED}✗ Failed to convert chat format{Style.RESET_ALL}') + + chat_rendered_successfully = self.downloader.render_chat( + render_json_path, + chat_video_path, + output_args, + video_duration=video_duration + ) # Merge video and chat if configured merged_video_path = None @@ -730,17 +755,26 @@ class TwitchArchiveManager: Manages multiple TwitchArchive instances for monitoring multiple streamers. """ - def __init__(self, specific_streamer: Optional[str] = None, verbose: bool = False): + def __init__(self, specific_streamer: Optional[str] = None, verbose: bool = False, + chat_only: bool = False, + use_chat_downloader_primary: bool = False, + use_chat_downloader_fallback: bool = True): """ Initialize the manager. Args: specific_streamer: If provided, only monitor this streamer (ignore enabled status) verbose: Enable verbose debug output + chat_only: Only download chat, skip video recording (test mode) + use_chat_downloader_primary: Use chat_downloader as primary chat source + use_chat_downloader_fallback: Enable chat_downloader fallback """ self.config_manager = ConfigManager() self.specific_streamer = specific_streamer self.verbose = verbose + self.chat_only = chat_only + self.use_chat_downloader_primary = use_chat_downloader_primary + self.use_chat_downloader_fallback = use_chat_downloader_fallback self.archivers: Dict[str, TwitchArchive] = {} self.shutdown_requested = False self.active_recordings: Dict[str, str] = {} # Track active recordings: {username: stream_id} @@ -783,6 +817,11 @@ class TwitchArchiveManager: TwitchArchive: Initialized archiver instance """ config = self.config_manager.load_streamer_config(username) + + # Apply command-line overrides for chat_downloader options + config['useChatDownloaderPrimary'] = self.use_chat_downloader_primary + config['useChatDownloaderFallback'] = self.use_chat_downloader_fallback + archiver = TwitchArchive(config) return archiver @@ -794,6 +833,8 @@ class TwitchArchiveManager: """ print(f'\n{Fore.CYAN}{"=" * 70}{Style.RESET_ALL}') print(f'{Fore.CYAN}TWITCH ARCHIVE - Multi-Streamer Mode{Style.RESET_ALL}') + if self.chat_only: + print(f'{Fore.YELLOW}🧪 TEST MODE: Chat-Only (Video Recording Disabled){Style.RESET_ALL}') print(f'{Fore.CYAN}{"=" * 70}{Style.RESET_ALL}\n') # Get streamers to monitor @@ -805,6 +846,14 @@ class TwitchArchiveManager: print(f'{Fore.CYAN}→ Or run with -u to create a new config{Style.RESET_ALL}') sys.exit(1) + if self.chat_only: + print(f'{Fore.YELLOW}📝 Chat-Only Mode Enabled:{Style.RESET_ALL}') + print(f'{Fore.CYAN} • Verbose logging: ON{Style.RESET_ALL}') + print(f'{Fore.CYAN} • Video recording: DISABLED{Style.RESET_ALL}') + print(f'{Fore.CYAN} • Chat download: ENABLED{Style.RESET_ALL}') + print(f'{Fore.CYAN} • VOD download: DISABLED{Style.RESET_ALL}') + print() + print(f'{Fore.GREEN}Monitoring {len(streamers)} streamer(s):{Style.RESET_ALL}') for streamer in streamers: print(f' • {Fore.CYAN}{streamer}{Style.RESET_ALL}') @@ -985,26 +1034,131 @@ class TwitchArchiveManager: chat_json_path = str(archiver.file_manager.chat_json_path / f"{PREFIX_CHAT}{filename_base}.json") # Send notification - archiver.notification_manager.send( - f"Stream Started - {archiver.username}", - f"Recording: {stream_info['title']}" - ) + if not self.chat_only: + archiver.notification_manager.send( + f"Stream Started - {archiver.username}", + f"Recording: {stream_info['title']}" + ) - # Start live chat download if enabled and VOD ID is available + # Start live chat download if enabled (with fallback support) live_chat_process = None - if archiver.downloadLiveCHAT and stream_info.get('archiveVideo') and stream_info['archiveVideo'].get('id'): - live_vod_id = stream_info['archiveVideo']['id'] - print(f'{Fore.CYAN}Live VOD ID detected: {live_vod_id}{Style.RESET_ALL}') - live_chat_process = archiver.downloader.start_live_chat_download(live_vod_id, chat_json_path) - elif archiver.downloadLiveCHAT: - print(f'{Fore.YELLOW}⚠ No VOD ID available yet for live chat download{Style.RESET_ALL}') + live_chat_method = 'failed' + if archiver.downloadLiveCHAT: + if self.verbose or self.chat_only: + print(f'\n{Fore.MAGENTA}[VERBOSE] Starting chat download process...{Style.RESET_ALL}') + print(f'{Fore.MAGENTA}[VERBOSE] downloadLiveCHAT: {archiver.downloadLiveCHAT}{Style.RESET_ALL}') + print(f'{Fore.MAGENTA}[VERBOSE] useChatDownloaderPrimary: {archiver.downloader.use_chat_downloader_primary}{Style.RESET_ALL}') + print(f'{Fore.MAGENTA}[VERBOSE] useChatDownloaderFallback: {archiver.downloader.use_chat_downloader_fallback}{Style.RESET_ALL}') + + # Get VOD ID if available + live_vod_id = None + if stream_info.get('archiveVideo') and stream_info['archiveVideo'].get('id'): + live_vod_id = stream_info['archiveVideo']['id'] + print(f'{Fore.CYAN}Live VOD ID detected: {live_vod_id}{Style.RESET_ALL}') + if self.verbose or self.chat_only: + print(f'{Fore.MAGENTA}[VERBOSE] VOD URL: https://www.twitch.tv/videos/{live_vod_id}{Style.RESET_ALL}') + else: + print(f'{Fore.YELLOW}⚠ No VOD ID available - will use fallback if configured{Style.RESET_ALL}') + if self.verbose or self.chat_only: + print(f'{Fore.MAGENTA}[VERBOSE] This happens when streamer has VODs disabled{Style.RESET_ALL}') + print(f'{Fore.MAGENTA}[VERBOSE] chat_downloader fallback will be used if enabled{Style.RESET_ALL}') + + # Try to start live chat download with fallback + try: + if self.verbose or self.chat_only: + print(f'{Fore.MAGENTA}[VERBOSE] Calling start_live_chat_download_with_fallback(){Style.RESET_ALL}') + print(f'{Fore.MAGENTA}[VERBOSE] Username: {archiver.username}{Style.RESET_ALL}') + print(f'{Fore.MAGENTA}[VERBOSE] VOD ID: {live_vod_id}{Style.RESET_ALL}') + print(f'{Fore.MAGENTA}[VERBOSE] Output path: {chat_json_path}{Style.RESET_ALL}') + + live_chat_process, live_chat_method = archiver.downloader.start_live_chat_download_with_fallback( + archiver.username, live_vod_id, chat_json_path + ) + + if self.verbose or self.chat_only: + print(f'{Fore.MAGENTA}[VERBOSE] Chat download method selected: {live_chat_method}{Style.RESET_ALL}') + print(f'{Fore.MAGENTA}[VERBOSE] Process handle: {live_chat_process}{Style.RESET_ALL}') + + # If chat_downloader is selected, start it in background thread now (before video recording) + if live_chat_method == 'chat_downloader' and not self.chat_only: + if self.verbose: + print(f'{Fore.MAGENTA}[VERBOSE] Starting chat_downloader in background thread...{Style.RESET_ALL}') + try: + print(f'{Fore.CYAN}Starting chat_downloader in background (concurrent with video)...{Style.RESET_ALL}') + archiver.downloader.start_chat_downloader_thread( + archiver.username, chat_json_path, + shutdown_check=lambda: self.shutdown_requested or archiver.shutdown_requested, + verbose=self.verbose + ) + except Exception as e: + print(f'{Fore.RED}✗ Failed to start chat thread: {e}{Style.RESET_ALL}') + import traceback + traceback.print_exc() + live_chat_method = 'failed' + + except Exception as e: + print(f'{Fore.RED}✗ Failed to start live chat download: {e}{Style.RESET_ALL}') + import traceback + traceback.print_exc() + live_chat_method = 'failed' - # Record livestream + # Record livestream (skip in chat-only mode) + if self.chat_only: + print(f'\n{Fore.YELLOW}🧪 Chat-Only Mode: Skipping video recording{Style.RESET_ALL}') + print(f'{Fore.CYAN}Waiting for chat download to complete...{Style.RESET_ALL}') + + # Start chat download based on method + if live_chat_method == 'chat_downloader': + if self.verbose: + print(f'{Fore.MAGENTA}[VERBOSE] Starting chat_downloader in background thread...{Style.RESET_ALL}') + try: + print(f'{Fore.CYAN}Using chat_downloader for live chat...{Style.RESET_ALL}') + archiver.downloader.start_chat_downloader_thread( + archiver.username, chat_json_path, + shutdown_check=lambda: self.shutdown_requested or archiver.shutdown_requested, + verbose=self.verbose or self.chat_only + ) + # Wait for completion + live_chat_downloaded = archiver.downloader.wait_for_chat_thread() + except Exception as e: + print(f'{Fore.RED}✗ chat_downloader failed: {e}{Style.RESET_ALL}') + import traceback + traceback.print_exc() + live_chat_downloaded = False + elif live_chat_method == 'twitch_downloader' and live_chat_process is not None: + if self.verbose: + print(f'{Fore.MAGENTA}[VERBOSE] Waiting for TwitchDownloaderCLI process...{Style.RESET_ALL}') + live_chat_downloaded = archiver.downloader.wait_for_chat_download(live_chat_process, chat_json_path) + else: + live_chat_downloaded = False + + # Report results + if live_chat_downloaded: + print(f'\n{Fore.GREEN}✓ Chat-Only Test Complete!{Style.RESET_ALL}') + print(f'{Fore.CYAN}Chat saved to: {chat_json_path}{Style.RESET_ALL}') + if os.path.exists(chat_json_path): + file_size = os.path.getsize(chat_json_path) + print(f'{Fore.CYAN}File size: {file_size / 1024:.2f} KB{Style.RESET_ALL}') + else: + print(f'\n{Fore.RED}✗ Chat download failed{Style.RESET_ALL}') + + return # Exit early, don't process video + + # Normal mode: Record livestream recording_successful = archiver.recorder.record(stream_info, live_raw_path) # Check if raw file exists (may exist even after interrupted recording) if not os.path.exists(live_raw_path): print(f'{Fore.RED}✗ No recording file found, skipping processing{Style.RESET_ALL}') + + # Still wait for chat if it's downloading + if live_chat_method == 'chat_downloader' and archiver.downloader.chat_thread is not None: + print(f'{Fore.CYAN}Waiting for chat download to finish...{Style.RESET_ALL}') + archiver.downloader.wait_for_chat_thread(timeout=30) + elif live_chat_method == 'twitch_downloader' and live_chat_process is not None: + print(f'{Fore.CYAN}Waiting for chat download to finish...{Style.RESET_ALL}') + archiver.downloader.wait_for_chat_download(live_chat_process, chat_json_path, timeout=30) + return # Get file size to check if anything was recorded @@ -1023,24 +1177,68 @@ class TwitchArchiveManager: live_chat_downloaded = False chat_rendered_successfully = False chat_video_path = None - if live_chat_process is not None: + + # Handle different chat download methods + if live_chat_method == 'twitch_downloader' and live_chat_process is not None: + # Wait for TwitchDownloaderCLI process + print(f'{Fore.CYAN}Waiting for live chat download to complete...{Style.RESET_ALL}') live_chat_downloaded = archiver.downloader.wait_for_chat_download(live_chat_process, chat_json_path) + elif live_chat_method == 'chat_downloader' and archiver.downloader.chat_thread is not None: + # Wait for chat_downloader thread + print(f'{Fore.CYAN}Waiting for live chat download to complete...{Style.RESET_ALL}') + try: + live_chat_downloaded = archiver.downloader.wait_for_chat_thread() + if live_chat_downloaded: + print(f'{Fore.GREEN}✓ Chat download thread completed successfully{Style.RESET_ALL}') + else: + print(f'{Fore.YELLOW}⚠ Chat download thread completed with errors or no messages{Style.RESET_ALL}') + except Exception as e: + print(f'{Fore.RED}✗ Error waiting for chat download thread: {e}{Style.RESET_ALL}') + import traceback + traceback.print_exc() + live_chat_downloaded = False # Render live chat if downloaded successfully if live_chat_downloaded: chat_video_path = str(archiver.file_manager.chat_mp4_path / f"{PREFIX_CHAT}{filename_base}.mp4") output_args = archiver.processor.build_chat_output_args() - # Get video duration to trim chat accordingly - ffmpeg_path = get_ffmpeg_executable(archiver.os_type) - video_duration = get_video_duration(live_proc_path, ffmpeg_path) - - chat_rendered_successfully = archiver.downloader.render_chat( - chat_json_path, - chat_video_path, - output_args, - video_duration=video_duration - ) + # Wait for chat file to be fully accessible (not locked) + print(f'{Fore.CYAN}Verifying chat file is ready for rendering...{Style.RESET_ALL}') + if not archiver.downloader.wait_for_file_access(chat_json_path, max_attempts=15, delay=0.5): + print(f'{Fore.RED}✗ Chat file is locked, skipping rendering{Style.RESET_ALL}') + chat_rendered_successfully = False + else: + # Get video duration first + ffmpeg_path = get_ffmpeg_executable(archiver.os_type) + video_duration = get_video_duration(live_proc_path, ffmpeg_path) + + if video_duration is None: + print(f'{Fore.YELLOW}⚠ Could not detect video duration from {live_proc_path}{Style.RESET_ALL}') + print(f'{Fore.YELLOW} Will use chat message timestamps instead{Style.RESET_ALL}') + else: + print(f'{Fore.CYAN}Video duration for chat rendering: {video_duration}s{Style.RESET_ALL}') + + # Convert chat format if chat_downloader was used + render_json_path = chat_json_path + if live_chat_method == 'chat_downloader': + converted_path = chat_json_path.replace('.json', '_converted.json') + print(f'{Fore.CYAN}Chat downloaded with chat_downloader, converting format...{Style.RESET_ALL}') + if archiver.downloader.convert_chat_downloader_to_twitch_format(chat_json_path, converted_path, video_duration): + render_json_path = converted_path + print(f'{Fore.GREEN}✓ Using converted chat file for rendering{Style.RESET_ALL}') + else: + print(f'{Fore.RED}✗ Format conversion failed, skipping rendering{Style.RESET_ALL}') + chat_rendered_successfully = False + render_json_path = None + + if render_json_path: + chat_rendered_successfully = archiver.downloader.render_chat( + render_json_path, + chat_video_path, + output_args, + video_duration=video_duration + ) # Merge video and chat if configured merged_video_path = None @@ -1225,6 +1423,10 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving -u, --username Monitor only this Twitch channel --verbose Enable verbose debug output --legacy Force legacy mode (use config.json) + --chat-only Test mode: Only download chat (skip video recording) + Automatically enables verbose logging + --use-chat-downloader-primary Use chat_downloader as primary chat source (for testing) + --no-chat-downloader-fallback Disable chat_downloader fallback {Fore.GREEN}LEGACY OPTIONS (when using --legacy):{Style.RESET_ALL} -q, --quality Stream quality: best/source, high/720p, @@ -1247,6 +1449,8 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving python twitch-archive.py # Monitor all enabled streamers python twitch-archive.py -u vinesauce # Monitor only vinesauce python twitch-archive.py -u hackerling --verbose # Monitor with debug output + python twitch-archive.py -u streamername --chat-only # Test chat download only (no video) + python twitch-archive.py --use-chat-downloader-primary # Test chat_downloader library python twitch-archive.py --legacy # Use old config.json mode {Fore.CYAN}{"=" * 70}{Style.RESET_ALL} @@ -1257,7 +1461,8 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving argv, "h:u:q:a:v:c:m:r:d:n:", ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", - "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose"] + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] ) except getopt.GetoptError as e: print(f'{Fore.RED}Error: {e}{Style.RESET_ALL}\n') @@ -1270,7 +1475,26 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving # Parse command line args legacy_overrides = {} verbose_mode = False + chat_only_mode = False + use_chat_downloader_primary = False + use_chat_downloader_fallback = True # Default to enabled for opt, arg in opts: + if opt in ('-h', '--help'): + print(help_msg) + sys.exit(0) + elif opt in ("-u", "--username"): + specific_streamer = arg + elif opt == "--verbose": + verbose_mode = True + elif opt == "--chat-only": + chat_only_mode = True + verbose_mode = True # Auto-enable verbose for chat-only mode + elif opt == "--legacy": + use_legacy_mode = True + elif opt == "--use-chat-downloader-primary": + use_chat_downloader_primary = True + elif opt == "--no-chat-downloader-fallback": + use_chat_downloader_fallback = False if opt in ('-h', '--help'): print(help_msg) sys.exit(0) @@ -1310,11 +1534,23 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving for key, value in legacy_overrides.items(): setattr(twitch_archive, key, value) + # Apply chat_downloader options + if hasattr(twitch_archive.downloader, 'use_chat_downloader_primary'): + twitch_archive.downloader.use_chat_downloader_primary = use_chat_downloader_primary + if hasattr(twitch_archive.downloader, 'use_chat_downloader_fallback'): + twitch_archive.downloader.use_chat_downloader_fallback = use_chat_downloader_fallback + # Start the archive system twitch_archive.run() else: # New multi-streamer mode - manager = TwitchArchiveManager(specific_streamer=specific_streamer, verbose=verbose_mode) + manager = TwitchArchiveManager( + specific_streamer=specific_streamer, + verbose=verbose_mode, + chat_only=chat_only_mode, + use_chat_downloader_primary=use_chat_downloader_primary, + use_chat_downloader_fallback=use_chat_downloader_fallback + ) manager.run()