Add chat_downloader support with fallback for live chat downloads

- Integrated chat_downloader as an optional dependency for downloading live chat.
- Implemented fallback logic to use chat_downloader when TwitchDownloaderCLI fails or is not available.
- Enhanced chat rendering process to handle different JSON formats from chat_downloader.
- Updated requirements.txt to include chat-downloader package.
- Modified start.bat to allow passing additional arguments for flexibility.
- Improved error handling and logging for chat download processes.
- Added command-line options for testing chat-only mode and configuring chat_downloader behavior.

modules/downloader.py

@@ -1,14 +1,26 @@
 """
 VOD and chat downloading functionality using TwitchDownloaderCLI.
+Includes fallback support for chat_downloader when VOD-based methods fail.
 """
 
 import os
 import subprocess
+import json
+import threading
+import time
 from typing import Dict, Any, Optional
 from colorama import Fore, Style
 
 from .utils import get_bin_path
 
+# Try to import chat_downloader (optional dependency)
+try:
+    from chat_downloader import ChatDownloader
+    CHAT_DOWNLOADER_AVAILABLE = True
+except ImportError:
+    CHAT_DOWNLOADER_AVAILABLE = False
+    ChatDownloader = None
+
 
 class ContentDownloader:
     """Handles VOD and chat downloading using TwitchDownloaderCLI."""
@@ -29,6 +41,24 @@ class ContentDownloader:
         self.download_vod = config.get('downloadVOD', True)
         self.download_chat = config.get('downloadCHAT', True)
         self.download_live_chat = config.get('downloadLiveCHAT', True)
+        self.use_chat_downloader_primary = config.get('useChatDownloaderPrimary', False)
+        self.use_chat_downloader_fallback = config.get('useChatDownloaderFallback', True)
+        
+        # Initialize chat_downloader if available
+        self.chat_downloader = None
+        if CHAT_DOWNLOADER_AVAILABLE:
+            try:
+                self.chat_downloader = ChatDownloader()
+            except Exception as e:
+                print(f'{Fore.YELLOW}⚠ Failed to initialize chat_downloader: {e}{Style.RESET_ALL}')
+        elif self.use_chat_downloader_primary or self.use_chat_downloader_fallback:
+            print(f'{Fore.YELLOW}⚠ chat_downloader not available but requested in config{Style.RESET_ALL}')
+            print(f'{Fore.YELLOW}  Install with: pip install chat-downloader{Style.RESET_ALL}')
+        
+        # Thread management for chat_downloader
+        self.chat_thread = None
+        self.chat_thread_success = False
+        self.chat_thread_error = None
     
     def download_vod(self, vod_info: Dict[str, Any], output_path: str) -> bool:
         """
@@ -156,6 +186,7 @@ class ContentDownloader:
             '--background-color', '#FF111111',
             '-w', '500',
             '-h', '1080',
+            '--framerate', '30',
             '--outline',
             '-f', 'Arial',
             '--font-size', '22',
@@ -166,6 +197,9 @@ class ContentDownloader:
             '--collision', 'Rename'
         ]
         
+        # Always start from beginning
+        chat_settings.extend(['-b', '0'])
+        
         # Trim chat to match video duration if provided
         if video_duration is not None and video_duration > 0:
             # Format duration as seconds with 1 decimal place
@@ -183,13 +217,37 @@ class ContentDownloader:
             # Build complete command
             full_cmd = [self.twitch_downloader_path, 'chatrender', '-i', json_path, '-o', video_path] + chat_settings
             
-            result = subprocess.call(full_cmd)
+            # Capture output to see what TwitchDownloaderCLI says
+            process = subprocess.Popen(
+                full_cmd,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                text=True,
+                encoding='utf-8',
+                errors='replace'
+            )
+            
+            # Stream output in real-time
+            for line in process.stdout:
+                print(line.rstrip())
+            
+            result = process.wait()
             
             if result != 0:
                 print(f'{Fore.RED}✗ Chat render failed with exit code: {result}{Style.RESET_ALL}')
                 return False
             
-            print(f'{Fore.GREEN}✓ Chat rendered{Style.RESET_ALL}')
+            # Verify the output file was created and has content
+            if not os.path.exists(video_path):
+                print(f'{Fore.RED}✗ Chat video file was not created{Style.RESET_ALL}')
+                return False
+            
+            file_size = os.path.getsize(video_path)
+            if file_size < 1024:  # Less than 1KB is suspicious
+                print(f'{Fore.RED}✗ Chat video file is too small ({file_size} bytes){Style.RESET_ALL}')
+                return False
+                
+            print(f'{Fore.GREEN}✓ Chat rendered ({file_size:,} bytes){Style.RESET_ALL}')
             return True
             
         except Exception as e:
@@ -302,3 +360,461 @@ class ContentDownloader:
         except Exception as e:
             print(f'{Fore.RED}✗ Error waiting for chat download: {str(e)}{Style.RESET_ALL}')
             return False
+    
+    def download_live_chat_with_chat_downloader(self, username: str, json_path: str, 
+                                               max_messages: Optional[int] = None,
+                                               timeout: Optional[float] = None,
+                                               shutdown_check: Optional[callable] = None,
+                                               verbose: bool = False) -> bool:
+        """
+        Download live chat using chat_downloader library as fallback.
+        This works even when VOD is disabled on the channel.
+        
+        Args:
+            username: Twitch username/channel name
+            json_path: Path to save chat JSON
+            max_messages: Maximum messages to download (None = unlimited)
+            timeout: Stop after this many seconds (None = until stream ends)
+            shutdown_check: Optional callback function that returns True when shutdown requested
+            verbose: Show chat message previews
+            
+        Returns:
+            bool: True if chat download succeeded, False otherwise
+        """
+        if not CHAT_DOWNLOADER_AVAILABLE or self.chat_downloader is None:
+            print(f'{Fore.RED}✗ chat_downloader not available{Style.RESET_ALL}')
+            print(f'{Fore.YELLOW}  Install with: pip install chat-downloader{Style.RESET_ALL}')
+            return False
+        
+        if not self.download_live_chat:
+            print(f'{Fore.YELLOW}⚠ downloadLiveCHAT is disabled in config{Style.RESET_ALL}')
+            return False
+        
+        print(f'\n{Fore.CYAN}Starting live chat download (chat_downloader)...{Style.RESET_ALL}')
+        print(f'{Fore.MAGENTA}[VERBOSE] chat_downloader library version: {ChatDownloader.__module__}{Style.RESET_ALL}')
+        
+        try:
+            # Construct Twitch stream URL
+            stream_url = f'https://www.twitch.tv/{username}'
+            print(f'{Fore.YELLOW}Downloading chat from: {stream_url}{Style.RESET_ALL}')
+            print(f'{Fore.MAGENTA}[VERBOSE] Output path: {json_path}{Style.RESET_ALL}')
+            print(f'{Fore.MAGENTA}[VERBOSE] Timeout: {timeout}s (None = unlimited){Style.RESET_ALL}')
+            print(f'{Fore.MAGENTA}[VERBOSE] Max messages: {max_messages} (None = unlimited){Style.RESET_ALL}')
+            
+            # Get chat messages
+            print(f'{Fore.CYAN}Connecting to Twitch chat...{Style.RESET_ALL}')
+            chat = self.chat_downloader.get_chat(
+                stream_url,
+                message_types=['text_message'],  # Basic text messages
+                output=json_path,
+                timeout=timeout,
+                max_messages=max_messages
+            )
+            
+            # The get_chat with output parameter writes to file automatically
+            # We just need to iterate to trigger the download
+            message_count = 0
+            print(f'{Fore.CYAN}Receiving chat messages (press Ctrl+C to stop)...{Style.RESET_ALL}')
+            try:
+                for message in chat:
+                    # Check for shutdown request
+                    if shutdown_check and shutdown_check():
+                        print(f'\n{Fore.YELLOW}⚠ Chat download stopped by shutdown request{Style.RESET_ALL}')
+                        break
+                    
+                    message_count += 1
+                    
+                    # Show progress every 100 messages
+                    if message_count % 100 == 0:
+                        print(f'{Fore.CYAN}  Downloaded {message_count} messages...{Style.RESET_ALL}', end='\r')
+                    
+                    # Show chat previews in verbose mode (every 10 messages)
+                    if verbose and message_count % 10 == 0:
+                        author = message.get('author', {}).get('name', 'Unknown')
+                        msg_text = message.get('message', 'N/A')
+                        # Truncate long messages
+                        if len(msg_text) > 60:
+                            msg_text = msg_text[:60] + '...'
+                        print(f'\n{Fore.GREEN}💬 {author}: {Fore.WHITE}{msg_text}{Style.RESET_ALL}')
+                    
+                    # Print sample message every 500 for extra debugging
+                    if message_count % 500 == 0:
+                        print(f'\n{Fore.MAGENTA}[VERBOSE] Sample message #{message_count}: {message.get("message", "N/A")[:50]}...{Style.RESET_ALL}')
+            except KeyboardInterrupt:
+                print(f'\n{Fore.YELLOW}⚠ Chat download interrupted by user{Style.RESET_ALL}')
+            except Exception as e:
+                # Stream might have ended
+                print(f'\n{Fore.YELLOW}Chat download stopped: {str(e)}{Style.RESET_ALL}')
+            
+            # Check if file was created
+            if os.path.exists(json_path):
+                file_size = os.path.getsize(json_path)
+                if file_size > 100:
+                    print(f'\n{Fore.GREEN}✓ Live chat downloaded ({message_count} messages, {file_size} bytes){Style.RESET_ALL}')
+                    return True
+                else:
+                    print(f'\n{Fore.RED}✗ Chat file too small ({file_size} bytes){Style.RESET_ALL}')
+                    return False
+            else:
+                print(f'\n{Fore.RED}✗ Chat file was not created{Style.RESET_ALL}')
+                return False
+                
+        except Exception as e:
+            print(f'{Fore.RED}✗ chat_downloader failed: {str(e)}{Style.RESET_ALL}')
+            import traceback
+            traceback.print_exc()
+            return False
+    
+    def start_chat_downloader_thread(self, username: str, json_path: str,
+                                    shutdown_check: Optional[callable] = None,
+                                    verbose: bool = False) -> threading.Thread:
+        """
+        Start chat_downloader in a background thread.
+        
+        Args:
+            username: Twitch username
+            json_path: Path to save chat JSON
+            shutdown_check: Callback to check for shutdown
+            verbose: Show chat previews
+            
+        Returns:
+            threading.Thread: The thread running the download
+        """
+        def download_thread():
+            try:
+                self.chat_thread_success = self.download_live_chat_with_chat_downloader(
+                    username, json_path, 
+                    shutdown_check=shutdown_check,
+                    verbose=verbose
+                )
+            except Exception as e:
+                self.chat_thread_error = e
+                self.chat_thread_success = False
+                print(f'\n{Fore.RED}✗ Chat thread error: {e}{Style.RESET_ALL}')
+                import traceback
+                traceback.print_exc()
+        
+        thread = threading.Thread(target=download_thread, daemon=True)
+        thread.start()
+        self.chat_thread = thread
+        return thread
+    
+    def wait_for_chat_thread(self, timeout: Optional[float] = None) -> bool:
+        """
+        Wait for chat download thread to complete.
+        
+        Args:
+            timeout: Maximum time to wait in seconds
+            
+        Returns:
+            bool: True if chat download succeeded
+        """
+        if self.chat_thread is None:
+            return False
+        
+        try:
+            self.chat_thread.join(timeout=timeout)
+            
+            # Give a moment for file to be fully written and closed
+            if self.chat_thread_success:
+                print(f'{Fore.CYAN}Ensuring chat file is fully written and closed...{Style.RESET_ALL}')
+                time.sleep(1.0)  # Increased from 0.5s
+                
+                # Force garbage collection to help close file handles
+                import gc
+                gc.collect()
+                time.sleep(0.5)
+            
+            return self.chat_thread_success
+        except Exception as e:
+            print(f'{Fore.RED}✗ Error waiting for chat thread: {e}{Style.RESET_ALL}')
+            return False
+        finally:
+            self.chat_thread = None
+    
+    def wait_for_file_access(self, file_path: str, max_attempts: int = 10, delay: float = 0.5) -> bool:
+        """
+        Wait for a file to be accessible (not locked by another process).
+        
+        Args:
+            file_path: Path to the file to check
+            max_attempts: Maximum number of attempts
+            delay: Delay between attempts in seconds
+            
+        Returns:
+            bool: True if file is accessible, False otherwise
+        """
+        for attempt in range(max_attempts):
+            try:
+                # Try to open the file for reading with shared access
+                with open(file_path, 'r', encoding='utf-8') as f:
+                    # Try to read a bit to ensure it's really accessible
+                    f.read(1)
+                print(f'{Fore.GREEN}✓ File is accessible and ready to use{Style.RESET_ALL}')
+                return True
+            except PermissionError:
+                if attempt < max_attempts - 1:
+                    print(f'{Fore.YELLOW}⚠ File still locked, waiting... (attempt {attempt + 1}/{max_attempts}){Style.RESET_ALL}')
+                    time.sleep(delay)
+                else:
+                    print(f'{Fore.RED}✗ File still locked after {max_attempts} attempts{Style.RESET_ALL}')
+                    return False
+            except Exception as e:
+                print(f'{Fore.RED}✗ Error checking file access: {e}{Style.RESET_ALL}')
+                return False
+        
+        return False
+    
+    def convert_chat_downloader_to_twitch_format(self, input_path: str, output_path: str, 
+                                                    video_duration: Optional[float] = None) -> bool:
+        """
+        Convert chat_downloader JSON format to TwitchDownloaderCLI format.
+        
+        Args:
+            input_path: Path to chat_downloader JSON file
+            output_path: Path to save converted TwitchDownloaderCLI format
+            video_duration: Optional video duration in seconds (overrides calculated duration)
+            
+        Returns:
+            bool: True if conversion succeeded, False otherwise
+        """
+        try:
+            from datetime import datetime, timezone as dt_timezone
+            print(f'{Fore.CYAN}Converting chat format for rendering...{Style.RESET_ALL}')
+            
+            # Read chat_downloader format (JSON array)
+            with open(input_path, 'r', encoding='utf-8') as f:
+                messages = json.load(f)
+            
+            # Ensure we have a list
+            if not isinstance(messages, list):
+                print(f'{Fore.RED}✗ Expected JSON array, got {type(messages)}{Style.RESET_ALL}')
+                return False
+            
+            print(f'{Fore.CYAN}Converting {len(messages)} messages...{Style.RESET_ALL}')
+            
+            # Track first message timestamp for offset calculation
+            first_timestamp = None
+            
+            # Convert to TwitchDownloaderCLI format
+            comments = []
+            for msg in messages:
+                # Skip non-dictionary items
+                if not isinstance(msg, dict):
+                    continue
+                
+                # Extract data from chat_downloader format
+                # Timestamp is in microseconds, convert to seconds
+                timestamp_us = msg.get('timestamp', 0)
+                timestamp_sec = timestamp_us / 1000000.0 if timestamp_us else 0.0
+                
+                # Track first timestamp for offset calculation
+                if first_timestamp is None:
+                    first_timestamp = timestamp_sec
+                
+                # Calculate offset from start of stream
+                content_offset_seconds = timestamp_sec - first_timestamp
+                
+                # Extract author info
+                author = msg.get('author', {})
+                author_name = author.get('display_name', author.get('name', 'Unknown'))
+                author_login = author.get('name', author_name.lower())
+                author_id = author.get('id', '')
+                
+                # Extract message text
+                message_text = msg.get('message', '')
+                
+                # Format timestamp as ISO 8601
+                dt = datetime.fromtimestamp(timestamp_sec, tz=dt_timezone.utc)
+                created_at = dt.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z'
+                
+                # Get user color (chat_downloader uses "colour" - British spelling)
+                user_color = msg.get('colour', msg.get('color', '#FFFFFF'))
+                if not user_color:
+                    user_color = '#FFFFFF'
+                
+                comment = {
+                    "_id": msg.get('message_id', f"msg_{len(comments)}"),
+                    "created_at": created_at,
+                    "updated_at": created_at,
+                    "channel_id": msg.get('channel_id', ''),
+                    "content_type": "video",
+                    "content_id": "",
+                    "content_offset_seconds": content_offset_seconds,
+                    "commenter": {
+                        "display_name": author_name,
+                        "name": author_login,
+                        "_id": str(author_id),
+                        "type": "user",
+                        "bio": None,
+                        "created_at": created_at,
+                        "updated_at": created_at,
+                        "logo": None
+                    },
+                    "source": "chat",
+                    "state": "published",
+                    "message": {
+                        "body": message_text,
+                        "bits_spent": 0,
+                        "fragments": [
+                            {
+                                "text": message_text,
+                                "emoticon": None
+                            }
+                        ],
+                        "is_action": False,
+                        "user_badges": [],
+                        "user_color": user_color,
+                        "user_notice_params": {}
+                    },
+                    "more_replies": False
+                }
+                
+                comments.append(comment)
+            
+            if not comments:
+                print(f'{Fore.RED}✗ No valid comments to convert{Style.RESET_ALL}')
+                return False
+            
+            # Use provided video duration, or calculate from last message
+            print(f'{Fore.MAGENTA}[DEBUG] Input video_duration parameter: {video_duration}{Style.RESET_ALL}')
+            if video_duration is None:
+                video_duration = int(comments[-1]["content_offset_seconds"]) if comments else 0
+                print(f'{Fore.MAGENTA}[DEBUG] No video duration provided, calculated from chat: {video_duration}s{Style.RESET_ALL}')
+            else:
+                video_duration = int(video_duration)
+                print(f'{Fore.MAGENTA}[DEBUG] Using provided video duration: {video_duration}s{Style.RESET_ALL}')
+            
+            # Debug output
+            print(f'{Fore.MAGENTA}[DEBUG] First message offset: {comments[0]["content_offset_seconds"]:.2f}s{Style.RESET_ALL}')
+            print(f'{Fore.MAGENTA}[DEBUG] Last message offset: {comments[-1]["content_offset_seconds"]:.2f}s{Style.RESET_ALL}')
+            print(f'{Fore.MAGENTA}[DEBUG] Calculated duration: {video_duration}s{Style.RESET_ALL}')
+            print(f'{Fore.MAGENTA}[DEBUG] Sample comment structure:{Style.RESET_ALL}')
+            print(f'{Fore.MAGENTA}  ID: {comments[0]["_id"]}{Style.RESET_ALL}')
+            print(f'{Fore.MAGENTA}  Offset: {comments[0]["content_offset_seconds"]}{Style.RESET_ALL}')
+            print(f'{Fore.MAGENTA}  Message: {comments[0]["message"]["body"][:50]}{Style.RESET_ALL}')
+            print(f'{Fore.MAGENTA}[DEBUG] Final JSON duration will be: {video_duration}s{Style.RESET_ALL}')
+            print(f'{Fore.MAGENTA}[DEBUG] Comment count: {len(comments)}{Style.RESET_ALL}')
+            
+            # Create final structure matching TwitchDownloaderCLI format
+            from datetime import datetime as dt
+            created_at_iso = dt.now().isoformat() + 'Z'
+            
+            output_data = {
+                "FileInfo": {
+                    "Version": {
+                        "Major": 1,
+                        "Minor": 4,
+                        "Patch": 0
+                    },
+                    "CreatedAt": created_at_iso,
+                    "UpdatedAt": "0001-01-01T00:00:00"
+                },
+                "streamer": {
+                    "name": comments[0]["channel_id"] if comments else "",
+                    "login": comments[0]["channel_id"] if comments else "",
+                    "id": int(comments[0]["channel_id"]) if comments and comments[0]["channel_id"].isdigit() else 0
+                },
+                "video": {
+                    "title": "Live Chat",
+                    "description": None,
+                    "id": None,
+                    "created_at": comments[0]["created_at"] if comments else created_at_iso,
+                    "start": 0,
+                    "end": video_duration,
+                    "length": video_duration,
+                    "viewCount": 0,
+                    "game": None,
+                    "chapters": []
+                },
+                "comments": comments,
+                "embeddedData": None
+            }
+            
+            # Write to output file
+            with open(output_path, 'w', encoding='utf-8') as f:
+                json.dump(output_data, f, indent=2, ensure_ascii=False)
+            
+            # Verify output
+            file_size = os.path.getsize(output_path)
+            print(f'{Fore.MAGENTA}[DEBUG] Converted JSON file: {file_size:,} bytes{Style.RESET_ALL}')
+            
+            print(f'{Fore.GREEN}✓ Chat format converted successfully{Style.RESET_ALL}')
+            return True
+            
+        except Exception as e:
+            print(f'{Fore.RED}✗ Chat format conversion failed: {e}{Style.RESET_ALL}')
+            import traceback
+            traceback.print_exc()
+            return False
+    
+    def start_live_chat_download_with_fallback(self, username: str, vod_id: Optional[str], 
+                                              json_path: str) -> tuple[Optional[subprocess.Popen], str]:
+        """
+        Start live chat download with automatic fallback.
+        
+        Tries TwitchDownloaderCLI first (if VOD ID available and not using chat_downloader as primary),
+        falls back to chat_downloader if that fails or if VOD ID is not available.
+        
+        Args:
+            username: Twitch username
+            vod_id: Optional VOD ID (may be None if VODs disabled)
+            json_path: Path to save chat JSON
+            
+        Returns:
+            tuple: (process_handle or None, method_used)
+                   method_used is one of: 'twitch_downloader', 'chat_downloader', 'failed'
+        """
+        print(f'\n{Fore.MAGENTA}[VERBOSE] === CHAT DOWNLOAD FALLBACK LOGIC ==={Style.RESET_ALL}')
+        print(f'{Fore.MAGENTA}[VERBOSE] Username: {username}{Style.RESET_ALL}')
+        print(f'{Fore.MAGENTA}[VERBOSE] VOD ID: {vod_id}{Style.RESET_ALL}')
+        print(f'{Fore.MAGENTA}[VERBOSE] Primary method: {"chat_downloader" if self.use_chat_downloader_primary else "TwitchDownloaderCLI"}{Style.RESET_ALL}')
+        print(f'{Fore.MAGENTA}[VERBOSE] Fallback enabled: {self.use_chat_downloader_fallback}{Style.RESET_ALL}')
+        
+        # Determine primary method
+        use_twitch_downloader_first = (
+            vod_id is not None and 
+            not self.use_chat_downloader_primary and
+            self.download_live_chat
+        )
+        
+        print(f'{Fore.MAGENTA}[VERBOSE] Will try TwitchDownloaderCLI first: {use_twitch_downloader_first}{Style.RESET_ALL}')
+        
+        # Try TwitchDownloaderCLI first if conditions met
+        if use_twitch_downloader_first:
+            print(f'{Fore.CYAN}Attempting live chat download with TwitchDownloaderCLI...{Style.RESET_ALL}')
+            print(f'{Fore.MAGENTA}[VERBOSE] Starting TwitchDownloaderCLI process...{Style.RESET_ALL}')
+            process = self.start_live_chat_download(vod_id, json_path)
+            if process is not None:
+                print(f'{Fore.GREEN}✓ TwitchDownloaderCLI started successfully{Style.RESET_ALL}')
+                print(f'{Fore.MAGENTA}[VERBOSE] Process PID: {process.pid}{Style.RESET_ALL}')
+                return (process, 'twitch_downloader')
+            else:
+                print(f'{Fore.YELLOW}⚠ TwitchDownloaderCLI failed to start{Style.RESET_ALL}')
+                print(f'{Fore.MAGENTA}[VERBOSE] Will try fallback method...{Style.RESET_ALL}')
+        
+        # Try chat_downloader as fallback (or as primary)
+        if self.use_chat_downloader_fallback or self.use_chat_downloader_primary:
+            if vod_id is None:
+                print(f'{Fore.YELLOW}⚠ No VOD ID available - using chat_downloader fallback{Style.RESET_ALL}')
+                print(f'{Fore.MAGENTA}[VERBOSE] This typically means VODs are disabled on this channel{Style.RESET_ALL}')
+            
+            if CHAT_DOWNLOADER_AVAILABLE and self.chat_downloader is not None:
+                print(f'{Fore.CYAN}Using chat_downloader as {"primary method" if self.use_chat_downloader_primary else "fallback"}{Style.RESET_ALL}')
+                print(f'{Fore.MAGENTA}[VERBOSE] chat_downloader is available and initialized{Style.RESET_ALL}')
+                # Return special marker for chat_downloader
+                return (None, 'chat_downloader')
+            else:
+                print(f'{Fore.RED}✗ chat_downloader not available for fallback{Style.RESET_ALL}')
+                print(f'{Fore.MAGENTA}[VERBOSE] CHAT_DOWNLOADER_AVAILABLE: {CHAT_DOWNLOADER_AVAILABLE}{Style.RESET_ALL}')
+                print(f'{Fore.MAGENTA}[VERBOSE] chat_downloader instance: {self.chat_downloader}{Style.RESET_ALL}')
+                print(f'{Fore.YELLOW}  Install with: pip install chat-downloader{Style.RESET_ALL}')
+        else:
+            print(f'{Fore.YELLOW}⚠ Fallback disabled in configuration{Style.RESET_ALL}')
+        
+        # Both methods failed or unavailable
+        if vod_id is None:
+            print(f'{Fore.RED}✗ No VOD ID and no fallback available - cannot download live chat{Style.RESET_ALL}')
+        
+        print(f'{Fore.MAGENTA}[VERBOSE] All chat download methods exhausted{Style.RESET_ALL}')
+        return (None, 'failed')

requirements.txt

@@ -3,3 +3,4 @@ python-dotenv==1.0.1
 pytz==2024.2
 requests==2.32.3
 streamlink==8.2.0
+chat-downloader>=0.2.8

start.bat

@@ -10,7 +10,8 @@ rem Activate the virtual environment
     pip install -r requirements.txt
 
     rem Run the desired command in the virtual environment
-    python twitch-archive.py -u %1
+    rem Pass username as -u and forward all additional arguments
+    python twitch-archive.py -u %1 %2 %3 %4 %5 %6 %7 %8 %9
 
 rem Deactivate the virtual environment
 call "%VENV_PATH%\Scripts\deactivate.bat"

twitch-archive.py

@@ -380,14 +380,21 @@ class TwitchArchive:
                 
                 # Start live chat download if enabled
                 live_chat_process = None
+                live_chat_method = None  # Track which method was used
                 chat_json_path = str(self.file_manager.chat_json_path / f"{PREFIX_CHAT}{filename_base}.json")
                 
-                if self.downloadLiveCHAT and is_live.get('archiveVideo') and is_live['archiveVideo'].get('id'):
+                if self.downloadLiveCHAT:
-                    live_vod_id = is_live['archiveVideo']['id']
+                    vod_id = is_live.get('archiveVideo', {}).get('id') if is_live.get('archiveVideo') else None
-                    print(f'{Fore.CYAN}Live VOD ID detected: {live_vod_id}{Style.RESET_ALL}')
+                    stream_url = f"https://twitch.tv/{self.username}"
-                    live_chat_process = self.downloader.start_live_chat_download(live_vod_id, chat_json_path)
+                    
-                elif self.downloadLiveCHAT:
+                    live_chat_process, live_chat_method = self.downloader.start_live_chat_download_with_fallback(
-                    print(f'{Fore.YELLOW}⚠ No VOD ID available yet for live chat download{Style.RESET_ALL}')
+                        vod_id=vod_id,
+                        stream_url=stream_url,
+                        json_path=chat_json_path,
+                        use_chat_downloader_primary=self.use_chat_downloader_primary,
+                        no_chat_downloader_fallback=self.no_chat_downloader_fallback,
+                        verbose=self.verbose
+                    )
                 
                 # Record the live stream
                 recording_completed = self.recorder.record(is_live, live_raw_path)
@@ -411,12 +418,30 @@ class TwitchArchive:
                     chat_video_path = str(self.file_manager.chat_mp4_path / f"{PREFIX_CHAT}{filename_base}.mp4")
                     output_args = self.processor.build_chat_output_args()
                     
-                    # Get video duration to trim chat accordingly
+                    # Wait for chat file to be fully accessible (not locked)
+                    print(f'{Fore.CYAN}Verifying chat file is ready for rendering...{Style.RESET_ALL}')
+                    if not self.downloader.wait_for_file_access(chat_json_path, max_attempts=15, delay=0.5):
+                        print(f'{Fore.RED}✗ Chat file is locked, skipping rendering{Style.RESET_ALL}')
+                        chat_rendered_successfully = False
+                    else:
+                        # Get video duration first (needed for chat conversion and trimming)
                         ffmpeg_path = get_ffmpeg_executable(self.os_type)
                         video_duration = get_video_duration(live_proc_path, ffmpeg_path)
+                        print(f'{Fore.CYAN}Video duration for chat rendering: {video_duration}s{Style.RESET_ALL}')
+                        
+                        # Convert chat format if needed (chat_downloader uses different JSON structure)
+                        render_json_path = chat_json_path
+                        if live_chat_method == 'chat_downloader':
+                            print(f'{Fore.CYAN}Converting chat format for rendering...{Style.RESET_ALL}')
+                            converted_path = chat_json_path.replace('.json', '_converted.json')
+                            if self.downloader.convert_chat_downloader_to_twitch_format(chat_json_path, converted_path, video_duration):
+                                render_json_path = converted_path
+                                print(f'{Fore.GREEN}✓ Chat format converted successfully{Style.RESET_ALL}')
+                            else:
+                                print(f'{Fore.RED}✗ Failed to convert chat format{Style.RESET_ALL}')
                         
                         chat_rendered_successfully = self.downloader.render_chat(
-                        chat_json_path, 
+                            render_json_path, 
                             chat_video_path, 
                             output_args,
                             video_duration=video_duration
@@ -730,17 +755,26 @@ class TwitchArchiveManager:
     Manages multiple TwitchArchive instances for monitoring multiple streamers.
     """
     
-    def __init__(self, specific_streamer: Optional[str] = None, verbose: bool = False):
+    def __init__(self, specific_streamer: Optional[str] = None, verbose: bool = False,
+                 chat_only: bool = False,
+                 use_chat_downloader_primary: bool = False,
+                 use_chat_downloader_fallback: bool = True):
         """
         Initialize the manager.
         
         Args:
             specific_streamer: If provided, only monitor this streamer (ignore enabled status)
             verbose: Enable verbose debug output
+            chat_only: Only download chat, skip video recording (test mode)
+            use_chat_downloader_primary: Use chat_downloader as primary chat source
+            use_chat_downloader_fallback: Enable chat_downloader fallback
         """
         self.config_manager = ConfigManager()
         self.specific_streamer = specific_streamer
         self.verbose = verbose
+        self.chat_only = chat_only
+        self.use_chat_downloader_primary = use_chat_downloader_primary
+        self.use_chat_downloader_fallback = use_chat_downloader_fallback
         self.archivers: Dict[str, TwitchArchive] = {}
         self.shutdown_requested = False
         self.active_recordings: Dict[str, str] = {}  # Track active recordings: {username: stream_id}
@@ -783,6 +817,11 @@ class TwitchArchiveManager:
             TwitchArchive: Initialized archiver instance
         """
         config = self.config_manager.load_streamer_config(username)
+        
+        # Apply command-line overrides for chat_downloader options
+        config['useChatDownloaderPrimary'] = self.use_chat_downloader_primary
+        config['useChatDownloaderFallback'] = self.use_chat_downloader_fallback
+        
         archiver = TwitchArchive(config)
         return archiver
     
@@ -794,6 +833,8 @@ class TwitchArchiveManager:
         """
         print(f'\n{Fore.CYAN}{"=" * 70}{Style.RESET_ALL}')
         print(f'{Fore.CYAN}TWITCH ARCHIVE - Multi-Streamer Mode{Style.RESET_ALL}')
+        if self.chat_only:
+            print(f'{Fore.YELLOW}🧪 TEST MODE: Chat-Only (Video Recording Disabled){Style.RESET_ALL}')
         print(f'{Fore.CYAN}{"=" * 70}{Style.RESET_ALL}\n')
         
         # Get streamers to monitor
@@ -805,6 +846,14 @@ class TwitchArchiveManager:
             print(f'{Fore.CYAN}→ Or run with -u <username> to create a new config{Style.RESET_ALL}')
             sys.exit(1)
         
+        if self.chat_only:
+            print(f'{Fore.YELLOW}📝 Chat-Only Mode Enabled:{Style.RESET_ALL}')
+            print(f'{Fore.CYAN}  • Verbose logging: ON{Style.RESET_ALL}')
+            print(f'{Fore.CYAN}  • Video recording: DISABLED{Style.RESET_ALL}')
+            print(f'{Fore.CYAN}  • Chat download: ENABLED{Style.RESET_ALL}')
+            print(f'{Fore.CYAN}  • VOD download: DISABLED{Style.RESET_ALL}')
+            print()
+        
         print(f'{Fore.GREEN}Monitoring {len(streamers)} streamer(s):{Style.RESET_ALL}')
         for streamer in streamers:
             print(f'  • {Fore.CYAN}{streamer}{Style.RESET_ALL}')
@@ -985,26 +1034,131 @@ class TwitchArchiveManager:
         chat_json_path = str(archiver.file_manager.chat_json_path / f"{PREFIX_CHAT}{filename_base}.json")
         
         # Send notification
+        if not self.chat_only:
             archiver.notification_manager.send(
                 f"Stream Started - {archiver.username}",
                 f"Recording: {stream_info['title']}"
             )
         
-        # Start live chat download if enabled and VOD ID is available
+        # Start live chat download if enabled (with fallback support)
         live_chat_process = None
-        if archiver.downloadLiveCHAT and stream_info.get('archiveVideo') and stream_info['archiveVideo'].get('id'):
+        live_chat_method = 'failed'
+        if archiver.downloadLiveCHAT:
+            if self.verbose or self.chat_only:
+                print(f'\n{Fore.MAGENTA}[VERBOSE] Starting chat download process...{Style.RESET_ALL}')
+                print(f'{Fore.MAGENTA}[VERBOSE] downloadLiveCHAT: {archiver.downloadLiveCHAT}{Style.RESET_ALL}')
+                print(f'{Fore.MAGENTA}[VERBOSE] useChatDownloaderPrimary: {archiver.downloader.use_chat_downloader_primary}{Style.RESET_ALL}')
+                print(f'{Fore.MAGENTA}[VERBOSE] useChatDownloaderFallback: {archiver.downloader.use_chat_downloader_fallback}{Style.RESET_ALL}')
+            
+            # Get VOD ID if available
+            live_vod_id = None
+            if stream_info.get('archiveVideo') and stream_info['archiveVideo'].get('id'):
                 live_vod_id = stream_info['archiveVideo']['id']
                 print(f'{Fore.CYAN}Live VOD ID detected: {live_vod_id}{Style.RESET_ALL}')
-            live_chat_process = archiver.downloader.start_live_chat_download(live_vod_id, chat_json_path)
+                if self.verbose or self.chat_only:
-        elif archiver.downloadLiveCHAT:
+                    print(f'{Fore.MAGENTA}[VERBOSE] VOD URL: https://www.twitch.tv/videos/{live_vod_id}{Style.RESET_ALL}')
-            print(f'{Fore.YELLOW}⚠ No VOD ID available yet for live chat download{Style.RESET_ALL}')
+            else:
+                print(f'{Fore.YELLOW}⚠ No VOD ID available - will use fallback if configured{Style.RESET_ALL}')
+                if self.verbose or self.chat_only:
+                    print(f'{Fore.MAGENTA}[VERBOSE] This happens when streamer has VODs disabled{Style.RESET_ALL}')
+                    print(f'{Fore.MAGENTA}[VERBOSE] chat_downloader fallback will be used if enabled{Style.RESET_ALL}')
             
-        # Record livestream
+            # Try to start live chat download with fallback
+            try:
+                if self.verbose or self.chat_only:
+                    print(f'{Fore.MAGENTA}[VERBOSE] Calling start_live_chat_download_with_fallback(){Style.RESET_ALL}')
+                    print(f'{Fore.MAGENTA}[VERBOSE] Username: {archiver.username}{Style.RESET_ALL}')
+                    print(f'{Fore.MAGENTA}[VERBOSE] VOD ID: {live_vod_id}{Style.RESET_ALL}')
+                    print(f'{Fore.MAGENTA}[VERBOSE] Output path: {chat_json_path}{Style.RESET_ALL}')
+                
+                live_chat_process, live_chat_method = archiver.downloader.start_live_chat_download_with_fallback(
+                    archiver.username, live_vod_id, chat_json_path
+                )
+                
+                if self.verbose or self.chat_only:
+                    print(f'{Fore.MAGENTA}[VERBOSE] Chat download method selected: {live_chat_method}{Style.RESET_ALL}')
+                    print(f'{Fore.MAGENTA}[VERBOSE] Process handle: {live_chat_process}{Style.RESET_ALL}')
+                
+                # If chat_downloader is selected, start it in background thread now (before video recording)
+                if live_chat_method == 'chat_downloader' and not self.chat_only:
+                    if self.verbose:
+                        print(f'{Fore.MAGENTA}[VERBOSE] Starting chat_downloader in background thread...{Style.RESET_ALL}')
+                    try:
+                        print(f'{Fore.CYAN}Starting chat_downloader in background (concurrent with video)...{Style.RESET_ALL}')
+                        archiver.downloader.start_chat_downloader_thread(
+                            archiver.username, chat_json_path,
+                            shutdown_check=lambda: self.shutdown_requested or archiver.shutdown_requested,
+                            verbose=self.verbose
+                        )
+                    except Exception as e:
+                        print(f'{Fore.RED}✗ Failed to start chat thread: {e}{Style.RESET_ALL}')
+                        import traceback
+                        traceback.print_exc()
+                        live_chat_method = 'failed'
+                    
+            except Exception as e:
+                print(f'{Fore.RED}✗ Failed to start live chat download: {e}{Style.RESET_ALL}')
+                import traceback
+                traceback.print_exc()
+                live_chat_method = 'failed'
+        
+        # Record livestream (skip in chat-only mode)
+        if self.chat_only:
+            print(f'\n{Fore.YELLOW}🧪 Chat-Only Mode: Skipping video recording{Style.RESET_ALL}')
+            print(f'{Fore.CYAN}Waiting for chat download to complete...{Style.RESET_ALL}')
+            
+            # Start chat download based on method
+            if live_chat_method == 'chat_downloader':
+                if self.verbose:
+                    print(f'{Fore.MAGENTA}[VERBOSE] Starting chat_downloader in background thread...{Style.RESET_ALL}')
+                try:
+                    print(f'{Fore.CYAN}Using chat_downloader for live chat...{Style.RESET_ALL}')
+                    archiver.downloader.start_chat_downloader_thread(
+                        archiver.username, chat_json_path,
+                        shutdown_check=lambda: self.shutdown_requested or archiver.shutdown_requested,
+                        verbose=self.verbose or self.chat_only
+                    )
+                    # Wait for completion
+                    live_chat_downloaded = archiver.downloader.wait_for_chat_thread()
+                except Exception as e:
+                    print(f'{Fore.RED}✗ chat_downloader failed: {e}{Style.RESET_ALL}')
+                    import traceback
+                    traceback.print_exc()
+                    live_chat_downloaded = False
+            elif live_chat_method == 'twitch_downloader' and live_chat_process is not None:
+                if self.verbose:
+                    print(f'{Fore.MAGENTA}[VERBOSE] Waiting for TwitchDownloaderCLI process...{Style.RESET_ALL}')
+                live_chat_downloaded = archiver.downloader.wait_for_chat_download(live_chat_process, chat_json_path)
+            else:
+                live_chat_downloaded = False
+            
+            # Report results
+            if live_chat_downloaded:
+                print(f'\n{Fore.GREEN}✓ Chat-Only Test Complete!{Style.RESET_ALL}')
+                print(f'{Fore.CYAN}Chat saved to: {chat_json_path}{Style.RESET_ALL}')
+                if os.path.exists(chat_json_path):
+                    file_size = os.path.getsize(chat_json_path)
+                    print(f'{Fore.CYAN}File size: {file_size / 1024:.2f} KB{Style.RESET_ALL}')
+            else:
+                print(f'\n{Fore.RED}✗ Chat download failed{Style.RESET_ALL}')
+            
+            return  # Exit early, don't process video
+        
+        # Normal mode: Record livestream
         recording_successful = archiver.recorder.record(stream_info, live_raw_path)
         
         # Check if raw file exists (may exist even after interrupted recording)
         if not os.path.exists(live_raw_path):
             print(f'{Fore.RED}✗ No recording file found, skipping processing{Style.RESET_ALL}')
+            
+            # Still wait for chat if it's downloading
+            if live_chat_method == 'chat_downloader' and archiver.downloader.chat_thread is not None:
+                print(f'{Fore.CYAN}Waiting for chat download to finish...{Style.RESET_ALL}')
+                archiver.downloader.wait_for_chat_thread(timeout=30)
+            elif live_chat_method == 'twitch_downloader' and live_chat_process is not None:
+                print(f'{Fore.CYAN}Waiting for chat download to finish...{Style.RESET_ALL}')
+                archiver.downloader.wait_for_chat_download(live_chat_process, chat_json_path, timeout=30)
+            
             return
         
         # Get file size to check if anything was recorded
@@ -1023,20 +1177,64 @@ class TwitchArchiveManager:
         live_chat_downloaded = False
         chat_rendered_successfully = False
         chat_video_path = None
-        if live_chat_process is not None:
+        
+        # Handle different chat download methods
+        if live_chat_method == 'twitch_downloader' and live_chat_process is not None:
+            # Wait for TwitchDownloaderCLI process
+            print(f'{Fore.CYAN}Waiting for live chat download to complete...{Style.RESET_ALL}')
             live_chat_downloaded = archiver.downloader.wait_for_chat_download(live_chat_process, chat_json_path)
+        elif live_chat_method == 'chat_downloader' and archiver.downloader.chat_thread is not None:
+            # Wait for chat_downloader thread
+            print(f'{Fore.CYAN}Waiting for live chat download to complete...{Style.RESET_ALL}')
+            try:
+                live_chat_downloaded = archiver.downloader.wait_for_chat_thread()
+                if live_chat_downloaded:
+                    print(f'{Fore.GREEN}✓ Chat download thread completed successfully{Style.RESET_ALL}')
+                else:
+                    print(f'{Fore.YELLOW}⚠ Chat download thread completed with errors or no messages{Style.RESET_ALL}')
+            except Exception as e:
+                print(f'{Fore.RED}✗ Error waiting for chat download thread: {e}{Style.RESET_ALL}')
+                import traceback
+                traceback.print_exc()
+                live_chat_downloaded = False
         
         # Render live chat if downloaded successfully
         if live_chat_downloaded:
             chat_video_path = str(archiver.file_manager.chat_mp4_path / f"{PREFIX_CHAT}{filename_base}.mp4")
             output_args = archiver.processor.build_chat_output_args()
             
-            # Get video duration to trim chat accordingly
+            # Wait for chat file to be fully accessible (not locked)
+            print(f'{Fore.CYAN}Verifying chat file is ready for rendering...{Style.RESET_ALL}')
+            if not archiver.downloader.wait_for_file_access(chat_json_path, max_attempts=15, delay=0.5):
+                print(f'{Fore.RED}✗ Chat file is locked, skipping rendering{Style.RESET_ALL}')
+                chat_rendered_successfully = False
+            else:
+                # Get video duration first
                 ffmpeg_path = get_ffmpeg_executable(archiver.os_type)
                 video_duration = get_video_duration(live_proc_path, ffmpeg_path)
                 
+                if video_duration is None:
+                    print(f'{Fore.YELLOW}⚠ Could not detect video duration from {live_proc_path}{Style.RESET_ALL}')
+                    print(f'{Fore.YELLOW}  Will use chat message timestamps instead{Style.RESET_ALL}')
+                else:
+                    print(f'{Fore.CYAN}Video duration for chat rendering: {video_duration}s{Style.RESET_ALL}')
+                
+                # Convert chat format if chat_downloader was used
+                render_json_path = chat_json_path
+                if live_chat_method == 'chat_downloader':
+                    converted_path = chat_json_path.replace('.json', '_converted.json')
+                    print(f'{Fore.CYAN}Chat downloaded with chat_downloader, converting format...{Style.RESET_ALL}')
+                    if archiver.downloader.convert_chat_downloader_to_twitch_format(chat_json_path, converted_path, video_duration):
+                        render_json_path = converted_path
+                        print(f'{Fore.GREEN}✓ Using converted chat file for rendering{Style.RESET_ALL}')
+                    else:
+                        print(f'{Fore.RED}✗ Format conversion failed, skipping rendering{Style.RESET_ALL}')
+                        chat_rendered_successfully = False
+                        render_json_path = None
+                
+                if render_json_path:
                     chat_rendered_successfully = archiver.downloader.render_chat(
-                chat_json_path, 
+                        render_json_path, 
                         chat_video_path, 
                         output_args,
                         video_duration=video_duration
@@ -1225,6 +1423,10 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving
     -u, --username <name>   Monitor only this Twitch channel
     --verbose               Enable verbose debug output
     --legacy                Force legacy mode (use config.json)
+    --chat-only             Test mode: Only download chat (skip video recording)
+                            Automatically enables verbose logging
+    --use-chat-downloader-primary   Use chat_downloader as primary chat source (for testing)
+    --no-chat-downloader-fallback   Disable chat_downloader fallback
 
 {Fore.GREEN}LEGACY OPTIONS (when using --legacy):{Style.RESET_ALL}
     -q, --quality <qual>    Stream quality: best/source, high/720p, 
@@ -1247,6 +1449,8 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving
     python twitch-archive.py                    # Monitor all enabled streamers
     python twitch-archive.py -u vinesauce       # Monitor only vinesauce
     python twitch-archive.py -u hackerling --verbose  # Monitor with debug output
+    python twitch-archive.py -u streamername --chat-only  # Test chat download only (no video)
+    python twitch-archive.py --use-chat-downloader-primary  # Test chat_downloader library
     python twitch-archive.py --legacy           # Use old config.json mode
 
 {Fore.CYAN}{"=" * 70}{Style.RESET_ALL}
@@ -1257,7 +1461,8 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving
             argv,
             "h:u:q:a:v:c:m:r:d:n:",
             ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", 
-             "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose"]
+             "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose",
+             "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"]
         )
     except getopt.GetoptError as e:
         print(f'{Fore.RED}Error: {e}{Style.RESET_ALL}\n')
@@ -1270,7 +1475,26 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving
     # Parse command line args
     legacy_overrides = {}
     verbose_mode = False
+    chat_only_mode = False
+    use_chat_downloader_primary = False
+    use_chat_downloader_fallback = True  # Default to enabled
     for opt, arg in opts:
+        if opt in ('-h', '--help'):
+            print(help_msg)
+            sys.exit(0)
+        elif opt in ("-u", "--username"):
+            specific_streamer = arg
+        elif opt == "--verbose":
+            verbose_mode = True
+        elif opt == "--chat-only":
+            chat_only_mode = True
+            verbose_mode = True  # Auto-enable verbose for chat-only mode
+        elif opt == "--legacy":
+            use_legacy_mode = True
+        elif opt == "--use-chat-downloader-primary":
+            use_chat_downloader_primary = True
+        elif opt == "--no-chat-downloader-fallback":
+            use_chat_downloader_fallback = False
         if opt in ('-h', '--help'):
             print(help_msg)
             sys.exit(0)
@@ -1310,11 +1534,23 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving
         for key, value in legacy_overrides.items():
             setattr(twitch_archive, key, value)
         
+        # Apply chat_downloader options
+        if hasattr(twitch_archive.downloader, 'use_chat_downloader_primary'):
+            twitch_archive.downloader.use_chat_downloader_primary = use_chat_downloader_primary
+        if hasattr(twitch_archive.downloader, 'use_chat_downloader_fallback'):
+            twitch_archive.downloader.use_chat_downloader_fallback = use_chat_downloader_fallback
+        
         # Start the archive system
         twitch_archive.run()
     else:
         # New multi-streamer mode
-        manager = TwitchArchiveManager(specific_streamer=specific_streamer, verbose=verbose_mode)
+        manager = TwitchArchiveManager(
+            specific_streamer=specific_streamer, 
+            verbose=verbose_mode,
+            chat_only=chat_only_mode,
+            use_chat_downloader_primary=use_chat_downloader_primary,
+            use_chat_downloader_fallback=use_chat_downloader_fallback
+        )
         manager.run()