feat: add standalone chat downloader script and batch file for testing

modules/downloader.py

@@ -8,6 +8,8 @@ import subprocess
 import json
 import threading
 import time
+import socket
+import re
 from typing import Dict, Any, Optional
 from colorama import Fore, Style
 
@@ -326,6 +328,118 @@ class ContentDownloader:
             print(f'{Fore.RED}✗ Failed to start live chat download: {str(e)}{Style.RESET_ALL}')
             return None
 
+    def _download_live_chat_via_irc(self, username: str, json_path: str,
+                                    max_messages: Optional[int] = None,
+                                    timeout: Optional[float] = None,
+                                    shutdown_check: Optional[callable] = None,
+                                    stream_monitor = None,
+                                    verbose: bool = False) -> bool:
+        """
+        Simple IRC-based fallback to capture Twitch chat when GraphQL methods fail.
+
+        This writes newline-delimited JSON objects with at least: timestamp (ms),
+        author (dict with `name`), and `message`.
+        """
+        try:
+            sock = socket.socket()
+            sock.connect(('irc.chat.twitch.tv', 6667))
+            sock.settimeout(1.0)
+
+            # Request tags & capabilities
+            sock.sendall(b'CAP REQ :twitch.tv/tags twitch.tv/commands twitch.tv/membership\r\n')
+            sock.sendall(b'PASS SCHMOOPIIE\r\n')
+            sock.sendall(b'NICK justinfan67420\r\n')
+            sock.sendall(f'JOIN #{username}\r\n'.encode('utf-8'))
+
+            messages_written = 0
+            start_time = time.time()
+
+            # Open file for streaming newline-delimited JSON
+            os.makedirs(os.path.dirname(json_path), exist_ok=True)
+            with open(json_path, 'w', encoding='utf-8') as out_f:
+                buffer = ''
+                while True:
+                    # Shutdown/timeouts
+                    if shutdown_check and shutdown_check():
+                        break
+                    if timeout and (time.time() - start_time) > timeout:
+                        break
+                    if stream_monitor:
+                        try:
+                            if not stream_monitor.is_user_live():
+                                break
+                        except Exception:
+                            pass
+
+                    try:
+                        data = sock.recv(4096).decode('utf-8', 'ignore')
+                    except socket.timeout:
+                        continue
+                    except Exception as e:
+                        print(f'{Fore.YELLOW}⚠ IRC recv error: {e}{Style.RESET_ALL}')
+                        break
+
+                    if not data:
+                        continue
+
+                    buffer += data
+                    lines = buffer.split('\r\n')
+                    buffer = lines.pop()  # remainder
+
+                    for line in lines:
+                        if not line:
+                            continue
+                        # Respond to PINGs
+                        if line.startswith('PING'):
+                            try:
+                                sock.sendall(b'PONG :tmi.twitch.tv\r\n')
+                            except Exception:
+                                pass
+                            continue
+
+                        # Extract PRIVMSG lines
+                        m = re.match(r'(?:@[^ ]+ )?:([^!]+)!.* PRIVMSG #[^ ]+ :(.+)', line)
+                        if not m:
+                            continue
+
+                        author = m.group(1)
+                        msg_text = m.group(2)
+                        timestamp_ms = int(time.time() * 1000)
+
+                        item = {
+                            'timestamp': timestamp_ms,
+                            'author': {'name': author},
+                            'message': msg_text
+                        }
+
+                        out_f.write(json.dumps(item, ensure_ascii=False) + '\n')
+                        out_f.flush()
+                        messages_written += 1
+
+                        if verbose and (messages_written % 10 == 0):
+                            print(f'\n{Fore.GREEN}💬 {author}: {Fore.WHITE}{msg_text}{Style.RESET_ALL}')
+
+                        if max_messages and messages_written >= max_messages:
+                            break
+
+                    if max_messages and messages_written >= max_messages:
+                        break
+
+            sock.close()
+
+            if messages_written > 0:
+                print(f'\n{Fore.GREEN}✓ IRC fallback captured {messages_written} messages{Style.RESET_ALL}')
+                return True
+            else:
+                print(f'\n{Fore.RED}✗ IRC fallback captured no messages{Style.RESET_ALL}')
+                return False
+
+        except Exception as e:
+            print(f'{Fore.RED}✗ IRC fallback failed: {e}{Style.RESET_ALL}')
+            import traceback
+            traceback.print_exc()
+            return False
+    
     def wait_for_chat_download(self, process: Optional[subprocess.Popen], 
                                json_path: str, timeout: int = 300) -> bool:
         """
@@ -403,8 +517,12 @@ class ContentDownloader:
             print(f'{Fore.MAGENTA}[VERBOSE] Timeout: {timeout}s (None = unlimited){Style.RESET_ALL}')
             print(f'{Fore.MAGENTA}[VERBOSE] Max messages: {max_messages} (None = unlimited){Style.RESET_ALL}')
             
-            # Get chat messages
+            # Get chat messages with a small retry loop to handle transient GQL/network issues
             print(f'{Fore.CYAN}Connecting to Twitch chat...{Style.RESET_ALL}')
+            chat = None
+            max_attempts = 3
+            for attempt in range(1, max_attempts + 1):
+                try:
                     chat = self.chat_downloader.get_chat(
                         stream_url,
                         message_types=['text_message'],  # Basic text messages
@@ -412,6 +530,32 @@ class ContentDownloader:
                         timeout=timeout,
                         max_messages=max_messages
                     )
+                    break
+                except Exception as e:
+                    # Provide a clearer, user-facing message for common failures
+                    print(f"{Fore.YELLOW}⚠ chat_downloader attempt {attempt}/{max_attempts} failed: {str(e)}{Style.RESET_ALL}")
+                    # On final attempt, dump traceback to help diagnose library internals
+                    if attempt >= max_attempts:
+                        print(f"{Fore.RED}✗ chat_downloader failed after {max_attempts} attempts. This may be caused by Twitch GraphQL changes or rate-limiting.{Style.RESET_ALL}")
+                        print(f"{Fore.YELLOW}  Try upgrading the chat-downloader package: pip install -U chat-downloader{Style.RESET_ALL}")
+                        import traceback
+                        traceback.print_exc()
+                        # Try IRC fallback before giving up
+                        print(f"{Fore.MAGENTA}[VERBOSE] Attempting IRC fallback for chat capture...{Style.RESET_ALL}")
+                        try:
+                            return self._download_live_chat_via_irc(username, json_path,
+                                                                    max_messages=max_messages,
+                                                                    timeout=timeout,
+                                                                    shutdown_check=shutdown_check,
+                                                                    stream_monitor=stream_monitor,
+                                                                    verbose=verbose)
+                        except Exception as fallback_err:
+                            print(f"{Fore.RED}✗ IRC fallback failed: {fallback_err}{Style.RESET_ALL}")
+                            traceback.print_exc()
+                            return False
+                    else:
+                        time.sleep(1)
+                        continue
             
             # The get_chat with output parameter writes to file automatically
             # We just need to iterate to trigger the download

run_chat_only.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python3
+"""
+Start chat downloader standalone for testing without recording video.
+
+Usage:
+  python run_chat_only.py --username vinesauce [--output path] [--max-messages N] [--timeout S] [--verbose]
+
+This script uses the project's `ConfigManager` and `FileManager` to create
+appropriate directories and then starts the chat downloader in a background
+thread. Press Ctrl+C to stop.
+"""
+import argparse
+import time
+from datetime import datetime
+import os
+
+from colorama import Fore, Style
+
+from modules.config import ConfigManager
+from modules.file_manager import FileManager
+from modules.utils import get_ffmpeg_executable, get_twitch_downloader_executable, detect_operating_system
+from modules.downloader import ContentDownloader
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Run chat downloader standalone for testing')
+    parser.add_argument('--username', '-u', required=True, help='Twitch username/channel name')
+    parser.add_argument('--output', '-o', help='Output JSON path (optional)')
+    parser.add_argument('--max-messages', type=int, default=None, help='Max messages to capture')
+    parser.add_argument('--timeout', type=float, default=None, help='Timeout in seconds')
+    parser.add_argument('--verbose', action='store_true', help='Show verbose/chat previews')
+    parser.add_argument('--foreground', action='store_true', help='Run downloader in foreground (blocking)')
+    parser.add_argument('--use-chat-downloader-primary', action='store_true', help='Use chat_downloader as primary method')
+    parser.add_argument('--use-chat-downloader-fallback', dest='use_chat_downloader_fallback', action='store_true', help='Allow chat_downloader as fallback (default)')
+    parser.add_argument('--no-chat-downloader-fallback', dest='use_chat_downloader_fallback', action='store_false', help='Disable chat_downloader fallback')
+    args = parser.parse_args()
+
+    cfg = ConfigManager()
+    config = cfg.load_streamer_config(args.username)
+
+    # Apply overrides from CLI
+    if args.use_chat_downloader_primary:
+        config['useChatDownloaderPrimary'] = True
+    if args.use_chat_downloader_fallback is not None:
+        config['useChatDownloaderFallback'] = bool(args.use_chat_downloader_fallback)
+
+    # Ensure directories exist (use configured archive root path)
+    fm = FileManager(root_path=config.get('root_path', 'archive'), username=args.username, config=config)
+    fm.initialize_directories()
+
+    # Build default output path if not provided
+    if args.output:
+        json_path = args.output
+    else:
+        ts = datetime.now().strftime('%Y%m%d_%Hh%Mm%Ss')
+        json_path = str(fm.chat_json_path / f"CHAT_TEST_{args.username}_{ts}.json")
+
+    # Initialize downloader
+    os_type = detect_operating_system()
+    twitch_downloader_path = get_twitch_downloader_executable(os_type)
+    ffmpeg_path = get_ffmpeg_executable(os_type)
+
+    downloader = ContentDownloader(twitch_downloader_path=twitch_downloader_path,
+                                   ffmpeg_path=ffmpeg_path,
+                                   config=config)
+
+    print(f"{Fore.CYAN}Starting standalone chat downloader for {args.username}{Style.RESET_ALL}")
+    print(f"Output path: {json_path}")
+
+    stop_requested = {'stop': False}
+
+    def shutdown_check():
+        return stop_requested['stop']
+
+    # Start thread
+    thread = downloader.start_chat_downloader_thread(
+        args.username,
+        json_path,
+        shutdown_check=shutdown_check,
+        stream_monitor=None,
+        verbose=args.verbose
+    )
+
+    try:
+        if args.foreground:
+            # Run download directly in foreground
+            print('Running in foreground; this will block until download completes or interrupted')
+            success = downloader.download_live_chat_with_chat_downloader(
+                args.username,
+                json_path,
+                max_messages=args.max_messages,
+                timeout=args.timeout,
+                shutdown_check=shutdown_check,
+                stream_monitor=None,
+                verbose=args.verbose
+            )
+            print('Done, success=' + str(success))
+        else:
+            # Wait for thread to finish or until interrupted
+            while thread.is_alive():
+                time.sleep(0.5)
+    except KeyboardInterrupt:
+        print('\nKeyboard interrupt received; stopping downloader...')
+        stop_requested['stop'] = True
+        thread.join(timeout=5)
+
+
+if __name__ == '__main__':
+    main()

start_chat_only.bat

@@ -0,0 +1,18 @@
+rem @echo off
+setlocal
+
+rem Set the path to your virtual environment
+set VENV_PATH=.\venv314
+
+rem Activate the virtual environment
+call "%VENV_PATH%\Scripts\activate.bat"
+rem Ensure required packages are installed
+pip install -r requirements.txt
+
+rem Run the standalone chat downloader script
+rem Usage: start_chat_only.bat <username> [--output path] [--max-messages N] [--timeout S] [--verbose] [--foreground] [--use-chat-downloader-primary]
+rem Pass username as -u and forward additional arguments (mirrors start.bat behavior)
+python run_chat_only.py -u %1 %2 %3 %4 %5 %6 %7 %8 %9
+
+rem Deactivate the virtual environment
+call "%VENV_PATH%\Scripts\deactivate.bat"