From 22a1f5b6006d7b652f50722db3e4572d193af544 Mon Sep 17 00:00:00 2001 From: MaddoScientisto Date: Sun, 15 Feb 2026 09:38:58 +0100 Subject: [PATCH] feat: add standalone chat downloader script and batch file for testing --- modules/downloader.py | 160 +++++++++++++++++++++++++++++++++++++++--- run_chat_only.py | 109 ++++++++++++++++++++++++++++ start_chat_only.bat | 18 +++++ 3 files changed, 279 insertions(+), 8 deletions(-) create mode 100644 run_chat_only.py create mode 100644 start_chat_only.bat diff --git a/modules/downloader.py b/modules/downloader.py index a1f166b..1a7ba00 100644 --- a/modules/downloader.py +++ b/modules/downloader.py @@ -8,6 +8,8 @@ import subprocess import json import threading import time +import socket +import re from typing import Dict, Any, Optional from colorama import Fore, Style @@ -325,6 +327,118 @@ class ContentDownloader: except Exception as e: print(f'{Fore.RED}✗ Failed to start live chat download: {str(e)}{Style.RESET_ALL}') return None + + def _download_live_chat_via_irc(self, username: str, json_path: str, + max_messages: Optional[int] = None, + timeout: Optional[float] = None, + shutdown_check: Optional[callable] = None, + stream_monitor = None, + verbose: bool = False) -> bool: + """ + Simple IRC-based fallback to capture Twitch chat when GraphQL methods fail. + + This writes newline-delimited JSON objects with at least: timestamp (ms), + author (dict with `name`), and `message`. + """ + try: + sock = socket.socket() + sock.connect(('irc.chat.twitch.tv', 6667)) + sock.settimeout(1.0) + + # Request tags & capabilities + sock.sendall(b'CAP REQ :twitch.tv/tags twitch.tv/commands twitch.tv/membership\r\n') + sock.sendall(b'PASS SCHMOOPIIE\r\n') + sock.sendall(b'NICK justinfan67420\r\n') + sock.sendall(f'JOIN #{username}\r\n'.encode('utf-8')) + + messages_written = 0 + start_time = time.time() + + # Open file for streaming newline-delimited JSON + os.makedirs(os.path.dirname(json_path), exist_ok=True) + with open(json_path, 'w', encoding='utf-8') as out_f: + buffer = '' + while True: + # Shutdown/timeouts + if shutdown_check and shutdown_check(): + break + if timeout and (time.time() - start_time) > timeout: + break + if stream_monitor: + try: + if not stream_monitor.is_user_live(): + break + except Exception: + pass + + try: + data = sock.recv(4096).decode('utf-8', 'ignore') + except socket.timeout: + continue + except Exception as e: + print(f'{Fore.YELLOW}⚠ IRC recv error: {e}{Style.RESET_ALL}') + break + + if not data: + continue + + buffer += data + lines = buffer.split('\r\n') + buffer = lines.pop() # remainder + + for line in lines: + if not line: + continue + # Respond to PINGs + if line.startswith('PING'): + try: + sock.sendall(b'PONG :tmi.twitch.tv\r\n') + except Exception: + pass + continue + + # Extract PRIVMSG lines + m = re.match(r'(?:@[^ ]+ )?:([^!]+)!.* PRIVMSG #[^ ]+ :(.+)', line) + if not m: + continue + + author = m.group(1) + msg_text = m.group(2) + timestamp_ms = int(time.time() * 1000) + + item = { + 'timestamp': timestamp_ms, + 'author': {'name': author}, + 'message': msg_text + } + + out_f.write(json.dumps(item, ensure_ascii=False) + '\n') + out_f.flush() + messages_written += 1 + + if verbose and (messages_written % 10 == 0): + print(f'\n{Fore.GREEN}💬 {author}: {Fore.WHITE}{msg_text}{Style.RESET_ALL}') + + if max_messages and messages_written >= max_messages: + break + + if max_messages and messages_written >= max_messages: + break + + sock.close() + + if messages_written > 0: + print(f'\n{Fore.GREEN}✓ IRC fallback captured {messages_written} messages{Style.RESET_ALL}') + return True + else: + print(f'\n{Fore.RED}✗ IRC fallback captured no messages{Style.RESET_ALL}') + return False + + except Exception as e: + print(f'{Fore.RED}✗ IRC fallback failed: {e}{Style.RESET_ALL}') + import traceback + traceback.print_exc() + return False def wait_for_chat_download(self, process: Optional[subprocess.Popen], json_path: str, timeout: int = 300) -> bool: @@ -403,15 +517,45 @@ class ContentDownloader: print(f'{Fore.MAGENTA}[VERBOSE] Timeout: {timeout}s (None = unlimited){Style.RESET_ALL}') print(f'{Fore.MAGENTA}[VERBOSE] Max messages: {max_messages} (None = unlimited){Style.RESET_ALL}') - # Get chat messages + # Get chat messages with a small retry loop to handle transient GQL/network issues print(f'{Fore.CYAN}Connecting to Twitch chat...{Style.RESET_ALL}') - chat = self.chat_downloader.get_chat( - stream_url, - message_types=['text_message'], # Basic text messages - output=json_path, - timeout=timeout, - max_messages=max_messages - ) + chat = None + max_attempts = 3 + for attempt in range(1, max_attempts + 1): + try: + chat = self.chat_downloader.get_chat( + stream_url, + message_types=['text_message'], # Basic text messages + output=json_path, + timeout=timeout, + max_messages=max_messages + ) + break + except Exception as e: + # Provide a clearer, user-facing message for common failures + print(f"{Fore.YELLOW}⚠ chat_downloader attempt {attempt}/{max_attempts} failed: {str(e)}{Style.RESET_ALL}") + # On final attempt, dump traceback to help diagnose library internals + if attempt >= max_attempts: + print(f"{Fore.RED}✗ chat_downloader failed after {max_attempts} attempts. This may be caused by Twitch GraphQL changes or rate-limiting.{Style.RESET_ALL}") + print(f"{Fore.YELLOW} Try upgrading the chat-downloader package: pip install -U chat-downloader{Style.RESET_ALL}") + import traceback + traceback.print_exc() + # Try IRC fallback before giving up + print(f"{Fore.MAGENTA}[VERBOSE] Attempting IRC fallback for chat capture...{Style.RESET_ALL}") + try: + return self._download_live_chat_via_irc(username, json_path, + max_messages=max_messages, + timeout=timeout, + shutdown_check=shutdown_check, + stream_monitor=stream_monitor, + verbose=verbose) + except Exception as fallback_err: + print(f"{Fore.RED}✗ IRC fallback failed: {fallback_err}{Style.RESET_ALL}") + traceback.print_exc() + return False + else: + time.sleep(1) + continue # The get_chat with output parameter writes to file automatically # We just need to iterate to trigger the download diff --git a/run_chat_only.py b/run_chat_only.py new file mode 100644 index 0000000..ff94189 --- /dev/null +++ b/run_chat_only.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +""" +Start chat downloader standalone for testing without recording video. + +Usage: + python run_chat_only.py --username vinesauce [--output path] [--max-messages N] [--timeout S] [--verbose] + +This script uses the project's `ConfigManager` and `FileManager` to create +appropriate directories and then starts the chat downloader in a background +thread. Press Ctrl+C to stop. +""" +import argparse +import time +from datetime import datetime +import os + +from colorama import Fore, Style + +from modules.config import ConfigManager +from modules.file_manager import FileManager +from modules.utils import get_ffmpeg_executable, get_twitch_downloader_executable, detect_operating_system +from modules.downloader import ContentDownloader + + +def main(): + parser = argparse.ArgumentParser(description='Run chat downloader standalone for testing') + parser.add_argument('--username', '-u', required=True, help='Twitch username/channel name') + parser.add_argument('--output', '-o', help='Output JSON path (optional)') + parser.add_argument('--max-messages', type=int, default=None, help='Max messages to capture') + parser.add_argument('--timeout', type=float, default=None, help='Timeout in seconds') + parser.add_argument('--verbose', action='store_true', help='Show verbose/chat previews') + parser.add_argument('--foreground', action='store_true', help='Run downloader in foreground (blocking)') + parser.add_argument('--use-chat-downloader-primary', action='store_true', help='Use chat_downloader as primary method') + parser.add_argument('--use-chat-downloader-fallback', dest='use_chat_downloader_fallback', action='store_true', help='Allow chat_downloader as fallback (default)') + parser.add_argument('--no-chat-downloader-fallback', dest='use_chat_downloader_fallback', action='store_false', help='Disable chat_downloader fallback') + args = parser.parse_args() + + cfg = ConfigManager() + config = cfg.load_streamer_config(args.username) + + # Apply overrides from CLI + if args.use_chat_downloader_primary: + config['useChatDownloaderPrimary'] = True + if args.use_chat_downloader_fallback is not None: + config['useChatDownloaderFallback'] = bool(args.use_chat_downloader_fallback) + + # Ensure directories exist (use configured archive root path) + fm = FileManager(root_path=config.get('root_path', 'archive'), username=args.username, config=config) + fm.initialize_directories() + + # Build default output path if not provided + if args.output: + json_path = args.output + else: + ts = datetime.now().strftime('%Y%m%d_%Hh%Mm%Ss') + json_path = str(fm.chat_json_path / f"CHAT_TEST_{args.username}_{ts}.json") + + # Initialize downloader + os_type = detect_operating_system() + twitch_downloader_path = get_twitch_downloader_executable(os_type) + ffmpeg_path = get_ffmpeg_executable(os_type) + + downloader = ContentDownloader(twitch_downloader_path=twitch_downloader_path, + ffmpeg_path=ffmpeg_path, + config=config) + + print(f"{Fore.CYAN}Starting standalone chat downloader for {args.username}{Style.RESET_ALL}") + print(f"Output path: {json_path}") + + stop_requested = {'stop': False} + + def shutdown_check(): + return stop_requested['stop'] + + # Start thread + thread = downloader.start_chat_downloader_thread( + args.username, + json_path, + shutdown_check=shutdown_check, + stream_monitor=None, + verbose=args.verbose + ) + + try: + if args.foreground: + # Run download directly in foreground + print('Running in foreground; this will block until download completes or interrupted') + success = downloader.download_live_chat_with_chat_downloader( + args.username, + json_path, + max_messages=args.max_messages, + timeout=args.timeout, + shutdown_check=shutdown_check, + stream_monitor=None, + verbose=args.verbose + ) + print('Done, success=' + str(success)) + else: + # Wait for thread to finish or until interrupted + while thread.is_alive(): + time.sleep(0.5) + except KeyboardInterrupt: + print('\nKeyboard interrupt received; stopping downloader...') + stop_requested['stop'] = True + thread.join(timeout=5) + + +if __name__ == '__main__': + main() diff --git a/start_chat_only.bat b/start_chat_only.bat new file mode 100644 index 0000000..e1c4d4a --- /dev/null +++ b/start_chat_only.bat @@ -0,0 +1,18 @@ +rem @echo off +setlocal + +rem Set the path to your virtual environment +set VENV_PATH=.\venv314 + +rem Activate the virtual environment +call "%VENV_PATH%\Scripts\activate.bat" +rem Ensure required packages are installed +pip install -r requirements.txt + +rem Run the standalone chat downloader script +rem Usage: start_chat_only.bat [--output path] [--max-messages N] [--timeout S] [--verbose] [--foreground] [--use-chat-downloader-primary] +rem Pass username as -u and forward additional arguments (mirrors start.bat behavior) +python run_chat_only.py -u %1 %2 %3 %4 %5 %6 %7 %8 %9 + +rem Deactivate the virtual environment +call "%VENV_PATH%\Scripts\deactivate.bat"