feat: add standalone chat downloader script and batch file for testing
This commit is contained in:
parent
0d3cdfd12c
commit
22a1f5b600
3 changed files with 279 additions and 8 deletions
|
|
@ -8,6 +8,8 @@ import subprocess
|
|||
import json
|
||||
import threading
|
||||
import time
|
||||
import socket
|
||||
import re
|
||||
from typing import Dict, Any, Optional
|
||||
from colorama import Fore, Style
|
||||
|
||||
|
|
@ -325,6 +327,118 @@ class ContentDownloader:
|
|||
except Exception as e:
|
||||
print(f'{Fore.RED}✗ Failed to start live chat download: {str(e)}{Style.RESET_ALL}')
|
||||
return None
|
||||
|
||||
def _download_live_chat_via_irc(self, username: str, json_path: str,
|
||||
max_messages: Optional[int] = None,
|
||||
timeout: Optional[float] = None,
|
||||
shutdown_check: Optional[callable] = None,
|
||||
stream_monitor = None,
|
||||
verbose: bool = False) -> bool:
|
||||
"""
|
||||
Simple IRC-based fallback to capture Twitch chat when GraphQL methods fail.
|
||||
|
||||
This writes newline-delimited JSON objects with at least: timestamp (ms),
|
||||
author (dict with `name`), and `message`.
|
||||
"""
|
||||
try:
|
||||
sock = socket.socket()
|
||||
sock.connect(('irc.chat.twitch.tv', 6667))
|
||||
sock.settimeout(1.0)
|
||||
|
||||
# Request tags & capabilities
|
||||
sock.sendall(b'CAP REQ :twitch.tv/tags twitch.tv/commands twitch.tv/membership\r\n')
|
||||
sock.sendall(b'PASS SCHMOOPIIE\r\n')
|
||||
sock.sendall(b'NICK justinfan67420\r\n')
|
||||
sock.sendall(f'JOIN #{username}\r\n'.encode('utf-8'))
|
||||
|
||||
messages_written = 0
|
||||
start_time = time.time()
|
||||
|
||||
# Open file for streaming newline-delimited JSON
|
||||
os.makedirs(os.path.dirname(json_path), exist_ok=True)
|
||||
with open(json_path, 'w', encoding='utf-8') as out_f:
|
||||
buffer = ''
|
||||
while True:
|
||||
# Shutdown/timeouts
|
||||
if shutdown_check and shutdown_check():
|
||||
break
|
||||
if timeout and (time.time() - start_time) > timeout:
|
||||
break
|
||||
if stream_monitor:
|
||||
try:
|
||||
if not stream_monitor.is_user_live():
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
data = sock.recv(4096).decode('utf-8', 'ignore')
|
||||
except socket.timeout:
|
||||
continue
|
||||
except Exception as e:
|
||||
print(f'{Fore.YELLOW}⚠ IRC recv error: {e}{Style.RESET_ALL}')
|
||||
break
|
||||
|
||||
if not data:
|
||||
continue
|
||||
|
||||
buffer += data
|
||||
lines = buffer.split('\r\n')
|
||||
buffer = lines.pop() # remainder
|
||||
|
||||
for line in lines:
|
||||
if not line:
|
||||
continue
|
||||
# Respond to PINGs
|
||||
if line.startswith('PING'):
|
||||
try:
|
||||
sock.sendall(b'PONG :tmi.twitch.tv\r\n')
|
||||
except Exception:
|
||||
pass
|
||||
continue
|
||||
|
||||
# Extract PRIVMSG lines
|
||||
m = re.match(r'(?:@[^ ]+ )?:([^!]+)!.* PRIVMSG #[^ ]+ :(.+)', line)
|
||||
if not m:
|
||||
continue
|
||||
|
||||
author = m.group(1)
|
||||
msg_text = m.group(2)
|
||||
timestamp_ms = int(time.time() * 1000)
|
||||
|
||||
item = {
|
||||
'timestamp': timestamp_ms,
|
||||
'author': {'name': author},
|
||||
'message': msg_text
|
||||
}
|
||||
|
||||
out_f.write(json.dumps(item, ensure_ascii=False) + '\n')
|
||||
out_f.flush()
|
||||
messages_written += 1
|
||||
|
||||
if verbose and (messages_written % 10 == 0):
|
||||
print(f'\n{Fore.GREEN}💬 {author}: {Fore.WHITE}{msg_text}{Style.RESET_ALL}')
|
||||
|
||||
if max_messages and messages_written >= max_messages:
|
||||
break
|
||||
|
||||
if max_messages and messages_written >= max_messages:
|
||||
break
|
||||
|
||||
sock.close()
|
||||
|
||||
if messages_written > 0:
|
||||
print(f'\n{Fore.GREEN}✓ IRC fallback captured {messages_written} messages{Style.RESET_ALL}')
|
||||
return True
|
||||
else:
|
||||
print(f'\n{Fore.RED}✗ IRC fallback captured no messages{Style.RESET_ALL}')
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
print(f'{Fore.RED}✗ IRC fallback failed: {e}{Style.RESET_ALL}')
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return False
|
||||
|
||||
def wait_for_chat_download(self, process: Optional[subprocess.Popen],
|
||||
json_path: str, timeout: int = 300) -> bool:
|
||||
|
|
@ -403,15 +517,45 @@ class ContentDownloader:
|
|||
print(f'{Fore.MAGENTA}[VERBOSE] Timeout: {timeout}s (None = unlimited){Style.RESET_ALL}')
|
||||
print(f'{Fore.MAGENTA}[VERBOSE] Max messages: {max_messages} (None = unlimited){Style.RESET_ALL}')
|
||||
|
||||
# Get chat messages
|
||||
# Get chat messages with a small retry loop to handle transient GQL/network issues
|
||||
print(f'{Fore.CYAN}Connecting to Twitch chat...{Style.RESET_ALL}')
|
||||
chat = self.chat_downloader.get_chat(
|
||||
stream_url,
|
||||
message_types=['text_message'], # Basic text messages
|
||||
output=json_path,
|
||||
timeout=timeout,
|
||||
max_messages=max_messages
|
||||
)
|
||||
chat = None
|
||||
max_attempts = 3
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
try:
|
||||
chat = self.chat_downloader.get_chat(
|
||||
stream_url,
|
||||
message_types=['text_message'], # Basic text messages
|
||||
output=json_path,
|
||||
timeout=timeout,
|
||||
max_messages=max_messages
|
||||
)
|
||||
break
|
||||
except Exception as e:
|
||||
# Provide a clearer, user-facing message for common failures
|
||||
print(f"{Fore.YELLOW}⚠ chat_downloader attempt {attempt}/{max_attempts} failed: {str(e)}{Style.RESET_ALL}")
|
||||
# On final attempt, dump traceback to help diagnose library internals
|
||||
if attempt >= max_attempts:
|
||||
print(f"{Fore.RED}✗ chat_downloader failed after {max_attempts} attempts. This may be caused by Twitch GraphQL changes or rate-limiting.{Style.RESET_ALL}")
|
||||
print(f"{Fore.YELLOW} Try upgrading the chat-downloader package: pip install -U chat-downloader{Style.RESET_ALL}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
# Try IRC fallback before giving up
|
||||
print(f"{Fore.MAGENTA}[VERBOSE] Attempting IRC fallback for chat capture...{Style.RESET_ALL}")
|
||||
try:
|
||||
return self._download_live_chat_via_irc(username, json_path,
|
||||
max_messages=max_messages,
|
||||
timeout=timeout,
|
||||
shutdown_check=shutdown_check,
|
||||
stream_monitor=stream_monitor,
|
||||
verbose=verbose)
|
||||
except Exception as fallback_err:
|
||||
print(f"{Fore.RED}✗ IRC fallback failed: {fallback_err}{Style.RESET_ALL}")
|
||||
traceback.print_exc()
|
||||
return False
|
||||
else:
|
||||
time.sleep(1)
|
||||
continue
|
||||
|
||||
# The get_chat with output parameter writes to file automatically
|
||||
# We just need to iterate to trigger the download
|
||||
|
|
|
|||
109
run_chat_only.py
Normal file
109
run_chat_only.py
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
Start chat downloader standalone for testing without recording video.
|
||||
|
||||
Usage:
|
||||
python run_chat_only.py --username vinesauce [--output path] [--max-messages N] [--timeout S] [--verbose]
|
||||
|
||||
This script uses the project's `ConfigManager` and `FileManager` to create
|
||||
appropriate directories and then starts the chat downloader in a background
|
||||
thread. Press Ctrl+C to stop.
|
||||
"""
|
||||
import argparse
|
||||
import time
|
||||
from datetime import datetime
|
||||
import os
|
||||
|
||||
from colorama import Fore, Style
|
||||
|
||||
from modules.config import ConfigManager
|
||||
from modules.file_manager import FileManager
|
||||
from modules.utils import get_ffmpeg_executable, get_twitch_downloader_executable, detect_operating_system
|
||||
from modules.downloader import ContentDownloader
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Run chat downloader standalone for testing')
|
||||
parser.add_argument('--username', '-u', required=True, help='Twitch username/channel name')
|
||||
parser.add_argument('--output', '-o', help='Output JSON path (optional)')
|
||||
parser.add_argument('--max-messages', type=int, default=None, help='Max messages to capture')
|
||||
parser.add_argument('--timeout', type=float, default=None, help='Timeout in seconds')
|
||||
parser.add_argument('--verbose', action='store_true', help='Show verbose/chat previews')
|
||||
parser.add_argument('--foreground', action='store_true', help='Run downloader in foreground (blocking)')
|
||||
parser.add_argument('--use-chat-downloader-primary', action='store_true', help='Use chat_downloader as primary method')
|
||||
parser.add_argument('--use-chat-downloader-fallback', dest='use_chat_downloader_fallback', action='store_true', help='Allow chat_downloader as fallback (default)')
|
||||
parser.add_argument('--no-chat-downloader-fallback', dest='use_chat_downloader_fallback', action='store_false', help='Disable chat_downloader fallback')
|
||||
args = parser.parse_args()
|
||||
|
||||
cfg = ConfigManager()
|
||||
config = cfg.load_streamer_config(args.username)
|
||||
|
||||
# Apply overrides from CLI
|
||||
if args.use_chat_downloader_primary:
|
||||
config['useChatDownloaderPrimary'] = True
|
||||
if args.use_chat_downloader_fallback is not None:
|
||||
config['useChatDownloaderFallback'] = bool(args.use_chat_downloader_fallback)
|
||||
|
||||
# Ensure directories exist (use configured archive root path)
|
||||
fm = FileManager(root_path=config.get('root_path', 'archive'), username=args.username, config=config)
|
||||
fm.initialize_directories()
|
||||
|
||||
# Build default output path if not provided
|
||||
if args.output:
|
||||
json_path = args.output
|
||||
else:
|
||||
ts = datetime.now().strftime('%Y%m%d_%Hh%Mm%Ss')
|
||||
json_path = str(fm.chat_json_path / f"CHAT_TEST_{args.username}_{ts}.json")
|
||||
|
||||
# Initialize downloader
|
||||
os_type = detect_operating_system()
|
||||
twitch_downloader_path = get_twitch_downloader_executable(os_type)
|
||||
ffmpeg_path = get_ffmpeg_executable(os_type)
|
||||
|
||||
downloader = ContentDownloader(twitch_downloader_path=twitch_downloader_path,
|
||||
ffmpeg_path=ffmpeg_path,
|
||||
config=config)
|
||||
|
||||
print(f"{Fore.CYAN}Starting standalone chat downloader for {args.username}{Style.RESET_ALL}")
|
||||
print(f"Output path: {json_path}")
|
||||
|
||||
stop_requested = {'stop': False}
|
||||
|
||||
def shutdown_check():
|
||||
return stop_requested['stop']
|
||||
|
||||
# Start thread
|
||||
thread = downloader.start_chat_downloader_thread(
|
||||
args.username,
|
||||
json_path,
|
||||
shutdown_check=shutdown_check,
|
||||
stream_monitor=None,
|
||||
verbose=args.verbose
|
||||
)
|
||||
|
||||
try:
|
||||
if args.foreground:
|
||||
# Run download directly in foreground
|
||||
print('Running in foreground; this will block until download completes or interrupted')
|
||||
success = downloader.download_live_chat_with_chat_downloader(
|
||||
args.username,
|
||||
json_path,
|
||||
max_messages=args.max_messages,
|
||||
timeout=args.timeout,
|
||||
shutdown_check=shutdown_check,
|
||||
stream_monitor=None,
|
||||
verbose=args.verbose
|
||||
)
|
||||
print('Done, success=' + str(success))
|
||||
else:
|
||||
# Wait for thread to finish or until interrupted
|
||||
while thread.is_alive():
|
||||
time.sleep(0.5)
|
||||
except KeyboardInterrupt:
|
||||
print('\nKeyboard interrupt received; stopping downloader...')
|
||||
stop_requested['stop'] = True
|
||||
thread.join(timeout=5)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
18
start_chat_only.bat
Normal file
18
start_chat_only.bat
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
rem @echo off
|
||||
setlocal
|
||||
|
||||
rem Set the path to your virtual environment
|
||||
set VENV_PATH=.\venv314
|
||||
|
||||
rem Activate the virtual environment
|
||||
call "%VENV_PATH%\Scripts\activate.bat"
|
||||
rem Ensure required packages are installed
|
||||
pip install -r requirements.txt
|
||||
|
||||
rem Run the standalone chat downloader script
|
||||
rem Usage: start_chat_only.bat <username> [--output path] [--max-messages N] [--timeout S] [--verbose] [--foreground] [--use-chat-downloader-primary]
|
||||
rem Pass username as -u and forward additional arguments (mirrors start.bat behavior)
|
||||
python run_chat_only.py -u %1 %2 %3 %4 %5 %6 %7 %8 %9
|
||||
|
||||
rem Deactivate the virtual environment
|
||||
call "%VENV_PATH%\Scripts\deactivate.bat"
|
||||
Loading…
Add table
Add a link
Reference in a new issue