From e078cada3bc3bfd6d42594cd477b45159d3e0bb0 Mon Sep 17 00:00:00 2001 From: MaddoScientisto Date: Mon, 9 Feb 2026 23:46:11 +0100 Subject: [PATCH] Refactor code structure for improved readability and maintainability --- modules/__init__.py | 6 + modules/config.py | 158 +++++ modules/constants.py | 46 ++ modules/downloader.py | 281 ++++++++ modules/file_manager.py | 237 +++++++ modules/notifications.py | 68 ++ modules/processor.py | 171 +++++ modules/recorder.py | 105 +++ modules/stream_monitor.py | 140 ++++ modules/utils.py | 231 +++++++ twitch-archive.py | 1385 ++++++------------------------------- 11 files changed, 1642 insertions(+), 1186 deletions(-) create mode 100644 modules/__init__.py create mode 100644 modules/config.py create mode 100644 modules/constants.py create mode 100644 modules/downloader.py create mode 100644 modules/file_manager.py create mode 100644 modules/notifications.py create mode 100644 modules/processor.py create mode 100644 modules/recorder.py create mode 100644 modules/stream_monitor.py create mode 100644 modules/utils.py diff --git a/modules/__init__.py b/modules/__init__.py new file mode 100644 index 0000000..47ca7db --- /dev/null +++ b/modules/__init__.py @@ -0,0 +1,6 @@ +""" +Twitch Archive Modules +Refactored components for the Twitch Archive system. +""" + +__version__ = '2.0.0' diff --git a/modules/config.py b/modules/config.py new file mode 100644 index 0000000..9b9ed72 --- /dev/null +++ b/modules/config.py @@ -0,0 +1,158 @@ +""" +Configuration management for Twitch Archive. + +Handles loading global and per-streamer configuration files. +""" + +import json +import pathlib +from typing import Dict, Any +from colorama import Fore, Style + +from .constants import DEFAULT_CONFIG + + +class ConfigManager: + """ + Manages global and per-streamer configurations. + + Loads global defaults from config/global.json and merges with per-streamer + configs from config/streamers/*.json. + """ + + def __init__(self): + """Initialize the configuration manager.""" + self.config_dir = pathlib.Path(__file__).parent.parent / "config" + self.streamers_dir = self.config_dir / "streamers" + self.global_config = self._load_global_config() + + def _load_global_config(self) -> Dict[str, Any]: + """ + Load global configuration from config/global.json. + + Returns: + dict: Global configuration with defaults + """ + global_file = self.config_dir / "global.json" + + # Start with DEFAULT_CONFIG as ultimate fallback + config = DEFAULT_CONFIG.copy() + + # Try to load global config + if global_file.exists(): + try: + with open(global_file, 'r', encoding='utf-8') as f: + user_config = json.load(f) + # Filter out comment fields and schema references + user_config = {k: v for k, v in user_config.items() + if not k.startswith('_') and k != '$schema'} + config.update(user_config) + print(f'{Fore.GREEN}✓ Global configuration loaded from config/global.json{Style.RESET_ALL}') + except json.JSONDecodeError as e: + print(f'{Fore.YELLOW}⚠ Warning: Invalid JSON in config/global.json: {e}{Style.RESET_ALL}') + print(f'{Fore.YELLOW} Using default configuration{Style.RESET_ALL}') + except Exception as e: + print(f'{Fore.YELLOW}⚠ Warning: Could not load config/global.json: {e}{Style.RESET_ALL}') + else: + print(f'{Fore.YELLOW}⚠ Warning: config/global.json not found{Style.RESET_ALL}') + print(f'{Fore.CYAN} → Create config/global.json with default settings{Style.RESET_ALL}') + + return config + + def load_streamer_config(self, username: str) -> Dict[str, Any]: + """ + Load configuration for a specific streamer. + + Merges global config with streamer-specific overrides. + + Args: + username: Twitch username + + Returns: + dict: Complete configuration for the streamer + """ + # Start with global config + config = self.global_config.copy() + + # Load streamer-specific config + streamer_file = self.streamers_dir / f"{username}.json" + + if streamer_file.exists(): + try: + with open(streamer_file, 'r', encoding='utf-8') as f: + streamer_config = json.load(f) + # Filter out comments and schema references + streamer_config = {k: v for k, v in streamer_config.items() + if not k.startswith('_') and k != '$schema'} + # Merge streamer config (overrides global) + config.update(streamer_config) + print(f'{Fore.GREEN}✓ Loaded config for {username}{Style.RESET_ALL}') + except json.JSONDecodeError as e: + print(f'{Fore.YELLOW}⚠ Warning: Invalid JSON in {streamer_file}: {e}{Style.RESET_ALL}') + except Exception as e: + print(f'{Fore.YELLOW}⚠ Warning: Could not load {streamer_file}: {e}{Style.RESET_ALL}') + else: + # Create default config for new streamer + print(f'{Fore.CYAN}→ Creating default config for new streamer: {username}{Style.RESET_ALL}') + self.create_default_streamer_config(username) + config['username'] = username + config['enabled'] = True + + # Ensure username is set + config['username'] = username + + return config + + def create_default_streamer_config(self, username: str) -> None: + """ + Create a default configuration file for a new streamer. + + Args: + username: Twitch username + """ + # Ensure streamers directory exists + self.streamers_dir.mkdir(parents=True, exist_ok=True) + + streamer_file = self.streamers_dir / f"{username}.json" + + default_config = { + "$schema": "../streamer.schema.json", + "username": username, + "enabled": True + } + + try: + with open(streamer_file, 'w', encoding='utf-8') as f: + json.dump(default_config, f, indent=2) + print(f'{Fore.GREEN}✓ Created config file: config/streamers/{username}.json{Style.RESET_ALL}') + print(f'{Fore.CYAN} → Edit the file to add custom settings or overrides{Style.RESET_ALL}') + except Exception as e: + print(f'{Fore.RED}✗ Could not create config file for {username}: {e}{Style.RESET_ALL}') + + def get_all_enabled_streamers(self) -> list: + """ + Get list of all enabled streamers. + + Returns: + list: List of usernames configured and enabled + """ + if not self.streamers_dir.exists(): + return [] + + enabled_streamers = [] + + for config_file in self.streamers_dir.glob("*.json"): + try: + with open(config_file, 'r', encoding='utf-8') as f: + config = json.load(f) + # Filter comments and schema references + config = {k: v for k, v in config.items() + if not k.startswith('_') and k != '$schema'} + + if config.get('enabled', False): + username = config.get('username') or config_file.stem + enabled_streamers.append(username) + except Exception as e: + print(f'{Fore.YELLOW}⚠ Warning: Could not read {config_file}: {e}{Style.RESET_ALL}') + + return enabled_streamers diff --git a/modules/constants.py b/modules/constants.py new file mode 100644 index 0000000..b5e0cb3 --- /dev/null +++ b/modules/constants.py @@ -0,0 +1,46 @@ +""" +Constants and default configuration values for Twitch Archive. +""" + +# API Endpoints +TWITCH_OAUTH_URL = "https://id.twitch.tv/oauth2/token" +TWITCH_API_URL = "https://api.twitch.tv/helix" +TWITCH_GQL_URL = "https://gql.twitch.tv/gql" +TWITCH_GQL_CLIENT_ID = "kimne78kx3ncx6brgo4mv6wki5h1ko" + +# File prefixes for different content types +PREFIX_LIVE = "LIVE_" +PREFIX_VOD = "VOD_" +PREFIX_CHAT = "CHAT_" +PREFIX_METADATA = "METADA_" # Note: keeping original typo for compatibility + +# Default configuration values +DEFAULT_CONFIG = { + 'username': 'your_twitch_username', + 'quality': 'best', + 'root_path': 'archive', + 'rclone_path': 'remote:path/to/streams', + 'refresh': 60.0, + 'streamlink_ttvlol': False, + 'notifications': False, + 'downloadMETADATA': True, + 'downloadVOD': True, + 'downloadCHAT': True, + 'downloadLiveCHAT': True, + 'vodTimeout': 300, + 'uploadCloud': True, + 'deleteFiles': False, + 'onlyRaw': False, + 'cleanRaw': True, + 'hls_segments': 3, + 'hls_segmentsVOD': 10, + # FFmpeg 8.0+ Enhancement Options + 'ffmpeg_hwaccel': 'auto', # Hardware acceleration: 'auto', 'nvenc', 'qsv', 'amf', 'vaapi', 'none' + 'ffmpeg_threads': 0, # Thread count (0 = auto-detect) + 'ffmpeg_audio_codec': 'aac', # Audio codec for audio-only streams + 'ffmpeg_audio_samplerate': 48000, # Audio sample rate (48000 recommended for broadcasts) + 'ffmpeg_audio_bitrate': '192k', # Audio bitrate + 'ffmpeg_error_recovery': True, # Enable error recovery for corrupted streams + 'ffmpeg_faststart': True, # Enable faststart for MP4 (better streaming compatibility) + 'ffmpeg_progress': False # Show encoding progress +} diff --git a/modules/downloader.py b/modules/downloader.py new file mode 100644 index 0000000..09d515a --- /dev/null +++ b/modules/downloader.py @@ -0,0 +1,281 @@ +""" +VOD and chat downloading functionality using TwitchDownloaderCLI. +""" + +import os +import subprocess +from typing import Dict, Any, Optional +from colorama import Fore, Style + +from .utils import get_bin_path + + +class ContentDownloader: + """Handles VOD and chat downloading using TwitchDownloaderCLI.""" + + def __init__(self, twitch_downloader_path: str, ffmpeg_path: str, config: dict): + """ + Initialize the content downloader. + + Args: + twitch_downloader_path: Path to TwitchDownloaderCLI executable + ffmpeg_path: Path to FFmpeg executable + config: Configuration dictionary + """ + self.twitch_downloader_path = twitch_downloader_path + self.ffmpeg_path = ffmpeg_path + self.quality = config.get('quality', 'best') + self.hls_segments_vod = config.get('hls_segmentsVOD', 10) + self.download_vod = config.get('downloadVOD', True) + self.download_chat = config.get('downloadCHAT', True) + self.download_live_chat = config.get('downloadLiveCHAT', True) + + def download_vod(self, vod_info: Dict[str, Any], output_path: str) -> bool: + """ + Download VOD using TwitchDownloaderCLI. + + Args: + vod_info: VOD metadata from Twitch API + output_path: Path where the VOD will be saved + + Returns: + bool: True if download succeeded, False otherwise + """ + if not self.download_vod: + return False + + print(f'\n{Fore.CYAN}Downloading VOD: {vod_info["title"]}{Style.RESET_ALL}') + + # Extract numeric VOD ID + vod_id = vod_info["id"] + if isinstance(vod_id, str) and vod_id.startswith('v'): + vod_id = vod_id[1:] + + vod_url = f"https://www.twitch.tv/videos/{vod_id}" + print(f'{Fore.YELLOW}VOD URL: {vod_url}{Style.RESET_ALL}') + + bin_path = get_bin_path() + cmd = [ + self.twitch_downloader_path, + 'videodownload', + '-u', vod_url, + '-q', self.quality, + '-t', str(self.hls_segments_vod), + '--ffmpeg-path', self.ffmpeg_path, + '--temp-path', os.path.join(bin_path, 'temp'), + '--collision', 'Rename', + '-o', output_path + ] + + try: + result = subprocess.call(cmd) + if result == 0: + print(f'{Fore.GREEN}✓ VOD downloaded{Style.RESET_ALL}') + return True + else: + print(f'{Fore.RED}✗ VOD download failed with exit code: {result}{Style.RESET_ALL}') + return False + except Exception as e: + print(f'{Fore.RED}✗ VOD download failed: {str(e)}{Style.RESET_ALL}') + return False + + def download_chat_json(self, vod_id: str, json_path: str) -> bool: + """ + Download chat JSON for a VOD. + + Args: + vod_id: VOD ID + json_path: Path to save chat JSON + + Returns: + bool: True if succeeded, False otherwise + """ + # Remove 'v' prefix if present + if isinstance(vod_id, str) and vod_id.startswith('v'): + vod_id = vod_id[1:] + + print(f'{Fore.YELLOW}Downloading chat JSON for VOD {vod_id}...{Style.RESET_ALL}') + + try: + result = subprocess.call([ + self.twitch_downloader_path, 'chatdownload', + '--id', vod_id, + '--embed-images', + '--collision', 'Rename', + '-o', json_path + ]) + + if result != 0: + print(f'{Fore.RED}✗ Chat JSON download failed with exit code: {result}{Style.RESET_ALL}') + return False + + if not os.path.exists(json_path): + print(f'{Fore.RED}✗ Chat JSON file was not created{Style.RESET_ALL}') + return False + + print(f'{Fore.GREEN}✓ Chat JSON downloaded{Style.RESET_ALL}') + return True + + except Exception as e: + print(f'{Fore.RED}✗ Chat download failed: {str(e)}{Style.RESET_ALL}') + return False + + def render_chat(self, json_path: str, video_path: str, output_args: str) -> bool: + """ + Render chat JSON as a video. + + Args: + json_path: Path to chat JSON file + video_path: Path to save rendered chat video + output_args: FFmpeg output arguments for encoding + + Returns: + bool: True if succeeded, False otherwise + """ + if not os.path.exists(json_path): + print(f'{Fore.RED}✗ Chat JSON file not found: {json_path}{Style.RESET_ALL}') + return False + + bin_path = get_bin_path() + + # Chat rendering settings + chat_settings = [ + '--background-color', '#FF111111', + '-w', '500', + '-h', '1080', + '--outline', + '-f', 'Arial', + '--font-size', '22', + '--update-rate', '1.0', + '--offline', + '--output-args', output_args, + '--ffmpeg-path', self.ffmpeg_path, + '--temp-path', os.path.join(bin_path, 'temp'), + '--collision', 'Rename' + ] + + try: + print(f'{Fore.YELLOW}Rendering chat video...{Style.RESET_ALL}') + + # Debug output + full_cmd = [self.twitch_downloader_path, 'chatrender', '-i', json_path, '-o', video_path] + chat_settings + print(f'{Fore.CYAN}DEBUG - Chat render command:{Style.RESET_ALL}') + print(f'{Fore.CYAN} Output args passed: {repr(output_args)}{Style.RESET_ALL}') + + result = subprocess.call(full_cmd) + + if result != 0: + print(f'{Fore.RED}✗ Chat render failed with exit code: {result}{Style.RESET_ALL}') + return False + + print(f'{Fore.GREEN}✓ Chat rendered{Style.RESET_ALL}') + return True + + except Exception as e: + print(f'{Fore.RED}✗ Chat rendering failed: {str(e)}{Style.RESET_ALL}') + return False + + def download_and_render_chat(self, vod_info: Dict[str, Any], json_path: str, + video_path: str, output_args: str) -> bool: + """ + Download chat logs and render them as video. + + Args: + vod_info: VOD metadata from Twitch API + json_path: Path to save chat JSON + video_path: Path to save rendered chat video + output_args: FFmpeg output arguments for encoding + + Returns: + bool: True if succeeded, False otherwise + """ + if not self.download_chat: + return False + + print(f'\n{Fore.CYAN}Downloading chat: {vod_info["title"]}{Style.RESET_ALL}') + + # Extract numeric VOD ID + vod_id = vod_info["id"] + + # Download chat JSON + if not self.download_chat_json(vod_id, json_path): + return False + + # Render chat video + return self.render_chat(json_path, video_path, output_args) + + def start_live_chat_download(self, vod_id: str, json_path: str) -> Optional[subprocess.Popen]: + """ + Start downloading live chat in the background while stream is recording. + + Args: + vod_id: The VOD/stream ID to download chat from + json_path: Path to save chat JSON + + Returns: + subprocess.Popen: The process handle, or None if failed to start + """ + if not self.download_live_chat: + return None + + print(f'\n{Fore.CYAN}Starting live chat download...{Style.RESET_ALL}') + + # Remove 'v' prefix if present + if isinstance(vod_id, str) and vod_id.startswith('v'): + vod_id = vod_id[1:] + + try: + cmd = [ + self.twitch_downloader_path, 'chatdownload', + '--id', vod_id, + '--embed-images', + '--collision', 'Rename', + '-o', json_path + ] + + print(f'{Fore.YELLOW}Live chat download started in background for VOD {vod_id}{Style.RESET_ALL}') + process = subprocess.Popen( + cmd, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL + ) + return process + + except Exception as e: + print(f'{Fore.RED}✗ Failed to start live chat download: {str(e)}{Style.RESET_ALL}') + return None + + def wait_for_chat_download(self, process: Optional[subprocess.Popen], + json_path: str, timeout: int = 300) -> bool: + """ + Wait for live chat download process to complete. + + Args: + process: The chat download process handle + json_path: Path where chat JSON should be saved + timeout: Maximum time to wait in seconds + + Returns: + bool: True if chat download succeeded, False otherwise + """ + if process is None: + return False + + try: + print(f'{Fore.YELLOW}Waiting for live chat download to complete...{Style.RESET_ALL}') + return_code = process.wait(timeout=timeout) + + if return_code == 0 and os.path.exists(json_path): + print(f'{Fore.GREEN}✓ Live chat JSON downloaded{Style.RESET_ALL}') + return True + else: + print(f'{Fore.RED}✗ Live chat download failed (exit code: {return_code}){Style.RESET_ALL}') + return False + + except subprocess.TimeoutExpired: + print(f'{Fore.YELLOW}⚠ Live chat download timed out, terminating...{Style.RESET_ALL}') + process.terminate() + return False + except Exception as e: + print(f'{Fore.RED}✗ Error waiting for chat download: {str(e)}{Style.RESET_ALL}') + return False diff --git a/modules/file_manager.py b/modules/file_manager.py new file mode 100644 index 0000000..8848b8a --- /dev/null +++ b/modules/file_manager.py @@ -0,0 +1,237 @@ +""" +Cloud storage and file management for Twitch Archive. +""" + +import os +import json +import pathlib +import subprocess +from typing import List +from colorama import Fore, Style + +from .constants import PREFIX_LIVE, PREFIX_VOD, PREFIX_CHAT, PREFIX_METADATA +from .utils import get_bin_path + + +class FileManager: + """Handles file operations, cloud uploads, and cleanup.""" + + def __init__(self, root_path: str, username: str, config: dict): + """ + Initialize the file manager. + + Args: + root_path: Root directory for archives + username: Twitch username + config: Configuration dictionary + """ + self.root_path = pathlib.Path(root_path) + self.username = username + self.upload_cloud = config.get('uploadCloud', True) + self.delete_files = config.get('deleteFiles', False) + self.clean_raw = config.get('cleanRaw', True) + self.download_vod = config.get('downloadVOD', True) + self.download_chat = config.get('downloadCHAT', True) + self.download_metadata = config.get('downloadMETADATA', True) + self.rclone_path = config.get('rclone_path', 'remote:path') + + # Initialize paths + self.raw_path = self.root_path / username / "video" / "raw" + self.video_path = self.root_path / username / "video" + self.chat_json_path = self.root_path / username / "chat" / "json" + self.chat_mp4_path = self.root_path / username / "chat" + self.metadata_path = self.root_path / username / "metadata" + self.log_file = self.root_path / ".log" + + def initialize_directories(self) -> None: + """Create all necessary directory structures.""" + for path in [self.raw_path, self.video_path, self.chat_json_path, + self.chat_mp4_path, self.metadata_path]: + path.mkdir(parents=True, exist_ok=True) + + # Create log file if it doesn't exist + if not self.log_file.exists(): + self.log_file.touch() + + def is_stream_processed(self, stream_id: str) -> bool: + """ + Check if a stream has already been processed. + + Args: + stream_id: Unique identifier for the stream + + Returns: + bool: True if already processed, False otherwise + """ + with open(self.log_file, 'r', encoding='utf-8') as f: + return stream_id in f.read() + + def mark_stream_processed(self, stream_id: str) -> None: + """Add stream to log file to prevent re-processing.""" + with open(self.log_file, 'a', encoding='utf-8') as f: + f.write(f"{stream_id}\n") + + def save_metadata(self, vod_info: dict, filename_base: str) -> None: + """ + Save VOD metadata to JSON file. + + Args: + vod_info: VOD metadata from Twitch API + filename_base: Base filename (without extension) + """ + if not self.download_metadata: + return + + metadata_path = self.metadata_path / f"{PREFIX_METADATA}{filename_base}.json" + + with open(metadata_path, 'w', encoding='utf-8') as f: + json.dump(vod_info, f, ensure_ascii=False, indent=4) + + print(f'{Fore.GREEN}✓ Metadata saved{Style.RESET_ALL}') + + def clean_raw_file(self, raw_path: str) -> None: + """ + Delete raw .ts file if configured. + + Args: + raw_path: Path to raw file + """ + if self.clean_raw and os.path.exists(raw_path): + print(f'{Fore.YELLOW}Deleting raw .ts file...{Style.RESET_ALL}') + os.remove(raw_path) + + def upload_to_cloud(self, filename_base: str, notification_callback=None) -> bool: + """ + Upload archived files to cloud storage using rclone. + + Args: + filename_base: Base filename (without prefixes/extensions) + notification_callback: Optional callback to send notifications + + Returns: + bool: True if upload succeeded or is disabled, False if failed + """ + if not self.upload_cloud: + return True + + print(f'\n{Fore.CYAN}Uploading to cloud storage...{Style.RESET_ALL}') + if notification_callback: + notification_callback(f'☁ Uploading - {filename_base}', 'Uploading files to cloud storage') + + # Create list of files to upload + bin_path = get_bin_path() + upload_list_path = os.path.join(bin_path, 'temp', 'upload.txt') + + # Ensure temp directory exists + os.makedirs(os.path.dirname(upload_list_path), exist_ok=True) + + files_to_upload = [ + f"{PREFIX_LIVE}{filename_base}.ts", + f"{PREFIX_LIVE}{filename_base}.mp4", + f"{PREFIX_LIVE}{filename_base}.mp3", + f"{PREFIX_VOD}{filename_base}.ts", + f"{PREFIX_VOD}{filename_base}.mp4", + f"{PREFIX_VOD}{filename_base}.mp3", + f"{PREFIX_METADATA}{filename_base}.json", + f"{PREFIX_CHAT}{filename_base}.json", + f"{PREFIX_CHAT}{filename_base}.mp4" + ] + + with open(upload_list_path, 'w') as f: + f.write('\n'.join(files_to_upload)) + + # Run rclone + try: + result = subprocess.call([ + 'rclone', 'copy', + str(self.root_path.resolve()), + self.rclone_path, + '--include-from', upload_list_path + ]) + + # Clean up upload list + if os.path.exists(upload_list_path): + os.remove(upload_list_path) + + if result == 0: + print(f'{Fore.GREEN}✓ Upload complete{Style.RESET_ALL}') + if notification_callback: + notification_callback(f'✓ Upload Success - {filename_base}', 'All files uploaded successfully') + return True + else: + print(f'{Fore.RED}✗ Upload failed (exit code: {result}){Style.RESET_ALL}') + print(f'{Fore.YELLOW}Files preserved locally due to upload failure{Style.RESET_ALL}') + if notification_callback: + notification_callback(f'✗ Upload Failed - {filename_base}', + f'Upload failed with code {result}. Files preserved locally.') + return False + + except Exception as e: + print(f'{Fore.RED}✗ Upload error: {str(e)}{Style.RESET_ALL}') + return False + + def delete_local_files(self, filename_base: str, live_raw_path: str, + live_proc_path: str, notification_callback=None) -> None: + """ + Delete local archive files after successful upload. + + Args: + filename_base: Base filename (without prefixes/extensions) + live_raw_path: Path to live raw file + live_proc_path: Path to live processed file + notification_callback: Optional callback to send notifications + """ + print(f'\n{Fore.RED}{"=" * 60}{Style.RESET_ALL}') + print(f'{Fore.RED}⚠ DELETING LOCAL FILES{Style.RESET_ALL}') + print(f'{Fore.RED}{"=" * 60}{Style.RESET_ALL}\n') + + if notification_callback: + notification_callback(f'🗑 Deleting - {filename_base}', + 'Deleting local files after successful upload') + + files_to_delete: List[str] = [] + + # Live files + if not self.clean_raw and os.path.exists(live_raw_path): + files_to_delete.append(live_raw_path) + if os.path.exists(live_proc_path): + files_to_delete.append(live_proc_path) + + # VOD files + if self.download_vod: + vod_raw = self.raw_path / f"{PREFIX_VOD}{filename_base}.ts" + vod_mp4 = self.video_path / f"{PREFIX_VOD}{filename_base}.mp4" + vod_mp3 = self.video_path / f"{PREFIX_VOD}{filename_base}.mp3" + + if not self.clean_raw and vod_raw.exists(): + files_to_delete.append(str(vod_raw)) + if vod_mp4.exists(): + files_to_delete.append(str(vod_mp4)) + if vod_mp3.exists(): + files_to_delete.append(str(vod_mp3)) + + # Chat files + if self.download_chat: + chat_json = self.chat_json_path / f"{PREFIX_CHAT}{filename_base}.json" + chat_mp4 = self.chat_mp4_path / f"{PREFIX_CHAT}{filename_base}.mp4" + + if chat_json.exists(): + files_to_delete.append(str(chat_json)) + if chat_mp4.exists(): + files_to_delete.append(str(chat_mp4)) + + # Metadata files + if self.download_metadata: + metadata = self.metadata_path / f"{PREFIX_METADATA}{filename_base}.json" + if metadata.exists(): + files_to_delete.append(str(metadata)) + + # Delete all files + for filepath in files_to_delete: + try: + print(f'{Fore.RED} Deleting: {os.path.basename(filepath)}{Style.RESET_ALL}') + os.remove(filepath) + except Exception as e: + print(f'{Fore.YELLOW} ⚠ Failed to delete {filepath}: {e}{Style.RESET_ALL}') + + print(f'{Fore.RED}\n✓ Cleanup complete{Style.RESET_ALL}') diff --git a/modules/notifications.py b/modules/notifications.py new file mode 100644 index 0000000..99a6e84 --- /dev/null +++ b/modules/notifications.py @@ -0,0 +1,68 @@ +""" +Email notification functionality for Twitch Archive. +""" + +import os +import socket +import smtplib +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +from colorama import Fore, Style + + +class NotificationManager: + """Handles email notifications via Gmail SMTP.""" + + def __init__(self, enabled: bool = False, username: str = ""): + """ + Initialize the notification manager. + + Args: + enabled: Whether notifications are enabled + username: Streamer username for notification subject + """ + self.enabled = enabled + self.username = username + + def send(self, subject: str, content: str) -> None: + """ + Send email notification via Gmail SMTP. + + Only sends if notifications are enabled in configuration. + Requires SENDER, RECEIVER, and PASSWD in .env file. + + Args: + subject: Email subject line + content: Email body content + """ + if not self.enabled: + return + + try: + sender = os.getenv("SENDER") + receiver = os.getenv("RECEIVER") + password = os.getenv("PASSWD") + + if not all([sender, receiver, password]): + print(f'{Fore.YELLOW}⚠ Notification skipped: Missing email credentials in .env{Style.RESET_ALL}') + return + + # Construct email + msg = MIMEMultipart() + msg['From'] = sender + msg['To'] = receiver + msg['Subject'] = f"{self.username} - {subject}" + + body = f"Stream: {self.username}\n\n{content}" + msg.attach(MIMEText(body, 'plain')) + + # Send via Gmail SMTP + with smtplib.SMTP('smtp.gmail.com', 587) as server: + server.starttls() + server.login(sender, password) + server.sendmail(sender, receiver, msg.as_string()) + + except socket.error as e: + print(f'{Fore.YELLOW}⚠ Notification failed: {str(e)}{Style.RESET_ALL}') + except Exception as e: + print(f'{Fore.YELLOW}⚠ Notification error: {str(e)}{Style.RESET_ALL}') diff --git a/modules/processor.py b/modules/processor.py new file mode 100644 index 0000000..0a696ee --- /dev/null +++ b/modules/processor.py @@ -0,0 +1,171 @@ +""" +Video/audio processing functionality using FFmpeg. +""" + +import os +import subprocess +from colorama import Fore, Style + +from .utils import detect_hardware_acceleration, get_hwaccel_encoder + + +class StreamProcessor: + """Handles video/audio processing using FFmpeg.""" + + def __init__(self, os_type: str, ffmpeg_path: str, config: dict): + """ + Initialize the stream processor. + + Args: + os_type: Operating system type ('windows' or 'linux') + ffmpeg_path: Path to FFmpeg executable + config: Configuration dictionary with FFmpeg settings + """ + self.os_type = os_type + self.ffmpeg_path = ffmpeg_path + self.quality = config.get('quality', 'best') + self.only_raw = config.get('onlyRaw', False) + self.ffmpeg_threads = config.get('ffmpeg_threads', 0) + self.ffmpeg_audio_codec = config.get('ffmpeg_audio_codec', 'aac') + self.ffmpeg_audio_samplerate = config.get('ffmpeg_audio_samplerate', 48000) + self.ffmpeg_audio_bitrate = config.get('ffmpeg_audio_bitrate', '192k') + self.ffmpeg_error_recovery = config.get('ffmpeg_error_recovery', True) + self.ffmpeg_faststart = config.get('ffmpeg_faststart', True) + self.ffmpeg_progress = config.get('ffmpeg_progress', False) + self.hwaccel_type = detect_hardware_acceleration( + config.get('ffmpeg_hwaccel', 'auto'), + os_type + ) + + def process_raw_stream(self, raw_path: str, output_path: str) -> None: + """ + Process raw .ts file into mp4/mp3 using ffmpeg. + + Args: + raw_path: Path to the raw .ts file + output_path: Path for the processed output file + """ + if not os.path.exists(raw_path): + print(f'{Fore.YELLOW}⚠ Raw file not found, skipping processing{Style.RESET_ALL}') + return + + if self.only_raw: + print(f'{Fore.CYAN}Keeping raw .ts file (onlyRaw mode){Style.RESET_ALL}') + return + + print(f'{Fore.YELLOW}Processing raw stream file...{Style.RESET_ALL}') + + # Build ffmpeg command based on quality + if self.quality == 'audio_only': + self._process_audio(raw_path, output_path) + else: + self._process_video(raw_path, output_path) + + print(f'{Fore.GREEN}✓ Stream processed successfully{Style.RESET_ALL}') + + def _process_audio(self, raw_path: str, output_path: str) -> None: + """Process audio-only stream.""" + # Audio-only conversion with modern AAC encoding + cmd = [ + self.ffmpeg_path, + '-i', raw_path, + '-vn', # No video + '-c:a', self.ffmpeg_audio_codec, + '-ar', str(self.ffmpeg_audio_samplerate), + '-ac', '2', # Stereo + '-b:a', self.ffmpeg_audio_bitrate, + ] + + # Add threading for faster encoding + if self.ffmpeg_threads > 0: + cmd.extend(['-threads', str(self.ffmpeg_threads)]) + + # Add faststart for better streaming compatibility + if self.ffmpeg_faststart and output_path.endswith(('.mp4', '.m4a')): + cmd.extend(['-movflags', '+faststart']) + + cmd.append(output_path) + + # Run FFmpeg + if self.ffmpeg_progress: + subprocess.call(cmd) + else: + subprocess.call(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) + + def _process_video(self, raw_path: str, output_path: str) -> None: + """Process video stream.""" + cmd = [ + self.ffmpeg_path, + '-y', # Overwrite output file + ] + + # Add hardware acceleration if enabled + if self.hwaccel_type and self.hwaccel_type != 'none': + print(f'{Fore.CYAN}Using hardware acceleration: {self.hwaccel_type}{Style.RESET_ALL}') + cmd.extend(['-hwaccel', 'auto']) + + cmd.extend([ + '-i', raw_path, + '-analyzeduration', '2147483647', + '-probesize', '2147483647', + ]) + + # Threading support + if self.ffmpeg_threads >= 0: + cmd.extend(['-threads', str(self.ffmpeg_threads)]) + + # Error recovery options for corrupted streams + if self.ffmpeg_error_recovery: + cmd.extend([ + '-fflags', '+genpts', + '-avoid_negative_ts', 'make_zero', + '-err_detect', 'ignore_err' + ]) + + # Stream copy (fast, no re-encoding) + cmd.extend([ + '-c:v', 'copy', + '-c:a', 'copy', + '-start_at_zero', + '-copyts', + ]) + + # Add faststart for MP4 files + if self.ffmpeg_faststart and output_path.endswith('.mp4'): + cmd.extend(['-movflags', '+faststart']) + + cmd.append(output_path) + + # Run FFmpeg + if self.ffmpeg_progress: + subprocess.call(cmd) + else: + subprocess.call(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) + + def build_chat_output_args(self) -> str: + """ + Build FFmpeg output arguments for chat rendering with hardware acceleration support. + + Returns: + str: Complete FFmpeg output arguments string with "{save_path}" placeholder + """ + if self.hwaccel_type and self.hwaccel_type != 'none': + encoder = get_hwaccel_encoder(self.hwaccel_type) + print(f'{Fore.CYAN}Using hardware-accelerated encoder for chat: {encoder}{Style.RESET_ALL}') + + if 'nvenc' in encoder: + result = f'-c:v {encoder} -preset p4 -cq 18 -pix_fmt yuv420p "{{save_path}}"' + elif 'qsv' in encoder: + result = f'-c:v {encoder} -global_quality 18 -pix_fmt yuv420p "{{save_path}}"' + elif 'amf' in encoder: + result = f'-c:v {encoder} -qp_i 18 -pix_fmt yuv420p "{{save_path}}"' + elif 'vaapi' in encoder: + result = f'-c:v {encoder} -qp 18 -pix_fmt yuv420p "{{save_path}}"' + else: + result = f'-c:v libx264 -preset veryfast -crf 18 -pix_fmt yuv420p "{{save_path}}"' + else: + # Default software encoding + result = f'-c:v libx264 -preset veryfast -crf 18 -pix_fmt yuv420p "{{save_path}}"' + + print(f'{Fore.CYAN}DEBUG - Generated output_args: {result}{Style.RESET_ALL}') + return result diff --git a/modules/recorder.py b/modules/recorder.py new file mode 100644 index 0000000..fa9c9d1 --- /dev/null +++ b/modules/recorder.py @@ -0,0 +1,105 @@ +""" +Live stream recording functionality for Twitch Archive. +""" + +import os +import subprocess +from typing import Dict, Any, Optional +from colorama import Fore, Style + + +class StreamRecorder: + """Handles live stream recording using streamlink.""" + + def __init__(self, username: str, quality: str, refresh: float, + hls_segments: int, streamlink_ttvlol: bool, shutdown_callback=None): + """ + Initialize the stream recorder. + + Args: + username: Twitch username to record + quality: Stream quality (e.g., 'best', '1080p60', 'audio_only') + refresh: Retry interval in seconds + hls_segments: Number of parallel HLS segments to download + streamlink_ttvlol: Enable ttv-lol proxy (deprecated) + shutdown_callback: Callable to check if shutdown was requested + """ + self.username = username + self.quality = quality + self.refresh = refresh + self.hls_segments = hls_segments + self.streamlink_ttvlol = streamlink_ttvlol + self.shutdown_callback = shutdown_callback + self.current_process = None + + def record(self, stream_info: Dict[str, Any], output_path: str) -> bool: + """ + Record a live Twitch stream using streamlink. + + Args: + stream_info: Stream metadata from Twitch API + output_path: Path where the raw .ts file will be saved + + Returns: + bool: True if recording completed normally, False if interrupted + """ + print(f'\n{Fore.CYAN}{"=" * 60}{Style.RESET_ALL}') + print(f'{Fore.GREEN}🔴 STREAM STARTED: {stream_info["title"]}{Style.RESET_ALL}') + print(f'{Fore.CYAN}{"=" * 60}{Style.RESET_ALL}\n') + + # Build streamlink command + cmd = [ + 'streamlink', + f'twitch.tv/{self.username}', + self.quality, + '--hls-live-restart', + '--retry-streams', str(int(self.refresh)), + '--force', + '-o', output_path + ] + + # Add segment threads for faster downloads (requires streamlink 5.0+) + if self.hls_segments > 1: + cmd.extend(['--stream-segment-threads', str(self.hls_segments)]) + + # Add ad-blocking if enabled (deprecated warning) + if self.streamlink_ttvlol: + print(f'{Fore.YELLOW}⚠ Warning: ttv-lol proxy option is deprecated in newer streamlink versions{Style.RESET_ALL}') + print(f'{Fore.YELLOW} Consider disabling streamlink_ttvlol in config or using alternative methods{Style.RESET_ALL}') + + # Add authentication if available + oauth_token = os.getenv("OAUTH-PRIVATE-TOKEN", "") + if oauth_token and oauth_token != "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx": + cmd.extend(['--twitch-api-header', f'Authorization=OAuth {oauth_token}']) + + # Show command being executed (hide OAuth token for security) + cmd_display = [c if 'OAuth' not in str(c) else 'Authorization=OAuth [HIDDEN]' for c in cmd] + print(f'{Fore.CYAN}Command: {" ".join(cmd_display)}{Style.RESET_ALL}') + + # Record the stream (this blocks until stream ends) + print(f'{Fore.YELLOW}Recording stream...{Style.RESET_ALL}') + try: + self.current_process = subprocess.Popen(cmd) + return_code = self.current_process.wait() + self.current_process = None + + if self.shutdown_callback and self.shutdown_callback(): + print(f'{Fore.YELLOW}✓ Recording stopped by user{Style.RESET_ALL}') + # Return True so processing continues - we still want to process what was recorded + return True + + print(f'{Fore.GREEN}✓ Stream recording complete{Style.RESET_ALL}') + return True + except Exception as e: + self.current_process = None + print(f'{Fore.RED}✗ Recording error: {str(e)}{Style.RESET_ALL}') + return False + + def stop(self) -> None: + """Stop the current recording process.""" + if self.current_process: + try: + self.current_process.terminate() + print(f'{Fore.YELLOW}Stopping recording process...{Style.RESET_ALL}') + except Exception: + pass diff --git a/modules/stream_monitor.py b/modules/stream_monitor.py new file mode 100644 index 0000000..6c1607a --- /dev/null +++ b/modules/stream_monitor.py @@ -0,0 +1,140 @@ +""" +Stream monitoring and API interaction for Twitch Archive. +""" + +import os +import sys +from typing import Dict, Optional, Any +import requests +from colorama import Fore, Style + +from .constants import TWITCH_OAUTH_URL, TWITCH_API_URL, TWITCH_GQL_URL, TWITCH_GQL_CLIENT_ID + + +class StreamMonitor: + """Handles Twitch API interactions for monitoring stream status.""" + + def __init__(self, username: str): + """ + Initialize the stream monitor. + + Args: + username: Twitch username to monitor + """ + self.username = username + self._oauth_token = None + + def get_oauth_token(self) -> str: + """ + Get OAuth token from Twitch API. + + Uses CLIENT-ID and CLIENT-SECRET from environment variables. + + Returns: + str: OAuth access token + + Raises: + SystemExit: If authentication fails + """ + if self._oauth_token: + return self._oauth_token + + try: + url = f"{TWITCH_OAUTH_URL}?client_id={os.getenv('CLIENT-ID')}&client_secret={os.getenv('CLIENT-SECRET')}&grant_type=client_credentials" + response = requests.post(url, timeout=15) + response.raise_for_status() + self._oauth_token = response.json()['access_token'] + return self._oauth_token + except requests.exceptions.RequestException as e: + print(f'{Fore.RED}✗ ERROR: Failed to authenticate with Twitch API{Style.RESET_ALL}') + print(f'{Fore.YELLOW} {str(e)}{Style.RESET_ALL}') + print(f'{Fore.CYAN} → Check your CLIENT-ID and CLIENT-SECRET in the .env file{Style.RESET_ALL}') + sys.exit(1) + except KeyError: + print(f'{Fore.RED}✗ ERROR: Invalid response from Twitch API{Style.RESET_ALL}') + print(f'{Fore.CYAN} → Verify your CLIENT-ID and CLIENT-SECRET are correct{Style.RESET_ALL}') + sys.exit(1) + + def validate_username(self) -> bool: + """ + Validate that the configured Twitch username exists. + + Returns: + bool: True if username exists, False otherwise + + Raises: + SystemExit: If username is invalid or doesn't exist + """ + try: + url = f'{TWITCH_API_URL}/users?login={self.username}' + headers = { + "Authorization": f"Bearer {self.get_oauth_token()}", + "Client-ID": os.getenv('CLIENT-ID') + } + response = requests.get(url, headers=headers, timeout=15) + response.raise_for_status() + data = response.json() + + if not data.get('data'): + print(f'{Fore.RED}✗ ERROR: Twitch user "{self.username}" not found{Style.RESET_ALL}') + print(f'{Fore.CYAN} → Check the username in your config file{Style.RESET_ALL}') + sys.exit(1) + + print(f'{Fore.GREEN}✓ Username "{self.username}" validated{Style.RESET_ALL}') + return True + + except requests.exceptions.RequestException as e: + print(f'{Fore.RED}✗ ERROR: Could not validate username{Style.RESET_ALL}') + print(f'{Fore.YELLOW} {str(e)}{Style.RESET_ALL}') + sys.exit(1) + + def check_stream_status(self) -> Optional[Dict[str, Any]]: + """ + Check if the configured user is currently live. + + Returns: + dict: Stream information if live, None if offline + + Raises: + SystemExit: If API request fails + """ + query = f'query{{user(login: "{self.username}") {{stream{{archiveVideo{{id}}title createdAt}}}}}}' + + try: + response = requests.post( + TWITCH_GQL_URL, + json={'query': query}, + headers={"Client-ID": TWITCH_GQL_CLIENT_ID}, + timeout=15 + ) + response.raise_for_status() + return response.json() + + except requests.exceptions.RequestException as e: + print(f'{Fore.RED}✗ ERROR: Failed to check stream status{Style.RESET_ALL}') + print(f'{Fore.YELLOW} {str(e)}{Style.RESET_ALL}') + sys.exit(1) + + def get_latest_vod(self) -> Optional[Dict[str, Any]]: + """ + Get the most recent VOD for the configured user. + + Returns: + dict: VOD information, or None if no VODs found + """ + query = f'query {{user(login: "{self.username}") {{videos(first: 1) {{edges {{node {{id title description recordedAt lengthSeconds animatedPreviewURL previewThumbnailURL(height: 1280, width: 720) thumbnailURLs(height: 1280, width: 720)}}}}}}}}}}' + + try: + response = requests.post( + TWITCH_GQL_URL, + json={'query': query}, + headers={"Client-ID": TWITCH_GQL_CLIENT_ID}, + timeout=15 + ) + response.raise_for_status() + return response.json() + + except requests.exceptions.RequestException as e: + print(f'{Fore.YELLOW}⚠ Warning: Could not fetch latest VOD{Style.RESET_ALL}') + print(f'{Fore.YELLOW} {str(e)}{Style.RESET_ALL}') + return None diff --git a/modules/utils.py b/modules/utils.py new file mode 100644 index 0000000..88ae7b4 --- /dev/null +++ b/modules/utils.py @@ -0,0 +1,231 @@ +""" +Utility functions and helpers for Twitch Archive. +""" + +import os +import sys +import pathlib +import subprocess +from typing import Optional +from colorama import Fore, Style + + +def detect_operating_system() -> str: + """ + Detect the current operating system. + + Returns: + str: 'windows' or 'linux' + + Raises: + SystemExit: If OS is not supported + """ + if sys.platform.startswith('win32'): + return 'windows' + elif sys.platform.startswith('linux'): + return 'linux' + else: + print(f'{Fore.RED}✗ ERROR: Unsupported operating system: {sys.platform}{Style.RESET_ALL}') + print(f'{Fore.YELLOW} This script only supports Windows and Linux{Style.RESET_ALL}') + sys.exit(1) + + +def get_bin_path() -> str: + """Get the path to the bin directory containing external tools.""" + return str(pathlib.Path(__file__).parent.parent.resolve() / "bin") + + +def get_ffmpeg_executable(os_type: str) -> str: + """ + Get the platform-specific ffmpeg executable path. + + Args: + os_type: Operating system type ('windows' or 'linux') + + Returns: + str: Path to ffmpeg executable + """ + bin_path = get_bin_path() + if os_type == 'windows': + return os.path.join(bin_path, 'ffmpeg.exe') + return os.path.join(bin_path, 'ffmpeg') + + +def get_twitch_downloader_executable(os_type: str) -> str: + """ + Get the platform-specific TwitchDownloaderCLI executable path. + + Args: + os_type: Operating system type ('windows' or 'linux') + + Returns: + str: Path to TwitchDownloaderCLI executable + """ + bin_path = get_bin_path() + if os_type == 'windows': + return os.path.join(bin_path, 'TwitchDownloaderCLI.exe') + return os.path.join(bin_path, 'TwitchDownloaderCLI') + + +def get_unique_filename(filepath: str) -> str: + """ + Generate a unique filename by appending a counter if file already exists. + + Args: + filepath: The desired file path + + Returns: + str: A unique file path (original or with _N suffix) + + Example: + If 'video.mp4' exists, returns 'video_1.mp4' + If 'video_1.mp4' also exists, returns 'video_2.mp4' + """ + if not os.path.exists(filepath): + return filepath + + # Split into components + directory = os.path.dirname(filepath) + filename = os.path.basename(filepath) + name, ext = os.path.splitext(filename) + + # Find next available counter + counter = 1 + while True: + new_filepath = os.path.join(directory, f"{name}_{counter}{ext}") + if not os.path.exists(new_filepath): + return new_filepath + counter += 1 + + +def verify_streamlink() -> bool: + """ + Verify that streamlink is available. + + Returns: + bool: True if streamlink is available, False otherwise + """ + try: + result = subprocess.run(['streamlink', '--version'], + capture_output=True, + text=True, + timeout=5) + if result.returncode == 0: + version = result.stdout.strip().split()[1] if len(result.stdout.split()) > 1 else 'unknown' + print(f'{Fore.GREEN}✓ Streamlink v{version} found{Style.RESET_ALL}') + return True + else: + raise FileNotFoundError() + except (FileNotFoundError, subprocess.TimeoutExpired, IndexError): + print(f'{Fore.RED}✗ ERROR: Streamlink not found{Style.RESET_ALL}') + print(f'{Fore.CYAN} → Install streamlink: pip install streamlink{Style.RESET_ALL}') + print(f'{Fore.CYAN} → Or download from: https://streamlink.github.io/{Style.RESET_ALL}') + return False + + +def verify_ffmpeg(os_type: str) -> bool: + """ + Verify that ffmpeg is available. + + Args: + os_type: Operating system type ('windows' or 'linux') + + Returns: + bool: True if ffmpeg is available, False otherwise + """ + try: + ffmpeg_path = get_ffmpeg_executable(os_type) + if os.path.exists(ffmpeg_path): + print(f'{Fore.GREEN}✓ FFmpeg found at {ffmpeg_path}{Style.RESET_ALL}') + return True + else: + print(f'{Fore.YELLOW}⚠ Warning: FFmpeg not found at {ffmpeg_path}{Style.RESET_ALL}') + print(f'{Fore.YELLOW} → Download FFmpeg and place it in the bin/ folder{Style.RESET_ALL}') + return False + except Exception as e: + print(f'{Fore.YELLOW}⚠ Warning: Could not verify FFmpeg: {e}{Style.RESET_ALL}') + return False + + +def verify_twitch_downloader(os_type: str) -> bool: + """ + Verify that TwitchDownloaderCLI is available. + + Args: + os_type: Operating system type ('windows' or 'linux') + + Returns: + bool: True if TwitchDownloaderCLI is available, False otherwise + """ + try: + downloader_path = get_twitch_downloader_executable(os_type) + if os.path.exists(downloader_path): + print(f'{Fore.GREEN}✓ TwitchDownloaderCLI found{Style.RESET_ALL}') + return True + else: + print(f'{Fore.YELLOW}⚠ Warning: TwitchDownloaderCLI not found at {downloader_path}{Style.RESET_ALL}') + print(f'{Fore.YELLOW} → Download from: https://github.com/lay295/TwitchDownloader/releases{Style.RESET_ALL}') + return False + except Exception as e: + print(f'{Fore.YELLOW}⚠ Warning: Could not verify TwitchDownloaderCLI: {e}{Style.RESET_ALL}') + return False + + +def detect_hardware_acceleration(hwaccel_config: str, os_type: str) -> Optional[str]: + """ + Detect available hardware acceleration based on config and system. + + Args: + hwaccel_config: Hardware acceleration configuration ('auto', 'nvenc', 'qsv', 'amf', 'vaapi', 'none') + os_type: Operating system type ('windows' or 'linux') + + Returns: + str: Hardware acceleration type or None + """ + # If user explicitly set to 'none', disable hardware acceleration + if hwaccel_config == 'none': + return 'none' + + # If user specified a particular type, use it + if hwaccel_config in ['nvenc', 'qsv', 'amf', 'vaapi']: + return hwaccel_config + + # Auto-detect: try to determine available hardware + if hwaccel_config == 'auto': + # On Windows, NVIDIA is most common + if os_type == 'windows': + # Could check for nvidia-smi, but just return 'auto' for ffmpeg to decide + return 'auto' + else: + # On Linux, VAAPI is common for Intel/AMD, or NVENC for NVIDIA + # Let ffmpeg auto-detect + return 'auto' + + return None + + +def get_hwaccel_encoder(hwaccel_type: str) -> str: + """ + Get the appropriate hardware-accelerated encoder for the given acceleration type. + + Args: + hwaccel_type: Type of hardware acceleration ('nvenc', 'qsv', 'amf', 'vaapi', 'auto', 'none') + + Returns: + str: FFmpeg encoder name (e.g., 'h264_nvenc', 'libx264') + """ + encoder_map = { + 'nvenc': 'h264_nvenc', # NVIDIA + 'qsv': 'h264_qsv', # Intel Quick Sync + 'amf': 'h264_amf', # AMD + 'vaapi': 'h264_vaapi', # Linux VA-API + } + + if hwaccel_type in encoder_map: + return encoder_map[hwaccel_type] + elif hwaccel_type == 'auto': + # Try NVENC first (most common), fall back to libx264 + # In real usage, auto will attempt to use what's available + return 'h264_nvenc' + else: + return 'libx264' # Software encoding fallback diff --git a/twitch-archive.py b/twitch-archive.py index dfab86e..8e2437a 100644 --- a/twitch-archive.py +++ b/twitch-archive.py @@ -13,6 +13,18 @@ Requirements: - External tools: streamlink, ffmpeg, TwitchDownloaderCLI, rclone (optional) - Configuration file: config.json (copy from config.sample.json) - Environment file: .env (for API credentials) + +Refactored Version 2.0: +This version has been split into multiple modules for better maintainability: +- modules/constants.py: Constants and default configuration +- modules/config.py: Configuration management +- modules/notifications.py: Email notifications +- modules/utils.py: Utility functions +- modules/stream_monitor.py: Stream monitoring and API +- modules/recorder.py: Live stream recording +- modules/processor.py: Video/audio processing +- modules/downloader.py: VOD and chat downloading +- modules/file_manager.py: File and cloud management """ # Standard library imports @@ -20,218 +32,29 @@ import os import sys import time import json -import socket -import smtplib -import pathlib -import subprocess -import getopt import signal +import getopt from typing import Dict, Optional, Any from datetime import datetime, timedelta # Third-party imports -import requests from colorama import Fore, Style from pytz import timezone from dotenv import load_dotenv, find_dotenv -from email.mime.multipart import MIMEMultipart -from email.mime.text import MIMEText -# ============================================================================ -# CONSTANTS - Configuration defaults and magic values -# ============================================================================ - -# API Endpoints -TWITCH_OAUTH_URL = "https://id.twitch.tv/oauth2/token" -TWITCH_API_URL = "https://api.twitch.tv/helix" -TWITCH_GQL_URL = "https://gql.twitch.tv/gql" -TWITCH_GQL_CLIENT_ID = "kimne78kx3ncx6brgo4mv6wki5h1ko" - -# File prefixes for different content types -PREFIX_LIVE = "LIVE_" -PREFIX_VOD = "VOD_" -PREFIX_CHAT = "CHAT_" -PREFIX_METADATA = "METADA_" # Note: keeping original typo for compatibility - -# Default configuration values -DEFAULT_CONFIG = { - 'username': 'your_twitch_username', - 'quality': 'best', - 'root_path': 'archive', - 'rclone_path': 'remote:path/to/streams', - 'refresh': 60.0, - 'streamlink_ttvlol': False, - 'notifications': False, - 'downloadMETADATA': True, - 'downloadVOD': True, - 'downloadCHAT': True, - 'downloadLiveCHAT': True, - 'vodTimeout': 300, - 'uploadCloud': True, - 'deleteFiles': False, - 'onlyRaw': False, - 'cleanRaw': True, - 'hls_segments': 3, - 'hls_segmentsVOD': 10, - # FFmpeg 8.0+ Enhancement Options - 'ffmpeg_hwaccel': 'auto', # Hardware acceleration: 'auto', 'nvenc', 'qsv', 'amf', 'vaapi', 'none' - 'ffmpeg_threads': 0, # Thread count (0 = auto-detect) - 'ffmpeg_audio_codec': 'aac', # Audio codec for audio-only streams - 'ffmpeg_audio_samplerate': 48000, # Audio sample rate (48000 recommended for broadcasts) - 'ffmpeg_audio_bitrate': '192k', # Audio bitrate - 'ffmpeg_error_recovery': True, # Enable error recovery for corrupted streams - 'ffmpeg_faststart': True, # Enable faststart for MP4 (better streaming compatibility) - 'ffmpeg_progress': False # Show encoding progress -} - -# ============================================================================ -# MAIN CLASS -# ============================================================================ - -class ConfigManager: - """ - Manages global and per-streamer configurations. - - Loads global defaults from config/global.json and merges with per-streamer - configs from config/streamers/*.json. - """ - - def __init__(self): - """Initialize the configuration manager.""" - self.config_dir = pathlib.Path(__file__).parent / "config" - self.streamers_dir = self.config_dir / "streamers" - self.global_config = self._load_global_config() - - def _load_global_config(self) -> Dict[str, Any]: - """ - Load global configuration from config/global.json. - - Returns: - dict: Global configuration with defaults - """ - global_file = self.config_dir / "global.json" - - # Start with DEFAULT_CONFIG as ultimate fallback - config = DEFAULT_CONFIG.copy() - - # Try to load global config - if global_file.exists(): - try: - with open(global_file, 'r', encoding='utf-8') as f: - user_config = json.load(f) - # Filter out comment fields and schema references - user_config = {k: v for k, v in user_config.items() - if not k.startswith('_') and k != '$schema'} - config.update(user_config) - print(f'{Fore.GREEN}✓ Global configuration loaded from config/global.json{Style.RESET_ALL}') - except json.JSONDecodeError as e: - print(f'{Fore.YELLOW}⚠ Warning: Invalid JSON in config/global.json: {e}{Style.RESET_ALL}') - print(f'{Fore.YELLOW} Using default configuration{Style.RESET_ALL}') - except Exception as e: - print(f'{Fore.YELLOW}⚠ Warning: Could not load config/global.json: {e}{Style.RESET_ALL}') - else: - print(f'{Fore.YELLOW}⚠ Warning: config/global.json not found{Style.RESET_ALL}') - print(f'{Fore.CYAN} → Create config/global.json with default settings{Style.RESET_ALL}') - - return config - - def load_streamer_config(self, username: str) -> Dict[str, Any]: - """ - Load configuration for a specific streamer. - - Merges global config with streamer-specific overrides. - - Args: - username: Twitch username - - Returns: - dict: Complete configuration for the streamer - """ - # Start with global config - config = self.global_config.copy() - - # Load streamer-specific config - streamer_file = self.streamers_dir / f"{username}.json" - - if streamer_file.exists(): - try: - with open(streamer_file, 'r', encoding='utf-8') as f: - streamer_config = json.load(f) - # Filter out comments and schema references - streamer_config = {k: v for k, v in streamer_config.items() - if not k.startswith('_') and k != '$schema'} - # Merge streamer config (overrides global) - config.update(streamer_config) - print(f'{Fore.GREEN}✓ Loaded config for {username}{Style.RESET_ALL}') - except json.JSONDecodeError as e: - print(f'{Fore.YELLOW}⚠ Warning: Invalid JSON in {streamer_file}: {e}{Style.RESET_ALL}') - except Exception as e: - print(f'{Fore.YELLOW}⚠ Warning: Could not load {streamer_file}: {e}{Style.RESET_ALL}') - else: - # Create default config for new streamer - print(f'{Fore.CYAN}→ Creating default config for new streamer: {username}{Style.RESET_ALL}') - self.create_default_streamer_config(username) - config['username'] = username - config['enabled'] = True - - # Ensure username is set - config['username'] = username - - return config - - def create_default_streamer_config(self, username: str) -> None: - """ - Create a default configuration file for a new streamer. - - Args: - username: Twitch username - """ - # Ensure streamers directory exists - self.streamers_dir.mkdir(parents=True, exist_ok=True) - - streamer_file = self.streamers_dir / f"{username}.json" - - default_config = { - "$schema": "./streamer.schema.json", - "username": username, - "enabled": True - } - - try: - with open(streamer_file, 'w', encoding='utf-8') as f: - json.dump(default_config, f, indent=2) - print(f'{Fore.GREEN}✓ Created config file: config/streamers/{username}.json{Style.RESET_ALL}') - print(f'{Fore.CYAN} → Edit the file to add custom settings or overrides{Style.RESET_ALL}') - except Exception as e: - print(f'{Fore.RED}✗ Could not create config file for {username}: {e}{Style.RESET_ALL}') - - def get_all_enabled_streamers(self) -> list: - """ - Get list of all enabled streamers. - - Returns: - list: List of usernames configured and enabled - """ - if not self.streamers_dir.exists(): - return [] - - enabled_streamers = [] - - for config_file in self.streamers_dir.glob("*.json"): - try: - with open(config_file, 'r', encoding='utf-8') as f: - config = json.load(f) - # Filter comments and schema references - config = {k: v for k, v in config.items() - if not k.startswith('_') and k != '$schema'} - - if config.get('enabled', False): - username = config.get('username') or config_file.stem - enabled_streamers.append(username) - except Exception as e: - print(f'{Fore.YELLOW}⚠ Warning: Could not read {config_file}: {e}{Style.RESET_ALL}') - - return enabled_streamers +# Local module imports +from modules.constants import DEFAULT_CONFIG, PREFIX_LIVE, PREFIX_VOD, PREFIX_CHAT, PREFIX_METADATA +from modules.config import ConfigManager +from modules.notifications import NotificationManager +from modules.utils import ( + detect_operating_system, get_ffmpeg_executable, get_twitch_downloader_executable, + get_unique_filename, verify_streamlink, verify_ffmpeg, verify_twitch_downloader +) +from modules.stream_monitor import StreamMonitor +from modules.recorder import StreamRecorder +from modules.processor import StreamProcessor +from modules.downloader import ContentDownloader +from modules.file_manager import FileManager class TwitchArchive: @@ -240,6 +63,9 @@ class TwitchArchive: Handles monitoring a Twitch channel, recording live streams, and downloading VODs, chat logs, and metadata. Can optionally upload to cloud storage. + + Refactored Version 2.0: This class now delegates most functionality to + specialized modules for better code organization. """ def __init__(self, config: Optional[Dict[str, Any]] = None): @@ -257,15 +83,22 @@ class TwitchArchive: for key, value in config.items(): setattr(self, key, value) - self.os = self._detect_operating_system() - self.paths_initialized = False + # Initialize system components + self.os_type = detect_operating_system() self.shutdown_requested = False - self.current_process = None self.current_stream_data = {} + + # Initialize component modules (created during run()) + self.stream_monitor = None + self.notification_manager = None + self.file_manager = None + self.recorder = None + self.processor = None + self.downloader = None def load_config(self) -> None: """ - Load configuration from config.json file. + Load configuration from config.json file (legacy support). Falls back to default configuration if file is not found or cannot be read. Filters out comment fields (starting with '_') from the config. @@ -299,56 +132,6 @@ class TwitchArchive: for key, value in config.items(): setattr(self, key, value) - def _detect_operating_system(self) -> str: - """ - Detect the current operating system. - - Returns: - str: 'windows' or 'linux' - - Raises: - SystemExit: If OS is not supported - """ - if sys.platform.startswith('win32'): - return 'windows' - elif sys.platform.startswith('linux'): - return 'linux' - else: - print(f'{Fore.RED}✗ ERROR: Unsupported operating system: {sys.platform}{Style.RESET_ALL}') - print(f'{Fore.YELLOW} This script only supports Windows and Linux{Style.RESET_ALL}') - sys.exit(1) - - - def _initialize_paths(self) -> None: - """ - Initialize all directory paths needed for archiving. - - Creates the directory structure: - - root_path/username/video/raw/ (for raw .ts files) - - root_path/username/video/ (for processed videos) - - root_path/username/chat/json/ (for chat JSON files) - - root_path/username/chat/ (for rendered chat videos) - - root_path/username/metadata/ (for stream metadata) - """ - # Convert all paths to absolute paths - self.raw_path = pathlib.Path(self.root_path, self.username, "video", "raw").absolute() - self.video_path = pathlib.Path(self.root_path, self.username, "video").absolute() - self.chatJSON_path = pathlib.Path(self.root_path, self.username, "chat", "json").absolute() - self.chatMP4_path = pathlib.Path(self.root_path, self.username, "chat").absolute() - self.metadata_path = pathlib.Path(self.root_path, self.username, "metadata").absolute() - - # Create directories if they don't exist - for path in [self.raw_path, self.video_path, self.chatJSON_path, - self.chatMP4_path, self.metadata_path]: - path.mkdir(parents=True, exist_ok=True) - - # Create log file if it doesn't exist - log_file = pathlib.Path(self.root_path, ".log") - if not log_file.exists(): - log_file.touch() - - self.paths_initialized = True - def _load_environment_variables(self) -> None: """ Load environment variables from .env file. @@ -370,6 +153,51 @@ class TwitchArchive: print(f'{Fore.CYAN} → Required: CLIENT-ID, CLIENT-SECRET{Style.RESET_ALL}') sys.exit(1) + def _initialize_components(self) -> None: + """Initialize all component modules.""" + # Stream monitoring + self.stream_monitor = StreamMonitor(self.username) + + # Notifications + self.notification_manager = NotificationManager( + enabled=self.notifications, + username=self.username + ) + + # File management + self.file_manager = FileManager( + root_path=self.root_path, + username=self.username, + config=vars(self) + ) + self.file_manager.initialize_directories() + + # Recording + self.recorder = StreamRecorder( + username=self.username, + quality=self.quality, + refresh=self.refresh, + hls_segments=self.hls_segments, + streamlink_ttvlol=self.streamlink_ttvlol, + shutdown_callback=lambda: self.shutdown_requested + ) + + # Processing + ffmpeg_path = get_ffmpeg_executable(self.os_type) + self.processor = StreamProcessor( + os_type=self.os_type, + ffmpeg_path=ffmpeg_path, + config=vars(self) + ) + + # Downloading + twitch_downloader_path = get_twitch_downloader_executable(self.os_type) + self.downloader = ContentDownloader( + twitch_downloader_path=twitch_downloader_path, + ffmpeg_path=ffmpeg_path, + config=vars(self) + ) + def _print_configuration_summary(self) -> None: """Print a summary of the current configuration to the console.""" print(f'\n{Fore.CYAN}{"=" * 60}{Style.RESET_ALL}') @@ -379,7 +207,7 @@ class TwitchArchive: # Basic settings print(f'Streamer: {Fore.GREEN}{self.username}{Style.RESET_ALL}') print(f'Quality: {Fore.GREEN}{self.quality}{Style.RESET_ALL}') - print(f'Storage: {Fore.GREEN}{pathlib.Path(self.root_path).resolve()}{Style.RESET_ALL}') + print(f'Storage: {Fore.GREEN}{os.path.abspath(self.root_path)}{Style.RESET_ALL}') print(f'Refresh rate: {Fore.GREEN}{self.refresh}s{Style.RESET_ALL}\n') # Feature toggles @@ -415,867 +243,29 @@ class TwitchArchive: # Load environment variables self._load_environment_variables() + # Initialize all component modules + self._initialize_components() + # Validate username - self._validate_username() + self.stream_monitor.validate_username() - # Initialize directory structure - self._initialize_paths() - - # Verify streamlink is available - self._verify_dependencies() + # Verify dependencies + if not verify_streamlink(): + sys.exit(1) + verify_ffmpeg(self.os_type) + if self.downloadVOD or self.downloadCHAT: + verify_twitch_downloader(self.os_type) # Print configuration summary self._print_configuration_summary() # Start monitoring print(f"Monitoring {Fore.GREEN}{self.username}{Style.RESET_ALL} every {Fore.GREEN}{self.refresh}s{Style.RESET_ALL}") - self.send_notification("TWITCH ARCHIVE STARTED", + self.notification_manager.send("TWITCH ARCHIVE STARTED", f"Monitoring {self.username} every {self.refresh} seconds.") # Begin the main monitoring loop self.loopcheck() - - - def _get_oauth_token(self) -> str: - """ - Get OAuth token from Twitch API. - - Uses CLIENT-ID and CLIENT-SECRET from environment variables. - - Returns: - str: OAuth access token - - Raises: - SystemExit: If authentication fails - """ - try: - url = f"{TWITCH_OAUTH_URL}?client_id={os.getenv('CLIENT-ID')}&client_secret={os.getenv('CLIENT-SECRET')}&grant_type=client_credentials" - response = requests.post(url, timeout=15) - response.raise_for_status() - return response.json()['access_token'] - except requests.exceptions.RequestException as e: - print(f'{Fore.RED}✗ ERROR: Failed to authenticate with Twitch API{Style.RESET_ALL}') - print(f'{Fore.YELLOW} {str(e)}{Style.RESET_ALL}') - print(f'{Fore.CYAN} → Check your CLIENT-ID and CLIENT-SECRET in the .env file{Style.RESET_ALL}') - sys.exit(1) - except KeyError: - print(f'{Fore.RED}✗ ERROR: Invalid response from Twitch API{Style.RESET_ALL}') - print(f'{Fore.CYAN} → Verify your CLIENT-ID and CLIENT-SECRET are correct{Style.RESET_ALL}') - sys.exit(1) - - def _validate_username(self) -> None: - """ - Validate that the configured Twitch username exists. - - Raises: - SystemExit: If username is invalid or doesn't exist - """ - try: - url = f'{TWITCH_API_URL}/users?login={self.username}' - headers = { - "Authorization": f"Bearer {self._get_oauth_token()}", - "Client-ID": os.getenv('CLIENT-ID') - } - response = requests.get(url, headers=headers, timeout=15) - response.raise_for_status() - data = response.json() - - if not data.get('data'): - print(f'{Fore.RED}✗ ERROR: Twitch user "{self.username}" not found{Style.RESET_ALL}') - print(f'{Fore.CYAN} → Check the username in your config.json file{Style.RESET_ALL}') - sys.exit(1) - - print(f'{Fore.GREEN}✓ Username "{self.username}" validated{Style.RESET_ALL}') - - except requests.exceptions.RequestException as e: - print(f'{Fore.RED}✗ ERROR: Could not validate username{Style.RESET_ALL}') - print(f'{Fore.YELLOW} {str(e)}{Style.RESET_ALL}') - sys.exit(1) - - def _verify_dependencies(self) -> None: - """ - Verify that required external dependencies are available. - - Raises: - SystemExit: If required dependencies are not found - """ - # Check for streamlink - try: - result = subprocess.run(['streamlink', '--version'], - capture_output=True, - text=True, - timeout=5) - if result.returncode == 0: - version = result.stdout.strip().split()[1] if len(result.stdout.split()) > 1 else 'unknown' - print(f'{Fore.GREEN}✓ Streamlink v{version} found{Style.RESET_ALL}') - else: - raise FileNotFoundError() - except (FileNotFoundError, subprocess.TimeoutExpired, IndexError): - print(f'{Fore.RED}✗ ERROR: Streamlink not found{Style.RESET_ALL}') - print(f'{Fore.CYAN} → Install streamlink: pip install streamlink{Style.RESET_ALL}') - print(f'{Fore.CYAN} → Or download from: https://streamlink.github.io/{Style.RESET_ALL}') - sys.exit(1) - - # Check for ffmpeg - try: - ffmpeg_path = self._get_ffmpeg_executable() - if os.path.exists(ffmpeg_path): - print(f'{Fore.GREEN}✓ FFmpeg found at {ffmpeg_path}{Style.RESET_ALL}') - else: - print(f'{Fore.YELLOW}⚠ Warning: FFmpeg not found at {ffmpeg_path}{Style.RESET_ALL}') - print(f'{Fore.YELLOW} → Download FFmpeg and place it in the bin/ folder{Style.RESET_ALL}') - except Exception as e: - print(f'{Fore.YELLOW}⚠ Warning: Could not verify FFmpeg: {e}{Style.RESET_ALL}') - - # Check for TwitchDownloaderCLI (if VOD or Chat download enabled) - if self.downloadVOD or self.downloadCHAT: - try: - downloader_path = self._get_twitch_downloader_executable() - if os.path.exists(downloader_path): - print(f'{Fore.GREEN}✓ TwitchDownloaderCLI found{Style.RESET_ALL}') - else: - print(f'{Fore.YELLOW}⚠ Warning: TwitchDownloaderCLI not found at {downloader_path}{Style.RESET_ALL}') - print(f'{Fore.YELLOW} → Download from: https://github.com/lay295/TwitchDownloader/releases{Style.RESET_ALL}') - except Exception as e: - print(f'{Fore.YELLOW}⚠ Warning: Could not verify TwitchDownloaderCLI: {e}{Style.RESET_ALL}') - - - - def _check_stream_status(self) -> Optional[Dict[str, Any]]: - """ - Check if the configured user is currently live. - - Returns: - dict: Stream information if live, None if offline - - Raises: - SystemExit: If API request fails - """ - query = f'query{{user(login: "{self.username}") {{stream{{archiveVideo{{id}}title createdAt}}}}}}' - - try: - response = requests.post( - TWITCH_GQL_URL, - json={'query': query}, - headers={"Client-ID": TWITCH_GQL_CLIENT_ID}, - timeout=15 - ) - response.raise_for_status() - return response.json() - - except requests.exceptions.RequestException as e: - print(f'{Fore.RED}✗ ERROR: Failed to check stream status{Style.RESET_ALL}') - print(f'{Fore.YELLOW} {str(e)}{Style.RESET_ALL}') - sys.exit(1) - - def _get_latest_vod(self) -> Optional[Dict[str, Any]]: - """ - Get the most recent VOD for the configured user. - - Returns: - dict: VOD information, or None if no VODs found - """ - query = f'query {{user(login: "{self.username}") {{videos(first: 1) {{edges {{node {{id title description recordedAt lengthSeconds animatedPreviewURL previewThumbnailURL(height: 1280, width: 720) thumbnailURLs(height: 1280, width: 720)}}}}}}}}}}' - - try: - response = requests.post( - TWITCH_GQL_URL, - json={'query': query}, - headers={"Client-ID": TWITCH_GQL_CLIENT_ID}, - timeout=15 - ) - response.raise_for_status() - return response.json() - - except requests.exceptions.RequestException as e: - print(f'{Fore.YELLOW}⚠ Warning: Could not fetch latest VOD{Style.RESET_ALL}') - print(f'{Fore.YELLOW} {str(e)}{Style.RESET_ALL}') - return None - - - def _get_unique_filename(self, filepath: str) -> str: - """ - Generate a unique filename by appending a counter if file already exists. - - Args: - filepath: The desired file path - - Returns: - str: A unique file path (original or with _N suffix) - - Example: - If 'video.mp4' exists, returns 'video_1.mp4' - If 'video_1.mp4' also exists, returns 'video_2.mp4' - """ - if not os.path.exists(filepath): - return filepath - - # Split into components - directory = os.path.dirname(filepath) - filename = os.path.basename(filepath) - name, ext = os.path.splitext(filename) - - # Find next available counter - counter = 1 - while True: - new_filepath = os.path.join(directory, f"{name}_{counter}{ext}") - if not os.path.exists(new_filepath): - return new_filepath - counter += 1 - - def send_notification(self, subject: str, content: str) -> None: - """ - Send email notification via Gmail SMTP. - - Only sends if notifications are enabled in configuration. - Requires SENDER, RECEIVER, and PASSWD in .env file. - - Args: - subject: Email subject line - content: Email body content - """ - if not self.notifications: - return - - try: - sender = os.getenv("SENDER") - receiver = os.getenv("RECEIVER") - password = os.getenv("PASSWD") - - if not all([sender, receiver, password]): - print(f'{Fore.YELLOW}⚠ Notification skipped: Missing email credentials in .env{Style.RESET_ALL}') - return - - # Construct email - msg = MIMEMultipart() - msg['From'] = sender - msg['To'] = receiver - msg['Subject'] = f"{self.username} - {subject}" - - body = f"Stream: {self.username}\n\n{content}" - msg.attach(MIMEText(body, 'plain')) - - # Send via Gmail SMTP - with smtplib.SMTP('smtp.gmail.com', 587) as server: - server.starttls() - server.login(sender, password) - server.sendmail(sender, receiver, msg.as_string()) - - except socket.error as e: - print(f'{Fore.YELLOW}⚠ Notification failed: {str(e)}{Style.RESET_ALL}') - except Exception as e: - print(f'{Fore.YELLOW}⚠ Notification error: {str(e)}{Style.RESET_ALL}') - - def _is_stream_already_processed(self, stream_id: str) -> bool: - """ - Check if a stream has already been processed. - - Args: - stream_id: Unique identifier for the stream - - Returns: - bool: True if already processed, False otherwise - """ - log_file = pathlib.Path(self.root_path, ".log") - with open(log_file, 'r', encoding='utf-8') as f: - return stream_id in f.read() - - def _mark_stream_as_processed(self, stream_id: str) -> None: - """Add stream to log file to prevent re-processing.""" - log_file = pathlib.Path(self.root_path, ".log") - with open(log_file, 'a', encoding='utf-8') as f: - f.write(f"{stream_id}\n") - - def _get_bin_path(self) -> str: - """Get the path to the bin directory containing external tools.""" - return str(pathlib.Path(__file__).parent.resolve() / "bin") - - def _get_ffmpeg_executable(self) -> str: - """Get the platform-specific ffmpeg executable path.""" - bin_path = self._get_bin_path() - if self.os == 'windows': - return os.path.join(bin_path, 'ffmpeg.exe') - return os.path.join(bin_path, 'ffmpeg') - - def _get_twitch_downloader_executable(self) -> str: - """Get the platform-specific TwitchDownloaderCLI executable path.""" - bin_path = self._get_bin_path() - if self.os == 'windows': - return os.path.join(bin_path, 'TwitchDownloaderCLI.exe') - return os.path.join(bin_path, 'TwitchDownloaderCLI') - - def _detect_hardware_acceleration(self) -> Optional[str]: - """ - Detect available hardware acceleration based on config and system. - - Returns: - str: Hardware acceleration type ('nvenc', 'qsv', 'amf', 'vaapi', 'none') or None - """ - hwaccel_config = getattr(self, 'ffmpeg_hwaccel', 'auto') - - # If user explicitly set to 'none', disable hardware acceleration - if hwaccel_config == 'none': - return 'none' - - # If user specified a particular type, use it - if hwaccel_config in ['nvenc', 'qsv', 'amf', 'vaapi']: - return hwaccel_config - - # Auto-detect: try to determine available hardware - if hwaccel_config == 'auto': - # On Windows, NVIDIA is most common - if self.os == 'windows': - # Could check for nvidia-smi, but just return 'auto' for ffmpeg to decide - return 'auto' - else: - # On Linux, VAAPI is common for Intel/AMD, or NVENC for NVIDIA - # Let ffmpeg auto-detect - return 'auto' - - return None - - def _record_livestream(self, stream_info: Dict[str, Any], output_path: str) -> bool: - """ - Record a live Twitch stream using streamlink. - - Args: - stream_info: Stream metadata from Twitch API - output_path: Path where the raw .ts file will be saved - - Returns: - bool: True if recording completed normally, False if interrupted - """ - print(f'\n{Fore.CYAN}{"=" * 60}{Style.RESET_ALL}') - print(f'{Fore.GREEN}🔴 STREAM STARTED: {stream_info["title"]}{Style.RESET_ALL}') - print(f'{Fore.CYAN}{"=" * 60}{Style.RESET_ALL}\n') - - # Build streamlink command - cmd = [ - 'streamlink', - f'twitch.tv/{self.username}', - self.quality, - '--hls-live-restart', - '--retry-streams', str(int(self.refresh)), - '--force', - '-o', output_path - ] - - # Add segment threads for faster downloads (requires streamlink 5.0+) - # This allows multiple segments to be downloaded in parallel - if self.hls_segments > 1: - cmd.extend(['--stream-segment-threads', str(self.hls_segments)]) - - # Add ad-blocking if enabled (Note: twitch-proxy-playlist was removed in newer streamlink versions) - # For ad-blocking, you may need to use alternative methods like --twitch-low-latency - # or rely on Twitch's own ad-free viewing for subscribers - if self.streamlink_ttvlol: - # The old --twitch-proxy-playlist option has been removed from streamlink - # Consider using alternative ad-blocking approaches or updating your method - print(f'{Fore.YELLOW}⚠ Warning: ttv-lol proxy option is deprecated in newer streamlink versions{Style.RESET_ALL}') - print(f'{Fore.YELLOW} Consider disabling streamlink_ttvlol in config or using alternative methods{Style.RESET_ALL}') - - # Add authentication if available - oauth_token = os.getenv("OAUTH-PRIVATE-TOKEN", "") - if oauth_token and oauth_token != "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx": - cmd.extend(['--twitch-api-header', f'Authorization=OAuth {oauth_token}']) - - # Show command being executed (hide OAuth token for security) - cmd_display = [c if 'OAuth' not in str(c) else 'Authorization=OAuth [HIDDEN]' for c in cmd] - print(f'{Fore.CYAN}Command: {" ".join(cmd_display)}{Style.RESET_ALL}') - - # Record the stream (this blocks until stream ends) - print(f'{Fore.YELLOW}Recording stream...{Style.RESET_ALL}') - try: - self.current_process = subprocess.Popen(cmd) - return_code = self.current_process.wait() - self.current_process = None - - if self.shutdown_requested: - print(f'{Fore.YELLOW}✓ Recording stopped by user{Style.RESET_ALL}') - # Return True so processing continues - we still want to process what was recorded - return True - - print(f'{Fore.GREEN}✓ Stream recording complete{Style.RESET_ALL}') - return True - except Exception as e: - self.current_process = None - print(f'{Fore.RED}✗ Recording error: {str(e)}{Style.RESET_ALL}') - return False - - def _process_raw_stream(self, raw_path: str, output_path: str) -> None: - """ - Process raw .ts file into mp4/mp3 using ffmpeg. - - Args: - raw_path: Path to the raw .ts file - output_path: Path for the processed output file - """ - if not os.path.exists(raw_path): - print(f'{Fore.YELLOW}⚠ Raw file not found, skipping processing{Style.RESET_ALL}') - return - - if self.onlyRaw: - print(f'{Fore.CYAN}Keeping raw .ts file (onlyRaw mode){Style.RESET_ALL}') - return - - print(f'{Fore.YELLOW}Processing raw stream file...{Style.RESET_ALL}') - - # Build ffmpeg command based on quality - if self.quality == 'audio_only': - # Audio-only conversion with modern AAC encoding - cmd = [ - self._get_ffmpeg_executable(), - '-i', raw_path, - '-vn', # No video - '-c:a', self.ffmpeg_audio_codec, # Audio codec (AAC recommended) - '-ar', str(self.ffmpeg_audio_samplerate), # Audio sample rate - '-ac', '2', # Audio channels (stereo) - '-b:a', self.ffmpeg_audio_bitrate, # Audio bitrate - ] - - # Add threading for faster encoding - if self.ffmpeg_threads > 0: - cmd.extend(['-threads', str(self.ffmpeg_threads)]) - - # Add faststart for better streaming compatibility (MP4/M4A) - if self.ffmpeg_faststart and output_path.endswith(('.mp4', '.m4a')): - cmd.extend(['-movflags', '+faststart']) - - cmd.append(output_path) - else: - # Video conversion with hardware acceleration support - cmd = [ - self._get_ffmpeg_executable(), - '-y', # Overwrite output file - ] - - # Add hardware acceleration if enabled - hwaccel_type = self._detect_hardware_acceleration() - if hwaccel_type and hwaccel_type != 'none': - print(f'{Fore.CYAN}Using hardware acceleration: {hwaccel_type}{Style.RESET_ALL}') - cmd.extend(['-hwaccel', 'auto']) - - cmd.extend([ - '-i', raw_path, - '-analyzeduration', '2147483647', - '-probesize', '2147483647', - ]) - - # Threading support - if self.ffmpeg_threads >= 0: - cmd.extend(['-threads', str(self.ffmpeg_threads)]) - - # Error recovery options for corrupted streams - if self.ffmpeg_error_recovery: - cmd.extend([ - '-fflags', '+genpts', # Generate missing timestamps - '-avoid_negative_ts', 'make_zero', # Handle timestamp issues - '-err_detect', 'ignore_err' # More tolerant of errors - ]) - - # Stream copy (fast, no re-encoding) - cmd.extend([ - '-c:v', 'copy', # Copy video codec - '-c:a', 'copy', # Copy audio codec - '-start_at_zero', - '-copyts', - ]) - - # Add faststart for MP4 files - if self.ffmpeg_faststart and output_path.endswith('.mp4'): - cmd.extend(['-movflags', '+faststart']) - - cmd.append(output_path) - - # Run ffmpeg with optional progress output - if self.ffmpeg_progress: - subprocess.call(cmd) - else: - subprocess.call(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) - - print(f'{Fore.GREEN}✓ Stream processed successfully{Style.RESET_ALL}') - - def _download_vod(self, vod_info: Dict[str, Any], output_path: str) -> bool: - """ - Download VOD using TwitchDownloaderCLI. - - Args: - vod_info: VOD metadata from Twitch API - output_path: Path where the VOD will be saved - - Returns: - bool: True if download succeeded, False otherwise - """ - if not self.downloadVOD: - return False - - print(f'\n{Fore.CYAN}Downloading VOD: {vod_info["title"]}{Style.RESET_ALL}') - - # Extract numeric VOD ID (TwitchDownloaderCLI expects just the number) - vod_id = vod_info["id"] - # Remove 'v' prefix if present (API sometimes returns "v123456789") - if isinstance(vod_id, str) and vod_id.startswith('v'): - vod_id = vod_id[1:] - - # Build URL format that TwitchDownloaderCLI accepts - vod_url = f"https://www.twitch.tv/videos/{vod_id}" - - print(f'{Fore.YELLOW}VOD URL: {vod_url}{Style.RESET_ALL}') - - bin_path = self._get_bin_path() - cmd = [ - self._get_twitch_downloader_executable(), - 'videodownload', - '-u', vod_url, - '-q', self.quality, - '-t', str(self.hls_segmentsVOD), - '--ffmpeg-path', self._get_ffmpeg_executable(), - '--temp-path', os.path.join(bin_path, 'temp'), - '--collision', 'Rename', - '-o', output_path - ] - - try: - result = subprocess.call(cmd) - if result == 0: - print(f'{Fore.GREEN}✓ VOD downloaded{Style.RESET_ALL}') - return True - else: - print(f'{Fore.RED}✗ VOD download failed with exit code: {result}{Style.RESET_ALL}') - return False - except Exception as e: - print(f'{Fore.RED}✗ VOD download failed: {str(e)}{Style.RESET_ALL}') - self.send_notification('VOD Download Error', f'Failed to download VOD: {str(e)}') - return False - - def _download_and_render_chat(self, vod_info: Dict[str, Any], json_path: str, video_path: str) -> bool: - """ - Download chat logs and render them as video. - - Args: - vod_info: VOD metadata from Twitch API - json_path: Path to save chat JSON - video_path: Path to save rendered chat video - - Returns: - bool: True if succeeded, False otherwise - """ - if not self.downloadCHAT: - return False - - print(f'\n{Fore.CYAN}Downloading chat: {vod_info["title"]}{Style.RESET_ALL}') - - # Extract numeric VOD ID - vod_id = vod_info["id"] - if isinstance(vod_id, str) and vod_id.startswith('v'): - vod_id = vod_id[1:] - - bin_path = self._get_bin_path() - downloader = self._get_twitch_downloader_executable() - - # Chat rendering settings - chat_settings = [ - '--background-color', '#FF111111', - '-w', '500', - '-h', '1080', - '--outline', - '-f', 'Arial', - '--font-size', '22', - '--update-rate', '1.0', - '--offline', - '--ffmpeg-path', self._get_ffmpeg_executable(), - '--temp-path', os.path.join(bin_path, 'temp'), - '--collision', 'Rename' - ] - - try: - # Download chat JSON - print(f'{Fore.YELLOW}Downloading chat JSON for VOD {vod_id}...{Style.RESET_ALL}') - result = subprocess.call([ - downloader, 'chatdownload', - '--id', vod_id, - '--embed-images', - '--collision', 'Rename', - '-o', json_path - ]) - - if result != 0: - print(f'{Fore.RED}✗ Chat JSON download failed with exit code: {result}{Style.RESET_ALL}') - return False - - # Verify JSON file was created - if not os.path.exists(json_path): - print(f'{Fore.RED}✗ Chat JSON file was not created{Style.RESET_ALL}') - return False - - print(f'{Fore.GREEN}✓ Chat JSON downloaded{Style.RESET_ALL}') - - # Render chat video - print(f'{Fore.YELLOW}Rendering chat video...{Style.RESET_ALL}') - result = subprocess.call([ - downloader, 'chatrender', - '-i', json_path, - '-o', video_path - ] + chat_settings) - - if result != 0: - print(f'{Fore.RED}✗ Chat render failed with exit code: {result}{Style.RESET_ALL}') - return False - - print(f'{Fore.GREEN}✓ Chat rendered{Style.RESET_ALL}') - return True - - except Exception as e: - print(f'{Fore.RED}✗ Chat processing failed: {str(e)}{Style.RESET_ALL}') - self.send_notification('Chat Download Error', - f'Failed to download/render chat: {str(e)}') - return False - - def _download_live_chat(self, vod_id: str, json_path: str) -> Optional[subprocess.Popen]: - """ - Start downloading live chat in the background while stream is recording. - - Args: - vod_id: The VOD/stream ID to download chat from - json_path: Path to save chat JSON - - Returns: - subprocess.Popen: The process handle, or None if failed to start - """ - if not self.downloadLiveCHAT: - return None - - print(f'\n{Fore.CYAN}Starting live chat download...{Style.RESET_ALL}') - - # Remove 'v' prefix if present - if isinstance(vod_id, str) and vod_id.startswith('v'): - vod_id = vod_id[1:] - - downloader = self._get_twitch_downloader_executable() - - try: - # Start chat download as background process - cmd = [ - downloader, 'chatdownload', - '--id', vod_id, - '--embed-images', - '--collision', 'Rename', - '-o', json_path - ] - - print(f'{Fore.YELLOW}Live chat download started in background for VOD {vod_id}{Style.RESET_ALL}') - process = subprocess.Popen( - cmd, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL - ) - return process - - except Exception as e: - print(f'{Fore.RED}✗ Failed to start live chat download: {str(e)}{Style.RESET_ALL}') - return None - - def _wait_for_chat_download(self, process: Optional[subprocess.Popen], json_path: str) -> bool: - """ - Download chat logs and render them as video. - - Args: - vod_info: VOD metadata from Twitch API - json_path: Path to save chat JSON - video_path: Path to save rendered chat video - - Returns: - bool: True if succeeded, False otherwise - """ - if not self.downloadCHAT: - return False - - print(f'\n{Fore.CYAN}Downloading chat: {vod_info["title"]}{Style.RESET_ALL}') - - # Extract numeric VOD ID - vod_id = vod_info["id"] - if isinstance(vod_id, str) and vod_id.startswith('v'): - vod_id = vod_id[1:] - - bin_path = self._get_bin_path() - downloader = self._get_twitch_downloader_executable() - - # Chat rendering settings - chat_settings = [ - '--background-color', '#FF111111', - '-w', '500', - '-h', '1080', - '--outline', - '-f', 'Arial', - '--font-size', '22', - '--update-rate', '1.0', - '--offline', - '--ffmpeg-path', self._get_ffmpeg_executable(), - '--temp-path', os.path.join(bin_path, 'temp'), - '--collision', 'Rename' - ] - - try: - # Download chat JSON - print(f'{Fore.YELLOW}Downloading chat JSON for VOD {vod_id}...{Style.RESET_ALL}') - result = subprocess.call([ - downloader, 'chatdownload', - '--id', vod_id, - '--embed-images', - '--collision', 'Rename', - '-o', json_path - ]) - - if result != 0: - print(f'{Fore.RED}✗ Chat JSON download failed with exit code: {result}{Style.RESET_ALL}') - return False - - # Verify JSON file was created - if not os.path.exists(json_path): - print(f'{Fore.RED}✗ Chat JSON file was not created{Style.RESET_ALL}') - return False - - print(f'{Fore.GREEN}✓ Chat JSON downloaded{Style.RESET_ALL}') - - # Render chat video - print(f'{Fore.YELLOW}Rendering chat video...{Style.RESET_ALL}') - result = subprocess.call([ - downloader, 'chatrender', - '-i', json_path, - '-o', video_path - ] + chat_settings) - - if result != 0: - print(f'{Fore.RED}✗ Chat render failed with exit code: {result}{Style.RESET_ALL}') - return False - - print(f'{Fore.GREEN}✓ Chat rendered{Style.RESET_ALL}') - return True - - except Exception as e: - print(f'{Fore.RED}✗ Chat processing failed: {str(e)}{Style.RESET_ALL}') - self.send_notification('Chat Download Error', - f'Failed to download/render chat: {str(e)}') - return False - - def _wait_for_chat_download(self, process: Optional[subprocess.Popen], json_path: str) -> bool: - """ - Wait for live chat download process to complete. - - Args: - process: The chat download process handle - json_path: Path where chat JSON should be saved - - Returns: - bool: True if chat download succeeded, False otherwise - """ - if process is None: - return False - - try: - print(f'{Fore.YELLOW}Waiting for live chat download to complete...{Style.RESET_ALL}') - return_code = process.wait(timeout=300) # 5 minute timeout - - if return_code == 0 and os.path.exists(json_path): - print(f'{Fore.GREEN}✓ Live chat JSON downloaded{Style.RESET_ALL}') - return True - else: - print(f'{Fore.RED}✗ Live chat download failed (exit code: {return_code}){Style.RESET_ALL}') - return False - - except subprocess.TimeoutExpired: - print(f'{Fore.YELLOW}⚠ Live chat download timed out, terminating...{Style.RESET_ALL}') - process.terminate() - return False - except Exception as e: - print(f'{Fore.RED}✗ Error waiting for chat download: {str(e)}{Style.RESET_ALL}') - return False - - def _render_chat(self, json_path: str, video_path: str) -> bool: - """ - Render chat JSON as a video. - - Args: - json_path: Path to chat JSON file - video_path: Path to save rendered chat video - - Returns: - bool: True if succeeded, False otherwise - """ - if not os.path.exists(json_path): - print(f'{Fore.RED}✗ Chat JSON file not found: {json_path}{Style.RESET_ALL}') - return False - - bin_path = self._get_bin_path() - downloader = self._get_twitch_downloader_executable() - - # Chat rendering settings - chat_settings = [ - '--background-color', '#FF111111', - '-w', '500', - '-h', '1080', - '--outline', - '-f', 'Arial', - '--font-size', '22', - '--update-rate', '1.0', - '--offline', - '--ffmpeg-path', self._get_ffmpeg_executable(), - '--temp-path', os.path.join(bin_path, 'temp'), - '--collision', 'Rename' - ] - - try: - print(f'{Fore.YELLOW}Rendering chat video...{Style.RESET_ALL}') - result = subprocess.call([ - downloader, 'chatrender', - '-i', json_path, - '-o', video_path - ] + chat_settings) - - if result != 0: - print(f'{Fore.RED}✗ Chat render failed with exit code: {result}{Style.RESET_ALL}') - return False - - print(f'{Fore.GREEN}✓ Chat rendered{Style.RESET_ALL}') - return True - - except Exception as e: - print(f'{Fore.RED}✗ Chat rendering failed: {str(e)}{Style.RESET_ALL}') - return False - - def _save_metadata(self, vod_info: Dict[str, Any], filename_base: str) -> None: - """ - Save VOD metadata to JSON file. - - Args: - vod_info: VOD metadata from Twitch API - filename_base: Base filename (without extension) - """ - if not self.downloadMETADATA: - return - - metadata_path = os.path.join(self.metadata_path, f"{PREFIX_METADATA}{filename_base}.json") - - with open(metadata_path, 'w', encoding='utf-8') as f: - json.dump(vod_info, f, ensure_ascii=False, indent=4) - - print(f'{Fore.GREEN}✓ Metadata saved{Style.RESET_ALL}') - - def _signal_handler(self, signum, frame): - """Handle interrupt signals gracefully.""" - if not self.shutdown_requested: - print(f'\n{Fore.YELLOW}{"=" * 60}{Style.RESET_ALL}') - print(f'{Fore.YELLOW}⚠ Shutdown requested. Stopping downloads and finalizing...{Style.RESET_ALL}') - print(f'{Fore.YELLOW}{"=" * 60}{Style.RESET_ALL}\n') - self.shutdown_requested = True - - # Stop current subprocess if running - if self.current_process: - try: - self.current_process.terminate() - print(f'{Fore.YELLOW}Stopping current download process...{Style.RESET_ALL}') - except Exception: - pass def _interruptible_sleep(self, seconds: float) -> bool: """ @@ -1294,6 +284,18 @@ class TwitchArchive: time.sleep(min(1.0, seconds - (time.time() - start_time))) return True + def _signal_handler(self, signum, frame): + """Handle interrupt signals gracefully.""" + if not self.shutdown_requested: + print(f'\n{Fore.YELLOW}{"=" * 60}{Style.RESET_ALL}') + print(f'{Fore.YELLOW}⚠ Shutdown requested. Stopping downloads and finalizing...{Style.RESET_ALL}') + print(f'{Fore.YELLOW}{"=" * 60}{Style.RESET_ALL}\n') + self.shutdown_requested = True + + # Stop current subprocess if running + if self.recorder: + self.recorder.stop() + def loopcheck(self) -> None: """ Main monitoring loop. @@ -1307,14 +309,13 @@ class TwitchArchive: """ # Set up signal handlers for graceful shutdown signal.signal(signal.SIGINT, self._signal_handler) - # SIGTERM is not available on Windows, handle gracefully if hasattr(signal, 'SIGTERM'): signal.signal(signal.SIGTERM, self._signal_handler) while not self.shutdown_requested: try: - # Check stream status - response = self._check_stream_status() + # Check stream status using StreamMonitor + response = self.stream_monitor.check_stream_status() is_live = response['data']['user']['stream'] # Stream is offline @@ -1337,7 +338,7 @@ class TwitchArchive: print(f'\n{Fore.GREEN}✓ {self.username} is LIVE!{Style.RESET_ALL}') print(f'{Fore.CYAN}Title: {is_live["title"]}{Style.RESET_ALL}') - # Create unique stream identifier based on stream start time + # Create unique stream identifier stream_id = f"{is_live['createdAt']} - {self.username} - {is_live['title']}" # Parse stream start time @@ -1345,73 +346,65 @@ class TwitchArchive: is_live["createdAt"], '%Y-%m-%dT%H:%M:%SZ' ).replace(tzinfo=timezone('UTC')).astimezone(tz=None).replace(tzinfo=None) - # Use CURRENT time for filename to ensure each recording is unique - # This allows recording a live stream multiple times (e.g., if script restarts) + # Use CURRENT time for filename current_time = datetime.now() filename_base = current_time.strftime('%Y%m%d_%Hh%Mm%Ss') - # Check if we've already recorded this stream session - if self._is_stream_already_processed(stream_id): + # Check if stream was already processed + if self.file_manager.is_stream_processed(stream_id): print(f'{Fore.YELLOW}⚠ Stream was previously recorded, but it\'s still live!{Style.RESET_ALL}') print(f'{Fore.GREEN}✓ Starting new recording with timestamp: {filename_base}{Style.RESET_ALL}') else: - # First time seeing this stream - mark it - self._mark_stream_as_processed(stream_id) + self.file_manager.mark_stream_processed(stream_id) print(f'{Fore.GREEN}✓ New stream detected - starting recording{Style.RESET_ALL}') # Determine file paths - live_raw_path = os.path.join(self.raw_path, f"{PREFIX_LIVE}{filename_base}.ts") + live_raw_path = str(self.file_manager.raw_path / f"{PREFIX_LIVE}{filename_base}.ts") live_proc_ext = '.mp3' if self.quality == 'audio_only' else '.mp4' - live_proc_path = os.path.join(self.video_path, f"{PREFIX_LIVE}{filename_base}{live_proc_ext}") + live_proc_path = str(self.file_manager.video_path / f"{PREFIX_LIVE}{filename_base}{live_proc_ext}") # Ensure unique filenames - live_raw_path = self._get_unique_filename(live_raw_path) - live_proc_path = self._get_unique_filename(live_proc_path) + live_raw_path = get_unique_filename(live_raw_path) + live_proc_path = get_unique_filename(live_proc_path) filename_base = os.path.splitext(os.path.basename(live_raw_path))[0].replace(PREFIX_LIVE, "") print(f'{Fore.CYAN}Output path: {live_raw_path}{Style.RESET_ALL}') # Send notification - self.send_notification(f'🔴 Stream Started - {filename_base}', - f'Title: {is_live["title"]}') + self.notification_manager.send(f'🔴 Stream Started - {filename_base}', + f'Title: {is_live["title"]}') - # Store current stream data for potential graceful shutdown - self.current_stream_data = { - 'filename_base': filename_base, - 'live_raw_path': live_raw_path, - 'live_proc_path': live_proc_path - } - - # Start live chat download if enabled and VOD ID is available + # Start live chat download if enabled live_chat_process = None - chat_json_path = os.path.join(self.chatJSON_path, f"{PREFIX_CHAT}{filename_base}.json") + chat_json_path = str(self.file_manager.chat_json_path / f"{PREFIX_CHAT}{filename_base}.json") if self.downloadLiveCHAT and is_live.get('archiveVideo') and is_live['archiveVideo'].get('id'): live_vod_id = is_live['archiveVideo']['id'] print(f'{Fore.CYAN}Live VOD ID detected: {live_vod_id}{Style.RESET_ALL}') - live_chat_process = self._download_live_chat(live_vod_id, chat_json_path) + live_chat_process = self.downloader.start_live_chat_download(live_vod_id, chat_json_path) elif self.downloadLiveCHAT: print(f'{Fore.YELLOW}⚠ No VOD ID available yet for live chat download{Style.RESET_ALL}') # Record the live stream - recording_completed = self._record_livestream(is_live, live_raw_path) + recording_completed = self.recorder.record(is_live, live_raw_path) # If shutdown was requested during recording, try to finalize if self.shutdown_requested: print(f'{Fore.YELLOW}Attempting to process any recorded content...{Style.RESET_ALL}') # Process the raw stream file - self._process_raw_stream(live_raw_path, live_proc_path) + self.processor.process_raw_stream(live_raw_path, live_proc_path) # Wait for live chat download if it was started live_chat_downloaded = False if live_chat_process is not None: - live_chat_downloaded = self._wait_for_chat_download(live_chat_process, chat_json_path) + live_chat_downloaded = self.downloader.wait_for_chat_download(live_chat_process, chat_json_path) # Render live chat if downloaded successfully if live_chat_downloaded: - chat_video_path = os.path.join(self.chatMP4_path, f"{PREFIX_CHAT}{filename_base}.mp4") - self._render_chat(chat_json_path, chat_video_path) + chat_video_path = str(self.file_manager.chat_mp4_path / f"{PREFIX_CHAT}{filename_base}.mp4") + output_args = self.processor.build_chat_output_args() + self.downloader.render_chat(chat_json_path, chat_video_path, output_args) # Skip VOD/chat download if shutdown was requested or vodTimeout is 0 vod_response = None @@ -1426,7 +419,7 @@ class TwitchArchive: vod_wait_start = time.time() while time.time() - vod_wait_start < self.vodTimeout and not self.shutdown_requested: - vod_response = self._get_latest_vod() + vod_response = self.stream_monitor.get_latest_vod() if vod_response and vod_response['data']['user']['videos']['edges']: current_vod = vod_response['data']['user']['videos']['edges'][0]['node'] @@ -1454,6 +447,7 @@ class TwitchArchive: print(f'{Fore.CYAN} → Live recording and chat (if enabled) were saved successfully{Style.RESET_ALL}') vod_response = None + # Process VOD if found if not self.shutdown_requested and vod_response and vod_response['data']['user']['videos']['edges']: current_vod = vod_response['data']['user']['videos']['edges'][0]['node'] vod_date = datetime.strptime( @@ -1466,33 +460,40 @@ class TwitchArchive: print(f'\n{Fore.GREEN}✓ Found matching VOD{Style.RESET_ALL}') # Save metadata - self._save_metadata(current_vod, filename_base) + self.file_manager.save_metadata(current_vod, filename_base) # Download VOD vod_ext = '.mp3' if self.quality == 'audio_only' else '.mp4' - vod_path = os.path.join(self.video_path, f"{PREFIX_VOD}{filename_base}{vod_ext}") - self._download_vod(current_vod, vod_path) + vod_path = str(self.file_manager.video_path / f"{PREFIX_VOD}{filename_base}{vod_ext}") + self.downloader.download_vod(current_vod, vod_path) # Download and render chat from VOD (if not already done via live chat) if not live_chat_downloaded: - chat_video_path = os.path.join(self.chatMP4_path, f"{PREFIX_CHAT}{filename_base}.mp4") - self._download_and_render_chat(current_vod, chat_json_path, chat_video_path) + chat_video_path = str(self.file_manager.chat_mp4_path / f"{PREFIX_CHAT}{filename_base}.mp4") + output_args = self.processor.build_chat_output_args() + self.downloader.download_and_render_chat(current_vod, chat_json_path, chat_video_path, output_args) else: print(f'{Fore.CYAN}Chat already downloaded from live stream, skipping VOD chat download{Style.RESET_ALL}') else: print(f'{Fore.YELLOW}⚠ No matching VOD found for this stream{Style.RESET_ALL}') # Clean up raw files if configured - if self.cleanRaw and os.path.exists(live_raw_path): - print(f'{Fore.YELLOW}Deleting raw .ts file...{Style.RESET_ALL}') - os.remove(live_raw_path) + self.file_manager.clean_raw_file(live_raw_path) # Upload to cloud if configured - upload_success = self._upload_to_cloud(filename_base) + upload_success = self.file_manager.upload_to_cloud( + filename_base, + notification_callback=self.notification_manager.send + ) # Delete local files if configured and upload succeeded if self.deleteFiles and upload_success: - self._delete_local_files(filename_base, live_raw_path, live_proc_path) + self.file_manager.delete_local_files( + filename_base, + live_raw_path, + live_proc_path, + notification_callback=self.notification_manager.send + ) # Done processing this stream if self.shutdown_requested: @@ -1504,12 +505,11 @@ class TwitchArchive: print(f'\n{Fore.GREEN}{"=" * 60}{Style.RESET_ALL}') print(f'{Fore.GREEN}✓ Stream processing complete{Style.RESET_ALL}') print(f'{Fore.GREEN}{"=" * 60}{Style.RESET_ALL}\n') - self.send_notification(f'✓ Complete - {filename_base}', - 'Stream processing finished. Resuming monitoring.') + self.notification_manager.send(f'✓ Complete - {filename_base}', + 'Stream processing finished. Resuming monitoring.') self._interruptible_sleep(self.refresh) except KeyboardInterrupt: - # Additional catch for any other KeyboardInterrupt not handled by signal if not self.shutdown_requested: self.shutdown_requested = True print(f'\n{Fore.YELLOW}{"=" * 60}{Style.RESET_ALL}') @@ -1522,10 +522,9 @@ class TwitchArchive: print(f'{Fore.RED}✗ ERROR: {str(e)}{Style.RESET_ALL}') print(f'{Fore.YELLOW}Waiting {self.refresh} seconds before retrying...{Style.RESET_ALL}') print(f'{Fore.RED}{"=" * 60}{Style.RESET_ALL}\n') - self.send_notification('⚠ Error - Recovery', - f'Error: {str(e)}\nRetrying after {self.refresh} seconds.') + self.notification_manager.send('⚠ Error - Recovery', + f'Error: {str(e)}\nRetrying after {self.refresh} seconds.') - # Check for shutdown during sleep if self.shutdown_requested: break self._interruptible_sleep(self.refresh) @@ -1758,15 +757,19 @@ class TwitchArchiveManager: try: archiver = self._initialize_archiver(username) - # Load environment and validate + # Load environment and initialize components archiver._load_environment_variables() - archiver._validate_username() - archiver._initialize_paths() + archiver._initialize_components() + + # Validate username through stream_monitor + archiver.stream_monitor.validate_username() self.archivers[username] = archiver print(f'{Fore.GREEN}✓ Initialized {username}{Style.RESET_ALL}') except Exception as e: print(f'{Fore.RED}✗ Failed to initialize {username}: {e}{Style.RESET_ALL}') + import traceback + traceback.print_exc() if not self.archivers: print(f'{Fore.RED}✗ No archivers could be initialized{Style.RESET_ALL}') @@ -1775,7 +778,11 @@ class TwitchArchiveManager: # Verify dependencies once (shared across all streamers) print(f'\n{Fore.CYAN}Verifying dependencies...{Style.RESET_ALL}') first_archiver = next(iter(self.archivers.values())) - first_archiver._verify_dependencies() + if not verify_streamlink(): + sys.exit(1) + verify_ffmpeg(first_archiver.os_type) + if first_archiver.downloadVOD or first_archiver.downloadCHAT: + verify_twitch_downloader(first_archiver.os_type) # Print configuration summary for each streamer for username, archiver in self.archivers.items(): @@ -1811,7 +818,7 @@ class TwitchArchiveManager: # Check stream status try: - response = archiver._check_stream_status() + response = archiver.stream_monitor.check_stream_status() # Debug: Print the full response (if verbose) if self.verbose: @@ -1853,7 +860,7 @@ class TwitchArchiveManager: self._process_stream(archiver, stream_data, stream_id) # Mark as processed in log (for record keeping) - archiver._mark_stream_as_processed(stream_id) + archiver.file_manager.mark_stream_processed(stream_id) # Remove from active recordings if username in self.active_recordings: @@ -1906,12 +913,12 @@ class TwitchArchiveManager: raw_extension = '.ts' proc_extension = '.mp3' if archiver.quality == 'audio_only' else '.mp4' - live_raw_path = str(archiver.raw_path / f"{filename_base}{raw_extension}") - live_proc_path = str(archiver.video_path / f"{filename_base}{proc_extension}") - chat_json_path = str(archiver.chatJSON_path / f"{PREFIX_CHAT}{filename_base}.json") + live_raw_path = str(archiver.file_manager.raw_path / f"{filename_base}{raw_extension}") + live_proc_path = str(archiver.file_manager.video_path / f"{filename_base}{proc_extension}") + chat_json_path = str(archiver.file_manager.chat_json_path / f"{PREFIX_CHAT}{filename_base}.json") # Send notification - archiver.send_notification( + archiver.notification_manager.send( f"Stream Started - {archiver.username}", f"Recording: {stream_info['title']}" ) @@ -1921,12 +928,12 @@ class TwitchArchiveManager: if archiver.downloadLiveCHAT and stream_info.get('archiveVideo') and stream_info['archiveVideo'].get('id'): live_vod_id = stream_info['archiveVideo']['id'] print(f'{Fore.CYAN}Live VOD ID detected: {live_vod_id}{Style.RESET_ALL}') - live_chat_process = archiver._download_live_chat(live_vod_id, chat_json_path) + live_chat_process = archiver.downloader.start_live_chat_download(live_vod_id, chat_json_path) elif archiver.downloadLiveCHAT: print(f'{Fore.YELLOW}⚠ No VOD ID available yet for live chat download{Style.RESET_ALL}') # Record livestream - recording_successful = archiver._record_livestream(stream_info, live_raw_path) + recording_successful = archiver.recorder.record(stream_info, live_raw_path) # Check if raw file exists (may exist even after interrupted recording) if not os.path.exists(live_raw_path): @@ -1943,17 +950,18 @@ class TwitchArchiveManager: # Process raw stream if not archiver.onlyRaw: - archiver._process_raw_stream(live_raw_path, live_proc_path) + archiver.processor.process_raw_stream(live_raw_path, live_proc_path) # Wait for live chat download if it was started live_chat_downloaded = False if live_chat_process is not None: - live_chat_downloaded = archiver._wait_for_chat_download(live_chat_process, chat_json_path) + live_chat_downloaded = archiver.downloader.wait_for_chat_download(live_chat_process, chat_json_path) # Render live chat if downloaded successfully if live_chat_downloaded: - chat_video_path = str(archiver.chatMP4_path / f"{PREFIX_CHAT}{filename_base}.mp4") - archiver._render_chat(chat_json_path, chat_video_path) + chat_video_path = str(archiver.file_manager.chat_mp4_path / f"{PREFIX_CHAT}{filename_base}.mp4") + output_args = archiver.processor.build_chat_output_args() + archiver.downloader.render_chat(chat_json_path, chat_video_path, output_args) # Wait for VOD and download it vod_response = None @@ -1973,7 +981,7 @@ class TwitchArchiveManager: print(f'\n{Fore.YELLOW}VOD check interrupted by shutdown{Style.RESET_ALL}') break - vod_response = archiver._get_latest_vod() + vod_response = archiver.stream_monitor.get_latest_vod() if vod_response and vod_response['data']['user']['videos']['edges']: current_vod = vod_response['data']['user']['videos']['edges'][0]['node'] @@ -2011,42 +1019,47 @@ class TwitchArchiveManager: # Save metadata if archiver.downloadMETADATA: - archiver._save_metadata(current_vod, filename_base) + archiver.file_manager.save_metadata(current_vod, filename_base) # Download VOD if archiver.downloadVOD: vod_ext = '.mp3' if archiver.quality == 'audio_only' else '.mp4' - vod_path = str(archiver.video_path / f"{PREFIX_VOD}{filename_base}{vod_ext}") - archiver._download_vod(current_vod, vod_path) + vod_path = str(archiver.file_manager.video_path / f"{PREFIX_VOD}{filename_base}{vod_ext}") + archiver.downloader.download_vod(current_vod, vod_path) # Download and render chat from VOD (if not already done via live chat) if archiver.downloadCHAT and not live_chat_downloaded: - chat_video_path = str(archiver.chatMP4_path / f"{PREFIX_CHAT}{filename_base}.mp4") - archiver._download_and_render_chat(current_vod, chat_json_path, chat_video_path) + chat_video_path = str(archiver.file_manager.chat_mp4_path / f"{PREFIX_CHAT}{filename_base}.mp4") + output_args = archiver.processor.build_chat_output_args() + archiver.downloader.download_and_render_chat(current_vod, chat_json_path, chat_video_path, output_args) elif live_chat_downloaded: print(f'{Fore.CYAN}Chat already downloaded from live stream, skipping VOD chat download{Style.RESET_ALL}') else: print(f'{Fore.YELLOW}⚠ No matching VOD found for this stream{Style.RESET_ALL}') elif archiver.downloadMETADATA: # Save what metadata we have from the live stream - archiver._save_metadata(stream_info, filename_base) + archiver.file_manager.save_metadata(stream_info, filename_base) # Clean up raw file if configured - if archiver.cleanRaw and os.path.exists(live_raw_path): - print(f'{Fore.YELLOW}Deleting raw .ts file...{Style.RESET_ALL}') - os.remove(live_raw_path) + archiver.file_manager.clean_raw_file(live_raw_path) # Upload to cloud if configured - upload_success = False - if archiver.uploadCloud: - upload_success = archiver._upload_to_cloud(filename_base) + upload_success = archiver.file_manager.upload_to_cloud( + filename_base, + notification_callback=archiver.notification_manager.send + ) # Delete files if configured if archiver.deleteFiles and upload_success: - archiver._delete_local_files(filename_base, live_raw_path, live_proc_path) + archiver.file_manager.delete_local_files( + filename_base, + live_raw_path, + live_proc_path, + notification_callback=archiver.notification_manager.send + ) # Send completion notification - archiver.send_notification( + archiver.notification_manager.send( f"Stream Archived - {archiver.username}", f"Completed: {stream_info['title']}" )