Add chat_downloader support with fallback for live chat downloads

- Integrated chat_downloader as an optional dependency for downloading live chat. - Implemented fallback logic to use chat_downloader when TwitchDownloaderCLI fails or is not available. - Enhanced chat rendering process to handle different JSON formats from chat_downloader. - Updated requirements.txt to include chat-downloader package. - Modified start.bat to allow passing additional arguments for flexibility. - Improved error handling and logging for chat download processes. - Added command-line options for testing chat-only mode and configuring chat_downloader behavior.
2026-02-10 23:42:22 +01:00 · 2026-02-10 23:42:22 +01:00 · b50a4bad02
commit b50a4bad02
parent cdef8cf9bb
4 changed files with 801 additions and 47 deletions
--- a/twitch-archive.py
+++ b/twitch-archive.py
@ -380,14 +380,21 @@ class TwitchArchive:
                
                # Start live chat download if enabled
                live_chat_process = None
+                live_chat_method = None  # Track which method was used
                chat_json_path = str(self.file_manager.chat_json_path / f"{PREFIX_CHAT}{filename_base}.json")
                
-                if self.downloadLiveCHAT and is_live.get('archiveVideo') and is_live['archiveVideo'].get('id'):
-                    live_vod_id = is_live['archiveVideo']['id']
-                    print(f'{Fore.CYAN}Live VOD ID detected: {live_vod_id}{Style.RESET_ALL}')
-                    live_chat_process = self.downloader.start_live_chat_download(live_vod_id, chat_json_path)
-                elif self.downloadLiveCHAT:
-                    print(f'{Fore.YELLOW}⚠ No VOD ID available yet for live chat download{Style.RESET_ALL}')
+                if self.downloadLiveCHAT:
+                    vod_id = is_live.get('archiveVideo', {}).get('id') if is_live.get('archiveVideo') else None
+                    stream_url = f"https://twitch.tv/{self.username}"
+                    
+                    live_chat_process, live_chat_method = self.downloader.start_live_chat_download_with_fallback(
+                        vod_id=vod_id,
+                        stream_url=stream_url,
+                        json_path=chat_json_path,
+                        use_chat_downloader_primary=self.use_chat_downloader_primary,
+                        no_chat_downloader_fallback=self.no_chat_downloader_fallback,
+                        verbose=self.verbose
+                    )
                
                # Record the live stream
                recording_completed = self.recorder.record(is_live, live_raw_path)
@ -411,16 +418,34 @@ class TwitchArchive:
                    chat_video_path = str(self.file_manager.chat_mp4_path / f"{PREFIX_CHAT}{filename_base}.mp4")
                    output_args = self.processor.build_chat_output_args()
                    
-                    # Get video duration to trim chat accordingly
-                    ffmpeg_path = get_ffmpeg_executable(self.os_type)
-                    video_duration = get_video_duration(live_proc_path, ffmpeg_path)
-                    
-                    chat_rendered_successfully = self.downloader.render_chat(
-                        chat_json_path, 
-                        chat_video_path, 
-                        output_args,
-                        video_duration=video_duration
-                    )
+                    # Wait for chat file to be fully accessible (not locked)
+                    print(f'{Fore.CYAN}Verifying chat file is ready for rendering...{Style.RESET_ALL}')
+                    if not self.downloader.wait_for_file_access(chat_json_path, max_attempts=15, delay=0.5):
+                        print(f'{Fore.RED}✗ Chat file is locked, skipping rendering{Style.RESET_ALL}')
+                        chat_rendered_successfully = False
+                    else:
+                        # Get video duration first (needed for chat conversion and trimming)
+                        ffmpeg_path = get_ffmpeg_executable(self.os_type)
+                        video_duration = get_video_duration(live_proc_path, ffmpeg_path)
+                        print(f'{Fore.CYAN}Video duration for chat rendering: {video_duration}s{Style.RESET_ALL}')
+                        
+                        # Convert chat format if needed (chat_downloader uses different JSON structure)
+                        render_json_path = chat_json_path
+                        if live_chat_method == 'chat_downloader':
+                            print(f'{Fore.CYAN}Converting chat format for rendering...{Style.RESET_ALL}')
+                            converted_path = chat_json_path.replace('.json', '_converted.json')
+                            if self.downloader.convert_chat_downloader_to_twitch_format(chat_json_path, converted_path, video_duration):
+                                render_json_path = converted_path
+                                print(f'{Fore.GREEN}✓ Chat format converted successfully{Style.RESET_ALL}')
+                            else:
+                                print(f'{Fore.RED}✗ Failed to convert chat format{Style.RESET_ALL}')
+                        
+                        chat_rendered_successfully = self.downloader.render_chat(
+                            render_json_path, 
+                            chat_video_path, 
+                            output_args,
+                            video_duration=video_duration
+                        )
                
                # Merge video and chat if configured
                merged_video_path = None
@ -730,17 +755,26 @@ class TwitchArchiveManager:
    Manages multiple TwitchArchive instances for monitoring multiple streamers.
    """
    
-    def __init__(self, specific_streamer: Optional[str] = None, verbose: bool = False):
+    def __init__(self, specific_streamer: Optional[str] = None, verbose: bool = False,
+                 chat_only: bool = False,
+                 use_chat_downloader_primary: bool = False,
+                 use_chat_downloader_fallback: bool = True):
        """
        Initialize the manager.
        
        Args:
            specific_streamer: If provided, only monitor this streamer (ignore enabled status)
            verbose: Enable verbose debug output
+            chat_only: Only download chat, skip video recording (test mode)
+            use_chat_downloader_primary: Use chat_downloader as primary chat source
+            use_chat_downloader_fallback: Enable chat_downloader fallback
        """
        self.config_manager = ConfigManager()
        self.specific_streamer = specific_streamer
        self.verbose = verbose
+        self.chat_only = chat_only
+        self.use_chat_downloader_primary = use_chat_downloader_primary
+        self.use_chat_downloader_fallback = use_chat_downloader_fallback
        self.archivers: Dict[str, TwitchArchive] = {}
        self.shutdown_requested = False
        self.active_recordings: Dict[str, str] = {}  # Track active recordings: {username: stream_id}
@ -783,6 +817,11 @@ class TwitchArchiveManager:
            TwitchArchive: Initialized archiver instance
        """
        config = self.config_manager.load_streamer_config(username)
+        
+        # Apply command-line overrides for chat_downloader options
+        config['useChatDownloaderPrimary'] = self.use_chat_downloader_primary
+        config['useChatDownloaderFallback'] = self.use_chat_downloader_fallback
+        
        archiver = TwitchArchive(config)
        return archiver
    
@ -794,6 +833,8 @@ class TwitchArchiveManager:
        """
        print(f'\n{Fore.CYAN}{"=" * 70}{Style.RESET_ALL}')
        print(f'{Fore.CYAN}TWITCH ARCHIVE - Multi-Streamer Mode{Style.RESET_ALL}')
+        if self.chat_only:
+            print(f'{Fore.YELLOW}🧪 TEST MODE: Chat-Only (Video Recording Disabled){Style.RESET_ALL}')
        print(f'{Fore.CYAN}{"=" * 70}{Style.RESET_ALL}\n')
        
        # Get streamers to monitor
@ -805,6 +846,14 @@ class TwitchArchiveManager:
            print(f'{Fore.CYAN}→ Or run with -u <username> to create a new config{Style.RESET_ALL}')
            sys.exit(1)
        
+        if self.chat_only:
+            print(f'{Fore.YELLOW}📝 Chat-Only Mode Enabled:{Style.RESET_ALL}')
+            print(f'{Fore.CYAN}  • Verbose logging: ON{Style.RESET_ALL}')
+            print(f'{Fore.CYAN}  • Video recording: DISABLED{Style.RESET_ALL}')
+            print(f'{Fore.CYAN}  • Chat download: ENABLED{Style.RESET_ALL}')
+            print(f'{Fore.CYAN}  • VOD download: DISABLED{Style.RESET_ALL}')
+            print()
+        
        print(f'{Fore.GREEN}Monitoring {len(streamers)} streamer(s):{Style.RESET_ALL}')
        for streamer in streamers:
            print(f'  • {Fore.CYAN}{streamer}{Style.RESET_ALL}')
@ -985,26 +1034,131 @@ class TwitchArchiveManager:
        chat_json_path = str(archiver.file_manager.chat_json_path / f"{PREFIX_CHAT}{filename_base}.json")
        
        # Send notification
-        archiver.notification_manager.send(
-            f"Stream Started - {archiver.username}",
-            f"Recording: {stream_info['title']}"
-        )
+        if not self.chat_only:
+            archiver.notification_manager.send(
+                f"Stream Started - {archiver.username}",
+                f"Recording: {stream_info['title']}"
+            )
        
-        # Start live chat download if enabled and VOD ID is available
+        # Start live chat download if enabled (with fallback support)
        live_chat_process = None
-        if archiver.downloadLiveCHAT and stream_info.get('archiveVideo') and stream_info['archiveVideo'].get('id'):
-            live_vod_id = stream_info['archiveVideo']['id']
-            print(f'{Fore.CYAN}Live VOD ID detected: {live_vod_id}{Style.RESET_ALL}')
-            live_chat_process = archiver.downloader.start_live_chat_download(live_vod_id, chat_json_path)
-        elif archiver.downloadLiveCHAT:
-            print(f'{Fore.YELLOW}⚠ No VOD ID available yet for live chat download{Style.RESET_ALL}')
+        live_chat_method = 'failed'
+        if archiver.downloadLiveCHAT:
+            if self.verbose or self.chat_only:
+                print(f'\n{Fore.MAGENTA}[VERBOSE] Starting chat download process...{Style.RESET_ALL}')
+                print(f'{Fore.MAGENTA}[VERBOSE] downloadLiveCHAT: {archiver.downloadLiveCHAT}{Style.RESET_ALL}')
+                print(f'{Fore.MAGENTA}[VERBOSE] useChatDownloaderPrimary: {archiver.downloader.use_chat_downloader_primary}{Style.RESET_ALL}')
+                print(f'{Fore.MAGENTA}[VERBOSE] useChatDownloaderFallback: {archiver.downloader.use_chat_downloader_fallback}{Style.RESET_ALL}')
+            
+            # Get VOD ID if available
+            live_vod_id = None
+            if stream_info.get('archiveVideo') and stream_info['archiveVideo'].get('id'):
+                live_vod_id = stream_info['archiveVideo']['id']
+                print(f'{Fore.CYAN}Live VOD ID detected: {live_vod_id}{Style.RESET_ALL}')
+                if self.verbose or self.chat_only:
+                    print(f'{Fore.MAGENTA}[VERBOSE] VOD URL: https://www.twitch.tv/videos/{live_vod_id}{Style.RESET_ALL}')
+            else:
+                print(f'{Fore.YELLOW}⚠ No VOD ID available - will use fallback if configured{Style.RESET_ALL}')
+                if self.verbose or self.chat_only:
+                    print(f'{Fore.MAGENTA}[VERBOSE] This happens when streamer has VODs disabled{Style.RESET_ALL}')
+                    print(f'{Fore.MAGENTA}[VERBOSE] chat_downloader fallback will be used if enabled{Style.RESET_ALL}')
+            
+            # Try to start live chat download with fallback
+            try:
+                if self.verbose or self.chat_only:
+                    print(f'{Fore.MAGENTA}[VERBOSE] Calling start_live_chat_download_with_fallback(){Style.RESET_ALL}')
+                    print(f'{Fore.MAGENTA}[VERBOSE] Username: {archiver.username}{Style.RESET_ALL}')
+                    print(f'{Fore.MAGENTA}[VERBOSE] VOD ID: {live_vod_id}{Style.RESET_ALL}')
+                    print(f'{Fore.MAGENTA}[VERBOSE] Output path: {chat_json_path}{Style.RESET_ALL}')
+                
+                live_chat_process, live_chat_method = archiver.downloader.start_live_chat_download_with_fallback(
+                    archiver.username, live_vod_id, chat_json_path
+                )
+                
+                if self.verbose or self.chat_only:
+                    print(f'{Fore.MAGENTA}[VERBOSE] Chat download method selected: {live_chat_method}{Style.RESET_ALL}')
+                    print(f'{Fore.MAGENTA}[VERBOSE] Process handle: {live_chat_process}{Style.RESET_ALL}')
+                
+                # If chat_downloader is selected, start it in background thread now (before video recording)
+                if live_chat_method == 'chat_downloader' and not self.chat_only:
+                    if self.verbose:
+                        print(f'{Fore.MAGENTA}[VERBOSE] Starting chat_downloader in background thread...{Style.RESET_ALL}')
+                    try:
+                        print(f'{Fore.CYAN}Starting chat_downloader in background (concurrent with video)...{Style.RESET_ALL}')
+                        archiver.downloader.start_chat_downloader_thread(
+                            archiver.username, chat_json_path,
+                            shutdown_check=lambda: self.shutdown_requested or archiver.shutdown_requested,
+                            verbose=self.verbose
+                        )
+                    except Exception as e:
+                        print(f'{Fore.RED}✗ Failed to start chat thread: {e}{Style.RESET_ALL}')
+                        import traceback
+                        traceback.print_exc()
+                        live_chat_method = 'failed'
+                    
+            except Exception as e:
+                print(f'{Fore.RED}✗ Failed to start live chat download: {e}{Style.RESET_ALL}')
+                import traceback
+                traceback.print_exc()
+                live_chat_method = 'failed'
        
-        # Record livestream
+        # Record livestream (skip in chat-only mode)
+        if self.chat_only:
+            print(f'\n{Fore.YELLOW}🧪 Chat-Only Mode: Skipping video recording{Style.RESET_ALL}')
+            print(f'{Fore.CYAN}Waiting for chat download to complete...{Style.RESET_ALL}')
+            
+            # Start chat download based on method
+            if live_chat_method == 'chat_downloader':
+                if self.verbose:
+                    print(f'{Fore.MAGENTA}[VERBOSE] Starting chat_downloader in background thread...{Style.RESET_ALL}')
+                try:
+                    print(f'{Fore.CYAN}Using chat_downloader for live chat...{Style.RESET_ALL}')
+                    archiver.downloader.start_chat_downloader_thread(
+                        archiver.username, chat_json_path,
+                        shutdown_check=lambda: self.shutdown_requested or archiver.shutdown_requested,
+                        verbose=self.verbose or self.chat_only
+                    )
+                    # Wait for completion
+                    live_chat_downloaded = archiver.downloader.wait_for_chat_thread()
+                except Exception as e:
+                    print(f'{Fore.RED}✗ chat_downloader failed: {e}{Style.RESET_ALL}')
+                    import traceback
+                    traceback.print_exc()
+                    live_chat_downloaded = False
+            elif live_chat_method == 'twitch_downloader' and live_chat_process is not None:
+                if self.verbose:
+                    print(f'{Fore.MAGENTA}[VERBOSE] Waiting for TwitchDownloaderCLI process...{Style.RESET_ALL}')
+                live_chat_downloaded = archiver.downloader.wait_for_chat_download(live_chat_process, chat_json_path)
+            else:
+                live_chat_downloaded = False
+            
+            # Report results
+            if live_chat_downloaded:
+                print(f'\n{Fore.GREEN}✓ Chat-Only Test Complete!{Style.RESET_ALL}')
+                print(f'{Fore.CYAN}Chat saved to: {chat_json_path}{Style.RESET_ALL}')
+                if os.path.exists(chat_json_path):
+                    file_size = os.path.getsize(chat_json_path)
+                    print(f'{Fore.CYAN}File size: {file_size / 1024:.2f} KB{Style.RESET_ALL}')
+            else:
+                print(f'\n{Fore.RED}✗ Chat download failed{Style.RESET_ALL}')
+            
+            return  # Exit early, don't process video
+        
+        # Normal mode: Record livestream
        recording_successful = archiver.recorder.record(stream_info, live_raw_path)
        
        # Check if raw file exists (may exist even after interrupted recording)
        if not os.path.exists(live_raw_path):
            print(f'{Fore.RED}✗ No recording file found, skipping processing{Style.RESET_ALL}')
+            
+            # Still wait for chat if it's downloading
+            if live_chat_method == 'chat_downloader' and archiver.downloader.chat_thread is not None:
+                print(f'{Fore.CYAN}Waiting for chat download to finish...{Style.RESET_ALL}')
+                archiver.downloader.wait_for_chat_thread(timeout=30)
+            elif live_chat_method == 'twitch_downloader' and live_chat_process is not None:
+                print(f'{Fore.CYAN}Waiting for chat download to finish...{Style.RESET_ALL}')
+                archiver.downloader.wait_for_chat_download(live_chat_process, chat_json_path, timeout=30)
+            
            return
        
        # Get file size to check if anything was recorded
@ -1023,24 +1177,68 @@ class TwitchArchiveManager:
        live_chat_downloaded = False
        chat_rendered_successfully = False
        chat_video_path = None
-        if live_chat_process is not None:
+        
+        # Handle different chat download methods
+        if live_chat_method == 'twitch_downloader' and live_chat_process is not None:
+            # Wait for TwitchDownloaderCLI process
+            print(f'{Fore.CYAN}Waiting for live chat download to complete...{Style.RESET_ALL}')
            live_chat_downloaded = archiver.downloader.wait_for_chat_download(live_chat_process, chat_json_path)
+        elif live_chat_method == 'chat_downloader' and archiver.downloader.chat_thread is not None:
+            # Wait for chat_downloader thread
+            print(f'{Fore.CYAN}Waiting for live chat download to complete...{Style.RESET_ALL}')
+            try:
+                live_chat_downloaded = archiver.downloader.wait_for_chat_thread()
+                if live_chat_downloaded:
+                    print(f'{Fore.GREEN}✓ Chat download thread completed successfully{Style.RESET_ALL}')
+                else:
+                    print(f'{Fore.YELLOW}⚠ Chat download thread completed with errors or no messages{Style.RESET_ALL}')
+            except Exception as e:
+                print(f'{Fore.RED}✗ Error waiting for chat download thread: {e}{Style.RESET_ALL}')
+                import traceback
+                traceback.print_exc()
+                live_chat_downloaded = False
        
        # Render live chat if downloaded successfully
        if live_chat_downloaded:
            chat_video_path = str(archiver.file_manager.chat_mp4_path / f"{PREFIX_CHAT}{filename_base}.mp4")
            output_args = archiver.processor.build_chat_output_args()
            
-            # Get video duration to trim chat accordingly
-            ffmpeg_path = get_ffmpeg_executable(archiver.os_type)
-            video_duration = get_video_duration(live_proc_path, ffmpeg_path)
-            
-            chat_rendered_successfully = archiver.downloader.render_chat(
-                chat_json_path, 
-                chat_video_path, 
-                output_args,
-                video_duration=video_duration
-            )
+            # Wait for chat file to be fully accessible (not locked)
+            print(f'{Fore.CYAN}Verifying chat file is ready for rendering...{Style.RESET_ALL}')
+            if not archiver.downloader.wait_for_file_access(chat_json_path, max_attempts=15, delay=0.5):
+                print(f'{Fore.RED}✗ Chat file is locked, skipping rendering{Style.RESET_ALL}')
+                chat_rendered_successfully = False
+            else:
+                # Get video duration first
+                ffmpeg_path = get_ffmpeg_executable(archiver.os_type)
+                video_duration = get_video_duration(live_proc_path, ffmpeg_path)
+                
+                if video_duration is None:
+                    print(f'{Fore.YELLOW}⚠ Could not detect video duration from {live_proc_path}{Style.RESET_ALL}')
+                    print(f'{Fore.YELLOW}  Will use chat message timestamps instead{Style.RESET_ALL}')
+                else:
+                    print(f'{Fore.CYAN}Video duration for chat rendering: {video_duration}s{Style.RESET_ALL}')
+                
+                # Convert chat format if chat_downloader was used
+                render_json_path = chat_json_path
+                if live_chat_method == 'chat_downloader':
+                    converted_path = chat_json_path.replace('.json', '_converted.json')
+                    print(f'{Fore.CYAN}Chat downloaded with chat_downloader, converting format...{Style.RESET_ALL}')
+                    if archiver.downloader.convert_chat_downloader_to_twitch_format(chat_json_path, converted_path, video_duration):
+                        render_json_path = converted_path
+                        print(f'{Fore.GREEN}✓ Using converted chat file for rendering{Style.RESET_ALL}')
+                    else:
+                        print(f'{Fore.RED}✗ Format conversion failed, skipping rendering{Style.RESET_ALL}')
+                        chat_rendered_successfully = False
+                        render_json_path = None
+                
+                if render_json_path:
+                    chat_rendered_successfully = archiver.downloader.render_chat(
+                        render_json_path, 
+                        chat_video_path, 
+                        output_args,
+                        video_duration=video_duration
+                    )
        
        # Merge video and chat if configured
        merged_video_path = None
@ -1225,6 +1423,10 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving
    -u, --username <name>   Monitor only this Twitch channel
    --verbose               Enable verbose debug output
    --legacy                Force legacy mode (use config.json)
+    --chat-only             Test mode: Only download chat (skip video recording)
+                            Automatically enables verbose logging
+    --use-chat-downloader-primary   Use chat_downloader as primary chat source (for testing)
+    --no-chat-downloader-fallback   Disable chat_downloader fallback

 {Fore.GREEN}LEGACY OPTIONS (when using --legacy):{Style.RESET_ALL}
    -q, --quality <qual>    Stream quality: best/source, high/720p, 
@ -1247,6 +1449,8 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving
    python twitch-archive.py                    # Monitor all enabled streamers
    python twitch-archive.py -u vinesauce       # Monitor only vinesauce
    python twitch-archive.py -u hackerling --verbose  # Monitor with debug output
+    python twitch-archive.py -u streamername --chat-only  # Test chat download only (no video)
+    python twitch-archive.py --use-chat-downloader-primary  # Test chat_downloader library
    python twitch-archive.py --legacy           # Use old config.json mode

 {Fore.CYAN}{"=" * 70}{Style.RESET_ALL}
@ -1257,7 +1461,8 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving
            argv,
            "h:u:q:a:v:c:m:r:d:n:",
            ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", 
-             "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose"]
+             "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose",
+             "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"]
        )
    except getopt.GetoptError as e:
        print(f'{Fore.RED}Error: {e}{Style.RESET_ALL}\n')
@ -1270,7 +1475,26 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving
    # Parse command line args
    legacy_overrides = {}
    verbose_mode = False
+    chat_only_mode = False
+    use_chat_downloader_primary = False
+    use_chat_downloader_fallback = True  # Default to enabled
    for opt, arg in opts:
+        if opt in ('-h', '--help'):
+            print(help_msg)
+            sys.exit(0)
+        elif opt in ("-u", "--username"):
+            specific_streamer = arg
+        elif opt == "--verbose":
+            verbose_mode = True
+        elif opt == "--chat-only":
+            chat_only_mode = True
+            verbose_mode = True  # Auto-enable verbose for chat-only mode
+        elif opt == "--legacy":
+            use_legacy_mode = True
+        elif opt == "--use-chat-downloader-primary":
+            use_chat_downloader_primary = True
+        elif opt == "--no-chat-downloader-fallback":
+            use_chat_downloader_fallback = False
        if opt in ('-h', '--help'):
            print(help_msg)
            sys.exit(0)
@ -1310,11 +1534,23 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving
        for key, value in legacy_overrides.items():
            setattr(twitch_archive, key, value)
        
+        # Apply chat_downloader options
+        if hasattr(twitch_archive.downloader, 'use_chat_downloader_primary'):
+            twitch_archive.downloader.use_chat_downloader_primary = use_chat_downloader_primary
+        if hasattr(twitch_archive.downloader, 'use_chat_downloader_fallback'):
+            twitch_archive.downloader.use_chat_downloader_fallback = use_chat_downloader_fallback
+        
        # Start the archive system
        twitch_archive.run()
    else:
        # New multi-streamer mode
-        manager = TwitchArchiveManager(specific_streamer=specific_streamer, verbose=verbose_mode)
+        manager = TwitchArchiveManager(
+            specific_streamer=specific_streamer, 
+            verbose=verbose_mode,
+            chat_only=chat_only_mode,
+            use_chat_downloader_primary=use_chat_downloader_primary,
+            use_chat_downloader_fallback=use_chat_downloader_fallback
+        )
        manager.run()