Refactor downloader and file manager for improved rclone integration and add healthcheck and smoke test options

- Renamed download flags in ContentDownloader for clarity.
- Enhanced FileManager with methods to build upload paths and verify existing files for rclone uploads.
- Updated StreamProcessor to return success status for stream processing.
- Added rclone smoke test and healthcheck functions to validate configuration and tool availability.
- Improved environment variable handling with a utility function.
- Updated TwitchArchive to incorporate new rclone verification and processing logic.
- Added unit tests for new functionality and refactored existing tests for clarity and coverage.

Co-authored-by: Copilot <copilot@github.com>

modules/downloader.py

@@ -40,9 +40,9 @@ class ContentDownloader:
         self.ffmpeg_path = ffmpeg_path
         self.quality = config.get('quality', 'best')
         self.hls_segments_vod = config.get('hls_segmentsVOD', 10)
-        self.download_vod = config.get('downloadVOD', True)
-        self.download_chat = config.get('downloadCHAT', True)
-        self.download_live_chat = config.get('downloadLiveCHAT', True)
+        self.download_vod_enabled = config.get('downloadVOD', True)
+        self.download_chat_enabled = config.get('downloadCHAT', True)
+        self.download_live_chat_enabled = config.get('downloadLiveCHAT', True)
         self.use_chat_downloader_primary = config.get('useChatDownloaderPrimary', False)
         self.use_chat_downloader_fallback = config.get('useChatDownloaderFallback', True)
         
@@ -73,7 +73,7 @@ class ContentDownloader:
         Returns:
             bool: True if download succeeded, False otherwise
         """
-        if not self.download_vod:
+        if not self.download_vod_enabled:
             return False
         
         print(f'\n{Fore.CYAN}Downloading VOD: {vod_info["title"]}{Style.RESET_ALL}')
@@ -272,7 +272,7 @@ class ContentDownloader:
         Returns:
             bool: True if succeeded, False otherwise
         """
-        if not self.download_chat:
+        if not self.download_chat_enabled:
             return False
         
         print(f'\n{Fore.CYAN}Downloading chat: {vod_info["title"]}{Style.RESET_ALL}')
@@ -298,7 +298,7 @@ class ContentDownloader:
         Returns:
             subprocess.Popen: The process handle, or None if failed to start
         """
-        if not self.download_live_chat:
+        if not self.download_live_chat_enabled:
             return None
         
         print(f'\n{Fore.CYAN}Starting live chat download...{Style.RESET_ALL}')
@@ -502,7 +502,7 @@ class ContentDownloader:
             print(f'{Fore.YELLOW}  Install with: pip install chat-downloader{Style.RESET_ALL}')
             return False
         
-        if not self.download_live_chat:
+        if not self.download_live_chat_enabled:
             print(f'{Fore.YELLOW}⚠ downloadLiveCHAT is disabled in config{Style.RESET_ALL}')
             return False
 

modules/file_manager.py

@@ -45,6 +45,128 @@ class FileManager:
         self.chat_mp4_path = self.root_path / username / "chat"
         self.metadata_path = self.root_path / username / "metadata"
         self.log_file = self.root_path / ".log"
+
+    def _to_rclone_relative_path(self, *parts: str) -> str:
+        """Build a POSIX-style relative path for rclone --files-from."""
+        return pathlib.PurePosixPath(*parts).as_posix()
+
+    def _build_upload_relative_paths(self, filename_base: str) -> List[str]:
+        """Build the candidate upload list relative to root_path for rclone."""
+        files_to_upload: List[str] = [
+            self._to_rclone_relative_path(self.username, 'metadata', f"{PREFIX_METADATA}{filename_base}.json"),
+            self._to_rclone_relative_path(self.username, 'chat', 'json', f"{PREFIX_CHAT}{filename_base}.json")
+        ]
+
+        if self.upload_pre_merge_video:
+            files_to_upload.extend([
+                self._to_rclone_relative_path(self.username, 'video', 'raw', f"{PREFIX_LIVE}{filename_base}.ts"),
+                self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_LIVE}{filename_base}.mp4"),
+                self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_LIVE}{filename_base}.mp3"),
+                self._to_rclone_relative_path(self.username, 'video', 'raw', f"{PREFIX_VOD}{filename_base}.ts"),
+                self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_VOD}{filename_base}.mp4"),
+                self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_VOD}{filename_base}.mp3")
+            ])
+
+        if self.upload_merged_video:
+            files_to_upload.extend([
+                self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_MERGED}{filename_base}.mp4"),
+                self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_MERGED}{filename_base}.mp3"),
+                self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}.mp4"),
+                self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}.mp3")
+            ])
+
+        if self.upload_chat_video:
+            files_to_upload.append(self._to_rclone_relative_path(self.username, 'chat', f"{PREFIX_CHAT}{filename_base}.mp4"))
+
+        return files_to_upload
+
+    def _get_existing_upload_relative_paths(self, relative_paths: List[str]) -> List[str]:
+        """Filter candidate upload paths to the files that actually exist."""
+        existing_paths: List[str] = []
+        for relative_path in relative_paths:
+            if (self.root_path / pathlib.PurePosixPath(relative_path)).exists():
+                existing_paths.append(relative_path)
+        return existing_paths
+
+    def _run_rclone_copy(self, relative_paths: List[str], description: str) -> bool:
+        """Run rclone copy for a set of paths relative to root_path."""
+        existing_paths = self._get_existing_upload_relative_paths(relative_paths)
+        missing_paths = [path for path in relative_paths if path not in existing_paths]
+
+        if not existing_paths:
+            print(f'{Fore.RED}✗ Upload skipped: no matching files found for {description}{Style.RESET_ALL}')
+            for missing_path in missing_paths:
+                print(f'{Fore.YELLOW}  Missing: {missing_path}{Style.RESET_ALL}')
+            return False
+
+        if missing_paths:
+            print(f'{Fore.YELLOW}⚠ Some configured upload files were not found and will be skipped{Style.RESET_ALL}')
+            for missing_path in missing_paths:
+                print(f'{Fore.YELLOW}  Missing: {missing_path}{Style.RESET_ALL}')
+
+        print(f'{Fore.CYAN}rclone source: {self.root_path.resolve()}{Style.RESET_ALL}')
+        print(f'{Fore.CYAN}rclone destination: {self.rclone_path}{Style.RESET_ALL}')
+        print(f'{Fore.CYAN}Files queued for upload: {len(existing_paths)}{Style.RESET_ALL}')
+
+        bin_path = get_bin_path()
+        upload_list_path = os.path.join(bin_path, 'temp', 'upload.txt')
+        os.makedirs(os.path.dirname(upload_list_path), exist_ok=True)
+
+        with open(upload_list_path, 'w', encoding='utf-8', newline='\n') as f:
+            f.write('\n'.join(existing_paths))
+            f.write('\n')
+
+        try:
+            cmd = [
+                'rclone', 'copy',
+                str(self.root_path.resolve()),
+                self.rclone_path,
+                '--files-from', upload_list_path,
+                '--progress'
+            ]
+
+            print(f'{Fore.CYAN}Running: {' '.join(cmd)}{Style.RESET_ALL}')
+
+            proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
+            if proc.stdout:
+                for line in proc.stdout:
+                    print(line, end='')
+            proc.wait()
+            return proc.returncode == 0
+        finally:
+            if os.path.exists(upload_list_path):
+                os.remove(upload_list_path)
+
+    def run_rclone_smoke_test(self) -> bool:
+        """Create and upload a tiny metadata file to verify rclone output and configuration."""
+        smoke_name = 'RCLONE_SMOKE_TEST'
+        smoke_relative_path = self._to_rclone_relative_path(
+            self.username,
+            'metadata',
+            f"{PREFIX_METADATA}{smoke_name}.json"
+        )
+        smoke_file_path = self.root_path / pathlib.PurePosixPath(smoke_relative_path)
+
+        smoke_payload = {
+            'type': 'rclone_smoke_test',
+            'username': self.username
+        }
+
+        smoke_file_path.parent.mkdir(parents=True, exist_ok=True)
+        with open(smoke_file_path, 'w', encoding='utf-8') as f:
+            json.dump(smoke_payload, f, indent=2)
+
+        print(f'{Fore.CYAN}Created smoke-test file: {smoke_file_path}{Style.RESET_ALL}')
+        try:
+            result = self._run_rclone_copy([smoke_relative_path], 'rclone smoke test')
+            if result:
+                print(f'{Fore.GREEN}✓ Rclone smoke test completed{Style.RESET_ALL}')
+            else:
+                print(f'{Fore.RED}✗ Rclone smoke test failed{Style.RESET_ALL}')
+            return result
+        finally:
+            if smoke_file_path.exists():
+                smoke_file_path.unlink()
     
     def initialize_directories(self) -> None:
         """Create all necessary directory structures."""
@@ -120,81 +242,24 @@ class FileManager:
         print(f'\n{Fore.CYAN}Uploading to cloud storage...{Style.RESET_ALL}')
         if notification_callback:
             notification_callback(f'☁ Uploading - {filename_base}', 'Uploading files to cloud storage')
-        
-        # Create list of files to upload
-        bin_path = get_bin_path()
-        upload_list_path = os.path.join(bin_path, 'temp', 'upload.txt')
-        
-        # Ensure temp directory exists
-        os.makedirs(os.path.dirname(upload_list_path), exist_ok=True)
-        
-        files_to_upload = []
 
-        # Build files list relative to root_path so rclone can read them with --files-from
-        # Metadata and chat JSON
-        files_to_upload.append(os.path.join(self.username, 'metadata', f"{PREFIX_METADATA}{filename_base}.json"))
-        files_to_upload.append(os.path.join(self.username, 'chat', 'json', f"{PREFIX_CHAT}{filename_base}.json"))
+        files_to_upload = self._build_upload_relative_paths(filename_base)
 
-        # Pre-merge videos (raw .ts in video/raw, mp4/mp3 in video)
-        if self.upload_pre_merge_video:
-            files_to_upload.extend([
-                os.path.join(self.username, 'video', 'raw', f"{PREFIX_LIVE}{filename_base}.ts"),
-                os.path.join(self.username, 'video', f"{PREFIX_LIVE}{filename_base}.mp4"),
-                os.path.join(self.username, 'video', f"{PREFIX_LIVE}{filename_base}.mp3"),
-                os.path.join(self.username, 'video', 'raw', f"{PREFIX_VOD}{filename_base}.ts"),
-                os.path.join(self.username, 'video', f"{PREFIX_VOD}{filename_base}.mp4"),
-                os.path.join(self.username, 'video', f"{PREFIX_VOD}{filename_base}.mp3")
-            ])
-
-        # Merged videos (in video folder)
-        if self.upload_merged_video:
-            files_to_upload.extend([
-                os.path.join(self.username, 'video', f"{PREFIX_MERGED}{filename_base}.mp4"),
-                os.path.join(self.username, 'video', f"{PREFIX_MERGED}{filename_base}.mp3"),
-                os.path.join(self.username, 'video', f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}.mp4"),
-                os.path.join(self.username, 'video', f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}.mp3")
-            ])
-
-        # Standalone chat video (in chat folder)
-        if self.upload_chat_video:
-            files_to_upload.append(os.path.join(self.username, 'chat', f"{PREFIX_CHAT}{filename_base}.mp4"))
-        
-        with open(upload_list_path, 'w') as f:
-            f.write('\n'.join(files_to_upload))
-        
-        # Run rclone using --files-from so the listed paths (relative to root_path) are uploaded.
         try:
-            cmd = [
-                'rclone', 'copy',
-                str(self.root_path.resolve()),
-                self.rclone_path,
-                '--files-from', upload_list_path
-            ]
+            result = self._run_rclone_copy(files_to_upload, f'archive batch {filename_base}')
 
-            # Stream rclone output to console so user can see progress/errors
-            proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
-            if proc.stdout:
-                for line in proc.stdout:
-                    print(line, end='')
-            proc.wait()
-            result = proc.returncode
-
-            # Clean up upload list
-            if os.path.exists(upload_list_path):
-                os.remove(upload_list_path)
-
-            if result == 0:
+            if result:
                 print(f'{Fore.GREEN}✓ Upload complete{Style.RESET_ALL}')
                 if notification_callback:
                     notification_callback(f'✓ Upload Success - {filename_base}', 'All files uploaded successfully')
                 return True
-            else:
-                print(f'{Fore.RED}✗ Upload failed (exit code: {result}){Style.RESET_ALL}')
-                print(f'{Fore.YELLOW}Files preserved locally due to upload failure{Style.RESET_ALL}')
-                if notification_callback:
-                    notification_callback(f'✗ Upload Failed - {filename_base}', 
-                                        f'Upload failed with code {result}. Files preserved locally.')
-                return False
+
+            print(f'{Fore.RED}✗ Upload failed{Style.RESET_ALL}')
+            print(f'{Fore.YELLOW}Files preserved locally due to upload failure{Style.RESET_ALL}')
+            if notification_callback:
+                notification_callback(f'✗ Upload Failed - {filename_base}', 
+                                    'Upload failed. Files preserved locally. Check rclone output above.')
+            return False
 
         except Exception as e:
             print(f'{Fore.RED}✗ Upload error: {str(e)}{Style.RESET_ALL}')

modules/processor.py

@@ -37,37 +37,78 @@ class StreamProcessor:
             os_type
         )
     
-    def process_raw_stream(self, raw_path: str, output_path: str) -> None:
+    def process_raw_stream(self, raw_path: str, output_path: str) -> bool:
         """
         Process raw .ts file into mp4/mp3 using ffmpeg.
         
         Args:
             raw_path: Path to the raw .ts file
             output_path: Path for the processed output file
+
+        Returns:
+            bool: True when conversion succeeded, False otherwise
         """
         if not os.path.exists(raw_path):
             print(f'{Fore.YELLOW}⚠ Raw file not found, skipping processing{Style.RESET_ALL}')
-            return
+            return False
         
         if self.only_raw:
             print(f'{Fore.CYAN}Keeping raw .ts file (onlyRaw mode){Style.RESET_ALL}')
-            return
+            return False
         
         print(f'{Fore.YELLOW}Processing raw stream file...{Style.RESET_ALL}')
         
         # Build ffmpeg command based on quality
         if self.quality == 'audio_only':
-            self._process_audio(raw_path, output_path)
+            result = self._process_audio(raw_path, output_path)
         else:
-            self._process_video(raw_path, output_path)
+            result = self._process_video(raw_path, output_path)
         
-        print(f'{Fore.GREEN}✓ Stream processed successfully{Style.RESET_ALL}')
+        if result:
+            print(f'{Fore.GREEN}✓ Stream processed successfully{Style.RESET_ALL}')
+        else:
+            print(f'{Fore.RED}✗ Stream processing failed{Style.RESET_ALL}')
+
+        return result
     
-    def _process_audio(self, raw_path: str, output_path: str) -> None:
+    def _run_ffmpeg_command(self, cmd: list, output_path: str) -> bool:
+        """Run FFmpeg while streaming its output to the terminal."""
+        print(f'{Fore.CYAN}Running FFmpeg: {' '.join(cmd)}{Style.RESET_ALL}')
+
+        process = subprocess.Popen(
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True,
+            encoding='utf-8',
+            errors='replace'
+        )
+
+        if process.stdout:
+            for line in process.stdout:
+                print(line, end='')
+
+        result = process.wait()
+        if result != 0:
+            print(f'{Fore.RED}✗ FFmpeg exited with code: {result}{Style.RESET_ALL}')
+            return False
+
+        if not os.path.exists(output_path):
+            print(f'{Fore.RED}✗ FFmpeg did not create output: {output_path}{Style.RESET_ALL}')
+            return False
+
+        if os.path.getsize(output_path) == 0:
+            print(f'{Fore.RED}✗ FFmpeg created an empty output file: {output_path}{Style.RESET_ALL}')
+            return False
+
+        return True
+
+    def _process_audio(self, raw_path: str, output_path: str) -> bool:
         """Process audio-only stream."""
         # Audio-only conversion with modern AAC encoding
         cmd = [
             self.ffmpeg_path,
+            '-y',
             '-i', raw_path,
             '-vn',  # No video
             '-c:a', self.ffmpeg_audio_codec,
@@ -85,14 +126,9 @@ class StreamProcessor:
             cmd.extend(['-movflags', '+faststart'])
         
         cmd.append(output_path)
-        
-        # Run FFmpeg
-        if self.ffmpeg_progress:
-            subprocess.call(cmd)
-        else:
-            subprocess.call(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
+        return self._run_ffmpeg_command(cmd, output_path)
     
-    def _process_video(self, raw_path: str, output_path: str) -> None:
+    def _process_video(self, raw_path: str, output_path: str) -> bool:
         """Process video stream."""
         cmd = [
             self.ffmpeg_path,
@@ -135,12 +171,7 @@ class StreamProcessor:
             cmd.extend(['-movflags', '+faststart'])
         
         cmd.append(output_path)
-        
-        # Run FFmpeg
-        if self.ffmpeg_progress:
-            subprocess.call(cmd)
-        else:
-            subprocess.call(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
+        return self._run_ffmpeg_command(cmd, output_path)
     
     def build_chat_output_args(self) -> str:
         """

modules/recorder.py

@@ -7,6 +7,8 @@ import subprocess
 from typing import Dict, Any, Optional
 from colorama import Fore, Style
 
+from .utils import get_env_value
+
 
 class StreamRecorder:
     """Handles live stream recording using streamlink."""
@@ -68,7 +70,7 @@ class StreamRecorder:
             print(f'{Fore.YELLOW}  Consider disabling streamlink_ttvlol in config or using alternative methods{Style.RESET_ALL}')
         
         # Add authentication if available
-        oauth_token = os.getenv("OAUTH-PRIVATE-TOKEN", "")
+        oauth_token = get_env_value("OAUTH-PRIVATE-TOKEN", "OAUTH_PRIVATE_TOKEN", default="")
         if oauth_token and oauth_token != "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx":
             cmd.extend(['--twitch-api-header', f'Authorization=OAuth {oauth_token}'])
         

modules/stream_monitor.py

@@ -9,6 +9,7 @@ import requests
 from colorama import Fore, Style
 
 from .constants import TWITCH_OAUTH_URL, TWITCH_API_URL, TWITCH_GQL_URL, TWITCH_GQL_CLIENT_ID
+from .utils import get_env_value
 
 
 class StreamMonitor:
@@ -40,7 +41,9 @@ class StreamMonitor:
             return self._oauth_token
         
         try:
-            url = f"{TWITCH_OAUTH_URL}?client_id={os.getenv('CLIENT-ID')}&client_secret={os.getenv('CLIENT-SECRET')}&grant_type=client_credentials"
+            client_id = get_env_value('CLIENT-ID', 'CLIENT_ID')
+            client_secret = get_env_value('CLIENT-SECRET', 'CLIENT_SECRET')
+            url = f"{TWITCH_OAUTH_URL}?client_id={client_id}&client_secret={client_secret}&grant_type=client_credentials"
             response = requests.post(url, timeout=15)
             response.raise_for_status()
             self._oauth_token = response.json()['access_token']
@@ -69,7 +72,7 @@ class StreamMonitor:
             url = f'{TWITCH_API_URL}/users?login={self.username}'
             headers = {
                 "Authorization": f"Bearer {self.get_oauth_token()}",
-                "Client-ID": os.getenv('CLIENT-ID')
+                "Client-ID": get_env_value('CLIENT-ID', 'CLIENT_ID')
             }
             response = requests.get(url, headers=headers, timeout=15)
             response.raise_for_status()

modules/utils.py

@@ -4,6 +4,7 @@ Utility functions and helpers for Twitch Archive.
 
 import os
 import sys
+import shutil
 import pathlib
 import subprocess
 from typing import Optional
@@ -35,6 +36,15 @@ def get_bin_path() -> str:
     return str(pathlib.Path(__file__).parent.parent.resolve() / "bin")
 
 
+def get_env_value(*names: str, default: Optional[str] = None) -> Optional[str]:
+    """Return the first non-empty environment variable from the provided names."""
+    for name in names:
+        value = os.getenv(name)
+        if value not in (None, ""):
+            return value
+    return default
+
+
 def get_ffmpeg_executable(os_type: str) -> str:
     """
     Get the platform-specific ffmpeg executable path.
@@ -48,6 +58,11 @@ def get_ffmpeg_executable(os_type: str) -> str:
     bin_path = get_bin_path()
     if os_type == 'windows':
         return os.path.join(bin_path, 'ffmpeg.exe')
+
+    system_ffmpeg = shutil.which('ffmpeg')
+    if system_ffmpeg:
+        return system_ffmpeg
+
     return os.path.join(bin_path, 'ffmpeg')
 
 
@@ -64,6 +79,11 @@ def get_twitch_downloader_executable(os_type: str) -> str:
     bin_path = get_bin_path()
     if os_type == 'windows':
         return os.path.join(bin_path, 'TwitchDownloaderCLI.exe')
+
+    system_twitch_downloader = shutil.which('TwitchDownloaderCLI')
+    if system_twitch_downloader:
+        return system_twitch_downloader
+
     return os.path.join(bin_path, 'TwitchDownloaderCLI')
 
 
@@ -164,6 +184,24 @@ def verify_streamlink() -> bool:
         return False
 
 
+def verify_rclone() -> bool:
+    """Verify that rclone is available on PATH."""
+    try:
+        result = subprocess.run(['rclone', 'version'],
+                              capture_output=True,
+                              text=True,
+                              timeout=5)
+        if result.returncode == 0:
+            version_line = result.stdout.strip().splitlines()[0] if result.stdout.strip() else 'unknown'
+            print(f'{Fore.GREEN}✓ Rclone found ({version_line}){Style.RESET_ALL}')
+            return True
+        raise FileNotFoundError()
+    except (FileNotFoundError, subprocess.TimeoutExpired, IndexError):
+        print(f'{Fore.RED}✗ ERROR: rclone not found{Style.RESET_ALL}')
+        print(f'{Fore.CYAN}  → Install rclone and ensure it is on PATH{Style.RESET_ALL}')
+        return False
+
+
 def verify_ffmpeg(os_type: str) -> bool:
     """
     Verify that ffmpeg is available.