Refactor downloader and file manager for improved rclone integration and add healthcheck and smoke test options
- Renamed download flags in ContentDownloader for clarity. - Enhanced FileManager with methods to build upload paths and verify existing files for rclone uploads. - Updated StreamProcessor to return success status for stream processing. - Added rclone smoke test and healthcheck functions to validate configuration and tool availability. - Improved environment variable handling with a utility function. - Updated TwitchArchive to incorporate new rclone verification and processing logic. - Added unit tests for new functionality and refactored existing tests for clarity and coverage. Co-authored-by: Copilot <copilot@github.com>
This commit is contained in:
parent
e92f36474a
commit
f97e0200d6
23 changed files with 1013 additions and 289 deletions
|
|
@ -34,6 +34,8 @@ import time
|
|||
import json
|
||||
import signal
|
||||
import getopt
|
||||
import pathlib
|
||||
import subprocess
|
||||
from typing import Dict, Optional, Any
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
|
|
@ -48,7 +50,8 @@ from modules.config import ConfigManager
|
|||
from modules.notifications import NotificationManager
|
||||
from modules.utils import (
|
||||
detect_operating_system, get_ffmpeg_executable, get_twitch_downloader_executable,
|
||||
get_unique_filename, get_video_duration, verify_streamlink, verify_ffmpeg, verify_twitch_downloader
|
||||
get_unique_filename, get_video_duration, verify_streamlink, verify_ffmpeg, verify_twitch_downloader,
|
||||
verify_rclone, get_env_value
|
||||
)
|
||||
from modules.stream_monitor import StreamMonitor
|
||||
from modules.recorder import StreamRecorder
|
||||
|
|
@ -147,10 +150,25 @@ class TwitchArchive:
|
|||
Raises:
|
||||
SystemExit: If .env file is not found
|
||||
"""
|
||||
if not load_dotenv(find_dotenv()):
|
||||
dotenv_loaded = load_dotenv(find_dotenv())
|
||||
has_required_env = bool(
|
||||
get_env_value('CLIENT-ID', 'CLIENT_ID') and
|
||||
get_env_value('CLIENT-SECRET', 'CLIENT_SECRET')
|
||||
)
|
||||
|
||||
if not dotenv_loaded and has_required_env:
|
||||
print(f'{Fore.GREEN}✓ Twitch API credentials loaded from process environment{Style.RESET_ALL}')
|
||||
return
|
||||
|
||||
if not dotenv_loaded and not has_required_env:
|
||||
print(f'{Fore.RED}✗ ERROR: .env file not found{Style.RESET_ALL}')
|
||||
print(f'{Fore.CYAN} → Create a .env file with your Twitch API credentials{Style.RESET_ALL}')
|
||||
print(f'{Fore.CYAN} → Required: CLIENT-ID, CLIENT-SECRET{Style.RESET_ALL}')
|
||||
print(f'{Fore.CYAN} → Create a .env file with your Twitch API credentials or pass them via environment variables{Style.RESET_ALL}')
|
||||
print(f'{Fore.CYAN} → Required: CLIENT-ID/CLIENT_ID and CLIENT-SECRET/CLIENT_SECRET{Style.RESET_ALL}')
|
||||
sys.exit(1)
|
||||
|
||||
if not has_required_env:
|
||||
print(f'{Fore.RED}✗ ERROR: Twitch API credentials are missing{Style.RESET_ALL}')
|
||||
print(f'{Fore.CYAN} → Required: CLIENT-ID/CLIENT_ID and CLIENT-SECRET/CLIENT_SECRET{Style.RESET_ALL}')
|
||||
sys.exit(1)
|
||||
|
||||
def _initialize_components(self) -> None:
|
||||
|
|
@ -259,6 +277,8 @@ class TwitchArchive:
|
|||
verify_ffmpeg(self.os_type)
|
||||
if self.downloadVOD or self.downloadCHAT:
|
||||
verify_twitch_downloader(self.os_type)
|
||||
if self.uploadCloud and not verify_rclone():
|
||||
sys.exit(1)
|
||||
|
||||
# Print configuration summary
|
||||
self._print_configuration_summary()
|
||||
|
|
@ -404,7 +424,7 @@ class TwitchArchive:
|
|||
print(f'{Fore.YELLOW}Attempting to process any recorded content...{Style.RESET_ALL}')
|
||||
|
||||
# Process the raw stream file
|
||||
self.processor.process_raw_stream(live_raw_path, live_proc_path)
|
||||
processing_succeeded = self.processor.process_raw_stream(live_raw_path, live_proc_path)
|
||||
|
||||
# Wait for live chat download if it was started
|
||||
live_chat_downloaded = False
|
||||
|
|
@ -426,7 +446,11 @@ class TwitchArchive:
|
|||
else:
|
||||
# Get video duration first (needed for chat conversion and trimming)
|
||||
ffmpeg_path = get_ffmpeg_executable(self.os_type)
|
||||
video_duration = get_video_duration(live_proc_path, ffmpeg_path)
|
||||
if not processing_succeeded or not os.path.exists(live_proc_path):
|
||||
print(f'{Fore.YELLOW}⚠ Processed video file is unavailable, skipping chat render{Style.RESET_ALL}')
|
||||
video_duration = None
|
||||
else:
|
||||
video_duration = get_video_duration(live_proc_path, ffmpeg_path)
|
||||
print(f'{Fore.CYAN}Video duration for chat rendering: {video_duration}s{Style.RESET_ALL}')
|
||||
|
||||
# Convert chat format if needed (chat_downloader uses different JSON structure)
|
||||
|
|
@ -561,7 +585,10 @@ class TwitchArchive:
|
|||
print(f'{Fore.YELLOW}⚠ No matching VOD found for this stream{Style.RESET_ALL}')
|
||||
|
||||
# Clean up raw files if configured
|
||||
self.file_manager.clean_raw_file(live_raw_path)
|
||||
if processing_succeeded:
|
||||
self.file_manager.clean_raw_file(live_raw_path)
|
||||
elif os.path.exists(live_raw_path):
|
||||
print(f'{Fore.YELLOW}⚠ Keeping raw file because conversion did not complete successfully{Style.RESET_ALL}')
|
||||
|
||||
# Upload to cloud if configured
|
||||
upload_success = self.file_manager.upload_to_cloud(
|
||||
|
|
@ -570,7 +597,7 @@ class TwitchArchive:
|
|||
)
|
||||
|
||||
# Delete local files if configured and upload succeeded
|
||||
if self.deleteFiles and upload_success:
|
||||
if self.deleteFiles and self.uploadCloud and upload_success:
|
||||
self.file_manager.delete_local_files(
|
||||
filename_base,
|
||||
live_raw_path,
|
||||
|
|
@ -890,6 +917,8 @@ class TwitchArchiveManager:
|
|||
verify_ffmpeg(first_archiver.os_type)
|
||||
if first_archiver.downloadVOD or first_archiver.downloadCHAT:
|
||||
verify_twitch_downloader(first_archiver.os_type)
|
||||
if any(archiver.uploadCloud for archiver in self.archivers.values()) and not verify_rclone():
|
||||
sys.exit(1)
|
||||
|
||||
# Print configuration summary for each streamer
|
||||
for username, archiver in self.archivers.items():
|
||||
|
|
@ -1018,7 +1047,7 @@ class TwitchArchiveManager:
|
|||
|
||||
# Generate timestamp and filename
|
||||
timestamp = datetime.now(timezone('UTC')).strftime("%Y%m%d_%Hh%Mm%Ss")
|
||||
filename_base = f"{PREFIX_LIVE}{archiver.username}_{timestamp}"
|
||||
filename_base = f"{archiver.username}_{timestamp}"
|
||||
|
||||
# Parse stream start time
|
||||
live_date = datetime.strptime(
|
||||
|
|
@ -1029,8 +1058,8 @@ class TwitchArchiveManager:
|
|||
raw_extension = '.ts'
|
||||
proc_extension = '.mp3' if archiver.quality == 'audio_only' else '.mp4'
|
||||
|
||||
live_raw_path = str(archiver.file_manager.raw_path / f"{filename_base}{raw_extension}")
|
||||
live_proc_path = str(archiver.file_manager.video_path / f"{filename_base}{proc_extension}")
|
||||
live_raw_path = str(archiver.file_manager.raw_path / f"{PREFIX_LIVE}{filename_base}{raw_extension}")
|
||||
live_proc_path = str(archiver.file_manager.video_path / f"{PREFIX_LIVE}{filename_base}{proc_extension}")
|
||||
chat_json_path = str(archiver.file_manager.chat_json_path / f"{PREFIX_CHAT}{filename_base}.json")
|
||||
|
||||
# Send notification
|
||||
|
|
@ -1172,8 +1201,9 @@ class TwitchArchiveManager:
|
|||
print(f'{Fore.CYAN}Processing recorded content ({file_size / (1024*1024):.2f} MB)...{Style.RESET_ALL}')
|
||||
|
||||
# Process raw stream
|
||||
processing_succeeded = False
|
||||
if not archiver.onlyRaw:
|
||||
archiver.processor.process_raw_stream(live_raw_path, live_proc_path)
|
||||
processing_succeeded = archiver.processor.process_raw_stream(live_raw_path, live_proc_path)
|
||||
|
||||
# Wait for live chat download if it was started
|
||||
live_chat_downloaded = False
|
||||
|
|
@ -1212,8 +1242,12 @@ class TwitchArchiveManager:
|
|||
chat_rendered_successfully = False
|
||||
else:
|
||||
# Get video duration first
|
||||
ffmpeg_path = get_ffmpeg_executable(archiver.os_type)
|
||||
video_duration = get_video_duration(live_proc_path, ffmpeg_path)
|
||||
if not processing_succeeded or not os.path.exists(live_proc_path):
|
||||
print(f'{Fore.YELLOW}⚠ Processed video file is unavailable, skipping chat render{Style.RESET_ALL}')
|
||||
video_duration = None
|
||||
else:
|
||||
ffmpeg_path = get_ffmpeg_executable(archiver.os_type)
|
||||
video_duration = get_video_duration(live_proc_path, ffmpeg_path)
|
||||
|
||||
if video_duration is None:
|
||||
print(f'{Fore.YELLOW}⚠ Could not detect video duration from {live_proc_path}{Style.RESET_ALL}')
|
||||
|
|
@ -1362,7 +1396,10 @@ class TwitchArchiveManager:
|
|||
archiver.file_manager.save_metadata(stream_info, filename_base)
|
||||
|
||||
# Clean up raw file if configured
|
||||
archiver.file_manager.clean_raw_file(live_raw_path)
|
||||
if processing_succeeded:
|
||||
archiver.file_manager.clean_raw_file(live_raw_path)
|
||||
elif os.path.exists(live_raw_path):
|
||||
print(f'{Fore.YELLOW}⚠ Keeping raw file because conversion did not complete successfully{Style.RESET_ALL}')
|
||||
|
||||
# Upload to cloud if configured
|
||||
upload_success = archiver.file_manager.upload_to_cloud(
|
||||
|
|
@ -1371,7 +1408,7 @@ class TwitchArchiveManager:
|
|||
)
|
||||
|
||||
# Delete files if configured
|
||||
if archiver.deleteFiles and upload_success:
|
||||
if archiver.deleteFiles and archiver.uploadCloud and upload_success:
|
||||
archiver.file_manager.delete_local_files(
|
||||
filename_base,
|
||||
live_raw_path,
|
||||
|
|
@ -1386,6 +1423,79 @@ class TwitchArchiveManager:
|
|||
)
|
||||
|
||||
|
||||
def run_rclone_smoke_test(specific_streamer: Optional[str] = None) -> int:
|
||||
"""Run a one-off rclone smoke test using the configured upload destination."""
|
||||
config_manager = ConfigManager()
|
||||
|
||||
if specific_streamer:
|
||||
username = specific_streamer
|
||||
else:
|
||||
enabled_streamers = config_manager.get_all_enabled_streamers()
|
||||
if not enabled_streamers:
|
||||
print(f'{Fore.RED}✗ No enabled streamers available for smoke test{Style.RESET_ALL}')
|
||||
print(f'{Fore.CYAN}→ Use -u <username> or enable a streamer config{Style.RESET_ALL}')
|
||||
return 1
|
||||
username = enabled_streamers[0]
|
||||
|
||||
config = config_manager.load_streamer_config(username)
|
||||
file_manager = FileManager(
|
||||
root_path=config.get('root_path', 'archive'),
|
||||
username=username,
|
||||
config=config
|
||||
)
|
||||
file_manager.initialize_directories()
|
||||
|
||||
print(f'\n{Fore.CYAN}{"=" * 70}{Style.RESET_ALL}')
|
||||
print(f'{Fore.CYAN}TWITCH ARCHIVE - Rclone Smoke Test{Style.RESET_ALL}')
|
||||
print(f'{Fore.CYAN}{"=" * 70}{Style.RESET_ALL}')
|
||||
print(f'{Fore.GREEN}Streamer: {username}{Style.RESET_ALL}')
|
||||
print(f'{Fore.GREEN}Remote: {config.get("rclone_path", "<not configured>")}{Style.RESET_ALL}\n')
|
||||
|
||||
return 0 if file_manager.run_rclone_smoke_test() else 1
|
||||
|
||||
|
||||
def run_healthcheck(specific_streamer: Optional[str] = None) -> int:
|
||||
"""Run a local readiness check suitable for Docker health checks."""
|
||||
config_manager = ConfigManager()
|
||||
|
||||
if specific_streamer:
|
||||
username = specific_streamer
|
||||
else:
|
||||
enabled_streamers = config_manager.get_all_enabled_streamers()
|
||||
username = enabled_streamers[0] if enabled_streamers else 'vinesauce'
|
||||
|
||||
config = config_manager.load_streamer_config(username)
|
||||
archive = TwitchArchive(config)
|
||||
|
||||
try:
|
||||
archive._load_environment_variables()
|
||||
except SystemExit:
|
||||
return 1
|
||||
|
||||
archive._initialize_components()
|
||||
|
||||
checks_ok = True
|
||||
if not verify_streamlink():
|
||||
checks_ok = False
|
||||
if not verify_ffmpeg(archive.os_type):
|
||||
checks_ok = False
|
||||
if (archive.downloadVOD or archive.downloadCHAT) and not verify_twitch_downloader(archive.os_type):
|
||||
checks_ok = False
|
||||
if archive.uploadCloud:
|
||||
if not verify_rclone():
|
||||
checks_ok = False
|
||||
rclone_config_path = os.getenv('RCLONE_CONFIG')
|
||||
if rclone_config_path and not os.path.exists(rclone_config_path):
|
||||
print(f'{Fore.RED}✗ ERROR: RCLONE_CONFIG points to a missing file: {rclone_config_path}{Style.RESET_ALL}')
|
||||
checks_ok = False
|
||||
|
||||
if not checks_ok:
|
||||
return 1
|
||||
|
||||
print(f'{Fore.GREEN}✓ Healthcheck OK for {username}{Style.RESET_ALL}')
|
||||
return 0
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# COMMAND-LINE INTERFACE
|
||||
# ============================================================================
|
||||
|
|
@ -1401,6 +1511,8 @@ def main(argv: list) -> None:
|
|||
"""
|
||||
specific_streamer = None
|
||||
use_legacy_mode = False
|
||||
rclone_smoke_test_mode = False
|
||||
healthcheck_mode = False
|
||||
|
||||
help_msg = f'''
|
||||
{Fore.CYAN}{"=" * 70}
|
||||
|
|
@ -1427,6 +1539,8 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving
|
|||
--legacy Force legacy mode (use config.json)
|
||||
--chat-only Test mode: Only download chat (skip video recording)
|
||||
Automatically enables verbose logging
|
||||
--healthcheck Validate config and tool availability, then exit
|
||||
--rclone-smoke-test Create a small test file and upload it with rclone
|
||||
--use-chat-downloader-primary Use chat_downloader as primary chat source (for testing)
|
||||
--no-chat-downloader-fallback Disable chat_downloader fallback
|
||||
|
||||
|
|
@ -1464,7 +1578,7 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving
|
|||
"h:u:q:a:v:c:m:r:d:n:",
|
||||
["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=",
|
||||
"metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose",
|
||||
"chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"]
|
||||
"chat-only", "healthcheck", "rclone-smoke-test", "use-chat-downloader-primary", "no-chat-downloader-fallback"]
|
||||
)
|
||||
except getopt.GetoptError as e:
|
||||
print(f'{Fore.RED}Error: {e}{Style.RESET_ALL}\n')
|
||||
|
|
@ -1491,6 +1605,10 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving
|
|||
elif opt == "--chat-only":
|
||||
chat_only_mode = True
|
||||
verbose_mode = True # Auto-enable verbose for chat-only mode
|
||||
elif opt == "--healthcheck":
|
||||
healthcheck_mode = True
|
||||
elif opt == "--rclone-smoke-test":
|
||||
rclone_smoke_test_mode = True
|
||||
elif opt == "--legacy":
|
||||
use_legacy_mode = True
|
||||
elif opt == "--use-chat-downloader-primary":
|
||||
|
|
@ -1523,6 +1641,12 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving
|
|||
legacy_overrides['deleteFiles'] = bool(int(arg))
|
||||
elif opt in ("-n", "--notifications"):
|
||||
legacy_overrides['notifications'] = bool(int(arg))
|
||||
|
||||
if rclone_smoke_test_mode:
|
||||
sys.exit(run_rclone_smoke_test(specific_streamer))
|
||||
|
||||
if healthcheck_mode:
|
||||
sys.exit(run_healthcheck(specific_streamer))
|
||||
|
||||
# Determine which mode to use
|
||||
if use_legacy_mode or (legacy_config_exists and not specific_streamer and not os.path.exists('config/global.json')):
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue