Refactor downloader and file manager for improved rclone integration and add healthcheck and smoke test options
- Renamed download flags in ContentDownloader for clarity. - Enhanced FileManager with methods to build upload paths and verify existing files for rclone uploads. - Updated StreamProcessor to return success status for stream processing. - Added rclone smoke test and healthcheck functions to validate configuration and tool availability. - Improved environment variable handling with a utility function. - Updated TwitchArchive to incorporate new rclone verification and processing logic. - Added unit tests for new functionality and refactored existing tests for clarity and coverage. Co-authored-by: Copilot <copilot@github.com>
This commit is contained in:
parent
e92f36474a
commit
f97e0200d6
23 changed files with 1013 additions and 289 deletions
|
|
@ -20,12 +20,27 @@ import sys
|
|||
import os
|
||||
import json
|
||||
import getopt
|
||||
import tempfile
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch, MagicMock, Mock
|
||||
|
||||
# Add parent directory to path for imports
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
from modules.constants import DEFAULT_CONFIG
|
||||
from modules.file_manager import FileManager
|
||||
from modules.downloader import ContentDownloader
|
||||
from modules.utils import get_ffmpeg_executable, get_twitch_downloader_executable
|
||||
|
||||
|
||||
def load_twitch_archive_module():
|
||||
"""Load the main script module for targeted regression tests."""
|
||||
module_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'twitch-archive.py')
|
||||
spec = importlib.util.spec_from_file_location('twitch_archive_main', module_path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
class TestCommandLineArgumentParsing(unittest.TestCase):
|
||||
|
|
@ -115,6 +130,34 @@ class TestCommandLineArgumentParsing(unittest.TestCase):
|
|||
|
||||
self.assertEqual(len(opts), 1)
|
||||
self.assertEqual(opts[0], ('--chat-only', ''))
|
||||
|
||||
def test_rclone_smoke_test_option(self):
|
||||
"""Test --rclone-smoke-test option parsing."""
|
||||
argv = ['--rclone-smoke-test']
|
||||
opts, args = getopt.getopt(
|
||||
argv,
|
||||
"hu:q:a:v:c:m:r:d:n:",
|
||||
["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=",
|
||||
"metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose",
|
||||
"chat-only", "healthcheck", "rclone-smoke-test", "use-chat-downloader-primary", "no-chat-downloader-fallback"]
|
||||
)
|
||||
|
||||
self.assertEqual(len(opts), 1)
|
||||
self.assertEqual(opts[0], ('--rclone-smoke-test', ''))
|
||||
|
||||
def test_healthcheck_option(self):
|
||||
"""Test --healthcheck option parsing."""
|
||||
argv = ['--healthcheck']
|
||||
opts, args = getopt.getopt(
|
||||
argv,
|
||||
"hu:q:a:v:c:m:r:d:n:",
|
||||
["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=",
|
||||
"metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose",
|
||||
"chat-only", "healthcheck", "rclone-smoke-test", "use-chat-downloader-primary", "no-chat-downloader-fallback"]
|
||||
)
|
||||
|
||||
self.assertEqual(len(opts), 1)
|
||||
self.assertEqual(opts[0], ('--healthcheck', ''))
|
||||
|
||||
def test_legacy_option(self):
|
||||
"""Test --legacy option parsing."""
|
||||
|
|
@ -439,6 +482,161 @@ class TestConfigLogic(unittest.TestCase):
|
|||
self.assertIn('$schema', default_config)
|
||||
|
||||
|
||||
class TestFileManagerUploadPaths(unittest.TestCase):
|
||||
"""Test rclone upload path preparation."""
|
||||
|
||||
def test_build_upload_relative_paths_uses_forward_slashes(self):
|
||||
"""Rclone --files-from entries must use POSIX separators on Windows."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
manager = FileManager(
|
||||
root_path=temp_dir,
|
||||
username='testuser',
|
||||
config={
|
||||
'uploadCloud': True,
|
||||
'uploadPreMergeVideo': True,
|
||||
'uploadMergedVideo': True,
|
||||
'uploadChatVideo': True
|
||||
}
|
||||
)
|
||||
|
||||
relative_paths = manager._build_upload_relative_paths('20260424_12h00m00s')
|
||||
|
||||
self.assertTrue(relative_paths)
|
||||
self.assertTrue(all('\\' not in path for path in relative_paths))
|
||||
self.assertIn('testuser/metadata/METADA_20260424_12h00m00s.json', relative_paths)
|
||||
self.assertIn('testuser/chat/json/CHAT_20260424_12h00m00s.json', relative_paths)
|
||||
|
||||
|
||||
class TestDownloaderConfiguration(unittest.TestCase):
|
||||
"""Regression tests for downloader config wiring."""
|
||||
|
||||
def test_download_vod_method_not_shadowed_by_boolean_flag(self):
|
||||
"""Config booleans must not overwrite callable downloader methods."""
|
||||
downloader = ContentDownloader(
|
||||
twitch_downloader_path='TwitchDownloaderCLI',
|
||||
ffmpeg_path='ffmpeg',
|
||||
config={
|
||||
'downloadVOD': True,
|
||||
'downloadCHAT': True,
|
||||
'downloadLiveCHAT': True
|
||||
}
|
||||
)
|
||||
|
||||
self.assertTrue(callable(downloader.download_vod))
|
||||
self.assertTrue(downloader.download_vod_enabled)
|
||||
|
||||
|
||||
class TestLinuxToolResolution(unittest.TestCase):
|
||||
"""Ensure Linux containers prefer their own installed toolchain."""
|
||||
|
||||
@patch('modules.utils.shutil.which')
|
||||
def test_linux_prefers_system_ffmpeg(self, mock_which):
|
||||
mock_which.return_value = '/usr/bin/ffmpeg'
|
||||
|
||||
self.assertEqual(get_ffmpeg_executable('linux'), '/usr/bin/ffmpeg')
|
||||
|
||||
@patch('modules.utils.shutil.which')
|
||||
def test_linux_prefers_system_twitch_downloader(self, mock_which):
|
||||
mock_which.return_value = '/usr/local/bin/TwitchDownloaderCLI'
|
||||
|
||||
self.assertEqual(get_twitch_downloader_executable('linux'), '/usr/local/bin/TwitchDownloaderCLI')
|
||||
|
||||
|
||||
class TestMultiStreamerCleanupRegression(unittest.TestCase):
|
||||
"""Regression tests for multi-streamer conversion and cleanup behavior."""
|
||||
|
||||
def setUp(self):
|
||||
self.module = load_twitch_archive_module()
|
||||
|
||||
def _build_archiver(self, temp_dir: str, upload_cloud: bool = True):
|
||||
archiver = MagicMock()
|
||||
archiver.username = 'maddoscientist0'
|
||||
archiver.os_type = 'linux'
|
||||
archiver.quality = 'best'
|
||||
archiver.downloadLiveCHAT = False
|
||||
archiver.downloadCHAT = False
|
||||
archiver.downloadVOD = False
|
||||
archiver.downloadMETADATA = False
|
||||
archiver.mergeVideoChat = False
|
||||
archiver.mergeChatLayout = 'side-by-side'
|
||||
archiver.onlyRaw = False
|
||||
archiver.vodTimeout = 0
|
||||
archiver.shutdown_requested = False
|
||||
archiver.deleteFiles = True
|
||||
archiver.uploadCloud = upload_cloud
|
||||
|
||||
archiver.notification_manager = MagicMock()
|
||||
archiver.recorder = MagicMock()
|
||||
archiver.processor = MagicMock()
|
||||
archiver.downloader = MagicMock()
|
||||
archiver.stream_monitor = MagicMock()
|
||||
archiver.file_manager = MagicMock()
|
||||
archiver.file_manager.raw_path = Path(temp_dir) / 'raw'
|
||||
archiver.file_manager.video_path = Path(temp_dir) / 'video'
|
||||
archiver.file_manager.chat_json_path = Path(temp_dir) / 'chat_json'
|
||||
archiver.file_manager.chat_mp4_path = Path(temp_dir) / 'chat'
|
||||
|
||||
os.makedirs(archiver.file_manager.raw_path, exist_ok=True)
|
||||
os.makedirs(archiver.file_manager.video_path, exist_ok=True)
|
||||
os.makedirs(archiver.file_manager.chat_json_path, exist_ok=True)
|
||||
os.makedirs(archiver.file_manager.chat_mp4_path, exist_ok=True)
|
||||
|
||||
return archiver
|
||||
|
||||
def test_process_stream_keeps_raw_when_conversion_fails(self):
|
||||
manager = self.module.TwitchArchiveManager(specific_streamer='maddoscientist0')
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
archiver = self._build_archiver(temp_dir)
|
||||
archiver.processor.process_raw_stream.return_value = False
|
||||
archiver.file_manager.upload_to_cloud.return_value = True
|
||||
|
||||
def write_raw_file(_stream_info, raw_path):
|
||||
with open(raw_path, 'wb') as handle:
|
||||
handle.write(b'x' * 4096)
|
||||
return True
|
||||
|
||||
archiver.recorder.record.side_effect = write_raw_file
|
||||
|
||||
stream_info = {
|
||||
'title': 'Test',
|
||||
'createdAt': '2026-04-25T09:14:01Z'
|
||||
}
|
||||
|
||||
manager._process_stream(archiver, stream_info, 'stream-id')
|
||||
|
||||
archiver.file_manager.clean_raw_file.assert_not_called()
|
||||
archiver.file_manager.delete_local_files.assert_called_once()
|
||||
|
||||
def test_process_stream_only_deletes_rendered_files_after_real_upload(self):
|
||||
manager = self.module.TwitchArchiveManager(specific_streamer='maddoscientist0')
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
archiver = self._build_archiver(temp_dir, upload_cloud=False)
|
||||
archiver.processor.process_raw_stream.return_value = True
|
||||
archiver.file_manager.upload_to_cloud.return_value = True
|
||||
|
||||
def write_raw_file(_stream_info, raw_path):
|
||||
with open(raw_path, 'wb') as handle:
|
||||
handle.write(b'x' * 4096)
|
||||
return True
|
||||
|
||||
archiver.recorder.record.side_effect = write_raw_file
|
||||
|
||||
stream_info = {
|
||||
'title': 'Test',
|
||||
'createdAt': '2026-04-25T09:14:01Z'
|
||||
}
|
||||
|
||||
manager._process_stream(archiver, stream_info, 'stream-id')
|
||||
|
||||
archiver.file_manager.clean_raw_file.assert_called_once()
|
||||
archiver.file_manager.delete_local_files.assert_not_called()
|
||||
|
||||
upload_filename_base = archiver.file_manager.upload_to_cloud.call_args.args[0]
|
||||
self.assertFalse(upload_filename_base.startswith('LIVE_'))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# Run tests with verbose output
|
||||
print("="*70)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue