Refactor downloader and file manager for improved rclone integration and add healthcheck and smoke test options
- Renamed download flags in ContentDownloader for clarity. - Enhanced FileManager with methods to build upload paths and verify existing files for rclone uploads. - Updated StreamProcessor to return success status for stream processing. - Added rclone smoke test and healthcheck functions to validate configuration and tool availability. - Improved environment variable handling with a utility function. - Updated TwitchArchive to incorporate new rclone verification and processing logic. - Added unit tests for new functionality and refactored existing tests for clarity and coverage. Co-authored-by: Copilot <copilot@github.com>
This commit is contained in:
parent
e92f36474a
commit
f97e0200d6
23 changed files with 1013 additions and 289 deletions
|
|
@ -45,6 +45,128 @@ class FileManager:
|
|||
self.chat_mp4_path = self.root_path / username / "chat"
|
||||
self.metadata_path = self.root_path / username / "metadata"
|
||||
self.log_file = self.root_path / ".log"
|
||||
|
||||
def _to_rclone_relative_path(self, *parts: str) -> str:
|
||||
"""Build a POSIX-style relative path for rclone --files-from."""
|
||||
return pathlib.PurePosixPath(*parts).as_posix()
|
||||
|
||||
def _build_upload_relative_paths(self, filename_base: str) -> List[str]:
|
||||
"""Build the candidate upload list relative to root_path for rclone."""
|
||||
files_to_upload: List[str] = [
|
||||
self._to_rclone_relative_path(self.username, 'metadata', f"{PREFIX_METADATA}{filename_base}.json"),
|
||||
self._to_rclone_relative_path(self.username, 'chat', 'json', f"{PREFIX_CHAT}{filename_base}.json")
|
||||
]
|
||||
|
||||
if self.upload_pre_merge_video:
|
||||
files_to_upload.extend([
|
||||
self._to_rclone_relative_path(self.username, 'video', 'raw', f"{PREFIX_LIVE}{filename_base}.ts"),
|
||||
self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_LIVE}{filename_base}.mp4"),
|
||||
self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_LIVE}{filename_base}.mp3"),
|
||||
self._to_rclone_relative_path(self.username, 'video', 'raw', f"{PREFIX_VOD}{filename_base}.ts"),
|
||||
self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_VOD}{filename_base}.mp4"),
|
||||
self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_VOD}{filename_base}.mp3")
|
||||
])
|
||||
|
||||
if self.upload_merged_video:
|
||||
files_to_upload.extend([
|
||||
self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_MERGED}{filename_base}.mp4"),
|
||||
self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_MERGED}{filename_base}.mp3"),
|
||||
self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}.mp4"),
|
||||
self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}.mp3")
|
||||
])
|
||||
|
||||
if self.upload_chat_video:
|
||||
files_to_upload.append(self._to_rclone_relative_path(self.username, 'chat', f"{PREFIX_CHAT}{filename_base}.mp4"))
|
||||
|
||||
return files_to_upload
|
||||
|
||||
def _get_existing_upload_relative_paths(self, relative_paths: List[str]) -> List[str]:
|
||||
"""Filter candidate upload paths to the files that actually exist."""
|
||||
existing_paths: List[str] = []
|
||||
for relative_path in relative_paths:
|
||||
if (self.root_path / pathlib.PurePosixPath(relative_path)).exists():
|
||||
existing_paths.append(relative_path)
|
||||
return existing_paths
|
||||
|
||||
def _run_rclone_copy(self, relative_paths: List[str], description: str) -> bool:
|
||||
"""Run rclone copy for a set of paths relative to root_path."""
|
||||
existing_paths = self._get_existing_upload_relative_paths(relative_paths)
|
||||
missing_paths = [path for path in relative_paths if path not in existing_paths]
|
||||
|
||||
if not existing_paths:
|
||||
print(f'{Fore.RED}✗ Upload skipped: no matching files found for {description}{Style.RESET_ALL}')
|
||||
for missing_path in missing_paths:
|
||||
print(f'{Fore.YELLOW} Missing: {missing_path}{Style.RESET_ALL}')
|
||||
return False
|
||||
|
||||
if missing_paths:
|
||||
print(f'{Fore.YELLOW}⚠ Some configured upload files were not found and will be skipped{Style.RESET_ALL}')
|
||||
for missing_path in missing_paths:
|
||||
print(f'{Fore.YELLOW} Missing: {missing_path}{Style.RESET_ALL}')
|
||||
|
||||
print(f'{Fore.CYAN}rclone source: {self.root_path.resolve()}{Style.RESET_ALL}')
|
||||
print(f'{Fore.CYAN}rclone destination: {self.rclone_path}{Style.RESET_ALL}')
|
||||
print(f'{Fore.CYAN}Files queued for upload: {len(existing_paths)}{Style.RESET_ALL}')
|
||||
|
||||
bin_path = get_bin_path()
|
||||
upload_list_path = os.path.join(bin_path, 'temp', 'upload.txt')
|
||||
os.makedirs(os.path.dirname(upload_list_path), exist_ok=True)
|
||||
|
||||
with open(upload_list_path, 'w', encoding='utf-8', newline='\n') as f:
|
||||
f.write('\n'.join(existing_paths))
|
||||
f.write('\n')
|
||||
|
||||
try:
|
||||
cmd = [
|
||||
'rclone', 'copy',
|
||||
str(self.root_path.resolve()),
|
||||
self.rclone_path,
|
||||
'--files-from', upload_list_path,
|
||||
'--progress'
|
||||
]
|
||||
|
||||
print(f'{Fore.CYAN}Running: {' '.join(cmd)}{Style.RESET_ALL}')
|
||||
|
||||
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
|
||||
if proc.stdout:
|
||||
for line in proc.stdout:
|
||||
print(line, end='')
|
||||
proc.wait()
|
||||
return proc.returncode == 0
|
||||
finally:
|
||||
if os.path.exists(upload_list_path):
|
||||
os.remove(upload_list_path)
|
||||
|
||||
def run_rclone_smoke_test(self) -> bool:
|
||||
"""Create and upload a tiny metadata file to verify rclone output and configuration."""
|
||||
smoke_name = 'RCLONE_SMOKE_TEST'
|
||||
smoke_relative_path = self._to_rclone_relative_path(
|
||||
self.username,
|
||||
'metadata',
|
||||
f"{PREFIX_METADATA}{smoke_name}.json"
|
||||
)
|
||||
smoke_file_path = self.root_path / pathlib.PurePosixPath(smoke_relative_path)
|
||||
|
||||
smoke_payload = {
|
||||
'type': 'rclone_smoke_test',
|
||||
'username': self.username
|
||||
}
|
||||
|
||||
smoke_file_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(smoke_file_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(smoke_payload, f, indent=2)
|
||||
|
||||
print(f'{Fore.CYAN}Created smoke-test file: {smoke_file_path}{Style.RESET_ALL}')
|
||||
try:
|
||||
result = self._run_rclone_copy([smoke_relative_path], 'rclone smoke test')
|
||||
if result:
|
||||
print(f'{Fore.GREEN}✓ Rclone smoke test completed{Style.RESET_ALL}')
|
||||
else:
|
||||
print(f'{Fore.RED}✗ Rclone smoke test failed{Style.RESET_ALL}')
|
||||
return result
|
||||
finally:
|
||||
if smoke_file_path.exists():
|
||||
smoke_file_path.unlink()
|
||||
|
||||
def initialize_directories(self) -> None:
|
||||
"""Create all necessary directory structures."""
|
||||
|
|
@ -120,81 +242,24 @@ class FileManager:
|
|||
print(f'\n{Fore.CYAN}Uploading to cloud storage...{Style.RESET_ALL}')
|
||||
if notification_callback:
|
||||
notification_callback(f'☁ Uploading - {filename_base}', 'Uploading files to cloud storage')
|
||||
|
||||
# Create list of files to upload
|
||||
bin_path = get_bin_path()
|
||||
upload_list_path = os.path.join(bin_path, 'temp', 'upload.txt')
|
||||
|
||||
# Ensure temp directory exists
|
||||
os.makedirs(os.path.dirname(upload_list_path), exist_ok=True)
|
||||
|
||||
files_to_upload = []
|
||||
|
||||
# Build files list relative to root_path so rclone can read them with --files-from
|
||||
# Metadata and chat JSON
|
||||
files_to_upload.append(os.path.join(self.username, 'metadata', f"{PREFIX_METADATA}{filename_base}.json"))
|
||||
files_to_upload.append(os.path.join(self.username, 'chat', 'json', f"{PREFIX_CHAT}{filename_base}.json"))
|
||||
files_to_upload = self._build_upload_relative_paths(filename_base)
|
||||
|
||||
# Pre-merge videos (raw .ts in video/raw, mp4/mp3 in video)
|
||||
if self.upload_pre_merge_video:
|
||||
files_to_upload.extend([
|
||||
os.path.join(self.username, 'video', 'raw', f"{PREFIX_LIVE}{filename_base}.ts"),
|
||||
os.path.join(self.username, 'video', f"{PREFIX_LIVE}{filename_base}.mp4"),
|
||||
os.path.join(self.username, 'video', f"{PREFIX_LIVE}{filename_base}.mp3"),
|
||||
os.path.join(self.username, 'video', 'raw', f"{PREFIX_VOD}{filename_base}.ts"),
|
||||
os.path.join(self.username, 'video', f"{PREFIX_VOD}{filename_base}.mp4"),
|
||||
os.path.join(self.username, 'video', f"{PREFIX_VOD}{filename_base}.mp3")
|
||||
])
|
||||
|
||||
# Merged videos (in video folder)
|
||||
if self.upload_merged_video:
|
||||
files_to_upload.extend([
|
||||
os.path.join(self.username, 'video', f"{PREFIX_MERGED}{filename_base}.mp4"),
|
||||
os.path.join(self.username, 'video', f"{PREFIX_MERGED}{filename_base}.mp3"),
|
||||
os.path.join(self.username, 'video', f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}.mp4"),
|
||||
os.path.join(self.username, 'video', f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}.mp3")
|
||||
])
|
||||
|
||||
# Standalone chat video (in chat folder)
|
||||
if self.upload_chat_video:
|
||||
files_to_upload.append(os.path.join(self.username, 'chat', f"{PREFIX_CHAT}{filename_base}.mp4"))
|
||||
|
||||
with open(upload_list_path, 'w') as f:
|
||||
f.write('\n'.join(files_to_upload))
|
||||
|
||||
# Run rclone using --files-from so the listed paths (relative to root_path) are uploaded.
|
||||
try:
|
||||
cmd = [
|
||||
'rclone', 'copy',
|
||||
str(self.root_path.resolve()),
|
||||
self.rclone_path,
|
||||
'--files-from', upload_list_path
|
||||
]
|
||||
result = self._run_rclone_copy(files_to_upload, f'archive batch {filename_base}')
|
||||
|
||||
# Stream rclone output to console so user can see progress/errors
|
||||
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
|
||||
if proc.stdout:
|
||||
for line in proc.stdout:
|
||||
print(line, end='')
|
||||
proc.wait()
|
||||
result = proc.returncode
|
||||
|
||||
# Clean up upload list
|
||||
if os.path.exists(upload_list_path):
|
||||
os.remove(upload_list_path)
|
||||
|
||||
if result == 0:
|
||||
if result:
|
||||
print(f'{Fore.GREEN}✓ Upload complete{Style.RESET_ALL}')
|
||||
if notification_callback:
|
||||
notification_callback(f'✓ Upload Success - {filename_base}', 'All files uploaded successfully')
|
||||
return True
|
||||
else:
|
||||
print(f'{Fore.RED}✗ Upload failed (exit code: {result}){Style.RESET_ALL}')
|
||||
print(f'{Fore.YELLOW}Files preserved locally due to upload failure{Style.RESET_ALL}')
|
||||
if notification_callback:
|
||||
notification_callback(f'✗ Upload Failed - {filename_base}',
|
||||
f'Upload failed with code {result}. Files preserved locally.')
|
||||
return False
|
||||
|
||||
print(f'{Fore.RED}✗ Upload failed{Style.RESET_ALL}')
|
||||
print(f'{Fore.YELLOW}Files preserved locally due to upload failure{Style.RESET_ALL}')
|
||||
if notification_callback:
|
||||
notification_callback(f'✗ Upload Failed - {filename_base}',
|
||||
'Upload failed. Files preserved locally. Check rclone output above.')
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
print(f'{Fore.RED}✗ Upload error: {str(e)}{Style.RESET_ALL}')
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue