Compare commits
No commits in common. "master" and "dotnet" have entirely different histories.
7 changed files with 25 additions and 170 deletions
|
|
@ -1,8 +1,8 @@
|
||||||
TWITCH_ARCHIVE_IMAGE=forgejo.maddoscientisto.net/maddo/twitchdownloader:latest
|
TWITCH_ARCHIVE_IMAGE=forgejo.maddoscientisto.net/maddo/twitch-archive:latest
|
||||||
TWITCH_ARCHIVE_CONTAINER_NAME=twitchdownloader
|
TWITCH_ARCHIVE_CONTAINER_NAME=twitch-archive
|
||||||
TWITCH_ARCHIVE_APP_ENV_FILE=/mnt/storage/AppData/twitchdownloader/config/.env.production
|
TWITCH_ARCHIVE_APP_ENV_FILE=./.env.production
|
||||||
TWITCH_ARCHIVE_ARCHIVE_BIND=/mnt/storage/AppData/twitchdownloader/archive
|
TWITCH_ARCHIVE_ARCHIVE_BIND=./archive
|
||||||
TWITCH_ARCHIVE_CONFIG_BIND=/mnt/storage/AppData/twitchdownloader/config
|
TWITCH_ARCHIVE_CONFIG_BIND=./config
|
||||||
TWITCH_ARCHIVE_ARGS=-u vinesauce
|
TWITCH_ARCHIVE_ARGS=-u vinesauce
|
||||||
TWITCH_ARCHIVE_HEALTHCHECK_STREAMER=vinesauce
|
TWITCH_ARCHIVE_HEALTHCHECK_STREAMER=vinesauce
|
||||||
TWITCH_ARCHIVE_RCLONE_CONFIG=/app/config/rclone.conf
|
TWITCH_ARCHIVE_RCLONE_CONFIG=/app/config/rclone.conf
|
||||||
|
|
|
||||||
|
|
@ -10,7 +10,7 @@ services:
|
||||||
command:
|
command:
|
||||||
- sh
|
- sh
|
||||||
- -lc
|
- -lc
|
||||||
- exec python -u twitch-archive.py ${TWITCH_ARCHIVE_ARGS:--u vinesauce --verbose}
|
- python twitch-archive.py ${TWITCH_ARCHIVE_ARGS:--u vinesauce --verbose}
|
||||||
volumes:
|
volumes:
|
||||||
- .:/app
|
- .:/app
|
||||||
- ${TWITCH_ARCHIVE_ARCHIVE_BIND:-./archive}:/app/archive
|
- ${TWITCH_ARCHIVE_ARCHIVE_BIND:-./archive}:/app/archive
|
||||||
|
|
|
||||||
|
|
@ -4,35 +4,24 @@ services:
|
||||||
container_name: ${TWITCH_ARCHIVE_CONTAINER_NAME:-twitch-archive}
|
container_name: ${TWITCH_ARCHIVE_CONTAINER_NAME:-twitch-archive}
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
init: true
|
init: true
|
||||||
|
env_file:
|
||||||
|
- ${TWITCH_ARCHIVE_APP_ENV_FILE:-./.env.production}
|
||||||
environment:
|
environment:
|
||||||
PYTHONUNBUFFERED: ${PYTHONUNBUFFERED:-1}
|
PYTHONUNBUFFERED: ${PYTHONUNBUFFERED:-1}
|
||||||
TZ: ${TZ:-UTC}
|
TZ: ${TZ:-UTC}
|
||||||
RCLONE_CONFIG: ${TWITCH_ARCHIVE_RCLONE_CONFIG:-/app/config/rclone.conf}
|
RCLONE_CONFIG: ${TWITCH_ARCHIVE_RCLONE_CONFIG:-/app/config/rclone.conf}
|
||||||
TWITCH_ARCHIVE_HEALTHCHECK_STREAMER: ${TWITCH_ARCHIVE_HEALTHCHECK_STREAMER:-vinesauce}
|
TWITCH_ARCHIVE_HEALTHCHECK_STREAMER: ${TWITCH_ARCHIVE_HEALTHCHECK_STREAMER:-vinesauce}
|
||||||
CLIENT_ID: ${CLIENT_ID:-}
|
|
||||||
CLIENT_SECRET: ${CLIENT_SECRET:-}
|
|
||||||
OAUTH_PRIVATE_TOKEN: ${OAUTH_PRIVATE_TOKEN:-}
|
|
||||||
SENDER: ${SENDER:-}
|
|
||||||
RECEIVER: ${RECEIVER:-}
|
|
||||||
PASSWD: ${PASSWD:-}
|
|
||||||
"CLIENT-ID":
|
|
||||||
"CLIENT-SECRET":
|
|
||||||
"OAUTH-PRIVATE-TOKEN":
|
|
||||||
command:
|
command:
|
||||||
- sh
|
- sh
|
||||||
- -lc
|
- -lc
|
||||||
- exec python -u twitch-archive.py ${TWITCH_ARCHIVE_ARGS:--u vinesauce}
|
- python twitch-archive.py ${TWITCH_ARCHIVE_ARGS:--u vinesauce}
|
||||||
volumes:
|
volumes:
|
||||||
- ${TWITCH_ARCHIVE_ARCHIVE_BIND:-./archive}:/app/archive
|
- ${TWITCH_ARCHIVE_ARCHIVE_BIND:-./archive}:/app/archive
|
||||||
- ${TWITCH_ARCHIVE_CONFIG_BIND:-./config}:/app/config
|
- ${TWITCH_ARCHIVE_CONFIG_BIND:-./config}:/app/config
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test:
|
test:
|
||||||
- CMD
|
- CMD-SHELL
|
||||||
- python
|
- python twitch-archive.py --healthcheck -u ${TWITCH_ARCHIVE_HEALTHCHECK_STREAMER:-vinesauce}
|
||||||
- twitch-archive.py
|
|
||||||
- --healthcheck
|
|
||||||
- -u
|
|
||||||
- ${TWITCH_ARCHIVE_HEALTHCHECK_STREAMER:-vinesauce}
|
|
||||||
interval: 30s
|
interval: 30s
|
||||||
timeout: 10s
|
timeout: 10s
|
||||||
retries: 3
|
retries: 3
|
||||||
|
|
|
||||||
|
|
@ -1,27 +1,6 @@
|
||||||
#!/usr/bin/env python3
|
#!/bin/sh
|
||||||
import os
|
set -eu
|
||||||
import sys
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
|
mkdir -p /app/archive /app/config /app/bin/temp
|
||||||
|
|
||||||
for path in ('/app/archive', '/app/config', '/app/bin/temp'):
|
exec "$@"
|
||||||
Path(path).mkdir(parents=True, exist_ok=True)
|
|
||||||
|
|
||||||
|
|
||||||
env_aliases = {
|
|
||||||
'CLIENT-ID': 'CLIENT_ID',
|
|
||||||
'CLIENT-SECRET': 'CLIENT_SECRET',
|
|
||||||
'OAUTH-PRIVATE-TOKEN': 'OAUTH_PRIVATE_TOKEN',
|
|
||||||
}
|
|
||||||
|
|
||||||
for source_name, target_name in env_aliases.items():
|
|
||||||
source_value = os.environ.get(source_name)
|
|
||||||
if source_value and not os.environ.get(target_name):
|
|
||||||
os.environ[target_name] = source_value
|
|
||||||
|
|
||||||
|
|
||||||
if len(sys.argv) < 2:
|
|
||||||
raise SystemExit('twitch-archive-entrypoint requires a command to run')
|
|
||||||
|
|
||||||
|
|
||||||
os.execvpe(sys.argv[1], sys.argv[1:], os.environ)
|
|
||||||
|
|
@ -612,10 +612,7 @@ class ContentDownloader:
|
||||||
|
|
||||||
# Show progress every 100 messages
|
# Show progress every 100 messages
|
||||||
if message_count % 100 == 0:
|
if message_count % 100 == 0:
|
||||||
if sys.stdout.isatty():
|
print(f'{Fore.CYAN} Downloaded {message_count} messages...{Style.RESET_ALL}', end='\r')
|
||||||
print(f'{Fore.CYAN} Downloaded {message_count} messages...{Style.RESET_ALL}', end='\r', flush=True)
|
|
||||||
else:
|
|
||||||
print(f'{Fore.CYAN} Downloaded {message_count} messages...{Style.RESET_ALL}', flush=True)
|
|
||||||
|
|
||||||
# Show chat previews in verbose mode (every 10 messages)
|
# Show chat previews in verbose mode (every 10 messages)
|
||||||
if verbose and message_count % 10 == 0:
|
if verbose and message_count % 10 == 0:
|
||||||
|
|
|
||||||
|
|
@ -741,6 +741,7 @@ class TestMultiStreamerCleanupRegression(unittest.TestCase):
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
stream_info = {
|
stream_info = {
|
||||||
'title': 'Test',
|
'title': 'Test',
|
||||||
'createdAt': '2026-04-25T09:14:01Z'
|
'createdAt': '2026-04-25T09:14:01Z'
|
||||||
|
|
@ -752,70 +753,6 @@ class TestMultiStreamerCleanupRegression(unittest.TestCase):
|
||||||
archiver.file_manager.delete_local_files.assert_not_called()
|
archiver.file_manager.delete_local_files.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
class TestEnvironmentLoadingRegression(unittest.TestCase):
|
|
||||||
"""Regression tests for process environment startup in containers."""
|
|
||||||
|
|
||||||
def setUp(self):
|
|
||||||
self.module = load_twitch_archive_module()
|
|
||||||
|
|
||||||
@patch.dict(os.environ, {'CLIENT-ID': 'portainer-client', 'CLIENT-SECRET': 'portainer-secret'}, clear=True)
|
|
||||||
@patch('dotenv.main.find_dotenv', return_value='')
|
|
||||||
@patch('dotenv.main.load_dotenv', return_value=False)
|
|
||||||
def test_load_environment_variables_accepts_process_environment_without_dotenv(self, _mock_load_dotenv, _mock_find_dotenv):
|
|
||||||
archive = self.module.TwitchArchive.__new__(self.module.TwitchArchive)
|
|
||||||
|
|
||||||
self.module.TwitchArchive._load_environment_variables(archive)
|
|
||||||
|
|
||||||
|
|
||||||
class TestHealthcheckRegression(unittest.TestCase):
|
|
||||||
"""Regression tests for Docker healthcheck behavior."""
|
|
||||||
|
|
||||||
def setUp(self):
|
|
||||||
self.module = load_twitch_archive_module()
|
|
||||||
|
|
||||||
@patch.object(os, 'getenv', side_effect=lambda name, default=None: default)
|
|
||||||
def test_run_healthcheck_fails_when_heartbeat_is_missing(self, _mock_getenv):
|
|
||||||
config_manager = MagicMock()
|
|
||||||
config_manager.get_all_enabled_streamers.return_value = ['maddoscientist0']
|
|
||||||
config_manager.load_streamer_config.return_value = {'downloadVOD': False, 'downloadCHAT': False, 'uploadCloud': False}
|
|
||||||
|
|
||||||
archive = MagicMock()
|
|
||||||
archive.os_type = 'linux'
|
|
||||||
archive.downloadVOD = False
|
|
||||||
archive.downloadCHAT = False
|
|
||||||
archive.uploadCloud = False
|
|
||||||
|
|
||||||
with patch.object(self.module, 'ConfigManager', return_value=config_manager), \
|
|
||||||
patch.object(self.module, 'TwitchArchive', return_value=archive), \
|
|
||||||
patch.object(self.module, 'verify_streamlink', return_value=True), \
|
|
||||||
patch.object(self.module, 'verify_ffmpeg', return_value=True), \
|
|
||||||
patch.object(self.module, 'has_fresh_healthcheck_heartbeat', return_value=False):
|
|
||||||
result = self.module.run_healthcheck('maddoscientist0')
|
|
||||||
|
|
||||||
self.assertEqual(result, 1)
|
|
||||||
|
|
||||||
@patch.object(os, 'getenv', side_effect=lambda name, default=None: default)
|
|
||||||
def test_run_healthcheck_succeeds_with_fresh_heartbeat(self, _mock_getenv):
|
|
||||||
config_manager = MagicMock()
|
|
||||||
config_manager.get_all_enabled_streamers.return_value = ['maddoscientist0']
|
|
||||||
config_manager.load_streamer_config.return_value = {'downloadVOD': False, 'downloadCHAT': False, 'uploadCloud': False}
|
|
||||||
|
|
||||||
archive = MagicMock()
|
|
||||||
archive.os_type = 'linux'
|
|
||||||
archive.downloadVOD = False
|
|
||||||
archive.downloadCHAT = False
|
|
||||||
archive.uploadCloud = False
|
|
||||||
|
|
||||||
with patch.object(self.module, 'ConfigManager', return_value=config_manager), \
|
|
||||||
patch.object(self.module, 'TwitchArchive', return_value=archive), \
|
|
||||||
patch.object(self.module, 'verify_streamlink', return_value=True), \
|
|
||||||
patch.object(self.module, 'verify_ffmpeg', return_value=True), \
|
|
||||||
patch.object(self.module, 'has_fresh_healthcheck_heartbeat', return_value=True):
|
|
||||||
result = self.module.run_healthcheck('maddoscientist0')
|
|
||||||
|
|
||||||
self.assertEqual(result, 0)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
# Run tests with verbose output
|
# Run tests with verbose output
|
||||||
print("="*70)
|
print("="*70)
|
||||||
|
|
|
||||||
|
|
@ -60,33 +60,6 @@ from modules.downloader import ContentDownloader
|
||||||
from modules.file_manager import FileManager
|
from modules.file_manager import FileManager
|
||||||
|
|
||||||
|
|
||||||
HEALTHCHECK_HEARTBEAT_PATH = os.getenv('TWITCH_ARCHIVE_HEARTBEAT_PATH', '/tmp/twitch-archive-heartbeat')
|
|
||||||
HEALTHCHECK_MAX_AGE_SECONDS = int(os.getenv('TWITCH_ARCHIVE_HEALTHCHECK_MAX_AGE', '180'))
|
|
||||||
|
|
||||||
|
|
||||||
def write_healthcheck_heartbeat() -> None:
|
|
||||||
"""Record a recent application heartbeat for Docker health checks."""
|
|
||||||
pathlib.Path(HEALTHCHECK_HEARTBEAT_PATH).touch()
|
|
||||||
|
|
||||||
|
|
||||||
def has_fresh_healthcheck_heartbeat(max_age_seconds: int = HEALTHCHECK_MAX_AGE_SECONDS) -> bool:
|
|
||||||
"""Return whether the application heartbeat file exists and is recent."""
|
|
||||||
try:
|
|
||||||
heartbeat_age = time.time() - os.path.getmtime(HEALTHCHECK_HEARTBEAT_PATH)
|
|
||||||
except OSError:
|
|
||||||
return False
|
|
||||||
|
|
||||||
return heartbeat_age <= max_age_seconds
|
|
||||||
|
|
||||||
|
|
||||||
def print_progress_line(message: str) -> None:
|
|
||||||
"""Use carriage returns only in an interactive terminal so Docker logs keep full lines."""
|
|
||||||
if sys.stdout.isatty():
|
|
||||||
print(message, end='\r', flush=True)
|
|
||||||
else:
|
|
||||||
print(message, flush=True)
|
|
||||||
|
|
||||||
|
|
||||||
class TwitchArchive:
|
class TwitchArchive:
|
||||||
"""
|
"""
|
||||||
Main class for the Twitch Archive system.
|
Main class for the Twitch Archive system.
|
||||||
|
|
@ -164,7 +137,7 @@ class TwitchArchive:
|
||||||
|
|
||||||
def _load_environment_variables(self) -> None:
|
def _load_environment_variables(self) -> None:
|
||||||
"""
|
"""
|
||||||
Load environment variables from process environment or an optional .env file.
|
Load environment variables from .env file.
|
||||||
|
|
||||||
Required variables:
|
Required variables:
|
||||||
- CLIENT-ID: Twitch API client ID
|
- CLIENT-ID: Twitch API client ID
|
||||||
|
|
@ -175,26 +148,16 @@ class TwitchArchive:
|
||||||
- PASSWD: Email password for sending notifications (if enabled)
|
- PASSWD: Email password for sending notifications (if enabled)
|
||||||
|
|
||||||
Raises:
|
Raises:
|
||||||
SystemExit: If required Twitch API credentials are unavailable
|
SystemExit: If .env file is not found
|
||||||
"""
|
"""
|
||||||
has_required_env = bool(
|
|
||||||
get_env_value('CLIENT-ID', 'CLIENT_ID') and
|
|
||||||
get_env_value('CLIENT-SECRET', 'CLIENT_SECRET')
|
|
||||||
)
|
|
||||||
|
|
||||||
if has_required_env:
|
|
||||||
print(f'{Fore.GREEN}✓ Twitch API credentials loaded from process environment{Style.RESET_ALL}')
|
|
||||||
return
|
|
||||||
|
|
||||||
dotenv_loaded = load_dotenv(find_dotenv())
|
dotenv_loaded = load_dotenv(find_dotenv())
|
||||||
has_required_env = bool(
|
has_required_env = bool(
|
||||||
get_env_value('CLIENT-ID', 'CLIENT_ID') and
|
get_env_value('CLIENT-ID', 'CLIENT_ID') and
|
||||||
get_env_value('CLIENT-SECRET', 'CLIENT_SECRET')
|
get_env_value('CLIENT-SECRET', 'CLIENT_SECRET')
|
||||||
)
|
)
|
||||||
|
|
||||||
if has_required_env:
|
if not dotenv_loaded and has_required_env:
|
||||||
if dotenv_loaded:
|
print(f'{Fore.GREEN}✓ Twitch API credentials loaded from process environment{Style.RESET_ALL}')
|
||||||
print(f'{Fore.GREEN}✓ Twitch API credentials loaded from .env file{Style.RESET_ALL}')
|
|
||||||
return
|
return
|
||||||
|
|
||||||
if not dotenv_loaded and not has_required_env:
|
if not dotenv_loaded and not has_required_env:
|
||||||
|
|
@ -372,19 +335,16 @@ class TwitchArchive:
|
||||||
signal.signal(signal.SIGINT, self._signal_handler)
|
signal.signal(signal.SIGINT, self._signal_handler)
|
||||||
if hasattr(signal, 'SIGTERM'):
|
if hasattr(signal, 'SIGTERM'):
|
||||||
signal.signal(signal.SIGTERM, self._signal_handler)
|
signal.signal(signal.SIGTERM, self._signal_handler)
|
||||||
|
|
||||||
write_healthcheck_heartbeat()
|
|
||||||
|
|
||||||
while not self.shutdown_requested:
|
while not self.shutdown_requested:
|
||||||
try:
|
try:
|
||||||
write_healthcheck_heartbeat()
|
|
||||||
# Check stream status using StreamMonitor
|
# Check stream status using StreamMonitor
|
||||||
response = self.stream_monitor.check_stream_status()
|
response = self.stream_monitor.check_stream_status()
|
||||||
is_live = response['data']['user']['stream']
|
is_live = response['data']['user']['stream']
|
||||||
|
|
||||||
# Stream is offline
|
# Stream is offline
|
||||||
if is_live is None:
|
if is_live is None:
|
||||||
print_progress_line(f'{Fore.CYAN}⏳ {self.username} is offline. Checking again in {self.refresh}s...{Style.RESET_ALL}')
|
print(f'{Fore.CYAN}⏳ {self.username} is offline. Checking again in {self.refresh}s...{Style.RESET_ALL}', end='\r')
|
||||||
if self.shutdown_requested:
|
if self.shutdown_requested:
|
||||||
break
|
break
|
||||||
self._interruptible_sleep(self.refresh)
|
self._interruptible_sleep(self.refresh)
|
||||||
|
|
@ -552,7 +512,7 @@ class TwitchArchive:
|
||||||
|
|
||||||
# Wait before checking again
|
# Wait before checking again
|
||||||
if not vod_found:
|
if not vod_found:
|
||||||
print_progress_line(f'{Fore.CYAN}VOD not found yet, waiting...{Style.RESET_ALL}')
|
print(f'{Fore.CYAN}VOD not found yet, waiting...{Style.RESET_ALL}', end='\r')
|
||||||
if not self._interruptible_sleep(min(10, self.vodTimeout - (time.time() - vod_wait_start))):
|
if not self._interruptible_sleep(min(10, self.vodTimeout - (time.time() - vod_wait_start))):
|
||||||
break
|
break
|
||||||
|
|
||||||
|
|
@ -971,8 +931,6 @@ class TwitchArchiveManager:
|
||||||
# Print configuration summary for each streamer
|
# Print configuration summary for each streamer
|
||||||
for username, archiver in self.archivers.items():
|
for username, archiver in self.archivers.items():
|
||||||
archiver._print_configuration_summary()
|
archiver._print_configuration_summary()
|
||||||
|
|
||||||
write_healthcheck_heartbeat()
|
|
||||||
|
|
||||||
print(f'\n{Fore.GREEN}🚀 Starting monitoring loop...{Style.RESET_ALL}\n')
|
print(f'\n{Fore.GREEN}🚀 Starting monitoring loop...{Style.RESET_ALL}\n')
|
||||||
|
|
||||||
|
|
@ -990,7 +948,6 @@ class TwitchArchiveManager:
|
||||||
|
|
||||||
while not self.shutdown_requested:
|
while not self.shutdown_requested:
|
||||||
current_time = time.time()
|
current_time = time.time()
|
||||||
write_healthcheck_heartbeat()
|
|
||||||
|
|
||||||
# Print periodic status every 60 seconds
|
# Print periodic status every 60 seconds
|
||||||
if current_time - last_status_print >= 60:
|
if current_time - last_status_print >= 60:
|
||||||
|
|
@ -1070,7 +1027,7 @@ class TwitchArchiveManager:
|
||||||
else:
|
else:
|
||||||
# Not live
|
# Not live
|
||||||
if self.verbose:
|
if self.verbose:
|
||||||
print_progress_line(f'{Fore.CYAN}[{username}] Offline - checking again in {archiver.refresh}s{Style.RESET_ALL}')
|
print(f'{Fore.CYAN}[{username}] Offline - checking again in {archiver.refresh}s{Style.RESET_ALL}', end='\r')
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f'{Fore.RED}[{username}] Error checking stream: {e}{Style.RESET_ALL}')
|
print(f'{Fore.RED}[{username}] Error checking stream: {e}{Style.RESET_ALL}')
|
||||||
|
|
@ -1373,7 +1330,7 @@ class TwitchArchiveManager:
|
||||||
|
|
||||||
# Wait before checking again
|
# Wait before checking again
|
||||||
if not vod_found:
|
if not vod_found:
|
||||||
print_progress_line(f'{Fore.CYAN}VOD not found yet, waiting...{Style.RESET_ALL}')
|
print(f'{Fore.CYAN}VOD not found yet, waiting...{Style.RESET_ALL}', end='\r')
|
||||||
time.sleep(min(10, archiver.vodTimeout - (time.time() - vod_wait_start)))
|
time.sleep(min(10, archiver.vodTimeout - (time.time() - vod_wait_start)))
|
||||||
|
|
||||||
if not vod_found:
|
if not vod_found:
|
||||||
|
|
@ -1551,10 +1508,6 @@ def run_healthcheck(specific_streamer: Optional[str] = None) -> int:
|
||||||
if not checks_ok:
|
if not checks_ok:
|
||||||
return 1
|
return 1
|
||||||
|
|
||||||
if not has_fresh_healthcheck_heartbeat():
|
|
||||||
print(f'{Fore.RED}✗ ERROR: Application heartbeat is missing or stale at {HEALTHCHECK_HEARTBEAT_PATH}{Style.RESET_ALL}')
|
|
||||||
return 1
|
|
||||||
|
|
||||||
print(f'{Fore.GREEN}✓ Healthcheck OK for {username}{Style.RESET_ALL}')
|
print(f'{Fore.GREEN}✓ Healthcheck OK for {username}{Style.RESET_ALL}')
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue