From cd3e37ff59aac2375cedc15587096a263b99d90e Mon Sep 17 00:00:00 2001 From: MaddoScientisto Date: Sat, 25 Apr 2026 16:19:01 +0200 Subject: [PATCH 1/4] Update environment variable loading to support process environment and improve error handling Co-authored-by: Copilot --- .env.production | 10 +++++----- docker-compose.yml | 2 -- test_twitch_archive_simple.py | 16 +++++++++++++++- twitch-archive.py | 18 ++++++++++++++---- 4 files changed, 34 insertions(+), 12 deletions(-) diff --git a/.env.production b/.env.production index b1c4197..3c04af7 100644 --- a/.env.production +++ b/.env.production @@ -1,8 +1,8 @@ -TWITCH_ARCHIVE_IMAGE=forgejo.maddoscientisto.net/maddo/twitch-archive:latest -TWITCH_ARCHIVE_CONTAINER_NAME=twitch-archive -TWITCH_ARCHIVE_APP_ENV_FILE=./.env.production -TWITCH_ARCHIVE_ARCHIVE_BIND=./archive -TWITCH_ARCHIVE_CONFIG_BIND=./config +TWITCH_ARCHIVE_IMAGE=forgejo.maddoscientisto.net/maddo/twitchdownloader:latest +TWITCH_ARCHIVE_CONTAINER_NAME=twitchdownloader +TWITCH_ARCHIVE_APP_ENV_FILE=/mnt/storage/AppData/twitchdownloader/config/.env.production +TWITCH_ARCHIVE_ARCHIVE_BIND=/mnt/storage/AppData/twitchdownloader/archive +TWITCH_ARCHIVE_CONFIG_BIND=/mnt/storage/AppData/twitchdownloader/config TWITCH_ARCHIVE_ARGS=-u vinesauce TWITCH_ARCHIVE_HEALTHCHECK_STREAMER=vinesauce TWITCH_ARCHIVE_RCLONE_CONFIG=/app/config/rclone.conf diff --git a/docker-compose.yml b/docker-compose.yml index 68bc572..e77d6d6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,8 +4,6 @@ services: container_name: ${TWITCH_ARCHIVE_CONTAINER_NAME:-twitch-archive} restart: unless-stopped init: true - env_file: - - ${TWITCH_ARCHIVE_APP_ENV_FILE:-./.env.production} environment: PYTHONUNBUFFERED: ${PYTHONUNBUFFERED:-1} TZ: ${TZ:-UTC} diff --git a/test_twitch_archive_simple.py b/test_twitch_archive_simple.py index 445e741..3d096e3 100644 --- a/test_twitch_archive_simple.py +++ b/test_twitch_archive_simple.py @@ -741,7 +741,6 @@ class TestMultiStreamerCleanupRegression(unittest.TestCase): } } } - stream_info = { 'title': 'Test', 'createdAt': '2026-04-25T09:14:01Z' @@ -753,6 +752,21 @@ class TestMultiStreamerCleanupRegression(unittest.TestCase): archiver.file_manager.delete_local_files.assert_not_called() +class TestEnvironmentLoadingRegression(unittest.TestCase): + """Regression tests for process environment startup in containers.""" + + def setUp(self): + self.module = load_twitch_archive_module() + + @patch.dict(os.environ, {'CLIENT-ID': 'portainer-client', 'CLIENT-SECRET': 'portainer-secret'}, clear=True) + @patch('dotenv.main.find_dotenv', return_value='') + @patch('dotenv.main.load_dotenv', return_value=False) + def test_load_environment_variables_accepts_process_environment_without_dotenv(self, _mock_load_dotenv, _mock_find_dotenv): + archive = self.module.TwitchArchive.__new__(self.module.TwitchArchive) + + self.module.TwitchArchive._load_environment_variables(archive) + + if __name__ == '__main__': # Run tests with verbose output print("="*70) diff --git a/twitch-archive.py b/twitch-archive.py index 707bdfe..16290db 100644 --- a/twitch-archive.py +++ b/twitch-archive.py @@ -137,7 +137,7 @@ class TwitchArchive: def _load_environment_variables(self) -> None: """ - Load environment variables from .env file. + Load environment variables from process environment or an optional .env file. Required variables: - CLIENT-ID: Twitch API client ID @@ -148,16 +148,26 @@ class TwitchArchive: - PASSWD: Email password for sending notifications (if enabled) Raises: - SystemExit: If .env file is not found + SystemExit: If required Twitch API credentials are unavailable """ + has_required_env = bool( + get_env_value('CLIENT-ID', 'CLIENT_ID') and + get_env_value('CLIENT-SECRET', 'CLIENT_SECRET') + ) + + if has_required_env: + print(f'{Fore.GREEN}✓ Twitch API credentials loaded from process environment{Style.RESET_ALL}') + return + dotenv_loaded = load_dotenv(find_dotenv()) has_required_env = bool( get_env_value('CLIENT-ID', 'CLIENT_ID') and get_env_value('CLIENT-SECRET', 'CLIENT_SECRET') ) - if not dotenv_loaded and has_required_env: - print(f'{Fore.GREEN}✓ Twitch API credentials loaded from process environment{Style.RESET_ALL}') + if has_required_env: + if dotenv_loaded: + print(f'{Fore.GREEN}✓ Twitch API credentials loaded from .env file{Style.RESET_ALL}') return if not dotenv_loaded and not has_required_env: From 9d5c707646cb103af07696a69906ec85c75d0ab1 Mon Sep 17 00:00:00 2001 From: MaddoScientisto Date: Sat, 25 Apr 2026 16:46:20 +0200 Subject: [PATCH 2/4] Implement Docker healthcheck functionality and improve progress message handling Co-authored-by: Copilot --- docker-compose.override.yml | 2 +- docker-compose.yml | 2 +- modules/downloader.py | 5 +++- test_twitch_archive_simple.py | 49 +++++++++++++++++++++++++++++++++++ twitch-archive.py | 45 +++++++++++++++++++++++++++++--- 5 files changed, 96 insertions(+), 7 deletions(-) diff --git a/docker-compose.override.yml b/docker-compose.override.yml index c084df9..0d4cfd4 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -10,7 +10,7 @@ services: command: - sh - -lc - - python twitch-archive.py ${TWITCH_ARCHIVE_ARGS:--u vinesauce --verbose} + - exec python -u twitch-archive.py ${TWITCH_ARCHIVE_ARGS:--u vinesauce --verbose} volumes: - .:/app - ${TWITCH_ARCHIVE_ARCHIVE_BIND:-./archive}:/app/archive diff --git a/docker-compose.yml b/docker-compose.yml index e77d6d6..43b68af 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,7 +12,7 @@ services: command: - sh - -lc - - python twitch-archive.py ${TWITCH_ARCHIVE_ARGS:--u vinesauce} + - exec python -u twitch-archive.py ${TWITCH_ARCHIVE_ARGS:--u vinesauce} volumes: - ${TWITCH_ARCHIVE_ARCHIVE_BIND:-./archive}:/app/archive - ${TWITCH_ARCHIVE_CONFIG_BIND:-./config}:/app/config diff --git a/modules/downloader.py b/modules/downloader.py index 87fe30d..7f825bc 100644 --- a/modules/downloader.py +++ b/modules/downloader.py @@ -612,7 +612,10 @@ class ContentDownloader: # Show progress every 100 messages if message_count % 100 == 0: - print(f'{Fore.CYAN} Downloaded {message_count} messages...{Style.RESET_ALL}', end='\r') + if sys.stdout.isatty(): + print(f'{Fore.CYAN} Downloaded {message_count} messages...{Style.RESET_ALL}', end='\r', flush=True) + else: + print(f'{Fore.CYAN} Downloaded {message_count} messages...{Style.RESET_ALL}', flush=True) # Show chat previews in verbose mode (every 10 messages) if verbose and message_count % 10 == 0: diff --git a/test_twitch_archive_simple.py b/test_twitch_archive_simple.py index 3d096e3..01f8c73 100644 --- a/test_twitch_archive_simple.py +++ b/test_twitch_archive_simple.py @@ -767,6 +767,55 @@ class TestEnvironmentLoadingRegression(unittest.TestCase): self.module.TwitchArchive._load_environment_variables(archive) +class TestHealthcheckRegression(unittest.TestCase): + """Regression tests for Docker healthcheck behavior.""" + + def setUp(self): + self.module = load_twitch_archive_module() + + @patch.object(os, 'getenv', side_effect=lambda name, default=None: default) + def test_run_healthcheck_fails_when_heartbeat_is_missing(self, _mock_getenv): + config_manager = MagicMock() + config_manager.get_all_enabled_streamers.return_value = ['maddoscientist0'] + config_manager.load_streamer_config.return_value = {'downloadVOD': False, 'downloadCHAT': False, 'uploadCloud': False} + + archive = MagicMock() + archive.os_type = 'linux' + archive.downloadVOD = False + archive.downloadCHAT = False + archive.uploadCloud = False + + with patch.object(self.module, 'ConfigManager', return_value=config_manager), \ + patch.object(self.module, 'TwitchArchive', return_value=archive), \ + patch.object(self.module, 'verify_streamlink', return_value=True), \ + patch.object(self.module, 'verify_ffmpeg', return_value=True), \ + patch.object(self.module, 'has_fresh_healthcheck_heartbeat', return_value=False): + result = self.module.run_healthcheck('maddoscientist0') + + self.assertEqual(result, 1) + + @patch.object(os, 'getenv', side_effect=lambda name, default=None: default) + def test_run_healthcheck_succeeds_with_fresh_heartbeat(self, _mock_getenv): + config_manager = MagicMock() + config_manager.get_all_enabled_streamers.return_value = ['maddoscientist0'] + config_manager.load_streamer_config.return_value = {'downloadVOD': False, 'downloadCHAT': False, 'uploadCloud': False} + + archive = MagicMock() + archive.os_type = 'linux' + archive.downloadVOD = False + archive.downloadCHAT = False + archive.uploadCloud = False + + with patch.object(self.module, 'ConfigManager', return_value=config_manager), \ + patch.object(self.module, 'TwitchArchive', return_value=archive), \ + patch.object(self.module, 'verify_streamlink', return_value=True), \ + patch.object(self.module, 'verify_ffmpeg', return_value=True), \ + patch.object(self.module, 'has_fresh_healthcheck_heartbeat', return_value=True): + result = self.module.run_healthcheck('maddoscientist0') + + self.assertEqual(result, 0) + + if __name__ == '__main__': # Run tests with verbose output print("="*70) diff --git a/twitch-archive.py b/twitch-archive.py index 16290db..ed2cbcb 100644 --- a/twitch-archive.py +++ b/twitch-archive.py @@ -60,6 +60,33 @@ from modules.downloader import ContentDownloader from modules.file_manager import FileManager +HEALTHCHECK_HEARTBEAT_PATH = os.getenv('TWITCH_ARCHIVE_HEARTBEAT_PATH', '/tmp/twitch-archive-heartbeat') +HEALTHCHECK_MAX_AGE_SECONDS = int(os.getenv('TWITCH_ARCHIVE_HEALTHCHECK_MAX_AGE', '180')) + + +def write_healthcheck_heartbeat() -> None: + """Record a recent application heartbeat for Docker health checks.""" + pathlib.Path(HEALTHCHECK_HEARTBEAT_PATH).touch() + + +def has_fresh_healthcheck_heartbeat(max_age_seconds: int = HEALTHCHECK_MAX_AGE_SECONDS) -> bool: + """Return whether the application heartbeat file exists and is recent.""" + try: + heartbeat_age = time.time() - os.path.getmtime(HEALTHCHECK_HEARTBEAT_PATH) + except OSError: + return False + + return heartbeat_age <= max_age_seconds + + +def print_progress_line(message: str) -> None: + """Use carriage returns only in an interactive terminal so Docker logs keep full lines.""" + if sys.stdout.isatty(): + print(message, end='\r', flush=True) + else: + print(message, flush=True) + + class TwitchArchive: """ Main class for the Twitch Archive system. @@ -345,16 +372,19 @@ class TwitchArchive: signal.signal(signal.SIGINT, self._signal_handler) if hasattr(signal, 'SIGTERM'): signal.signal(signal.SIGTERM, self._signal_handler) + + write_healthcheck_heartbeat() while not self.shutdown_requested: try: + write_healthcheck_heartbeat() # Check stream status using StreamMonitor response = self.stream_monitor.check_stream_status() is_live = response['data']['user']['stream'] # Stream is offline if is_live is None: - print(f'{Fore.CYAN}⏳ {self.username} is offline. Checking again in {self.refresh}s...{Style.RESET_ALL}', end='\r') + print_progress_line(f'{Fore.CYAN}⏳ {self.username} is offline. Checking again in {self.refresh}s...{Style.RESET_ALL}') if self.shutdown_requested: break self._interruptible_sleep(self.refresh) @@ -522,7 +552,7 @@ class TwitchArchive: # Wait before checking again if not vod_found: - print(f'{Fore.CYAN}VOD not found yet, waiting...{Style.RESET_ALL}', end='\r') + print_progress_line(f'{Fore.CYAN}VOD not found yet, waiting...{Style.RESET_ALL}') if not self._interruptible_sleep(min(10, self.vodTimeout - (time.time() - vod_wait_start))): break @@ -941,6 +971,8 @@ class TwitchArchiveManager: # Print configuration summary for each streamer for username, archiver in self.archivers.items(): archiver._print_configuration_summary() + + write_healthcheck_heartbeat() print(f'\n{Fore.GREEN}🚀 Starting monitoring loop...{Style.RESET_ALL}\n') @@ -958,6 +990,7 @@ class TwitchArchiveManager: while not self.shutdown_requested: current_time = time.time() + write_healthcheck_heartbeat() # Print periodic status every 60 seconds if current_time - last_status_print >= 60: @@ -1037,7 +1070,7 @@ class TwitchArchiveManager: else: # Not live if self.verbose: - print(f'{Fore.CYAN}[{username}] Offline - checking again in {archiver.refresh}s{Style.RESET_ALL}', end='\r') + print_progress_line(f'{Fore.CYAN}[{username}] Offline - checking again in {archiver.refresh}s{Style.RESET_ALL}') except Exception as e: print(f'{Fore.RED}[{username}] Error checking stream: {e}{Style.RESET_ALL}') @@ -1340,7 +1373,7 @@ class TwitchArchiveManager: # Wait before checking again if not vod_found: - print(f'{Fore.CYAN}VOD not found yet, waiting...{Style.RESET_ALL}', end='\r') + print_progress_line(f'{Fore.CYAN}VOD not found yet, waiting...{Style.RESET_ALL}') time.sleep(min(10, archiver.vodTimeout - (time.time() - vod_wait_start))) if not vod_found: @@ -1518,6 +1551,10 @@ def run_healthcheck(specific_streamer: Optional[str] = None) -> int: if not checks_ok: return 1 + if not has_fresh_healthcheck_heartbeat(): + print(f'{Fore.RED}✗ ERROR: Application heartbeat is missing or stale at {HEALTHCHECK_HEARTBEAT_PATH}{Style.RESET_ALL}') + return 1 + print(f'{Fore.GREEN}✓ Healthcheck OK for {username}{Style.RESET_ALL}') return 0 From 4083fb4d789f65e1d01ba77eeb17b8d4a4460acf Mon Sep 17 00:00:00 2001 From: MaddoScientisto Date: Sat, 25 Apr 2026 17:04:44 +0200 Subject: [PATCH 3/4] Add support for CLIENT-ID, CLIENT-SECRET, and OAUTH-PRIVATE-TOKEN environment variables Co-authored-by: Copilot --- docker-compose.yml | 9 +++++++++ docker/entrypoint.sh | 21 +++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 43b68af..afc45a1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,15 @@ services: TZ: ${TZ:-UTC} RCLONE_CONFIG: ${TWITCH_ARCHIVE_RCLONE_CONFIG:-/app/config/rclone.conf} TWITCH_ARCHIVE_HEALTHCHECK_STREAMER: ${TWITCH_ARCHIVE_HEALTHCHECK_STREAMER:-vinesauce} + CLIENT_ID: ${CLIENT_ID:-} + CLIENT_SECRET: ${CLIENT_SECRET:-} + OAUTH_PRIVATE_TOKEN: ${OAUTH_PRIVATE_TOKEN:-} + SENDER: ${SENDER:-} + RECEIVER: ${RECEIVER:-} + PASSWD: ${PASSWD:-} + "CLIENT-ID": + "CLIENT-SECRET": + "OAUTH-PRIVATE-TOKEN": command: - sh - -lc diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index d327347..b378a45 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -3,4 +3,25 @@ set -eu mkdir -p /app/archive /app/config /app/bin/temp +if [ -z "${CLIENT_ID:-}" ]; then + client_id_from_hyphenated="$(printenv 'CLIENT-ID' 2>/dev/null || true)" + if [ -n "$client_id_from_hyphenated" ]; then + export CLIENT_ID="$client_id_from_hyphenated" + fi +fi + +if [ -z "${CLIENT_SECRET:-}" ]; then + client_secret_from_hyphenated="$(printenv 'CLIENT-SECRET' 2>/dev/null || true)" + if [ -n "$client_secret_from_hyphenated" ]; then + export CLIENT_SECRET="$client_secret_from_hyphenated" + fi +fi + +if [ -z "${OAUTH_PRIVATE_TOKEN:-}" ]; then + oauth_token_from_hyphenated="$(printenv 'OAUTH-PRIVATE-TOKEN' 2>/dev/null || true)" + if [ -n "$oauth_token_from_hyphenated" ]; then + export OAUTH_PRIVATE_TOKEN="$oauth_token_from_hyphenated" + fi +fi + exec "$@" \ No newline at end of file From 708464bd865caedd89ec0f6f1bb00674fb278f4a Mon Sep 17 00:00:00 2001 From: MaddoScientisto Date: Sat, 25 Apr 2026 17:12:39 +0200 Subject: [PATCH 4/4] Refactor entrypoint script to use Python and streamline environment variable handling Co-authored-by: Copilot --- docker-compose.yml | 8 ++++++-- docker/entrypoint.sh | 44 ++++++++++++++++++++++---------------------- 2 files changed, 28 insertions(+), 24 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index afc45a1..de56f80 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,8 +27,12 @@ services: - ${TWITCH_ARCHIVE_CONFIG_BIND:-./config}:/app/config healthcheck: test: - - CMD-SHELL - - python twitch-archive.py --healthcheck -u ${TWITCH_ARCHIVE_HEALTHCHECK_STREAMER:-vinesauce} + - CMD + - python + - twitch-archive.py + - --healthcheck + - -u + - ${TWITCH_ARCHIVE_HEALTHCHECK_STREAMER:-vinesauce} interval: 30s timeout: 10s retries: 3 diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index b378a45..fe404da 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -1,27 +1,27 @@ -#!/bin/sh -set -eu +#!/usr/bin/env python3 +import os +import sys +from pathlib import Path -mkdir -p /app/archive /app/config /app/bin/temp -if [ -z "${CLIENT_ID:-}" ]; then - client_id_from_hyphenated="$(printenv 'CLIENT-ID' 2>/dev/null || true)" - if [ -n "$client_id_from_hyphenated" ]; then - export CLIENT_ID="$client_id_from_hyphenated" - fi -fi +for path in ('/app/archive', '/app/config', '/app/bin/temp'): + Path(path).mkdir(parents=True, exist_ok=True) -if [ -z "${CLIENT_SECRET:-}" ]; then - client_secret_from_hyphenated="$(printenv 'CLIENT-SECRET' 2>/dev/null || true)" - if [ -n "$client_secret_from_hyphenated" ]; then - export CLIENT_SECRET="$client_secret_from_hyphenated" - fi -fi -if [ -z "${OAUTH_PRIVATE_TOKEN:-}" ]; then - oauth_token_from_hyphenated="$(printenv 'OAUTH-PRIVATE-TOKEN' 2>/dev/null || true)" - if [ -n "$oauth_token_from_hyphenated" ]; then - export OAUTH_PRIVATE_TOKEN="$oauth_token_from_hyphenated" - fi -fi +env_aliases = { + 'CLIENT-ID': 'CLIENT_ID', + 'CLIENT-SECRET': 'CLIENT_SECRET', + 'OAUTH-PRIVATE-TOKEN': 'OAUTH_PRIVATE_TOKEN', +} -exec "$@" \ No newline at end of file +for source_name, target_name in env_aliases.items(): + source_value = os.environ.get(source_name) + if source_value and not os.environ.get(target_name): + os.environ[target_name] = source_value + + +if len(sys.argv) < 2: + raise SystemExit('twitch-archive-entrypoint requires a command to run') + + +os.execvpe(sys.argv[1], sys.argv[1:], os.environ) \ No newline at end of file