maddo/TwitchDownloader
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Refactor configuration values in global.json and global.schema.json to use boolean types for better clarity and consistency; update default values in twitch-archive.py accordingly.

Browse source
This commit is contained in:
MaddoScientisto 2026-02-09 23:03:05 +01:00
commit efb320eb05
3 changed files with 292 additions and 144 deletions
Download patch file Download diff file Expand all files Collapse all files

26
config/global.json.example
View file

@ -4,25 +4,25 @@
"root_path": "archive", "root_path": "archive",
"rclone_path": "remote:path/to/streams", "rclone_path": "remote:path/to/streams",
"refresh": 60.0, "refresh": 60.0,
"notifications": 0, "notifications": false,
"downloadMETADATA": 1, "downloadMETADATA": true,
"downloadVOD": 1, "downloadVOD": true,
"downloadCHAT": 1, "downloadCHAT": true,
"downloadLiveCHAT": 1, "downloadLiveCHAT": true,
"vodTimeout": 300, "vodTimeout": 300,
"uploadCloud": 1, "uploadCloud": true,
"deleteFiles": 0, "deleteFiles": false,
"onlyRaw": 0, "onlyRaw": false,
"cleanRaw": 1, "cleanRaw": true,
"hls_segments": 3, "hls_segments": 3,
"hls_segmentsVOD": 10, "hls_segmentsVOD": 10,
"streamlink_ttvlol": 0, "streamlink_ttvlol": false,
"ffmpeg_hwaccel": "auto", "ffmpeg_hwaccel": "auto",
"ffmpeg_threads": 0, "ffmpeg_threads": 0,
"ffmpeg_audio_codec": "aac", "ffmpeg_audio_codec": "aac",
"ffmpeg_audio_samplerate": 48000, "ffmpeg_audio_samplerate": 48000,
"ffmpeg_audio_bitrate": "192k", "ffmpeg_audio_bitrate": "192k",
"ffmpeg_error_recovery": 1, "ffmpeg_error_recovery": true,
"ffmpeg_faststart": 1, "ffmpeg_faststart": true,
"ffmpeg_progress": 0 "ffmpeg_progress": false
} }

89
config/global.schema.json
View file

@ -28,34 +28,29 @@
"description": "Time between status checks in seconds (60.0 recommended for multiple streamers)" "description": "Time between status checks in seconds (60.0 recommended for multiple streamers)"
}, },
"notifications": { "notifications": {
"type": "integer", "type": "boolean",
"enum": [0, 1], "default": false,
"default": 0, "description": "Email notifications: false = disabled, true = enabled"
"description": "Email notifications: 0 = disabled, 1 = enabled"
}, },
"downloadMETADATA": { "downloadMETADATA": {
"type": "integer", "type": "boolean",
"enum": [0, 1], "default": true,
"default": 1, "description": "Download stream metadata: false = disabled, true = enabled"
"description": "Download stream metadata: 0 = disabled, 1 = enabled"
}, },
"downloadVOD": { "downloadVOD": {
"type": "integer", "type": "boolean",
"enum": [0, 1], "default": true,
"default": 1, "description": "Download VODs after stream ends: false = disabled, true = enabled"
"description": "Download VODs after stream ends: 0 = disabled, 1 = enabled"
}, },
"downloadCHAT": { "downloadCHAT": {
"type": "integer", "type": "boolean",
"enum": [0, 1], "default": true,
"default": 1, "description": "Download and render chat from VOD: false = disabled, true = enabled"
"description": "Download and render chat from VOD: 0 = disabled, 1 = enabled"
}, },
"downloadLiveCHAT": { "downloadLiveCHAT": {
"type": "integer", "type": "boolean",
"enum": [0, 1], "default": true,
"default": 1, "description": "Download chat during live stream: false = disabled, true = enabled"
"description": "Download chat during live stream: 0 = disabled, 1 = enabled"
}, },
"vodTimeout": { "vodTimeout": {
"type": "integer", "type": "integer",
@ -64,28 +59,24 @@
"description": "Seconds to wait for VOD to appear after stream ends" "description": "Seconds to wait for VOD to appear after stream ends"
}, },
"uploadCloud": { "uploadCloud": {
"type": "integer", "type": "boolean",
"enum": [0, 1], "default": true,
"default": 1, "description": "Upload to rclone remote: false = disabled, true = enabled"
"description": "Upload to rclone remote: 0 = disabled, 1 = enabled"
}, },
"deleteFiles": { "deleteFiles": {
"type": "integer", "type": "boolean",
"enum": [0, 1], "default": false,
"default": 0, "description": "Delete local files after upload: false = disabled, true = enabled (BE CAREFUL)"
"description": "Delete local files after upload: 0 = disabled, 1 = enabled (BE CAREFUL)"
}, },
"onlyRaw": { "onlyRaw": {
"type": "integer", "type": "boolean",
"enum": [0, 1], "default": false,
"default": 0, "description": "Keep only raw .ts files: false = convert to mp3/mp4, true = keep raw only"
"description": "Keep only raw .ts files: 0 = convert to mp3/mp4, 1 = keep raw only"
}, },
"cleanRaw": { "cleanRaw": {
"type": "integer", "type": "boolean",
"enum": [0, 1], "default": true,
"default": 1, "description": "Delete raw .ts files after processing: false = keep, true = delete"
"description": "Delete raw .ts files after processing: 0 = keep, 1 = delete"
}, },
"hls_segments": { "hls_segments": {
"type": "integer", "type": "integer",
@ -102,9 +93,8 @@
"description": "Number of parallel download threads for VODs (1-10)" "description": "Number of parallel download threads for VODs (1-10)"
}, },
"streamlink_ttvlol": { "streamlink_ttvlol": {
"type": "integer", "type": "boolean",
"enum": [0, 1], "default": false,
"default": 0,
"description": "DEPRECATED: Ad-blocking with ttvlol (--twitch-proxy-playlist removed in newer streamlink)" "description": "DEPRECATED: Ad-blocking with ttvlol (--twitch-proxy-playlist removed in newer streamlink)"
}, },
"ffmpeg_hwaccel": { "ffmpeg_hwaccel": {
@ -138,22 +128,19 @@
"description": "Audio bitrate (e.g., 128k, 192k, 256k, 320k)" "description": "Audio bitrate (e.g., 128k, 192k, 256k, 320k)"
}, },
"ffmpeg_error_recovery": { "ffmpeg_error_recovery": {
"type": "integer", "type": "boolean",
"enum": [0, 1], "default": true,
"default": 1, "description": "Enable error recovery for corrupted/incomplete streams: false = disabled, true = enabled"
"description": "Enable error recovery for corrupted/incomplete streams: 0 = disabled, 1 = enabled"
}, },
"ffmpeg_faststart": { "ffmpeg_faststart": {
"type": "integer", "type": "boolean",
"enum": [0, 1], "default": true,
"default": 1, "description": "Enable MP4 faststart flag for better streaming/playback: false = disabled, true = enabled"
"description": "Enable MP4 faststart flag for better streaming/playback: 0 = disabled, 1 = enabled"
}, },
"ffmpeg_progress": { "ffmpeg_progress": {
"type": "integer", "type": "boolean",
"enum": [0, 1], "default": false,
"default": 0, "description": "Show FFmpeg encoding progress: false = silent, true = verbose output"
"description": "Show FFmpeg encoding progress: 0 = silent, 1 = verbose output"
} }
} }
} }

319
twitch-archive.py
View file

@ -60,17 +60,17 @@ DEFAULT_CONFIG = {
'root_path': 'archive', 'root_path': 'archive',
'rclone_path': 'remote:path/to/streams', 'rclone_path': 'remote:path/to/streams',
'refresh': 60.0, 'refresh': 60.0,
'streamlink_ttvlol': 0, 'streamlink_ttvlol': False,
'notifications': 0, 'notifications': False,
'downloadMETADATA': 1, 'downloadMETADATA': True,
'downloadVOD': 1, 'downloadVOD': True,
'downloadCHAT': 1, 'downloadCHAT': True,
'downloadLiveCHAT': 1, 'downloadLiveCHAT': True,
'vodTimeout': 300, 'vodTimeout': 300,
'uploadCloud': 1, 'uploadCloud': True,
'deleteFiles': 0, 'deleteFiles': False,
'onlyRaw': 0, 'onlyRaw': False,
'cleanRaw': 1, 'cleanRaw': True,
'hls_segments': 3, 'hls_segments': 3,
'hls_segmentsVOD': 10, 'hls_segmentsVOD': 10,
# FFmpeg 8.0+ Enhancement Options # FFmpeg 8.0+ Enhancement Options
@ -79,9 +79,9 @@ DEFAULT_CONFIG = {
'ffmpeg_audio_codec': 'aac', # Audio codec for audio-only streams 'ffmpeg_audio_codec': 'aac', # Audio codec for audio-only streams
'ffmpeg_audio_samplerate': 48000, # Audio sample rate (48000 recommended for broadcasts) 'ffmpeg_audio_samplerate': 48000, # Audio sample rate (48000 recommended for broadcasts)
'ffmpeg_audio_bitrate': '192k', # Audio bitrate 'ffmpeg_audio_bitrate': '192k', # Audio bitrate
'ffmpeg_error_recovery': 1, # Enable error recovery for corrupted streams (0/1) 'ffmpeg_error_recovery': True, # Enable error recovery for corrupted streams
'ffmpeg_faststart': 1, # Enable faststart for MP4 (better streaming compatibility) (0/1) 'ffmpeg_faststart': True, # Enable faststart for MP4 (better streaming compatibility)
'ffmpeg_progress': 0 # Show encoding progress (0/1) 'ffmpeg_progress': False # Show encoding progress
} }
# ============================================================================ # ============================================================================
@ -390,9 +390,9 @@ class TwitchArchive:
self._print_toggle('Cloud upload', self.uploadCloud) self._print_toggle('Cloud upload', self.uploadCloud)
# Warning messages # Warning messages
if self.deleteFiles == 1: if self.deleteFiles:
print(f'\n{Fore.RED}⚠ WARNING: Files will be DELETED after processing{Style.RESET_ALL}') print(f'\n{Fore.RED}⚠ WARNING: Files will be DELETED after processing{Style.RESET_ALL}')
if self.uploadCloud == 0: if not self.uploadCloud:
print(f'{Fore.RED}⚠ CRITICAL: Files will be deleted WITHOUT cloud backup!{Style.RESET_ALL}') print(f'{Fore.RED}⚠ CRITICAL: Files will be deleted WITHOUT cloud backup!{Style.RESET_ALL}')
print(f'{Fore.YELLOW} Press CTRL+C to stop and change configuration{Style.RESET_ALL}') print(f'{Fore.YELLOW} Press CTRL+C to stop and change configuration{Style.RESET_ALL}')
else: else:
@ -400,9 +400,9 @@ class TwitchArchive:
print(f'\n{Fore.CYAN}{"=" * 60}{Style.RESET_ALL}\n') print(f'\n{Fore.CYAN}{"=" * 60}{Style.RESET_ALL}\n')
def _print_toggle(self, label: str, value: int) -> None: def _print_toggle(self, label: str, value: bool) -> None:
"""Helper method to print a configuration toggle in a consistent format.""" """Helper method to print a configuration toggle in a consistent format."""
status = f'{Fore.GREEN}Enabled{Style.RESET_ALL}' if value == 1 else f'{Fore.RED}Disabled{Style.RESET_ALL}' status = f'{Fore.GREEN}Enabled{Style.RESET_ALL}' if value else f'{Fore.RED}Disabled{Style.RESET_ALL}'
print(f'{label}: {status}') print(f'{label}: {status}')
def run(self) -> None: def run(self) -> None:
@ -528,7 +528,7 @@ class TwitchArchive:
print(f'{Fore.YELLOW}⚠ Warning: Could not verify FFmpeg: {e}{Style.RESET_ALL}') print(f'{Fore.YELLOW}⚠ Warning: Could not verify FFmpeg: {e}{Style.RESET_ALL}')
# Check for TwitchDownloaderCLI (if VOD or Chat download enabled) # Check for TwitchDownloaderCLI (if VOD or Chat download enabled)
if self.downloadVOD == 1 or self.downloadCHAT == 1: if self.downloadVOD or self.downloadCHAT:
try: try:
downloader_path = self._get_twitch_downloader_executable() downloader_path = self._get_twitch_downloader_executable()
if os.path.exists(downloader_path): if os.path.exists(downloader_path):
@ -634,7 +634,7 @@ class TwitchArchive:
subject: Email subject line subject: Email subject line
content: Email body content content: Email body content
""" """
if self.notifications != 1: if not self.notifications:
return return
try: try:
@ -768,7 +768,7 @@ class TwitchArchive:
# Add ad-blocking if enabled (Note: twitch-proxy-playlist was removed in newer streamlink versions) # Add ad-blocking if enabled (Note: twitch-proxy-playlist was removed in newer streamlink versions)
# For ad-blocking, you may need to use alternative methods like --twitch-low-latency # For ad-blocking, you may need to use alternative methods like --twitch-low-latency
# or rely on Twitch's own ad-free viewing for subscribers # or rely on Twitch's own ad-free viewing for subscribers
if self.streamlink_ttvlol == 1: if self.streamlink_ttvlol:
# The old --twitch-proxy-playlist option has been removed from streamlink # The old --twitch-proxy-playlist option has been removed from streamlink
# Consider using alternative ad-blocking approaches or updating your method # Consider using alternative ad-blocking approaches or updating your method
print(f'{Fore.YELLOW}⚠ Warning: ttv-lol proxy option is deprecated in newer streamlink versions{Style.RESET_ALL}') print(f'{Fore.YELLOW}⚠ Warning: ttv-lol proxy option is deprecated in newer streamlink versions{Style.RESET_ALL}')
@ -792,7 +792,8 @@ class TwitchArchive:
if self.shutdown_requested: if self.shutdown_requested:
print(f'{Fore.YELLOW}✓ Recording stopped by user{Style.RESET_ALL}') print(f'{Fore.YELLOW}✓ Recording stopped by user{Style.RESET_ALL}')
return False # Return True so processing continues - we still want to process what was recorded
return True
print(f'{Fore.GREEN}✓ Stream recording complete{Style.RESET_ALL}') print(f'{Fore.GREEN}✓ Stream recording complete{Style.RESET_ALL}')
return True return True
@ -813,7 +814,7 @@ class TwitchArchive:
print(f'{Fore.YELLOW}⚠ Raw file not found, skipping processing{Style.RESET_ALL}') print(f'{Fore.YELLOW}⚠ Raw file not found, skipping processing{Style.RESET_ALL}')
return return
if self.onlyRaw == 1: if self.onlyRaw:
print(f'{Fore.CYAN}Keeping raw .ts file (onlyRaw mode){Style.RESET_ALL}') print(f'{Fore.CYAN}Keeping raw .ts file (onlyRaw mode){Style.RESET_ALL}')
return return
@ -837,7 +838,7 @@ class TwitchArchive:
cmd.extend(['-threads', str(self.ffmpeg_threads)]) cmd.extend(['-threads', str(self.ffmpeg_threads)])
# Add faststart for better streaming compatibility (MP4/M4A) # Add faststart for better streaming compatibility (MP4/M4A)
if self.ffmpeg_faststart == 1 and output_path.endswith(('.mp4', '.m4a')): if self.ffmpeg_faststart and output_path.endswith(('.mp4', '.m4a')):
cmd.extend(['-movflags', '+faststart']) cmd.extend(['-movflags', '+faststart'])
cmd.append(output_path) cmd.append(output_path)
@ -865,7 +866,7 @@ class TwitchArchive:
cmd.extend(['-threads', str(self.ffmpeg_threads)]) cmd.extend(['-threads', str(self.ffmpeg_threads)])
# Error recovery options for corrupted streams # Error recovery options for corrupted streams
if self.ffmpeg_error_recovery == 1: if self.ffmpeg_error_recovery:
cmd.extend([ cmd.extend([
'-fflags', '+genpts', # Generate missing timestamps '-fflags', '+genpts', # Generate missing timestamps
'-avoid_negative_ts', 'make_zero', # Handle timestamp issues '-avoid_negative_ts', 'make_zero', # Handle timestamp issues
@ -881,13 +882,13 @@ class TwitchArchive:
]) ])
# Add faststart for MP4 files # Add faststart for MP4 files
if self.ffmpeg_faststart == 1 and output_path.endswith('.mp4'): if self.ffmpeg_faststart and output_path.endswith('.mp4'):
cmd.extend(['-movflags', '+faststart']) cmd.extend(['-movflags', '+faststart'])
cmd.append(output_path) cmd.append(output_path)
# Run ffmpeg with optional progress output # Run ffmpeg with optional progress output
if self.ffmpeg_progress == 1: if self.ffmpeg_progress:
subprocess.call(cmd) subprocess.call(cmd)
else: else:
subprocess.call(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) subprocess.call(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
@ -905,7 +906,7 @@ class TwitchArchive:
Returns: Returns:
bool: True if download succeeded, False otherwise bool: True if download succeeded, False otherwise
""" """
if self.downloadVOD != 1: if not self.downloadVOD:
return False return False
print(f'\n{Fore.CYAN}Downloading VOD: {vod_info["title"]}{Style.RESET_ALL}') print(f'\n{Fore.CYAN}Downloading VOD: {vod_info["title"]}{Style.RESET_ALL}')
@ -959,7 +960,7 @@ class TwitchArchive:
Returns: Returns:
bool: True if succeeded, False otherwise bool: True if succeeded, False otherwise
""" """
if self.downloadCHAT != 1: if not self.downloadCHAT:
return False return False
print(f'\n{Fore.CYAN}Downloading chat: {vod_info["title"]}{Style.RESET_ALL}') print(f'\n{Fore.CYAN}Downloading chat: {vod_info["title"]}{Style.RESET_ALL}')
@ -1041,7 +1042,7 @@ class TwitchArchive:
Returns: Returns:
subprocess.Popen: The process handle, or None if failed to start subprocess.Popen: The process handle, or None if failed to start
""" """
if self.downloadLiveCHAT != 1: if not self.downloadLiveCHAT:
return None return None
print(f'\n{Fore.CYAN}Starting live chat download...{Style.RESET_ALL}') print(f'\n{Fore.CYAN}Starting live chat download...{Style.RESET_ALL}')
@ -1086,7 +1087,7 @@ class TwitchArchive:
Returns: Returns:
bool: True if succeeded, False otherwise bool: True if succeeded, False otherwise
""" """
if self.downloadCHAT != 1: if not self.downloadCHAT:
return False return False
print(f'\n{Fore.CYAN}Downloading chat: {vod_info["title"]}{Style.RESET_ALL}') print(f'\n{Fore.CYAN}Downloading chat: {vod_info["title"]}{Style.RESET_ALL}')
@ -1250,7 +1251,7 @@ class TwitchArchive:
vod_info: VOD metadata from Twitch API vod_info: VOD metadata from Twitch API
filename_base: Base filename (without extension) filename_base: Base filename (without extension)
""" """
if self.downloadMETADATA != 1: if not self.downloadMETADATA:
return return
metadata_path = os.path.join(self.metadata_path, f"{PREFIX_METADATA}{filename_base}.json") metadata_path = os.path.join(self.metadata_path, f"{PREFIX_METADATA}{filename_base}.json")
@ -1385,11 +1386,11 @@ class TwitchArchive:
live_chat_process = None live_chat_process = None
chat_json_path = os.path.join(self.chatJSON_path, f"{PREFIX_CHAT}{filename_base}.json") chat_json_path = os.path.join(self.chatJSON_path, f"{PREFIX_CHAT}{filename_base}.json")
if self.downloadLiveCHAT == 1 and is_live.get('archiveVideo') and is_live['archiveVideo'].get('id'): if self.downloadLiveCHAT and is_live.get('archiveVideo') and is_live['archiveVideo'].get('id'):
live_vod_id = is_live['archiveVideo']['id'] live_vod_id = is_live['archiveVideo']['id']
print(f'{Fore.CYAN}Live VOD ID detected: {live_vod_id}{Style.RESET_ALL}') print(f'{Fore.CYAN}Live VOD ID detected: {live_vod_id}{Style.RESET_ALL}')
live_chat_process = self._download_live_chat(live_vod_id, chat_json_path) live_chat_process = self._download_live_chat(live_vod_id, chat_json_path)
elif self.downloadLiveCHAT == 1: elif self.downloadLiveCHAT:
print(f'{Fore.YELLOW}⚠ No VOD ID available yet for live chat download{Style.RESET_ALL}') print(f'{Fore.YELLOW}⚠ No VOD ID available yet for live chat download{Style.RESET_ALL}')
# Record the live stream # Record the live stream
@ -1482,7 +1483,7 @@ class TwitchArchive:
print(f'{Fore.YELLOW}⚠ No matching VOD found for this stream{Style.RESET_ALL}') print(f'{Fore.YELLOW}⚠ No matching VOD found for this stream{Style.RESET_ALL}')
# Clean up raw files if configured # Clean up raw files if configured
if self.cleanRaw == 1 and os.path.exists(live_raw_path): if self.cleanRaw and os.path.exists(live_raw_path):
print(f'{Fore.YELLOW}Deleting raw .ts file...{Style.RESET_ALL}') print(f'{Fore.YELLOW}Deleting raw .ts file...{Style.RESET_ALL}')
os.remove(live_raw_path) os.remove(live_raw_path)
@ -1490,7 +1491,7 @@ class TwitchArchive:
upload_success = self._upload_to_cloud(filename_base) upload_success = self._upload_to_cloud(filename_base)
# Delete local files if configured and upload succeeded # Delete local files if configured and upload succeeded
if self.deleteFiles == 1 and upload_success: if self.deleteFiles and upload_success:
self._delete_local_files(filename_base, live_raw_path, live_proc_path) self._delete_local_files(filename_base, live_raw_path, live_proc_path)
# Done processing this stream # Done processing this stream
@ -1542,7 +1543,7 @@ class TwitchArchive:
Returns: Returns:
bool: True if upload succeeded or is disabled, False if failed bool: True if upload succeeded or is disabled, False if failed
""" """
if self.uploadCloud != 1: if not self.uploadCloud:
return True # Consider upload "successful" if disabled return True # Consider upload "successful" if disabled
print(f'\n{Fore.CYAN}Uploading to cloud storage...{Style.RESET_ALL}') print(f'\n{Fore.CYAN}Uploading to cloud storage...{Style.RESET_ALL}')
@ -1618,18 +1619,18 @@ class TwitchArchive:
files_to_delete = [] files_to_delete = []
# Live files # Live files
if self.cleanRaw == 0 and os.path.exists(live_raw_path): if not self.cleanRaw and os.path.exists(live_raw_path):
files_to_delete.append(live_raw_path) files_to_delete.append(live_raw_path)
if os.path.exists(live_proc_path): if os.path.exists(live_proc_path):
files_to_delete.append(live_proc_path) files_to_delete.append(live_proc_path)
# VOD files # VOD files
if self.downloadVOD == 1: if self.downloadVOD:
vod_raw = os.path.join(self.raw_path, f"{PREFIX_VOD}{filename_base}.ts") vod_raw = os.path.join(self.raw_path, f"{PREFIX_VOD}{filename_base}.ts")
vod_mp4 = os.path.join(self.video_path, f"{PREFIX_VOD}{filename_base}.mp4") vod_mp4 = os.path.join(self.video_path, f"{PREFIX_VOD}{filename_base}.mp4")
vod_mp3 = os.path.join(self.video_path, f"{PREFIX_VOD}{filename_base}.mp3") vod_mp3 = os.path.join(self.video_path, f"{PREFIX_VOD}{filename_base}.mp3")
if self.cleanRaw == 0 and os.path.exists(vod_raw): if not self.cleanRaw and os.path.exists(vod_raw):
files_to_delete.append(vod_raw) files_to_delete.append(vod_raw)
if os.path.exists(vod_mp4): if os.path.exists(vod_mp4):
files_to_delete.append(vod_mp4) files_to_delete.append(vod_mp4)
@ -1637,7 +1638,7 @@ class TwitchArchive:
files_to_delete.append(vod_mp3) files_to_delete.append(vod_mp3)
# Chat files # Chat files
if self.downloadCHAT == 1: if self.downloadCHAT:
chat_json = os.path.join(self.chatJSON_path, f"{PREFIX_CHAT}{filename_base}.json") chat_json = os.path.join(self.chatJSON_path, f"{PREFIX_CHAT}{filename_base}.json")
chat_mp4 = os.path.join(self.chatMP4_path, f"{PREFIX_CHAT}{filename_base}.mp4") chat_mp4 = os.path.join(self.chatMP4_path, f"{PREFIX_CHAT}{filename_base}.mp4")
@ -1647,7 +1648,7 @@ class TwitchArchive:
files_to_delete.append(chat_mp4) files_to_delete.append(chat_mp4)
# Metadata files # Metadata files
if self.downloadMETADATA == 1: if self.downloadMETADATA:
metadata = os.path.join(self.metadata_path, f"{PREFIX_METADATA}{filename_base}.json") metadata = os.path.join(self.metadata_path, f"{PREFIX_METADATA}{filename_base}.json")
if os.path.exists(metadata): if os.path.exists(metadata):
files_to_delete.append(metadata) files_to_delete.append(metadata)
@ -1672,17 +1673,20 @@ class TwitchArchiveManager:
Manages multiple TwitchArchive instances for monitoring multiple streamers. Manages multiple TwitchArchive instances for monitoring multiple streamers.
""" """
def __init__(self, specific_streamer: Optional[str] = None): def __init__(self, specific_streamer: Optional[str] = None, verbose: bool = False):
""" """
Initialize the manager. Initialize the manager.
Args: Args:
specific_streamer: If provided, only monitor this streamer (ignore enabled status) specific_streamer: If provided, only monitor this streamer (ignore enabled status)
verbose: Enable verbose debug output
""" """
self.config_manager = ConfigManager() self.config_manager = ConfigManager()
self.specific_streamer = specific_streamer self.specific_streamer = specific_streamer
self.verbose = verbose
self.archivers: Dict[str, TwitchArchive] = {} self.archivers: Dict[str, TwitchArchive] = {}
self.shutdown_requested = False self.shutdown_requested = False
self.active_recordings: Dict[str, str] = {} # Track active recordings: {username: stream_id}
# Setup signal handlers # Setup signal handlers
signal.signal(signal.SIGTERM, self._signal_handler) signal.signal(signal.SIGTERM, self._signal_handler)
@ -1705,7 +1709,7 @@ class TwitchArchiveManager:
list: List of streamer usernames to monitor list: List of streamer usernames to monitor
""" """
if self.specific_streamer: if self.specific_streamer:
# Monitor only the specified streamer # Monitor only the specified streamer (ignore enabled flag)
return [self.specific_streamer] return [self.specific_streamer]
else: else:
# Monitor all enabled streamers # Monitor all enabled streamers
@ -1789,10 +1793,17 @@ class TwitchArchiveManager:
Checks each streamer's status and processes streams as needed. Checks each streamer's status and processes streams as needed.
""" """
last_check = {} last_check = {}
last_status_print = time.time()
while not self.shutdown_requested: while not self.shutdown_requested:
current_time = time.time() current_time = time.time()
# Print periodic status every 60 seconds
if current_time - last_status_print >= 60:
status_line = " | ".join([f"{username}: checking" for username in self.archivers.keys()])
print(f'{Fore.CYAN}[Status] {status_line}{Style.RESET_ALL}')
last_status_print = current_time
for username, archiver in self.archivers.items(): for username, archiver in self.archivers.items():
# Check if enough time has passed since last check for this streamer # Check if enough time has passed since last check for this streamer
if username not in last_check or (current_time - last_check[username]) >= archiver.refresh: if username not in last_check or (current_time - last_check[username]) >= archiver.refresh:
@ -1800,27 +1811,68 @@ class TwitchArchiveManager:
# Check stream status # Check stream status
try: try:
stream_info = archiver._check_stream_status() response = archiver._check_stream_status()
if stream_info: # Debug: Print the full response (if verbose)
# Stream is live if self.verbose:
stream_id = stream_info['archiveVideo']['id'] print(f'\n{Fore.MAGENTA}[DEBUG {username}] API Response: {response}{Style.RESET_ALL}')
if not archiver._is_stream_already_processed(stream_id): stream_data = response['data']['user']['stream'] if response else None
print(f'\n{Fore.GREEN}[{username}] Stream detected!{Style.RESET_ALL}')
print(f'{Fore.CYAN}Title: {stream_info["title"]}{Style.RESET_ALL}')
# Process the stream if self.verbose:
self._process_stream(archiver, stream_info, stream_id) print(f'{Fore.MAGENTA}[DEBUG {username}] Stream data: {stream_data}{Style.RESET_ALL}')
# Mark as processed if stream_data:
archiver._mark_stream_as_processed(stream_id) # Stream is live - check if it has required data
if stream_data.get('archiveVideo') and stream_data['archiveVideo'].get('id'):
# Create composite stream ID like single-streamer mode
# This prevents duplicate recordings in the same session
stream_id = f"{stream_data['createdAt']} - {username} - {stream_data.get('title', 'Untitled')}"
if self.verbose:
print(f'{Fore.MAGENTA}[DEBUG {username}] VOD ID: {stream_data["archiveVideo"]["id"]}{Style.RESET_ALL}')
print(f'{Fore.MAGENTA}[DEBUG {username}] Composite Stream ID: {stream_id}{Style.RESET_ALL}')
# Check if we're currently recording this stream
currently_recording = username in self.active_recordings and self.active_recordings[username] == stream_id
if self.verbose:
print(f'{Fore.MAGENTA}[DEBUG {username}] Currently recording: {currently_recording}{Style.RESET_ALL}')
print(f'{Fore.MAGENTA}[DEBUG {username}] Active recordings: {self.active_recordings}{Style.RESET_ALL}')
# Record if not currently recording (ignore .log file - always record if live)
if not currently_recording:
print(f'\n{Fore.GREEN}[{username}] Stream detected!{Style.RESET_ALL}')
print(f'{Fore.CYAN}Title: {stream_data.get("title", "No title")}{Style.RESET_ALL}')
print(f'{Fore.CYAN}Started at: {stream_data["createdAt"]}{Style.RESET_ALL}')
# Mark as currently recording
self.active_recordings[username] = stream_id
# Process the stream (this blocks until stream ends)
self._process_stream(archiver, stream_data, stream_id)
# Mark as processed in log (for record keeping)
archiver._mark_stream_as_processed(stream_id)
# Remove from active recordings
if username in self.active_recordings:
del self.active_recordings[username]
else:
if self.verbose:
print(f'{Fore.CYAN}[{username}] Currently recording this stream, skipping duplicate...{Style.RESET_ALL}')
else:
# Stream is live but VOD ID not available yet
print(f'{Fore.YELLOW}[{username}] Stream is live but VOD ID not ready yet (title: {stream_data.get("title", "No title")}){Style.RESET_ALL}')
else: else:
# Not live - check for new VODs if needed # Not live
pass if self.verbose:
print(f'{Fore.CYAN}[{username}] Offline - checking again in {archiver.refresh}s{Style.RESET_ALL}', end='\r')
except Exception as e: except Exception as e:
print(f'{Fore.RED}[{username}] Error checking stream: {e}{Style.RESET_ALL}') print(f'{Fore.RED}[{username}] Error checking stream: {e}{Style.RESET_ALL}')
import traceback
traceback.print_exc()
# Sleep briefly before next iteration # Sleep briefly before next iteration
time.sleep(1) time.sleep(1)
@ -1845,12 +1897,18 @@ class TwitchArchiveManager:
timestamp = datetime.now(timezone('UTC')).strftime("%Y%m%d_%Hh%Mm%Ss") timestamp = datetime.now(timezone('UTC')).strftime("%Y%m%d_%Hh%Mm%Ss")
filename_base = f"{PREFIX_LIVE}{archiver.username}_{timestamp}" filename_base = f"{PREFIX_LIVE}{archiver.username}_{timestamp}"
# Parse stream start time
live_date = datetime.strptime(
stream_info["createdAt"], '%Y-%m-%dT%H:%M:%SZ'
).replace(tzinfo=timezone('UTC')).astimezone(tz=None).replace(tzinfo=None)
# Define paths # Define paths
raw_extension = '.ts' raw_extension = '.ts'
proc_extension = '.mp3' if archiver.quality == 'audio_only' else '.mp4' proc_extension = '.mp3' if archiver.quality == 'audio_only' else '.mp4'
live_raw_path = str(archiver.raw_path / f"{filename_base}{raw_extension}") live_raw_path = str(archiver.raw_path / f"{filename_base}{raw_extension}")
live_proc_path = str(archiver.video_path / f"{filename_base}{proc_extension}") live_proc_path = str(archiver.video_path / f"{filename_base}{proc_extension}")
chat_json_path = str(archiver.chatJSON_path / f"{PREFIX_CHAT}{filename_base}.json")
# Send notification # Send notification
archiver.send_notification( archiver.send_notification(
@ -1858,35 +1916,133 @@ class TwitchArchiveManager:
f"Recording: {stream_info['title']}" f"Recording: {stream_info['title']}"
) )
# Start live chat download if enabled and VOD ID is available
live_chat_process = None
if archiver.downloadLiveCHAT and stream_info.get('archiveVideo') and stream_info['archiveVideo'].get('id'):
live_vod_id = stream_info['archiveVideo']['id']
print(f'{Fore.CYAN}Live VOD ID detected: {live_vod_id}{Style.RESET_ALL}')
live_chat_process = archiver._download_live_chat(live_vod_id, chat_json_path)
elif archiver.downloadLiveCHAT:
print(f'{Fore.YELLOW}⚠ No VOD ID available yet for live chat download{Style.RESET_ALL}')
# Record livestream # Record livestream
recording_successful = archiver._record_livestream(stream_info, live_raw_path) recording_successful = archiver._record_livestream(stream_info, live_raw_path)
if not recording_successful: # Check if raw file exists (may exist even after interrupted recording)
if not os.path.exists(live_raw_path):
print(f'{Fore.RED}✗ No recording file found, skipping processing{Style.RESET_ALL}')
return return
# Get file size to check if anything was recorded
file_size = os.path.getsize(live_raw_path)
if file_size < 1024: # Less than 1KB means essentially nothing was recorded
print(f'{Fore.RED}✗ Recording file too small ({file_size} bytes), skipping processing{Style.RESET_ALL}')
return
print(f'{Fore.CYAN}Processing recorded content ({file_size / (1024*1024):.2f} MB)...{Style.RESET_ALL}')
# Process raw stream # Process raw stream
if archiver.onlyRaw != 1: if not archiver.onlyRaw:
archiver._process_raw_stream(live_raw_path, live_proc_path) archiver._process_raw_stream(live_raw_path, live_proc_path)
# Clean up raw file if configured # Wait for live chat download if it was started
if archiver.cleanRaw == 1 and os.path.exists(live_raw_path): live_chat_downloaded = False
os.remove(live_raw_path) if live_chat_process is not None:
live_chat_downloaded = archiver._wait_for_chat_download(live_chat_process, chat_json_path)
# Save metadata # Render live chat if downloaded successfully
if archiver.downloadMETADATA == 1: if live_chat_downloaded:
archiver._save_metadata(stream_info, filename_base) chat_video_path = str(archiver.chatMP4_path / f"{PREFIX_CHAT}{filename_base}.mp4")
archiver._render_chat(chat_json_path, chat_video_path)
# Wait for VOD and download it # Wait for VOD and download it
if archiver.downloadVOD == 1 and archiver.vodTimeout > 0: vod_response = None
# This would need the full VOD logic from loopcheck if archiver.vodTimeout == 0:
pass print(f'{Fore.CYAN}VOD check disabled (vodTimeout=0). Skipping VOD download.{Style.RESET_ALL}')
elif archiver.shutdown_requested:
print(f'{Fore.YELLOW}Skipping VOD download due to shutdown request{Style.RESET_ALL}')
else:
# Try to match stream with VOD (with timeout)
print(f'{Fore.CYAN}Waiting for VOD to become available (timeout: {archiver.vodTimeout}s)...{Style.RESET_ALL}')
vod_found = False
vod_wait_start = time.time()
while time.time() - vod_wait_start < archiver.vodTimeout:
# Check for shutdown request
if archiver.shutdown_requested:
print(f'\n{Fore.YELLOW}VOD check interrupted by shutdown{Style.RESET_ALL}')
break
vod_response = archiver._get_latest_vod()
if vod_response and vod_response['data']['user']['videos']['edges']:
current_vod = vod_response['data']['user']['videos']['edges'][0]['node']
vod_date = datetime.strptime(
current_vod["recordedAt"], '%Y-%m-%dT%H:%M:%SZ'
).replace(tzinfo=timezone('UTC')).astimezone(tz=None).replace(tzinfo=None)
# Check if VOD matches the stream (within 1 minute tolerance)
time_tolerance = timedelta(minutes=1)
if (live_date - time_tolerance) <= vod_date <= (live_date + time_tolerance):
vod_found = True
break
# Wait before checking again
if not vod_found:
print(f'{Fore.CYAN}VOD not found yet, waiting...{Style.RESET_ALL}', end='\r')
time.sleep(min(10, archiver.vodTimeout - (time.time() - vod_wait_start)))
if not vod_found:
print(f'\n{Fore.YELLOW}⚠ VOD not found after {archiver.vodTimeout}s - streamer may have VODs disabled{Style.RESET_ALL}')
print(f'{Fore.CYAN} → Live recording and chat (if enabled) were saved successfully{Style.RESET_ALL}')
vod_response = None
# Process VOD if found
if vod_response and vod_response['data']['user']['videos']['edges']:
current_vod = vod_response['data']['user']['videos']['edges'][0]['node']
vod_date = datetime.strptime(
current_vod["recordedAt"], '%Y-%m-%dT%H:%M:%SZ'
).replace(tzinfo=timezone('UTC')).astimezone(tz=None).replace(tzinfo=None)
# Check if VOD matches the stream (within 1 minute tolerance)
time_tolerance = timedelta(minutes=1)
if (live_date - time_tolerance) <= vod_date <= (live_date + time_tolerance):
print(f'\n{Fore.GREEN}✓ Found matching VOD{Style.RESET_ALL}')
# Save metadata
if archiver.downloadMETADATA:
archiver._save_metadata(current_vod, filename_base)
# Download VOD
if archiver.downloadVOD:
vod_ext = '.mp3' if archiver.quality == 'audio_only' else '.mp4'
vod_path = str(archiver.video_path / f"{PREFIX_VOD}{filename_base}{vod_ext}")
archiver._download_vod(current_vod, vod_path)
# Download and render chat from VOD (if not already done via live chat)
if archiver.downloadCHAT and not live_chat_downloaded:
chat_video_path = str(archiver.chatMP4_path / f"{PREFIX_CHAT}{filename_base}.mp4")
archiver._download_and_render_chat(current_vod, chat_json_path, chat_video_path)
elif live_chat_downloaded:
print(f'{Fore.CYAN}Chat already downloaded from live stream, skipping VOD chat download{Style.RESET_ALL}')
else:
print(f'{Fore.YELLOW}⚠ No matching VOD found for this stream{Style.RESET_ALL}')
elif archiver.downloadMETADATA:
# Save what metadata we have from the live stream
archiver._save_metadata(stream_info, filename_base)
# Clean up raw file if configured
if archiver.cleanRaw and os.path.exists(live_raw_path):
print(f'{Fore.YELLOW}Deleting raw .ts file...{Style.RESET_ALL}')
os.remove(live_raw_path)
# Upload to cloud if configured # Upload to cloud if configured
if archiver.uploadCloud == 1: upload_success = False
archiver._upload_to_cloud(filename_base) if archiver.uploadCloud:
upload_success = archiver._upload_to_cloud(filename_base)
# Delete files if configured # Delete files if configured
if archiver.deleteFiles == 1: if archiver.deleteFiles and upload_success:
archiver._delete_local_files(filename_base, live_raw_path, live_proc_path) archiver._delete_local_files(filename_base, live_raw_path, live_proc_path)
# Send completion notification # Send completion notification
@ -1933,6 +2089,7 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving
{Fore.GREEN}OPTIONS:{Style.RESET_ALL} {Fore.GREEN}OPTIONS:{Style.RESET_ALL}
-h, --help Display this help information -h, --help Display this help information
-u, --username <name> Monitor only this Twitch channel -u, --username <name> Monitor only this Twitch channel
--verbose Enable verbose debug output
--legacy Force legacy mode (use config.json) --legacy Force legacy mode (use config.json)
{Fore.GREEN}LEGACY OPTIONS (when using --legacy):{Style.RESET_ALL} {Fore.GREEN}LEGACY OPTIONS (when using --legacy):{Style.RESET_ALL}
@ -1955,6 +2112,7 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving
{Fore.CYAN}EXAMPLES:{Style.RESET_ALL} {Fore.CYAN}EXAMPLES:{Style.RESET_ALL}
python twitch-archive.py # Monitor all enabled streamers python twitch-archive.py # Monitor all enabled streamers
python twitch-archive.py -u vinesauce # Monitor only vinesauce python twitch-archive.py -u vinesauce # Monitor only vinesauce
python twitch-archive.py -u hackerling --verbose # Monitor with debug output
python twitch-archive.py --legacy # Use old config.json mode python twitch-archive.py --legacy # Use old config.json mode
{Fore.CYAN}{"=" * 70}{Style.RESET_ALL} {Fore.CYAN}{"=" * 70}{Style.RESET_ALL}
@ -1965,7 +2123,7 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving
argv, argv,
"h:u:q:a:v:c:m:r:d:n:", "h:u:q:a:v:c:m:r:d:n:",
["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=",
"metadata=", "upload=", "delete=", "notifications=", "legacy"] "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose"]
) )
except getopt.GetoptError as e: except getopt.GetoptError as e:
print(f'{Fore.RED}Error: {e}{Style.RESET_ALL}\n') print(f'{Fore.RED}Error: {e}{Style.RESET_ALL}\n')
@ -1977,31 +2135,34 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving
# Parse command line args # Parse command line args
legacy_overrides = {} legacy_overrides = {}
verbose_mode = False
for opt, arg in opts: for opt, arg in opts:
if opt in ('-h', '--help'): if opt in ('-h', '--help'):
print(help_msg) print(help_msg)
sys.exit(0) sys.exit(0)
elif opt in ("-u", "--username"): elif opt in ("-u", "--username"):
specific_streamer = arg specific_streamer = arg
elif opt == "--verbose":
verbose_mode = True
elif opt == "--legacy": elif opt == "--legacy":
use_legacy_mode = True use_legacy_mode = True
# Legacy options (only used in legacy mode) # Legacy options (only used in legacy mode)
elif opt in ("-q", "--quality"): elif opt in ("-q", "--quality"):
legacy_overrides['quality'] = arg legacy_overrides['quality'] = arg
elif opt in ("-a", "--ttv-lol"): elif opt in ("-a", "--ttv-lol"):
legacy_overrides['streamlink_ttvlol'] = int(arg) legacy_overrides['streamlink_ttvlol'] = bool(int(arg))
elif opt in ("-v", "--vod"): elif opt in ("-v", "--vod"):
legacy_overrides['downloadVOD'] = int(arg) legacy_overrides['downloadVOD'] = bool(int(arg))
elif opt in ("-c", "--chat"): elif opt in ("-c", "--chat"):
legacy_overrides['downloadCHAT'] = int(arg) legacy_overrides['downloadCHAT'] = bool(int(arg))
elif opt in ("-m", "--metadata"): elif opt in ("-m", "--metadata"):
legacy_overrides['downloadMETADATA'] = int(arg) legacy_overrides['downloadMETADATA'] = bool(int(arg))
elif opt in ("-r", "--upload"): elif opt in ("-r", "--upload"):
legacy_overrides['uploadCloud'] = int(arg) legacy_overrides['uploadCloud'] = bool(int(arg))
elif opt in ("-d", "--delete"): elif opt in ("-d", "--delete"):
legacy_overrides['deleteFiles'] = int(arg) legacy_overrides['deleteFiles'] = bool(int(arg))
elif opt in ("-n", "--notifications"): elif opt in ("-n", "--notifications"):
legacy_overrides['notifications'] = int(arg) legacy_overrides['notifications'] = bool(int(arg))
# Determine which mode to use # Determine which mode to use
if use_legacy_mode or (legacy_config_exists and not specific_streamer and not os.path.exists('config/global.json')): if use_legacy_mode or (legacy_config_exists and not specific_streamer and not os.path.exists('config/global.json')):
@ -2019,7 +2180,7 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving
twitch_archive.run() twitch_archive.run()
else: else:
# New multi-streamer mode # New multi-streamer mode
manager = TwitchArchiveManager(specific_streamer=specific_streamer) manager = TwitchArchiveManager(specific_streamer=specific_streamer, verbose=verbose_mode)
manager.run() manager.run()