From 80255b201254de44863d230c7ab01b8bd0ecd05c Mon Sep 17 00:00:00 2001 From: MaddoScientisto Date: Mon, 9 Feb 2026 21:29:47 +0100 Subject: [PATCH] Updated script --- .gitignore | 7 +- .python-version | 1 + IMPROVEMENTS.md | 267 +++++++ README.md | 2 +- bin/TwitchDownloaderCLI | 4 +- bin/TwitchDownloaderCLI.exe | 4 +- config.sample.json | 51 ++ loopstart.bat | 23 + requirements.txt | 8 +- start.bat | 9 +- twitch-archive.py | 1484 ++++++++++++++++++++++++++++------- 11 files changed, 1568 insertions(+), 292 deletions(-) create mode 100644 .python-version create mode 100644 IMPROVEMENTS.md create mode 100644 config.sample.json create mode 100644 loopstart.bat diff --git a/.gitignore b/.gitignore index d4b4b9c..896b29f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# User configuration file (contains personal settings) +config.json + # Environments .env archive @@ -8,4 +11,6 @@ subprocess_time.py env2/** venv3/** .gitignore -bin/** \ No newline at end of file +bin/** +\n+# Ignore any virtual environment directories starting with 'venv' (venv, venv3, venv314, etc.) +venv*/ \ No newline at end of file diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..da71773 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.14.3 diff --git a/IMPROVEMENTS.md b/IMPROVEMENTS.md new file mode 100644 index 0000000..577ca76 --- /dev/null +++ b/IMPROVEMENTS.md @@ -0,0 +1,267 @@ +# Twitch Archive - Code Improvements + +## Overview +This document outlines the improvements made to make the `twitch-archive.py` script more user-friendly for programmers who aren't familiar with Python. + +## Key Improvements + +### 1. **Better Code Organization** +- **Module-level docstring**: Added comprehensive documentation at the top explaining what the script does +- **Constants section**: All magic values and API endpoints moved to well-named constants at the top +- **Logical grouping**: Code is now organized into clear sections with visual separators + +### 2. **Enhanced Documentation** +- **Type hints**: Added type annotations to all methods (e.g., `def run(self) -> None:`) +- **Docstrings**: Every method now has detailed documentation explaining: + - What it does + - What parameters it accepts + - What it returns + - Potential errors/exceptions +- **Inline comments**: Added explanatory comments throughout the code + +### 3. **Improved Method Names** +- Old: `sendNotif()` → New: `send_notification()` (more descriptive) +- Old: `get_OS()` → New: `_detect_operating_system()` (clearer purpose) +- Old: `correct_user()` → New: `_validate_username()` (describes action) +- Added underscore prefix to private methods (Python convention) + +### 4. **Modular Design** +The massive `loopcheck()` method was broken down into smaller, focused methods: + +- `_is_stream_already_processed()` - Check if stream was already recorded +- `_mark_stream_as_processed()` - Log stream to prevent duplicates +- `_record_livestream()` - Handle stream recording with streamlink +- `_process_raw_stream()` - Convert .ts files with ffmpeg +- `_download_vod()` - Download VOD with TwitchDownloaderCLI +- `_download_and_render_chat()` - Download and render chat logs +- `_save_metadata()` - Save stream metadata +- `_upload_to_cloud()` - Upload to cloud storage with rclone +- `_delete_local_files()` - Clean up local files after upload + +### 5. **Better Error Handling** +- **Specific exceptions**: Catch and handle specific exception types +- **User-friendly messages**: Error messages now explain what went wrong and how to fix it +- **Graceful degradation**: Script continues running even when non-critical operations fail +- **Visual feedback**: Uses colored output (✓, ✗, ⚠) to indicate status + +### 6. **Improved User Feedback** + +#### Before: +``` +Configuration: +Root path: E:\dev\Twitch-Archive-2\archive +Refresh rate: 60.0 +Email notifications: Enabled +``` + +#### After: +``` +============================================================ +TWITCH ARCHIVE - Configuration Summary +============================================================ + +Streamer: vinesauce +Quality: best +Storage: E:\dev\Twitch-Archive-2\archive +Refresh rate: 60s + +Email notifications: Enabled ✓ +Metadata download: Enabled ✓ +VOD download: Enabled ✓ +Chat download & render: Enabled ✓ +Cloud upload: Enabled ✓ + +✓ Files will be preserved locally +============================================================ +``` + +### 7. **Constants for Magic Values** +```python +# Before: Scattered throughout code +response = requests.post('https://gql.twitch.tv/gql', ...) + +# After: Named constants at top +TWITCH_GQL_URL = "https://gql.twitch.tv/gql" +response = requests.post(TWITCH_GQL_URL, ...) +``` + +### 8. **Cleaner Configuration Loading** +- Uses `DEFAULT_CONFIG` dictionary instead of inline defaults +- Better error messages when config.json is missing or invalid +- Filters comment fields more elegantly +- Validates JSON syntax and provides helpful error messages + +### 9. **Path Handling Improvements** +- Uses `pathlib.Path` consistently for cross-platform compatibility +- Centralized path initialization in `_initialize_paths()` method +- All paths are absolute to avoid confusion + +### 10. **Better Help Text** +The command-line help is now formatted with colors and clear sections: +``` +============================================================ +TWITCH ARCHIVE - Automated Stream Recording & Archiving +============================================================ + +USAGE: + python twitch-archive.py [OPTIONS] + +OPTIONS: + -h, --help Display this help information + -u, --username Twitch channel username to monitor + ... + +TIPS: + • Configure settings in config.json + • Set up API credentials in .env file + ... +``` + +### 11. **Safer File Operations** +- Checks if files exist before attempting to delete them +- Groups all file deletion in one method for easier tracking +- Prevents deletion if upload fails (data safety) + +### 12. **Code Readability Enhancements** +- Consistent indentation and spacing +- Logical variable names (e.g., `filename_base` instead of `live_raw_filename`) +- Removed redundant code (e.g., duplicate API calls) +- Consistent string formatting (f-strings everywhere) + +## How These Improvements Help Non-Python Programmers + +1. **Clearer Intent**: Type hints and docstrings make it obvious what each function does without needing to read the implementation + +2. **Easier Debugging**: Modular functions mean you can test individual pieces separately + +3. **Better Error Messages**: Instead of cryptic Python errors, users get helpful messages like: + ``` + ✗ ERROR: Twitch user "invaliduser" not found + → Check the username in your config.json file + ``` + +4. **Self-Documenting**: The code reads more like pseudocode with clear method names and constants + +5. **Standard Conventions**: Follows PEP 8 (Python style guide) making it easier to understand for anyone familiar with coding standards + +6. **Visual Organization**: Section headers and consistent formatting make it easy to navigate + +7. **Extensibility**: Each feature is isolated, making it easy to add new features or modify existing ones + +## Maintenance Benefits + +- **Easier Updates**: Modular design means changing one feature doesn't affect others +- **Testable**: Each method can be unit tested independently +- **Understandable**: Future developers can quickly understand what each part does +- **Documented Decisions**: Docstrings explain *why* things are done a certain way + +## Example: Before vs After + +### Before (Complex loopcheck): +```python +def loopcheck(self): + while True: + try: + is_live = self.check_user()['data']['user']['stream'] + if is_live is not None: + is_live_ready = self.check_user()['data']['user']['stream']['title'] + if is_live_ready is not None: + bin_path = str(pathlib.Path(__file__).parent.resolve())+"/bin" + live_date = datetime.strptime(is_live["createdAt"],'%Y-%m-%dT%H:%M:%SZ').replace(...) + # ... 200+ more lines of mixed logic ... +``` + +### After (Clean and modular): +```python +def loopcheck(self) -> None: + """ + Main monitoring loop. + + Continuously checks if the streamer is live, and when they are: + 1. Records the live stream + 2. Downloads the VOD + 3. Downloads and renders chat + 4. Uploads everything to cloud storage (if enabled) + 5. Optionally deletes local files after upload + """ + while True: + try: + response = self._check_stream_status() + is_live = response['data']['user']['stream'] + + if is_live is None: + time.sleep(self.refresh) + continue + + # Each step is a well-named method + stream_id = self._create_stream_id(is_live) + if self._is_stream_already_processed(stream_id): + continue + + self._record_livestream(...) + self._process_raw_stream(...) + self._download_vod(...) + # ... clear logical flow ... +``` + +## Backward Compatibility + +All improvements maintain backward compatibility: +- Same configuration file format +- Same command-line arguments +- Same file output structure +- Same external tool dependencies + +## Testing Recommendations + +After these improvements, test: +1. ✓ Configuration loading (valid and invalid config.json) +2. ✓ Username validation +3. ✓ Live stream recording +4. ✓ VOD downloading +5. ✓ Chat downloading +6. ✓ Cloud upload +7. ✓ File deletion safety +8. ✓ Error recovery + +## Future Enhancement Opportunities + +Now that the code is more maintainable, future enhancements could include: +- Configuration validation with schemas +- Logging to file instead of print statements +- Progress bars for downloads +- Multi-channel monitoring +- Web interface for configuration +- Database for tracking streams +- Automated testing suite + +## Streamlink Compatibility Update (February 2026) + +### Deprecated Options Removed +Updated the script to work with streamlink 5.0+ by removing deprecated command-line options: + +1. **`--twitch-disable-reruns`** - This option has been completely removed from streamlink + - The script now works without this option + - Twitch's rerun detection is handled differently in newer versions + +2. **`--twitch-proxy-playlist`** - The ttvlol ad-blocking proxy option has been removed + - If `streamlink_ttvlol` is enabled in config, the script will show a warning + - Users should disable this option or use alternative ad-blocking methods + - Consider using Twitch Turbo subscription for ad-free viewing + +3. **`--hls-segment-threads`** - Renamed to `--stream-segment-threads` + - The script already uses the correct new option name + - This applies to all segmented stream types (HLS, DASH, etc.) + - Valid values: 1-10 threads (default: 1) + +### What You Need to Do +1. Update streamlink to the latest version: + ```bash + pip install --upgrade streamlink + ``` + +2. Update your config.json: + - Set `streamlink_ttvlol` to `0` if you were using ad-blocking + - The other settings remain the same + +3. Run the script normally - it will work with the latest streamlink version diff --git a/README.md b/README.md index 188094b..fe0de47 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ CLIENT-ID=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx CLIENT-SECRET=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx OAUTH-PRIVATE-TOKEN=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx # optional to record without ADS or download sub-only VODS ``` -8. if you want to enable/disable more available options, edit `twitch-archive.py` +8. Copy `config.sample.json` to `config.json` and edit it with your settings (username, quality, paths, etc.) 9. run `Python twitch-archive.py` or for multiple streamers `Python twitch-archive.py -u streamer` diff --git a/bin/TwitchDownloaderCLI b/bin/TwitchDownloaderCLI index a74e3ce..e0338fd 100644 --- a/bin/TwitchDownloaderCLI +++ b/bin/TwitchDownloaderCLI @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a6b8d54355f465c500cb0374a354ca2808d89918061d330c9c683af561247ce9 -size 52832567 +oid sha256:f53ae4689ddd4ef8ff024b501a2b126b1b6bc83b1341b3b767e6dafb7af5324c +size 71051572 diff --git a/bin/TwitchDownloaderCLI.exe b/bin/TwitchDownloaderCLI.exe index 11d5639..f7a595a 100644 --- a/bin/TwitchDownloaderCLI.exe +++ b/bin/TwitchDownloaderCLI.exe @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:bcf0bd5de0d3589876515650466cfcd4e12eb95ede8c17ee9aca53b705d17392 -size 51579268 +oid sha256:c2f08e759ef110ed420bbb8419fd7b1cbcae59c80e964f270d7ff6ea1e05472c +size 68441306 diff --git a/config.sample.json b/config.sample.json new file mode 100644 index 0000000..4961184 --- /dev/null +++ b/config.sample.json @@ -0,0 +1,51 @@ +{ + "_comment": "Copy this file to config.json and edit with your settings", + + "username": "your_twitch_username", + "_username_comment": "Twitch streamer username to monitor and archive", + + "quality": "best", + "_quality_comment": "Quality options: best/source, high/720p, medium/540p, low/360p, audio_only", + + "root_path": "archive", + "_root_path_comment": "Path where this script saves everything (livestream, VODs, chat, metadata)", + + "rclone_path": "remote:path/to/streams", + "_rclone_path_comment": "Path to rclone remote storage (e.g., MyDrive:Backups/streams)", + + "refresh": 60.0, + "_refresh_comment": "Time between checking in seconds (5.0 is recommended), avoid less than 1.0", + + "streamlink_ttvlol": 0, + "_streamlink_ttvlol_comment": "0 = disable, 1 = enable blocking ads with ttvlol (DEPRECATED: --twitch-proxy-playlist removed in newer streamlink versions)", + + "notifications": 0, + "_notifications_comment": "0 = disable, 1 = enable email notifications", + + "downloadMETADATA": 1, + "_downloadMETADATA_comment": "0 = disable, 1 = enable metadata downloading", + + "downloadVOD": 1, + "_downloadVOD_comment": "0 = disable, 1 = enable VOD downloading after stream finished", + + "downloadCHAT": 1, + "_downloadCHAT_comment": "0 = disable, 1 = enable chat downloading and rendering", + + "uploadCloud": 1, + "_uploadCloud_comment": "0 = disable, 1 = enable upload to remote cloud", + + "deleteFiles": 0, + "_deleteFiles_comment": "0 = disable, 1 = enable deleting files after upload (BE CAREFUL WITH THIS OPTION)", + + "onlyRaw": 0, + "_onlyRaw_comment": "0 = convert ts files to mp3/mp4, 1 = keep only raw ts files for recording", + + "cleanRaw": 1, + "_cleanRaw_comment": "0 = keep raw .ts files, 1 = delete raw .ts files after processing", + + "hls_segments": 3, + "_hls_segments_comment": "Number of threads for live stream downloading (1-10, recommended 2-3)", + + "hls_segmentsVOD": 10, + "_hls_segmentsVOD_comment": "Number of threads for VOD downloading (1-10)" +} diff --git a/loopstart.bat b/loopstart.bat new file mode 100644 index 0000000..3f591f5 --- /dev/null +++ b/loopstart.bat @@ -0,0 +1,23 @@ +rem @echo off +setlocal + +rem Set the path to your virtual environment +set VENV_PATH=.\venv314 +rem Activate the virtual environment + call "%VENV_PATH%\Scripts\activate.bat" + rem pyenv-venv activate twitcharchive + + pip install -r requirements.txt + +:loop + + rem Run the desired command in the virtual environment + python twitch-archive.py -u %1 + + @REM rem if the program exits with a non-zero error code, go back to the beginning of the loop + @REM if %errorlevel% neq 0 ( + goto loop + @REM ) + +rem Deactivate the virtual environment +call "%VENV_PATH%\Scripts\deactivate.bat" diff --git a/requirements.txt b/requirements.txt index d4172c3..1aa42ed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ colorama==0.4.6 -python-dotenv==0.21.0 -pytz==2022.6 -requests==2.28.1 -streamlink==5.1.0 \ No newline at end of file +python-dotenv==1.0.1 +pytz==2024.2 +requests==2.32.3 +streamlink==8.2.0 \ No newline at end of file diff --git a/start.bat b/start.bat index 11965de..79b9174 100644 --- a/start.bat +++ b/start.bat @@ -2,22 +2,15 @@ rem @echo off setlocal rem Set the path to your virtual environment -set VENV_PATH=.\venv3 +set VENV_PATH=.\venv314 rem Activate the virtual environment call "%VENV_PATH%\Scripts\activate.bat" rem pyenv-venv activate twitcharchive pip install -r requirements.txt -:loop - rem Run the desired command in the virtual environment python twitch-archive.py -u %1 - @REM rem if the program exits with a non-zero error code, go back to the beginning of the loop - @REM if %errorlevel% neq 0 ( - goto loop - @REM ) - rem Deactivate the virtual environment call "%VENV_PATH%\Scripts\deactivate.bat" diff --git a/twitch-archive.py b/twitch-archive.py index 4854a0d..1f66e0d 100644 --- a/twitch-archive.py +++ b/twitch-archive.py @@ -1,308 +1,1244 @@ -import requests, os, time, json, sys, subprocess, getopt, smtplib, pathlib, socket -from colorama import Fore, Style +""" +Twitch Archive - Automated Twitch Stream Recording & Archiving System + +This script monitors a Twitch channel and automatically: +- Records live streams as they happen +- Downloads VODs (Video on Demand) after the stream ends +- Downloads and renders chat logs +- Saves stream metadata +- Uploads everything to cloud storage (optional) + +Requirements: +- Python 3.7+ +- External tools: streamlink, ffmpeg, TwitchDownloaderCLI, rclone (optional) +- Configuration file: config.json (copy from config.sample.json) +- Environment file: .env (for API credentials) +""" + +# Standard library imports +import os +import sys +import time +import json +import socket +import smtplib +import pathlib +import subprocess +import getopt +import signal +from typing import Dict, Optional, Any from datetime import datetime, timedelta + +# Third-party imports +import requests +from colorama import Fore, Style from pytz import timezone from dotenv import load_dotenv, find_dotenv from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText + +# ============================================================================ +# CONSTANTS - Configuration defaults and magic values +# ============================================================================ + +# API Endpoints +TWITCH_OAUTH_URL = "https://id.twitch.tv/oauth2/token" +TWITCH_API_URL = "https://api.twitch.tv/helix" +TWITCH_GQL_URL = "https://gql.twitch.tv/gql" +TWITCH_GQL_CLIENT_ID = "kimne78kx3ncx6brgo4mv6wki5h1ko" + +# File prefixes for different content types +PREFIX_LIVE = "LIVE_" +PREFIX_VOD = "VOD_" +PREFIX_CHAT = "CHAT_" +PREFIX_METADATA = "METADA_" # Note: keeping original typo for compatibility + +# Default configuration values +DEFAULT_CONFIG = { + 'username': 'your_twitch_username', + 'quality': 'best', + 'root_path': 'archive', + 'rclone_path': 'remote:path/to/streams', + 'refresh': 60.0, + 'streamlink_ttvlol': 0, + 'notifications': 0, + 'downloadMETADATA': 1, + 'downloadVOD': 1, + 'downloadCHAT': 1, + 'uploadCloud': 1, + 'deleteFiles': 0, + 'onlyRaw': 0, + 'cleanRaw': 1, + 'hls_segments': 3, + 'hls_segmentsVOD': 10 +} + +# ============================================================================ +# MAIN CLASS +# ============================================================================ + class TwitchArchive: + """ + Main class for the Twitch Archive system. + + Handles monitoring a Twitch channel, recording live streams, and downloading + VODs, chat logs, and metadata. Can optionally upload to cloud storage. + """ + def __init__(self): - # user configuration - self.username = "vinesauce" # Twitch streamer username - self.quality = "best" # Qualities options: best/source high/720p medium/540p low/360p audio_only - # global configuration - self.root_path = r"archive" # Path where this script saves everything (livestream,VODs,chat,metadata) - self.rclone_path = "MaddoNAS:Mediaroot/media/streams" # Path to rclone remote storage - self.refresh = 60.0 # Time between checking (5.0 is recommended), avoid less than 1.0 - self.streamlink_ttvlol = 0 # 0 - disable blocking ads with ttvlol, 1 - enable blocking ads with ttvlol, Uses this repo: https://github.com/2bc4/streamlink-ttvlol to block ads with ttvlol. Follows the steps on the repo to enable it. - self.notifications = 0 # 0 - disable email notification of current seccion, 1 - enable email notification of current seccion - self.downloadMETADATA = 1 # 0 - disable metadata downloading, 1 - enable metadata downloading - self.downloadVOD = 1 # 0 - disable VOD downloading after stream finished, 1 - enable VOD downloading after stream finished (this option downloads the latest public vod) - self.downloadCHAT = 1 # 0 - disable chat downloading and rendering, 1 - enable chat downloading and rendering - self.uploadCloud = 1 # 0 - disable upload to remote cloud, 1 - enable upload to remote cloud - self.deleteFiles = 0 # 0 - disable the deleting of files from current seccion after being uploaded to the cloud, 1 - enable the deleting files of files from current seccion after being uploaded to the cloud (BE CAREFUL WITH THIS OPTION) - self.onlyRaw = 0 # 0 - disable the converting of ts files to mp3/mp4, 1 - enable the converting of ts files to mp3/mp4 (only works for recording, the vod will still be downloaded to mp3/mp4) - self.cleanRaw = 1 # 0 - disable the deleting of raw (.ts) files, 1 - enable the deleteing of raw (.ts) files (if upload enable they will be deleted before) - self.hls_segments = 3 # 1-10 for live stream, it's possible to use multiple threads to potentially increase the throughput. 2-3 is enough - self.hls_segmentsVOD = 10 # 1-10 for downloading vod, it's possible to use multiple threads to potentially increase the throughput + """Initialize the TwitchArchive with configuration settings.""" + self.load_config() + self.os = self._detect_operating_system() + self.paths_initialized = False + self.shutdown_requested = False + self.current_process = None + self.current_stream_data = {} - def run(self): - self.os = self.get_OS() + def load_config(self) -> None: + """ + Load configuration from config.json file. - if load_dotenv(find_dotenv()): load_dotenv(find_dotenv()) + Falls back to default configuration if file is not found or cannot be read. + Filters out comment fields (starting with '_') from the config. + """ + config_file = os.path.join(os.path.dirname(__file__), 'config.json') + + # Start with default configuration + config = DEFAULT_CONFIG.copy() + + # Try to load and merge user configuration + if os.path.exists(config_file): + try: + with open(config_file, 'r', encoding='utf-8') as f: + user_config = json.load(f) + # Filter out comment fields (those starting with '_') + user_config = {k: v for k, v in user_config.items() if not k.startswith('_')} + # Merge user config with defaults (user config takes precedence) + config.update(user_config) + print(f'{Fore.GREEN}✓ Configuration loaded from config.json{Style.RESET_ALL}') + except json.JSONDecodeError as e: + print(f'{Fore.YELLOW}⚠ Warning: Invalid JSON in config.json: {e}{Style.RESET_ALL}') + print(f'{Fore.YELLOW} Using default configuration{Style.RESET_ALL}') + except Exception as e: + print(f'{Fore.YELLOW}⚠ Warning: Could not load config.json: {e}{Style.RESET_ALL}') + print(f'{Fore.YELLOW} Using default configuration{Style.RESET_ALL}') else: - print(f'{Fore.RED}\033[1mCREATE .env file with variables{Style.RESET_ALL}') - quit() - - self.correct_user() - - print('Twitch-Archive') - print('Configuration:') - print(f'Root path: {Fore.GREEN}' + str(pathlib.Path(self.root_path).resolve()) + f'{Style.RESET_ALL}') - print(f'Refresh rate: {Fore.GREEN} {str(self.refresh)}{Style.RESET_ALL}') - if self.notifications == 1: print(f'Email notifications: {Fore.GREEN}Enabled{Style.RESET_ALL}') - else: print(f'Email notifications: {Fore.RED}Disabled{Style.RESET_ALL}') - if self.downloadMETADATA == 1: print(f'Metada downloading {Fore.GREEN}Enabled{Style.RESET_ALL}') - else: print(f'Metada downloading: {Fore.RED}Disabled{Style.RESET_ALL}') - if self.downloadVOD == 1: print(f'VOD downloading {Fore.GREEN}Enabled{Style.RESET_ALL}') - else: print(f'VOD downloading: {Fore.RED}Disabled{Style.RESET_ALL}') - if self.downloadCHAT == 1: print(f'Chat downloading {Fore.GREEN}Enabled{Style.RESET_ALL}') - else: print(f'Chat downloading: {Fore.RED}Disabled{Style.RESET_ALL}') - if self.uploadCloud == 1: print(f'Upload to cloud storage: {Fore.GREEN}Enabled{Style.RESET_ALL}') - else: print(f'Upload to cloud storage: {Fore.RED}Disabled{Style.RESET_ALL}') - if self.deleteFiles == 1: print(f'{Fore.RED}'+'\033[1m'+f'CAREFUL FILES ARE CONFIGURATED TO BE DELETED{Style.RESET_ALL}') - else: print(f'{Fore.GREEN}'+'\033[1m'+f'Files will NOT be deleted{Style.RESET_ALL}') - if self.uploadCloud == 0 and self.deleteFiles == 1: print(f'{Fore.RED}'+'\033[1m'+f'FILES WILL BE DELETED AND NO UPLOADED {Style.RESET_ALL}{Fore.GREEN}\n"CTRL + C"{Style.RESET_ALL}{Fore.RED}'+'\033[1m'+f' TO STOP AND CHANGED CONFIGURATION{Style.RESET_ALL}') - - self.raw_path = str(pathlib.Path(os.path.join(self.root_path,self.username,"video", "raw")).absolute()) - self.video_path = str(pathlib.Path(os.path.join(self.root_path, self.username, "video")).absolute()) - self.chatJSON_path = str(pathlib.Path(os.path.join(self.root_path, self.username, "chat", "json")).absolute()) - self.chatMP4_path = str(pathlib.Path(os.path.join(self.root_path, self.username, "chat")).absolute()) - self.metadata_path = str(pathlib.Path(os.path.join(self.root_path, self.username, "metadata")).absolute()) - - if(os.path.isdir(self.raw_path) is False): os.makedirs(self.raw_path) - if(os.path.isdir(self.video_path) is False): os.makedirs(self.video_path) - if(os.path.isdir(self.chatJSON_path) is False): os.makedirs(self.chatJSON_path) - if(os.path.isdir(self.chatMP4_path) is False): os.makedirs(self.chatMP4_path) - if(os.path.isdir(self.metadata_path) is False): os.makedirs(self.metadata_path) - if not os.path.exists(os.path.join(self.root_path, ".log")): - with open(os.path.join(self.root_path, ".log"), 'w'): pass - - print(f"Checking for {Fore.GREEN}{self.username}{Style.RESET_ALL} every {Fore.GREEN}{self.refresh}{Style.RESET_ALL} seconds. Record with {Fore.GREEN}{self.quality}{Style.RESET_ALL} quality.") - self.sendNotif("TWITCH ARCHIVE", f"Checking for {self.username} every {self.refresh} seconds. Record with {self.quality} quality.") - self.loopcheck() - - def get_OS(self): + print(f'{Fore.YELLOW}⚠ Warning: config.json not found{Style.RESET_ALL}') + print(f'{Fore.CYAN} → Copy config.sample.json to config.json and edit it with your settings{Style.RESET_ALL}') + + # Set all configuration values as instance attributes + for key, value in config.items(): + setattr(self, key, value) + + def _detect_operating_system(self) -> str: + """ + Detect the current operating system. + + Returns: + str: 'windows' or 'linux' + + Raises: + SystemExit: If OS is not supported + """ if sys.platform.startswith('win32'): return 'windows' elif sys.platform.startswith('linux'): return 'linux' else: - print(f'{Fore.RED}\033[1mOS no supported{Style.RESET_ALL}') - quit() + print(f'{Fore.RED}✗ ERROR: Unsupported operating system: {sys.platform}{Style.RESET_ALL}') + print(f'{Fore.YELLOW} This script only supports Windows and Linux{Style.RESET_ALL}') + sys.exit(1) - def get_oauth_token(self): - try: - return requests.post(f"https://id.twitch.tv/oauth2/token?client_id={os.getenv('CLIENT-ID')}&client_secret={os.getenv('CLIENT-SECRET')}&grant_type=client_credentials").json()['access_token'] - except: - print(f'{Fore.RED}\033[1mCheck your client-id and secret{Style.RESET_ALL}') - quit() + + def _initialize_paths(self) -> None: + """ + Initialize all directory paths needed for archiving. + + Creates the directory structure: + - root_path/username/video/raw/ (for raw .ts files) + - root_path/username/video/ (for processed videos) + - root_path/username/chat/json/ (for chat JSON files) + - root_path/username/chat/ (for rendered chat videos) + - root_path/username/metadata/ (for stream metadata) + """ + # Convert all paths to absolute paths + self.raw_path = pathlib.Path(self.root_path, self.username, "video", "raw").absolute() + self.video_path = pathlib.Path(self.root_path, self.username, "video").absolute() + self.chatJSON_path = pathlib.Path(self.root_path, self.username, "chat", "json").absolute() + self.chatMP4_path = pathlib.Path(self.root_path, self.username, "chat").absolute() + self.metadata_path = pathlib.Path(self.root_path, self.username, "metadata").absolute() + + # Create directories if they don't exist + for path in [self.raw_path, self.video_path, self.chatJSON_path, + self.chatMP4_path, self.metadata_path]: + path.mkdir(parents=True, exist_ok=True) + + # Create log file if it doesn't exist + log_file = pathlib.Path(self.root_path, ".log") + if not log_file.exists(): + log_file.touch() + + self.paths_initialized = True - def correct_user(self): - try: - url = f'https://api.twitch.tv/helix/users?login={self.username}' - response = requests.get(url,headers = {"Authorization" : "Bearer " + self.get_oauth_token(), "Client-ID": os.getenv('CLIENT-ID')}, timeout = 15).json() - if response['data'] == []: - print(f'{Fore.RED}\033[1mUse a correct username{Style.RESET_ALL}') - quit() - except requests.exceptions.RequestException as e: - print(e) + def _load_environment_variables(self) -> None: + """ + Load environment variables from .env file. + + Required variables: + - CLIENT-ID: Twitch API client ID + - CLIENT-SECRET: Twitch API client secret + - OAUTH-PRIVATE-TOKEN: Optional, for accessing subscriber-only streams + - SENDER: Email address for notifications (if enabled) + - RECEIVER: Email address to receive notifications (if enabled) + - PASSWD: Email password for sending notifications (if enabled) + + Raises: + SystemExit: If .env file is not found + """ + if not load_dotenv(find_dotenv()): + print(f'{Fore.RED}✗ ERROR: .env file not found{Style.RESET_ALL}') + print(f'{Fore.CYAN} → Create a .env file with your Twitch API credentials{Style.RESET_ALL}') + print(f'{Fore.CYAN} → Required: CLIENT-ID, CLIENT-SECRET{Style.RESET_ALL}') + sys.exit(1) + + def _print_configuration_summary(self) -> None: + """Print a summary of the current configuration to the console.""" + print(f'\n{Fore.CYAN}{"=" * 60}{Style.RESET_ALL}') + print(f'{Fore.CYAN}TWITCH ARCHIVE - Configuration Summary{Style.RESET_ALL}') + print(f'{Fore.CYAN}{"=" * 60}{Style.RESET_ALL}\n') + + # Basic settings + print(f'Streamer: {Fore.GREEN}{self.username}{Style.RESET_ALL}') + print(f'Quality: {Fore.GREEN}{self.quality}{Style.RESET_ALL}') + print(f'Storage: {Fore.GREEN}{pathlib.Path(self.root_path).resolve()}{Style.RESET_ALL}') + print(f'Refresh rate: {Fore.GREEN}{self.refresh}s{Style.RESET_ALL}\n') + + # Feature toggles + self._print_toggle('Email notifications', self.notifications) + self._print_toggle('Metadata download', self.downloadMETADATA) + self._print_toggle('VOD download', self.downloadVOD) + self._print_toggle('Chat download & render', self.downloadCHAT) + self._print_toggle('Cloud upload', self.uploadCloud) + + # Warning messages + if self.deleteFiles == 1: + print(f'\n{Fore.RED}⚠ WARNING: Files will be DELETED after processing{Style.RESET_ALL}') + if self.uploadCloud == 0: + print(f'{Fore.RED}⚠ CRITICAL: Files will be deleted WITHOUT cloud backup!{Style.RESET_ALL}') + print(f'{Fore.YELLOW} Press CTRL+C to stop and change configuration{Style.RESET_ALL}') + else: + print(f'\n{Fore.GREEN}✓ Files will be preserved locally{Style.RESET_ALL}') + + print(f'\n{Fore.CYAN}{"=" * 60}{Style.RESET_ALL}\n') + + def _print_toggle(self, label: str, value: int) -> None: + """Helper method to print a configuration toggle in a consistent format.""" + status = f'{Fore.GREEN}Enabled{Style.RESET_ALL}' if value == 1 else f'{Fore.RED}Disabled{Style.RESET_ALL}' + print(f'{label}: {status}') - def check_user(self): - query = 'query{user(login: "' + self.username + '") {stream{archiveVideo{id}title createdAt}}}' - try: - response = requests.post('https://gql.twitch.tv/gql',json={'query': query},headers={"Client-ID": "kimne78kx3ncx6brgo4mv6wki5h1ko"}) - return json.loads(response.text) - except requests.exceptions.RequestException as e: - print(e) - quit() + def run(self) -> None: + """ + Main entry point for the application. + + Initializes environment, validates configuration, creates necessary + directories, and starts the monitoring loop. + """ + # Load environment variables + self._load_environment_variables() + + # Validate username + self._validate_username() + + # Initialize directory structure + self._initialize_paths() + + # Verify streamlink is available + self._verify_dependencies() + + # Print configuration summary + self._print_configuration_summary() + + # Start monitoring + print(f"Monitoring {Fore.GREEN}{self.username}{Style.RESET_ALL} every {Fore.GREEN}{self.refresh}s{Style.RESET_ALL}") + self.send_notification("TWITCH ARCHIVE STARTED", + f"Monitoring {self.username} every {self.refresh} seconds.") + + # Begin the main monitoring loop + self.loopcheck() - def get_vod(self): - query = 'query {user(login: "' + self.username + '") {videos(first: 1) {edges {node {id title description recordedAt lengthSeconds animatedPreviewURL previewThumbnailURL(height: 1280, width: 720) thumbnailURLs(height: 1280, width: 720)}}}}}' - try: - response = requests.post('https://gql.twitch.tv/gql', json={'query': query}, headers={"Client-ID": "kimne78kx3ncx6brgo4mv6wki5h1ko"}) - return json.loads(response.text) - except requests.exceptions.RequestException as e: - print(e) - def sendNotif(self, subject, content): - if self.notifications == 1: + def _get_oauth_token(self) -> str: + """ + Get OAuth token from Twitch API. + + Uses CLIENT-ID and CLIENT-SECRET from environment variables. + + Returns: + str: OAuth access token + + Raises: + SystemExit: If authentication fails + """ + try: + url = f"{TWITCH_OAUTH_URL}?client_id={os.getenv('CLIENT-ID')}&client_secret={os.getenv('CLIENT-SECRET')}&grant_type=client_credentials" + response = requests.post(url, timeout=15) + response.raise_for_status() + return response.json()['access_token'] + except requests.exceptions.RequestException as e: + print(f'{Fore.RED}✗ ERROR: Failed to authenticate with Twitch API{Style.RESET_ALL}') + print(f'{Fore.YELLOW} {str(e)}{Style.RESET_ALL}') + print(f'{Fore.CYAN} → Check your CLIENT-ID and CLIENT-SECRET in the .env file{Style.RESET_ALL}') + sys.exit(1) + except KeyError: + print(f'{Fore.RED}✗ ERROR: Invalid response from Twitch API{Style.RESET_ALL}') + print(f'{Fore.CYAN} → Verify your CLIENT-ID and CLIENT-SECRET are correct{Style.RESET_ALL}') + sys.exit(1) + + def _validate_username(self) -> None: + """ + Validate that the configured Twitch username exists. + + Raises: + SystemExit: If username is invalid or doesn't exist + """ + try: + url = f'{TWITCH_API_URL}/users?login={self.username}' + headers = { + "Authorization": f"Bearer {self._get_oauth_token()}", + "Client-ID": os.getenv('CLIENT-ID') + } + response = requests.get(url, headers=headers, timeout=15) + response.raise_for_status() + data = response.json() + + if not data.get('data'): + print(f'{Fore.RED}✗ ERROR: Twitch user "{self.username}" not found{Style.RESET_ALL}') + print(f'{Fore.CYAN} → Check the username in your config.json file{Style.RESET_ALL}') + sys.exit(1) + + print(f'{Fore.GREEN}✓ Username "{self.username}" validated{Style.RESET_ALL}') + + except requests.exceptions.RequestException as e: + print(f'{Fore.RED}✗ ERROR: Could not validate username{Style.RESET_ALL}') + print(f'{Fore.YELLOW} {str(e)}{Style.RESET_ALL}') + sys.exit(1) + + def _verify_dependencies(self) -> None: + """ + Verify that required external dependencies are available. + + Raises: + SystemExit: If required dependencies are not found + """ + # Check for streamlink + try: + result = subprocess.run(['streamlink', '--version'], + capture_output=True, + text=True, + timeout=5) + if result.returncode == 0: + version = result.stdout.strip().split()[1] if len(result.stdout.split()) > 1 else 'unknown' + print(f'{Fore.GREEN}✓ Streamlink v{version} found{Style.RESET_ALL}') + else: + raise FileNotFoundError() + except (FileNotFoundError, subprocess.TimeoutExpired, IndexError): + print(f'{Fore.RED}✗ ERROR: Streamlink not found{Style.RESET_ALL}') + print(f'{Fore.CYAN} → Install streamlink: pip install streamlink{Style.RESET_ALL}') + print(f'{Fore.CYAN} → Or download from: https://streamlink.github.io/{Style.RESET_ALL}') + sys.exit(1) + + # Check for ffmpeg + try: + ffmpeg_path = self._get_ffmpeg_executable() + if os.path.exists(ffmpeg_path): + print(f'{Fore.GREEN}✓ FFmpeg found at {ffmpeg_path}{Style.RESET_ALL}') + else: + print(f'{Fore.YELLOW}⚠ Warning: FFmpeg not found at {ffmpeg_path}{Style.RESET_ALL}') + print(f'{Fore.YELLOW} → Download FFmpeg and place it in the bin/ folder{Style.RESET_ALL}') + except Exception as e: + print(f'{Fore.YELLOW}⚠ Warning: Could not verify FFmpeg: {e}{Style.RESET_ALL}') + + # Check for TwitchDownloaderCLI (if VOD or Chat download enabled) + if self.downloadVOD == 1 or self.downloadCHAT == 1: try: - sender = os.getenv("SENDER") - receiver = os.getenv("RECEIVER") - msg = MIMEMultipart() - msg['From'] = sender - msg['To'] = receiver - msg['Subject'] = self.username + " _ " + subject - body = "Current seccion is for " + self.username + "\n\n\n\n" + content - msg.attach(MIMEText((body), 'plain')) - server = smtplib.SMTP('smtp.gmail.com', 587) - server.starttls() - server.login(sender, os.getenv("PASSWD")) - txt = msg.as_string() - server.sendmail(sender, receiver, txt) - server.quit() - except socket.error as e: - print(e) + downloader_path = self._get_twitch_downloader_executable() + if os.path.exists(downloader_path): + print(f'{Fore.GREEN}✓ TwitchDownloaderCLI found{Style.RESET_ALL}') + else: + print(f'{Fore.YELLOW}⚠ Warning: TwitchDownloaderCLI not found at {downloader_path}{Style.RESET_ALL}') + print(f'{Fore.YELLOW} → Download from: https://github.com/lay295/TwitchDownloader/releases{Style.RESET_ALL}') + except Exception as e: + print(f'{Fore.YELLOW}⚠ Warning: Could not verify TwitchDownloaderCLI: {e}{Style.RESET_ALL}') - def loopcheck(self): + + + def _check_stream_status(self) -> Optional[Dict[str, Any]]: + """ + Check if the configured user is currently live. + + Returns: + dict: Stream information if live, None if offline + + Raises: + SystemExit: If API request fails + """ + query = f'query{{user(login: "{self.username}") {{stream{{archiveVideo{{id}}title createdAt}}}}}}' + + try: + response = requests.post( + TWITCH_GQL_URL, + json={'query': query}, + headers={"Client-ID": TWITCH_GQL_CLIENT_ID}, + timeout=15 + ) + response.raise_for_status() + return response.json() + + except requests.exceptions.RequestException as e: + print(f'{Fore.RED}✗ ERROR: Failed to check stream status{Style.RESET_ALL}') + print(f'{Fore.YELLOW} {str(e)}{Style.RESET_ALL}') + sys.exit(1) + + def _get_latest_vod(self) -> Optional[Dict[str, Any]]: + """ + Get the most recent VOD for the configured user. + + Returns: + dict: VOD information, or None if no VODs found + """ + query = f'query {{user(login: "{self.username}") {{videos(first: 1) {{edges {{node {{id title description recordedAt lengthSeconds animatedPreviewURL previewThumbnailURL(height: 1280, width: 720) thumbnailURLs(height: 1280, width: 720)}}}}}}}}}}' + + try: + response = requests.post( + TWITCH_GQL_URL, + json={'query': query}, + headers={"Client-ID": TWITCH_GQL_CLIENT_ID}, + timeout=15 + ) + response.raise_for_status() + return response.json() + + except requests.exceptions.RequestException as e: + print(f'{Fore.YELLOW}⚠ Warning: Could not fetch latest VOD{Style.RESET_ALL}') + print(f'{Fore.YELLOW} {str(e)}{Style.RESET_ALL}') + return None + + + def _get_unique_filename(self, filepath: str) -> str: + """ + Generate a unique filename by appending a counter if file already exists. + + Args: + filepath: The desired file path + + Returns: + str: A unique file path (original or with _N suffix) + + Example: + If 'video.mp4' exists, returns 'video_1.mp4' + If 'video_1.mp4' also exists, returns 'video_2.mp4' + """ + if not os.path.exists(filepath): + return filepath + + # Split into components + directory = os.path.dirname(filepath) + filename = os.path.basename(filepath) + name, ext = os.path.splitext(filename) + + # Find next available counter + counter = 1 while True: - is_live = self.check_user()['data']['user']['stream'] - if is_live is not None: - is_live_ready = self.check_user()['data']['user']['stream']['title'] - if is_live_ready is not None: - bin_path = str(pathlib.Path(__file__).parent.resolve())+"/bin" - live_date = datetime.strptime(is_live["createdAt"],'%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=timezone('UTC')).astimezone(tz=None).replace(tzinfo=None) - live_raw_filename = datetime.strftime(live_date,'%Y%m%d_%Hh%Mm%Ss') - live_raw_path = os.path.join(self.raw_path, "LIVE_" + live_raw_filename + ".ts") - live_proc_path = os.path.join(self.video_path, "LIVE_" + live_raw_filename + ".mp4") - - with open(os.path.join(self.root_path, ".log"), encoding="utf-8") as logs: - logs = logs.read() - log_id = is_live["createdAt"] + " - " + self.username + " - " + is_live["title"] - if log_id in logs: - time.sleep(self.refresh) + new_filepath = os.path.join(directory, f"{name}_{counter}{ext}") + if not os.path.exists(new_filepath): + return new_filepath + counter += 1 - with open(os.path.join(self.root_path, ".log"), "r+", encoding="utf-8") as logs: - log_id = is_live["createdAt"] + " - " + self.username + " - " + is_live["title"] - for line in logs: - if log_id in line: - time.sleep(self.refresh) - else: - logs.write(is_live["createdAt"] + " - " + self.username + " - " + is_live["title"] +"\n") - - if self.streamlink_ttvlol == 1:ttvlol = ['--twitch-proxy-playlist=https://api.ttv.lol'] - else: ttvlol = ''.split() + def send_notification(self, subject: str, content: str) -> None: + """ + Send email notification via Gmail SMTP. + + Only sends if notifications are enabled in configuration. + Requires SENDER, RECEIVER, and PASSWD in .env file. + + Args: + subject: Email subject line + content: Email body content + """ + if self.notifications != 1: + return + + try: + sender = os.getenv("SENDER") + receiver = os.getenv("RECEIVER") + password = os.getenv("PASSWD") + + if not all([sender, receiver, password]): + print(f'{Fore.YELLOW}⚠ Notification skipped: Missing email credentials in .env{Style.RESET_ALL}') + return + + # Construct email + msg = MIMEMultipart() + msg['From'] = sender + msg['To'] = receiver + msg['Subject'] = f"{self.username} - {subject}" + + body = f"Stream: {self.username}\n\n{content}" + msg.attach(MIMEText(body, 'plain')) + + # Send via Gmail SMTP + with smtplib.SMTP('smtp.gmail.com', 587) as server: + server.starttls() + server.login(sender, password) + server.sendmail(sender, receiver, msg.as_string()) + + except socket.error as e: + print(f'{Fore.YELLOW}⚠ Notification failed: {str(e)}{Style.RESET_ALL}') + except Exception as e: + print(f'{Fore.YELLOW}⚠ Notification error: {str(e)}{Style.RESET_ALL}') - if self.quality == 'audio_only': - live_proc_path = os.path.join(self.video_path, "LIVE_" + live_raw_filename + ".mp3") - - if (os.getenv("OAUTH-PRIVATE-TOKEN") != "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"): - auth = ['--twitch-api-header', 'Authorization=OAuth ' + os.getenv('OAUTH-PRIVATE-TOKEN')] - else: auth = ''.split() - self.sendNotif('Stream - ' + live_raw_filename, 'Streamer went live: ' + is_live["title"]) - subprocess.call(['streamlink', 'twitch.tv/'+ self.username, self.quality, '--hls-segment-threads', str(self.hls_segments), '--hls-live-restart', '--retry-streams', str(self.refresh), '--twitch-disable-reruns', '-o', live_raw_path] + ttvlol + auth) - if(os.path.exists(live_raw_path) is True and self.onlyRaw == 0): - ffmpeg_settings = ['-y', '-i', live_raw_path, '-analyzeduration', '2147483647', '-probesize', '2147483647', '-c:v', 'copy', '-c:a', 'copy', '-start_at_zero', '-copyts', live_proc_path] - if self.quality == 'audio_only': ffmpeg_settings = ['-i', live_raw_path, '-vn', '-ar', '44100', '-ac', '2', '-b:a', '192k', live_proc_path] - if self.os == 'windows': subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/ffmpeg.exe']+ ffmpeg_settings, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) - elif self.os == 'linux': subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/ffmpeg']+ ffmpeg_settings, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) - else: - print("Skip fixing. File not found.") - - current_vod = self.get_vod()['data']['user']['videos']['edges'][0]['node'] - live_date_min = live_date - timedelta(minutes=1) - live_date_max = live_date + timedelta(minutes=1) - - vod_date = datetime.strptime(current_vod["recordedAt"],'%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=timezone('UTC')).astimezone(tz=None).replace(tzinfo=None) - - if live_date_min <= vod_date <= live_date_max: - vod_proc_path = os.path.join(self.video_path, "VOD_" + live_raw_filename + ".mp4") - chat_json_path = os.path.join(self.chatJSON_path, "CHAT_" + live_raw_filename + ".json") - chat_video_path = os.path.join(self.chatMP4_path, "CHAT_" + live_raw_filename + ".mp4") - if self.downloadMETADATA == 1: - self.sendNotif('Metadata - ' + live_raw_filename,'Downloading and saving metadata:\n' + json.dumps(current_vod, indent=4)) - with open(os.path.join(self.metadata_path, "METADA_" + live_raw_filename + ".json"), 'w', encoding='utf-8') as f: - json.dump(current_vod, f, ensure_ascii=False, indent=4) - - if self.quality == 'audio_only': - vod_proc_path = os.path.join(self.video_path, "VOD_" + live_raw_filename + ".mp3") - - if self.downloadVOD == 1: - print('Downloading VOD: ' + current_vod["title"]) - self.sendNotif('VOD - ' + live_raw_filename,'Downloading VOD: ' + current_vod["title"]) - try: - if self.os == 'windows':subprocess.call([bin_path+"/TwitchDownloaderCLI.exe", 'videoDownload', '-u', str(current_vod["id"]), '-q', self.quality, "-t", str(self.hls_segmentsVOD), "--ffmpeg-path", bin_path +"/ffmpeg.exe", '--temp-path', bin_path+"/temp", "-o", vod_proc_path]) - elif self.os == 'linux':subprocess.call([bin_path+"/TwitchDownloaderCLI", 'videoDownload', '-u', str(current_vod["id"]), '-q', self.quality, "-t", str(self.hls_segmentsVOD), "--ffmpeg-path", bin_path +"/ffmpeg.exe", '--temp-path', bin_path+"/temp", "-o", vod_proc_path]) - - except Exception as e: - print('Error', 'A ERROR has ocurred and the VOD will not be downloaded.\n') - self.sendNotif('ERROR - ' + live_raw_filename, 'A ERROR has ocurred and the VOD will not be downloaded.\n') - - if self.downloadCHAT == 1: - print('Downloading and rendering CHAT: ' + current_vod["title"]) - self.sendNotif('CHAT - ' + live_raw_filename,'Downloading JSON and rendering chat logs from VOD:\n' + current_vod["title"]) - chat_settings = ["--background-color", "#FF111111", "-w", "500", "-h", "1080", "--outline", "true", "-f", "Arial", "--font-size", "22", "--update-rate", "1.0", "--offline", "--ffmpeg-path", f"{bin_path}/ffmpeg.exe", "--temp-path", f"{bin_path}/temp"] - try: - if self.os == 'windows': - subprocess.call([bin_path + '/TwitchDownloaderCLI.exe', 'chatdownload', '--id', current_vod["id"], '-o', chat_json_path, '-E']) - print('Rendering CHAT: ' + current_vod["title"]) - subprocess.call([bin_path + '/TwitchDownloaderCLI.exe', 'chatrender', '-i', chat_json_path, '-o', chat_video_path] + chat_settings) - elif self.os == 'linux': - subprocess.call([bin_path + '/TwitchDownloaderCLI', 'chatdownload', '--id', current_vod["id"], '-o', chat_json_path, '-E']) - subprocess.call([bin_path + '/TwitchDownloaderCLI', 'chatrender', '-i', chat_json_path, '-o', chat_video_path] + chat_settings) - except Exception as e: - self.sendNotif('ERROR - ' + live_raw_filename, "A ERROR has ocurred and chat will need to be downloaded and rendered manually.\n") - print("A ERROR has ocurred and chat will need to be downloaded and rendered manually\n") - else: - print('not VOD associated with stream found') - if self.cleanRaw == 1: - print('Deleting raw files') - if(os.path.exists(live_raw_path) is True): os.remove(live_raw_path) - if self.uploadCloud == 1: - with open(str(pathlib.Path(__file__).parent.resolve())+"/bin/temp/upload.txt", "a") as myfile: - myfile.write("LIVE_" + live_raw_filename + ".ts\n"+"VOD_" + live_raw_filename + ".ts\n"+"LIVE_" + live_raw_filename + ".mp4\n"+"VOD_" + live_raw_filename + ".mp4\n"+"METADATA_" + live_raw_filename + ".json\n"+"CHAT_" + live_raw_filename + ".json\n"+"CHAT_" + live_raw_filename + ".mp4\n") - print('Uploading files') - self.sendNotif("UPLOADING - " + live_raw_filename, 'The files are being uploaded') - subprocess.call(['rclone', 'copy', str(pathlib.Path(self.root_path).resolve()), self.rclone_path, '--include-from', bin_path + '/temp/upload.txt']) - os.remove(str(pathlib.Path(__file__).parent.resolve())+"/bin/temp/upload.txt") - if self.deleteFiles == 1: - self.sendNotif("DELETING - " + live_raw_filename, "Deleting the files from current seccion.") - print(f'{Fore.RED}DELETING FILES{Style.RESET_ALL}') - if self.cleanRaw == 0: - print(f'{Fore.RED}Deleting ' + live_raw_path + f'{Style.RESET_ALL}') - os.remove(live_raw_path) - print(f'{Fore.RED}Deleting ' + live_proc_path + f'{Style.RESET_ALL}') - os.remove(live_proc_path) - if self.downloadVOD == 1: - if(os.path.exists(os.path.join(self.raw_path, "VOD_" + live_raw_filename + ".ts")) is True): - if self.cleanRaw == 0: - print(f'{Fore.RED}Deleting ' + os.path.join(self.raw_path, "VOD_" + live_raw_filename + ".ts") + f'{Style.RESET_ALL}') - os.remove(os.path.join(self.raw_path, "VOD_" + live_raw_filename + ".ts")) - if(os.path.exists(os.path.join(self.video_path, "VOD_" + live_raw_filename + ".mp4")) is True): - print(f'{Fore.RED}Deleting ' + os.path.join(self.video_path, "VOD_" + live_raw_filename + ".mp4") + f'{Style.RESET_ALL}') - os.remove(os.path.join(self.video_path, "VOD_" + live_raw_filename + ".mp4")) - if self.downloadCHAT == 1: - if(os.path.exists(os.path.join(self.chatJSON_path, "CHAT_"+live_raw_filename + ".json")) is True): - print(f'{Fore.RED}Deleting ' + os.path.join(self.chatJSON_path, "CHAT_"+live_raw_filename + ".json") + f'{Style.RESET_ALL}') - os.remove(os.path.join(self.chatJSON_path, "CHAT_"+live_raw_filename + ".json")) - if(os.path.exists(os.path.join(self.chatMP4_path, "CHAT_"+live_raw_filename + ".mp4")) is True): - print(f'{Fore.RED}Deleting ' + os.path.join(self.chatMP4_path, "CHAT_"+live_raw_filename + ".mp4") + f'{Style.RESET_ALL}') - os.remove(os.path.join(self.chatMP4_path, "CHAT_"+live_raw_filename + ".mp4")) - if self.downloadMETADATA == 1: - if(os.path.exists(os.path.join(self.metadata_path, "METADA_"+live_raw_filename+".json")) is True): - print(f'{Fore.RED}Deleting ' + os.path.join(self.metadata_path, "METADA_"+live_raw_filename+".json") + f'{Style.RESET_ALL}') - os.remove(os.path.join(self.metadata_path, "METADA_"+live_raw_filename+".json")) - print('CURRENT SECCION HAVE FINISHED GOING BACK TO CHECKING') - self.sendNotif("SECCION DONE - " + live_raw_filename, 'CURRENT SECCION HAVE FINISHED GOING BACK TO CHECKING') - time.sleep(self.refresh) - else: time.sleep(self.refresh) - else: time.sleep(self.refresh) -def main(argv): - twitch_archive = TwitchArchive() - help_msg = ''' - Twitch-Archive - Python script to record twitch live stream, download the VOD, metadata, chat and render it, and uploads them to any cloud storage. + def _is_stream_already_processed(self, stream_id: str) -> bool: + """ + Check if a stream has already been processed. + + Args: + stream_id: Unique identifier for the stream + + Returns: + bool: True if already processed, False otherwise + """ + log_file = pathlib.Path(self.root_path, ".log") + with open(log_file, 'r', encoding='utf-8') as f: + return stream_id in f.read() - -h, --help Display this information - -u, --username Twitch channel username - -q, --quality best/source high/720p medium/480p worst/360p - -a, --ttv-lol <1/0> Block ads with ttv-lol https://github.com/2bc4/streamlink-ttvlol - -v, --vod <1/0> Download vod - -c, --chat <1/0> Download chat and render it - -m, --metadata <1/0> Download metadata - -r, --upload <1/0> Upload to cloud storage - -d, --delete <1/0> Delete all files after upload (CAREFUL with this arg) - -n, --notifications <1/0> Receive email notification of the proccess through gmail + def _mark_stream_as_processed(self, stream_id: str) -> None: + """Add stream to log file to prevent re-processing.""" + log_file = pathlib.Path(self.root_path, ".log") + with open(log_file, 'a', encoding='utf-8') as f: + f.write(f"{stream_id}\n") + + def _get_bin_path(self) -> str: + """Get the path to the bin directory containing external tools.""" + return str(pathlib.Path(__file__).parent.resolve() / "bin") + + def _get_ffmpeg_executable(self) -> str: + """Get the platform-specific ffmpeg executable path.""" + bin_path = self._get_bin_path() + if self.os == 'windows': + return os.path.join(bin_path, 'ffmpeg.exe') + return os.path.join(bin_path, 'ffmpeg') + + def _get_twitch_downloader_executable(self) -> str: + """Get the platform-specific TwitchDownloaderCLI executable path.""" + bin_path = self._get_bin_path() + if self.os == 'windows': + return os.path.join(bin_path, 'TwitchDownloaderCLI.exe') + return os.path.join(bin_path, 'TwitchDownloaderCLI') + + def _record_livestream(self, stream_info: Dict[str, Any], output_path: str) -> bool: + """ + Record a live Twitch stream using streamlink. + + Args: + stream_info: Stream metadata from Twitch API + output_path: Path where the raw .ts file will be saved + + Returns: + bool: True if recording completed normally, False if interrupted + """ + print(f'\n{Fore.CYAN}{"=" * 60}{Style.RESET_ALL}') + print(f'{Fore.GREEN}🔴 STREAM STARTED: {stream_info["title"]}{Style.RESET_ALL}') + print(f'{Fore.CYAN}{"=" * 60}{Style.RESET_ALL}\n') + + # Build streamlink command + cmd = [ + 'streamlink', + f'twitch.tv/{self.username}', + self.quality, + '--hls-live-restart', + '--retry-streams', str(int(self.refresh)), + '--force', + '-o', output_path + ] + + # Add segment threads for faster downloads (requires streamlink 5.0+) + # This allows multiple segments to be downloaded in parallel + if self.hls_segments > 1: + cmd.extend(['--stream-segment-threads', str(self.hls_segments)]) + + # Add ad-blocking if enabled (Note: twitch-proxy-playlist was removed in newer streamlink versions) + # For ad-blocking, you may need to use alternative methods like --twitch-low-latency + # or rely on Twitch's own ad-free viewing for subscribers + if self.streamlink_ttvlol == 1: + # The old --twitch-proxy-playlist option has been removed from streamlink + # Consider using alternative ad-blocking approaches or updating your method + print(f'{Fore.YELLOW}⚠ Warning: ttv-lol proxy option is deprecated in newer streamlink versions{Style.RESET_ALL}') + print(f'{Fore.YELLOW} Consider disabling streamlink_ttvlol in config or using alternative methods{Style.RESET_ALL}') + + # Add authentication if available + oauth_token = os.getenv("OAUTH-PRIVATE-TOKEN", "") + if oauth_token and oauth_token != "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx": + cmd.extend(['--twitch-api-header', f'Authorization=OAuth {oauth_token}']) + + # Show command being executed (hide OAuth token for security) + cmd_display = [c if 'OAuth' not in str(c) else 'Authorization=OAuth [HIDDEN]' for c in cmd] + print(f'{Fore.CYAN}Command: {" ".join(cmd_display)}{Style.RESET_ALL}') + + # Record the stream (this blocks until stream ends) + print(f'{Fore.YELLOW}Recording stream...{Style.RESET_ALL}') + try: + self.current_process = subprocess.Popen(cmd) + return_code = self.current_process.wait() + self.current_process = None + + if self.shutdown_requested: + print(f'{Fore.YELLOW}✓ Recording stopped by user{Style.RESET_ALL}') + return False + + print(f'{Fore.GREEN}✓ Stream recording complete{Style.RESET_ALL}') + return True + except Exception as e: + self.current_process = None + print(f'{Fore.RED}✗ Recording error: {str(e)}{Style.RESET_ALL}') + return False + + def _process_raw_stream(self, raw_path: str, output_path: str) -> None: + """ + Process raw .ts file into mp4/mp3 using ffmpeg. + + Args: + raw_path: Path to the raw .ts file + output_path: Path for the processed output file + """ + if not os.path.exists(raw_path): + print(f'{Fore.YELLOW}⚠ Raw file not found, skipping processing{Style.RESET_ALL}') + return + + if self.onlyRaw == 1: + print(f'{Fore.CYAN}Keeping raw .ts file (onlyRaw mode){Style.RESET_ALL}') + return + + print(f'{Fore.YELLOW}Processing raw stream file...{Style.RESET_ALL}') + + # Build ffmpeg command based on quality + if self.quality == 'audio_only': + # Audio-only conversion + cmd = [ + self._get_ffmpeg_executable(), + '-i', raw_path, + '-vn', # No video + '-ar', '44100', # Audio sample rate + '-ac', '2', # Audio channels (stereo) + '-b:a', '192k', # Audio bitrate + output_path + ] + else: + # Video conversion (copy streams for speed) + cmd = [ + self._get_ffmpeg_executable(), + '-y', # Overwrite output file + '-i', raw_path, + '-analyzeduration', '2147483647', + '-probesize', '2147483647', + '-c:v', 'copy', # Copy video codec (fast) + '-c:a', 'copy', # Copy audio codec (fast) + '-start_at_zero', + '-copyts', + output_path + ] + + # Run ffmpeg (suppress output) + subprocess.call(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) + print(f'{Fore.GREEN}✓ Stream processed successfully{Style.RESET_ALL}') + + def _download_vod(self, vod_info: Dict[str, Any], output_path: str) -> bool: + """ + Download VOD using TwitchDownloaderCLI. + + Args: + vod_info: VOD metadata from Twitch API + output_path: Path where the VOD will be saved + + Returns: + bool: True if download succeeded, False otherwise + """ + if self.downloadVOD != 1: + return False + + print(f'\n{Fore.CYAN}Downloading VOD: {vod_info["title"]}{Style.RESET_ALL}') + + # Extract numeric VOD ID (TwitchDownloaderCLI expects just the number) + vod_id = vod_info["id"] + # Remove 'v' prefix if present (API sometimes returns "v123456789") + if isinstance(vod_id, str) and vod_id.startswith('v'): + vod_id = vod_id[1:] + + # Build URL format that TwitchDownloaderCLI accepts + vod_url = f"https://www.twitch.tv/videos/{vod_id}" + + print(f'{Fore.YELLOW}VOD URL: {vod_url}{Style.RESET_ALL}') + + bin_path = self._get_bin_path() + cmd = [ + self._get_twitch_downloader_executable(), + 'videodownload', + '-u', vod_url, + '-q', self.quality, + '-t', str(self.hls_segmentsVOD), + '--ffmpeg-path', self._get_ffmpeg_executable(), + '--temp-path', os.path.join(bin_path, 'temp'), + '--collision', 'Rename', + '-o', output_path + ] + + try: + result = subprocess.call(cmd) + if result == 0: + print(f'{Fore.GREEN}✓ VOD downloaded{Style.RESET_ALL}') + return True + else: + print(f'{Fore.RED}✗ VOD download failed with exit code: {result}{Style.RESET_ALL}') + return False + except Exception as e: + print(f'{Fore.RED}✗ VOD download failed: {str(e)}{Style.RESET_ALL}') + self.send_notification('VOD Download Error', f'Failed to download VOD: {str(e)}') + return False + + def _download_and_render_chat(self, vod_info: Dict[str, Any], json_path: str, video_path: str) -> bool: + """ + Download chat logs and render them as video. + + Args: + vod_info: VOD metadata from Twitch API + json_path: Path to save chat JSON + video_path: Path to save rendered chat video + + Returns: + bool: True if succeeded, False otherwise + """ + if self.downloadCHAT != 1: + return False + + print(f'\n{Fore.CYAN}Downloading chat: {vod_info["title"]}{Style.RESET_ALL}') + + # Extract numeric VOD ID + vod_id = vod_info["id"] + if isinstance(vod_id, str) and vod_id.startswith('v'): + vod_id = vod_id[1:] + + bin_path = self._get_bin_path() + downloader = self._get_twitch_downloader_executable() + + # Chat rendering settings + chat_settings = [ + '--background-color', '#FF111111', + '-w', '500', + '-h', '1080', + '--outline', + '-f', 'Arial', + '--font-size', '22', + '--update-rate', '1.0', + '--offline', + '--ffmpeg-path', self._get_ffmpeg_executable(), + '--temp-path', os.path.join(bin_path, 'temp'), + '--collision', 'Rename' + ] + + try: + # Download chat JSON + print(f'{Fore.YELLOW}Downloading chat JSON for VOD {vod_id}...{Style.RESET_ALL}') + result = subprocess.call([ + downloader, 'chatdownload', + '--id', vod_id, + '--embed-images', + '--collision', 'Rename', + '-o', json_path + ]) + + if result != 0: + print(f'{Fore.RED}✗ Chat JSON download failed with exit code: {result}{Style.RESET_ALL}') + return False + + # Verify JSON file was created + if not os.path.exists(json_path): + print(f'{Fore.RED}✗ Chat JSON file was not created{Style.RESET_ALL}') + return False + + print(f'{Fore.GREEN}✓ Chat JSON downloaded{Style.RESET_ALL}') + + # Render chat video + print(f'{Fore.YELLOW}Rendering chat video...{Style.RESET_ALL}') + result = subprocess.call([ + downloader, 'chatrender', + '-i', json_path, + '-o', video_path + ] + chat_settings) + + if result != 0: + print(f'{Fore.RED}✗ Chat render failed with exit code: {result}{Style.RESET_ALL}') + return False + + print(f'{Fore.GREEN}✓ Chat rendered{Style.RESET_ALL}') + return True + + except Exception as e: + print(f'{Fore.RED}✗ Chat processing failed: {str(e)}{Style.RESET_ALL}') + self.send_notification('Chat Download Error', + f'Failed to download/render chat: {str(e)}') + return False + + def _save_metadata(self, vod_info: Dict[str, Any], filename_base: str) -> None: + """ + Save VOD metadata to JSON file. + + Args: + vod_info: VOD metadata from Twitch API + filename_base: Base filename (without extension) + """ + if self.downloadMETADATA != 1: + return + + metadata_path = os.path.join(self.metadata_path, f"{PREFIX_METADATA}{filename_base}.json") + + with open(metadata_path, 'w', encoding='utf-8') as f: + json.dump(vod_info, f, ensure_ascii=False, indent=4) + + print(f'{Fore.GREEN}✓ Metadata saved{Style.RESET_ALL}') + + def _signal_handler(self, signum, frame): + """Handle interrupt signals gracefully.""" + if not self.shutdown_requested: + print(f'\n{Fore.YELLOW}{"=" * 60}{Style.RESET_ALL}') + print(f'{Fore.YELLOW}⚠ Shutdown requested. Stopping downloads and finalizing...{Style.RESET_ALL}') + print(f'{Fore.YELLOW}{"=" * 60}{Style.RESET_ALL}\n') + self.shutdown_requested = True + + # Stop current subprocess if running + if self.current_process: + try: + self.current_process.terminate() + print(f'{Fore.YELLOW}Stopping current download process...{Style.RESET_ALL}') + except Exception: + pass + + def _interruptible_sleep(self, seconds: float) -> bool: + """ + Sleep for the specified duration, but check for shutdown periodically. + + Args: + seconds: Number of seconds to sleep + + Returns: + bool: True if sleep completed, False if interrupted by shutdown + """ + start_time = time.time() + while time.time() - start_time < seconds: + if self.shutdown_requested: + return False + time.sleep(min(1.0, seconds - (time.time() - start_time))) + return True + + def loopcheck(self) -> None: + """ + Main monitoring loop. + + Continuously checks if the streamer is live, and when they are: + 1. Records the live stream + 2. Downloads the VOD + 3. Downloads and renders chat + 4. Uploads everything to cloud storage (if enabled) + 5. Optionally deletes local files after upload + """ + # Set up signal handlers for graceful shutdown + signal.signal(signal.SIGINT, self._signal_handler) + # SIGTERM is not available on Windows, handle gracefully + if hasattr(signal, 'SIGTERM'): + signal.signal(signal.SIGTERM, self._signal_handler) + + while not self.shutdown_requested: + try: + # Check stream status + response = self._check_stream_status() + is_live = response['data']['user']['stream'] + + # Stream is offline + if is_live is None: + print(f'{Fore.CYAN}⏳ {self.username} is offline. Checking again in {self.refresh}s...{Style.RESET_ALL}', end='\r') + if self.shutdown_requested: + break + self._interruptible_sleep(self.refresh) + continue + + # Stream is live but not ready yet + if not is_live.get('title'): + print(f'{Fore.YELLOW}⚠ Stream detected but no title yet. Waiting...{Style.RESET_ALL}') + if self.shutdown_requested: + break + self._interruptible_sleep(self.refresh) + continue + + # Stream is live and ready! + print(f'\n{Fore.GREEN}✓ {self.username} is LIVE!{Style.RESET_ALL}') + print(f'{Fore.CYAN}Title: {is_live["title"]}{Style.RESET_ALL}') + + # Create unique stream identifier based on stream start time + stream_id = f"{is_live['createdAt']} - {self.username} - {is_live['title']}" + + # Parse stream start time + live_date = datetime.strptime( + is_live["createdAt"], '%Y-%m-%dT%H:%M:%SZ' + ).replace(tzinfo=timezone('UTC')).astimezone(tz=None).replace(tzinfo=None) + + # Use CURRENT time for filename to ensure each recording is unique + # This allows recording a live stream multiple times (e.g., if script restarts) + current_time = datetime.now() + filename_base = current_time.strftime('%Y%m%d_%Hh%Mm%Ss') + + # Check if we've already recorded this stream session + if self._is_stream_already_processed(stream_id): + print(f'{Fore.YELLOW}⚠ Stream was previously recorded, but it\'s still live!{Style.RESET_ALL}') + print(f'{Fore.GREEN}✓ Starting new recording with timestamp: {filename_base}{Style.RESET_ALL}') + else: + # First time seeing this stream - mark it + self._mark_stream_as_processed(stream_id) + print(f'{Fore.GREEN}✓ New stream detected - starting recording{Style.RESET_ALL}') + + # Determine file paths + live_raw_path = os.path.join(self.raw_path, f"{PREFIX_LIVE}{filename_base}.ts") + live_proc_ext = '.mp3' if self.quality == 'audio_only' else '.mp4' + live_proc_path = os.path.join(self.video_path, f"{PREFIX_LIVE}{filename_base}{live_proc_ext}") + + # Ensure unique filenames + live_raw_path = self._get_unique_filename(live_raw_path) + live_proc_path = self._get_unique_filename(live_proc_path) + filename_base = os.path.splitext(os.path.basename(live_raw_path))[0].replace(PREFIX_LIVE, "") + + print(f'{Fore.CYAN}Output path: {live_raw_path}{Style.RESET_ALL}') + + # Send notification + self.send_notification(f'🔴 Stream Started - {filename_base}', + f'Title: {is_live["title"]}') + + # Store current stream data for potential graceful shutdown + self.current_stream_data = { + 'filename_base': filename_base, + 'live_raw_path': live_raw_path, + 'live_proc_path': live_proc_path + } + + # Record the live stream + recording_completed = self._record_livestream(is_live, live_raw_path) + + # If shutdown was requested during recording, try to finalize + if self.shutdown_requested: + print(f'{Fore.YELLOW}Attempting to process any recorded content...{Style.RESET_ALL}') + + # Process the raw stream file + self._process_raw_stream(live_raw_path, live_proc_path) + + # Skip VOD/chat download if shutdown was requested + vod_response = None + if self.shutdown_requested: + print(f'{Fore.YELLOW}Skipping VOD and chat download due to shutdown request{Style.RESET_ALL}') + else: + # Try to match stream with VOD + vod_response = self._get_latest_vod() + + if not self.shutdown_requested and vod_response and vod_response['data']['user']['videos']['edges']: + current_vod = vod_response['data']['user']['videos']['edges'][0]['node'] + vod_date = datetime.strptime( + current_vod["recordedAt"], '%Y-%m-%dT%H:%M:%SZ' + ).replace(tzinfo=timezone('UTC')).astimezone(tz=None).replace(tzinfo=None) + + # Check if VOD matches the stream (within 1 minute tolerance) + time_tolerance = timedelta(minutes=1) + if (live_date - time_tolerance) <= vod_date <= (live_date + time_tolerance): + print(f'\n{Fore.GREEN}✓ Found matching VOD{Style.RESET_ALL}') + + # Save metadata + self._save_metadata(current_vod, filename_base) + + # Download VOD + vod_ext = '.mp3' if self.quality == 'audio_only' else '.mp4' + vod_path = os.path.join(self.video_path, f"{PREFIX_VOD}{filename_base}{vod_ext}") + self._download_vod(current_vod, vod_path) + + # Download and render chat + chat_json_path = os.path.join(self.chatJSON_path, f"{PREFIX_CHAT}{filename_base}.json") + chat_video_path = os.path.join(self.chatMP4_path, f"{PREFIX_CHAT}{filename_base}.mp4") + self._download_and_render_chat(current_vod, chat_json_path, chat_video_path) + else: + print(f'{Fore.YELLOW}⚠ No matching VOD found for this stream{Style.RESET_ALL}') + + # Clean up raw files if configured + if self.cleanRaw == 1 and os.path.exists(live_raw_path): + print(f'{Fore.YELLOW}Deleting raw .ts file...{Style.RESET_ALL}') + os.remove(live_raw_path) + + # Upload to cloud if configured + upload_success = self._upload_to_cloud(filename_base) + + # Delete local files if configured and upload succeeded + if self.deleteFiles == 1 and upload_success: + self._delete_local_files(filename_base, live_raw_path, live_proc_path) + + # Done processing this stream + if self.shutdown_requested: + print(f'\n{Fore.YELLOW}{"=" * 60}{Style.RESET_ALL}') + print(f'{Fore.YELLOW}✓ Stream processing stopped by user{Style.RESET_ALL}') + print(f'{Fore.YELLOW}{"=" * 60}{Style.RESET_ALL}\n') + break + else: + print(f'\n{Fore.GREEN}{"=" * 60}{Style.RESET_ALL}') + print(f'{Fore.GREEN}✓ Stream processing complete{Style.RESET_ALL}') + print(f'{Fore.GREEN}{"=" * 60}{Style.RESET_ALL}\n') + self.send_notification(f'✓ Complete - {filename_base}', + 'Stream processing finished. Resuming monitoring.') + self._interruptible_sleep(self.refresh) + + except KeyboardInterrupt: + # Additional catch for any other KeyboardInterrupt not handled by signal + if not self.shutdown_requested: + self.shutdown_requested = True + print(f'\n{Fore.YELLOW}{"=" * 60}{Style.RESET_ALL}') + print(f'{Fore.YELLOW}⚠ Interrupted. Cleaning up...{Style.RESET_ALL}') + print(f'{Fore.YELLOW}{"=" * 60}{Style.RESET_ALL}\n') + break + + except Exception as e: + print(f'\n{Fore.RED}{"=" * 60}{Style.RESET_ALL}') + print(f'{Fore.RED}✗ ERROR: {str(e)}{Style.RESET_ALL}') + print(f'{Fore.YELLOW}Waiting {self.refresh} seconds before retrying...{Style.RESET_ALL}') + print(f'{Fore.RED}{"=" * 60}{Style.RESET_ALL}\n') + self.send_notification('⚠ Error - Recovery', + f'Error: {str(e)}\nRetrying after {self.refresh} seconds.') + + # Check for shutdown during sleep + if self.shutdown_requested: + break + self._interruptible_sleep(self.refresh) + + # Final cleanup message + print(f'{Fore.GREEN}✓ Monitoring stopped cleanly{Style.RESET_ALL}') + + def _upload_to_cloud(self, filename_base: str) -> bool: + """ + Upload archived files to cloud storage using rclone. + + Args: + filename_base: Base filename (without prefixes/extensions) + + Returns: + bool: True if upload succeeded or is disabled, False if failed + """ + if self.uploadCloud != 1: + return True # Consider upload "successful" if disabled + + print(f'\n{Fore.CYAN}Uploading to cloud storage...{Style.RESET_ALL}') + self.send_notification(f'☁ Uploading - {filename_base}', 'Uploading files to cloud storage') + + # Create list of files to upload + bin_path = self._get_bin_path() + upload_list_path = os.path.join(bin_path, 'temp', 'upload.txt') + + # Ensure temp directory exists + os.makedirs(os.path.dirname(upload_list_path), exist_ok=True) + + files_to_upload = [ + f"{PREFIX_LIVE}{filename_base}.ts", + f"{PREFIX_LIVE}{filename_base}.mp4", + f"{PREFIX_LIVE}{filename_base}.mp3", + f"{PREFIX_VOD}{filename_base}.ts", + f"{PREFIX_VOD}{filename_base}.mp4", + f"{PREFIX_VOD}{filename_base}.mp3", + f"{PREFIX_METADATA}{filename_base}.json", + f"{PREFIX_CHAT}{filename_base}.json", + f"{PREFIX_CHAT}{filename_base}.mp4" + ] + + with open(upload_list_path, 'w') as f: + f.write('\n'.join(files_to_upload)) + + # Run rclone + try: + result = subprocess.call([ + 'rclone', 'copy', + str(pathlib.Path(self.root_path).resolve()), + self.rclone_path, + '--include-from', upload_list_path + ]) + + # Clean up upload list + if os.path.exists(upload_list_path): + os.remove(upload_list_path) + + if result == 0: + print(f'{Fore.GREEN}✓ Upload complete{Style.RESET_ALL}') + self.send_notification(f'✓ Upload Success - {filename_base}', + 'All files uploaded successfully') + return True + else: + print(f'{Fore.RED}✗ Upload failed (exit code: {result}){Style.RESET_ALL}') + print(f'{Fore.YELLOW}Files preserved locally due to upload failure{Style.RESET_ALL}') + self.send_notification(f'✗ Upload Failed - {filename_base}', + f'Upload failed with code {result}. Files preserved locally.') + return False + + except Exception as e: + print(f'{Fore.RED}✗ Upload error: {str(e)}{Style.RESET_ALL}') + return False + + def _delete_local_files(self, filename_base: str, live_raw_path: str, live_proc_path: str) -> None: + """ + Delete local archive files after successful upload. + + Args: + filename_base: Base filename (without prefixes/extensions) + live_raw_path: Path to live raw file + live_proc_path: Path to live processed file + """ + print(f'\n{Fore.RED}{"=" * 60}{Style.RESET_ALL}') + print(f'{Fore.RED}⚠ DELETING LOCAL FILES{Style.RESET_ALL}') + print(f'{Fore.RED}{"=" * 60}{Style.RESET_ALL}\n') + + self.send_notification(f'🗑 Deleting - {filename_base}', + 'Deleting local files after successful upload') + + files_to_delete = [] + + # Live files + if self.cleanRaw == 0 and os.path.exists(live_raw_path): + files_to_delete.append(live_raw_path) + if os.path.exists(live_proc_path): + files_to_delete.append(live_proc_path) + + # VOD files + if self.downloadVOD == 1: + vod_raw = os.path.join(self.raw_path, f"{PREFIX_VOD}{filename_base}.ts") + vod_mp4 = os.path.join(self.video_path, f"{PREFIX_VOD}{filename_base}.mp4") + vod_mp3 = os.path.join(self.video_path, f"{PREFIX_VOD}{filename_base}.mp3") + + if self.cleanRaw == 0 and os.path.exists(vod_raw): + files_to_delete.append(vod_raw) + if os.path.exists(vod_mp4): + files_to_delete.append(vod_mp4) + if os.path.exists(vod_mp3): + files_to_delete.append(vod_mp3) + + # Chat files + if self.downloadCHAT == 1: + chat_json = os.path.join(self.chatJSON_path, f"{PREFIX_CHAT}{filename_base}.json") + chat_mp4 = os.path.join(self.chatMP4_path, f"{PREFIX_CHAT}{filename_base}.mp4") + + if os.path.exists(chat_json): + files_to_delete.append(chat_json) + if os.path.exists(chat_mp4): + files_to_delete.append(chat_mp4) + + # Metadata files + if self.downloadMETADATA == 1: + metadata = os.path.join(self.metadata_path, f"{PREFIX_METADATA}{filename_base}.json") + if os.path.exists(metadata): + files_to_delete.append(metadata) + + # Delete all files + for filepath in files_to_delete: + try: + print(f'{Fore.RED} Deleting: {os.path.basename(filepath)}{Style.RESET_ALL}') + os.remove(filepath) + except Exception as e: + print(f'{Fore.YELLOW} ⚠ Failed to delete {filepath}: {e}{Style.RESET_ALL}') + + print(f'{Fore.RED}\n✓ Cleanup complete{Style.RESET_ALL}') + + +# ============================================================================ +# COMMAND-LINE INTERFACE +# ============================================================================ + +def main(argv: list) -> None: + """ + Main entry point for command-line execution. + + Parses command-line arguments and starts the archive system. + + Args: + argv: Command-line arguments + """ + twitch_archive = TwitchArchive() + + help_msg = f''' +{Fore.CYAN}{"=" * 70} +TWITCH ARCHIVE - Automated Stream Recording & Archiving +{"=" * 70}{Style.RESET_ALL} + +{Fore.GREEN}USAGE:{Style.RESET_ALL} + python twitch-archive.py [OPTIONS] + +{Fore.GREEN}OPTIONS:{Style.RESET_ALL} + -h, --help Display this help information + -u, --username Twitch channel username to monitor + -q, --quality Stream quality: best/source, high/720p, + medium/480p, low/360p, audio_only + -a, --ttv-lol <0|1> Enable ad-blocking (1) or disable (0) + -v, --vod <0|1> Download VODs after stream ends + -c, --chat <0|1> Download and render chat + -m, --metadata <0|1> Download stream metadata + -r, --upload <0|1> Upload to cloud storage via rclone + -d, --delete <0|1> Delete local files after upload (CAREFUL!) + -n, --notifications <0|1> Send email notifications + +{Fore.YELLOW}TIPS:{Style.RESET_ALL} + • Configure settings in config.json (copy from config.sample.json) + • Set up API credentials in .env file + • Most users only need to edit config.json, no command-line args needed + +{Fore.CYAN}{"=" * 70}{Style.RESET_ALL} ''' + try: - opts, args = getopt.getopt(argv,"h:u:q:v:c:m:r:d:n",["username=","quality=","vod=","chat=","metadata=","upload=","delete=","notifications="]) - except getopt.GetoptError: - print (help_msg) + opts, args = getopt.getopt( + argv, + "h:u:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications="] + ) + except getopt.GetoptError as e: + print(f'{Fore.RED}Error: {e}{Style.RESET_ALL}\n') + print(help_msg) sys.exit(2) + for opt, arg in opts: if opt in ('-h', '--help'): print(help_msg) - sys.exit() - elif opt in ("-u", "--username"): twitch_archive.username = arg - elif opt in ("-q", "--quality"): twitch_archive.quality = arg - elif opt in ("-a", "--ttv-lol"): twitch_archive.streamlink_ttvlol = int(arg) - elif opt in ("-v", "--vod"): twitch_archive.downloadVOD = int(arg) - elif opt in ("-c", "--chat"): twitch_archive.downloadCHAT = int(arg) - elif opt in ("-m", "--metadata"): twitch_archive.downloadMETADATA = int(arg) - elif opt in ("-r", "--upload"): twitch_archive.uploadCloud = int(arg) - elif opt in ("-d", "--delete"): twitch_archive.deleteFiles = int(arg) - elif opt in ("-n", "--notifications"): twitch_archive.notifications = int(arg) + sys.exit(0) + elif opt in ("-u", "--username"): + twitch_archive.username = arg + elif opt in ("-q", "--quality"): + twitch_archive.quality = arg + elif opt in ("-a", "--ttv-lol"): + twitch_archive.streamlink_ttvlol = int(arg) + elif opt in ("-v", "--vod"): + twitch_archive.downloadVOD = int(arg) + elif opt in ("-c", "--chat"): + twitch_archive.downloadCHAT = int(arg) + elif opt in ("-m", "--metadata"): + twitch_archive.downloadMETADATA = int(arg) + elif opt in ("-r", "--upload"): + twitch_archive.uploadCloud = int(arg) + elif opt in ("-d", "--delete"): + twitch_archive.deleteFiles = int(arg) + elif opt in ("-n", "--notifications"): + twitch_archive.notifications = int(arg) + + # Start the archive system twitch_archive.run() + + if __name__ == "__main__": - main(sys.argv[1:]) \ No newline at end of file + try: + main(sys.argv[1:]) + except KeyboardInterrupt: + # Suppress stack trace for clean exit + print(f'\n{Fore.GREEN}✓ Graceful shutdown complete{Style.RESET_ALL}') + sys.exit(0) \ No newline at end of file