diff --git a/.gitignore b/.gitignore index 653eee7..7bf1e2b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ # Environments -.env \ No newline at end of file +.env +# Tests +test.py \ No newline at end of file diff --git a/README.md b/README.md index 3fef5ec..b1f79ca 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,14 @@ # Twitch Archive Inspired by https://github.com/EnterGin/Auto-Stream-Recording-Twitch -Python script to check, download live stream, VOD, chat and upload them to any cloud service supported by rclone. +Python script to check, download live stream, VOD, chat and upload them to any cloud storage supported by rclone. ## Requirements - [Python 3](https://www.python.org/downloads/) - [Streamlink](https://github.com/streamlink/streamlink) ## Getting started -1. Install Python 3 -2. Install Streamlink -3. If you want to upload to any cloud service using rclone, [configure rclone](https://rclone.org/docs/#configure) (Doesnt need to download, the `rclone.exe` is avalible in [bin/rclone.exe](https://github.com/piero0920/Twitch-Archive/blob/main/bin/rclone.exe)). +1. Install Python 3.x +2. Install Streamlink 5.1.x +3. If you want to upload to any cloud storage using rclone, [configure rclone](https://rclone.org/docs/#configure). 4. `git clone https://github.com/piero0920/Twitch-Archive.git` 5. `cd Twitch-Archive` 6. `pip install -r requirements.txt` diff --git a/bin/upload.bat b/bin/upload.bat index e4389ad..5c2fb8b 100644 --- a/bin/upload.bat +++ b/bin/upload.bat @@ -5,4 +5,4 @@ if "%root_path:~-1%" == "\" set "root_path_1=%root_path:~0,-1%" for %%f in ("%root_path_1%") do set "root_path_name=%%~nxf" FOR /F "eol=# tokens=*" %%i in (%~dp0\..\.env) do SET %%i CD %~dp0 -rclone.exe copy ../%root_path%/%user% %remote%/%root_path_name%/%user% --progress \ No newline at end of file +rclone.exe copy %root_path%/%user% %remote%/%root_path_name%/%user% --progress \ No newline at end of file diff --git a/extra.md b/extra.md index dd41c67..c43a0d3 100644 --- a/extra.md +++ b/extra.md @@ -30,7 +30,7 @@ This option will Download the latest public VOD, if the streamer hasn't publishe Using a simple api request downloads the `.json` metadata of the latest VOD and saves it to: `/root_path/streamer_username/metadata/metada_yyyymmdd_hhmmss.json` ### Upload to the cloud -Using [rclone](https://rclone.org/) after everything being downloaded and rendered , it will upload every file from the `root_path/streamer` folder to any cloud service supported by rclone such as [Google Drive, Mega, One Drive, etc.](https://rclone.org/overview/#features) +Using [rclone](https://rclone.org/) after everything being downloaded and rendered , it will upload every file from the `root_path/streamer` folder to any cloud storage supported by rclone such as [Google Drive, Mega, One Drive, etc.](https://rclone.org/overview/#features) The destination path to where it will be uploaded it has to be stated in the `.env` file. Example: ```env diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 6d5fe5d..0000000 --- a/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -colorama==0.4.6 -python-dotenv==0.21.0 -python_dateutil==2.8.2 -pytz==2022.6 -requests==2.28.1 diff --git a/twitch-archive b/twitch-archive deleted file mode 100644 index a034c66..0000000 --- a/twitch-archive +++ /dev/null @@ -1,309 +0,0 @@ -import requests, os, time, json, sys, subprocess, getopt, smtplib, pathlib -from colorama import Fore, Style -from datetime import datetime, timedelta -from pytz import timezone -from dateutil import parser -from dotenv import load_dotenv, find_dotenv -from email.mime.multipart import MIMEMultipart -from email.mime.text import MIMEText -load_dotenv(find_dotenv()) -class TwitchArchive: - def __init__(self): - # user configuration - self.username = "KalathrasLolweapon" # Twitch streamer username - self.quality = "best" # Qualities options: best/source high/720p medium/540p low/360p - # global configuration - self.root_path = r"archive" # Path where this script saves everything (livestream,VODs,chat,metadata) - self.timezone = "US/Eastern" # Timezones, you can see a list of the format timezone here: https://gist.github.com/heyalexej/8bf688fd67d7199be4a1682b3eec7568 - self.refresh = 5.0 # Time between checking (5.0 is recommended) - self.notifications = 0 # 0 - disable email notification of current seccion, 1 - enable email notification of current seccion - self.downloadMETADATA = 1 # 0 - disable metadata downloading, 1 - enable metadata downloading - self.downloadVOD = 1 # 0 - disable VOD downloading after stream finished, 1 - enable VOD downloading after stream finished (this option downloads the latest public vod) - self.downloadCHAT = 1 # 0 - disable chat downloading and rendering, 1 - enable chat downloading and rendering - self.uploadCloud = 0 # 0 - disable upload to remote cloud, 1 - enable upload to remote cloud - self.deleteFiles = 0 # 0 - disable the deleting of files from current seccion after being uploaded to the cloud, 1 - enable the deleting files of files from current seccion after being uploaded to the cloud (BE CAREFUL WITH THIS OPTION) - self.streamlink_debug = 0 # 0 - disable streamlink debug display, 1 - enable streamlink debug display - self.hls_segments = 3 # 1-10 for live stream, it's possible to use multiple threads to potentially increase the throughput. 2-3 is enough - self.hls_segmentsVOD = 10 # 1-10 for downloading vod, it's possible to use multiple threads to potentially increase the throughput - - def run(self): - print('Twitch-Archive') - print('Configuration:') - print(f'Root path: {Fore.GREEN}' + str(pathlib.Path(self.root_path).resolve()) + f'{Style.RESET_ALL}') - print(f'Timezone: {Fore.GREEN}{self.timezone}{Style.RESET_ALL}') - print(f'Refresh rate: {Fore.GREEN} {str(self.refresh)}{Style.RESET_ALL}') - if self.notifications == 1: print(f'Email notifications: {Fore.GREEN}Enabled{Style.RESET_ALL}') - else: print(f'Email notifications: {Fore.RED}Disabled{Style.RESET_ALL}') - if self.downloadMETADATA == 1: print(f'Metada downloading {Fore.GREEN}Enabled{Style.RESET_ALL}') - else: print(f'Metada downloading: {Fore.RED}Disabled{Style.RESET_ALL}') - if self.downloadVOD == 1: print(f'VOD downloading {Fore.GREEN}Enabled{Style.RESET_ALL}') - else: print(f'VOD downloading: {Fore.RED}Disabled{Style.RESET_ALL}') - if self.downloadCHAT == 1: print(f'Chat downloading {Fore.GREEN}Enabled{Style.RESET_ALL}') - else: print(f'Chat downloading: {Fore.RED}Disabled{Style.RESET_ALL}') - if self.uploadCloud == 1: print(f'Upload to Google Drive: {Fore.GREEN}Enabled{Style.RESET_ALL}') - else: print(f'Upload to cloud service: {Fore.RED}Disabled{Style.RESET_ALL}') - if self.deleteFiles == 1: print(f'{Fore.RED}'+'\033[1m'+f'CAREFUL FILES ARE CONFIGURATED TO BE DELETED{Style.RESET_ALL}') - else: print(f'{Fore.GREEN}'+'\033[1m'+f'Files will NOT be deleted{Style.RESET_ALL}') - if self.uploadCloud == 0 and self.deleteFiles == 1: print(f'{Fore.RED}'+'\033[1m'+f'FILES WILL BE DELETED AND NO UPLOADED {Style.RESET_ALL}{Fore.GREEN}\n"CTRL + C"{Style.RESET_ALL}{Fore.RED}'+'\033[1m'+f' TO STOP AND CHANGED CONFIGURATION{Style.RESET_ALL}') - - self.oauth_token = self.get_oauth_token() - self.get_channel_id() - if self.streamlink_debug == 1: self.debug_cmd = "--loglevel trace".split() - else: self.debug_cmd = "".split() - - self.recorded_path = str(pathlib.Path(os.path.join(self.root_path,self.username,"video", "recorded")).absolute()) - self.processed_path = str(pathlib.Path(os.path.join(self.root_path, self.username, "video", "processed")).absolute()) - self.chatJSON_path = str(pathlib.Path(os.path.join(self.root_path, self.username, "chat", "json")).absolute()) - self.chatMP4_path = str(pathlib.Path(os.path.join(self.root_path, self.username, "chat", "rendered")).absolute()) - self.metadata_path = str(pathlib.Path(os.path.join(self.root_path, self.username, "metadata")).absolute()) - - if(os.path.isdir(self.recorded_path) is False): os.makedirs(self.recorded_path) - if(os.path.isdir(self.processed_path) is False): os.makedirs(self.processed_path) - if(os.path.isdir(self.chatJSON_path) is False): os.makedirs(self.chatJSON_path) - if(os.path.isdir(self.chatMP4_path) is False): os.makedirs(self.chatMP4_path) - if(os.path.isdir(self.metadata_path) is False): os.makedirs(self.metadata_path) - - try: - video_list = [f for f in os.listdir(self.recorded_path) if os.path.isfile(os.path.join(self.recorded_path, f))] - if(len(video_list) > 0): - print('Fixing previously recorded files.') - for f in video_list: - recorded_filename = os.path.join(self.recorded_path, f) - stream_dir_path = self.processed_path - if(os.path.isdir(stream_dir_path) is False): - os.makedirs(stream_dir_path) - print('Fixing ' + recorded_filename + '.') - try: - subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/ffmpeg' , '-y', '-i', recorded_filename, '-analyzeduration', '2147483647', '-probesize', '2147483647', '-c:v', 'copy', '-c:a', 'copy', '-start_at_zero', '-copyts', os.path.join(stream_dir_path, f[:-2]+"mp4")], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) - except Exception as e: - print(e) - elif(os.path.exists(os.path.join(stream_dir_path, f)) is False): - print('Fixing ' + recorded_filename + '.') - try: - subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/ffmpeg', '-y', '-i', recorded_filename, '-analyzeduration', '2147483647', '-probesize', '2147483647', '-c:v', 'copy', '-c:a', 'copy', '-start_at_zero', '-copyts', os.path.join(stream_dir_path, f[:-2]+"mp4")], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) - except Exception as e: - print(e) - except Exception as e: - print(e) - - print(f"Checking for {Fore.GREEN}{self.username}{Style.RESET_ALL} every {Fore.GREEN}{self.refresh}{Style.RESET_ALL} seconds. Record with {Fore.GREEN}{self.quality}{Style.RESET_ALL} quality.") - self.sendNotif("TWITCH ARCHIVE", f"Checking for {self.username} every {self.refresh} seconds. Record with {self.quality} quality.") - self.loopcheck() - - def get_oauth_token(self): - try: - return requests.post(f"https://id.twitch.tv/oauth2/token?client_id={os.environ.get('CLIENT-ID')}&client_secret={os.environ.get('CLIENT-SECRET')}&grant_type=client_credentials").json()['access_token'] - except: - return None - - def get_channel_id(self): - self.getting_channel_id_error = 0 - self.user_not_found = 0 - if self.oauth_token == None: - self.getting_channel_id_error = 1 - return - url = 'https://api.twitch.tv/helix/users?login=' + self.username - try: - r = requests.get(url, headers = {"Authorization" : "Bearer " + self.oauth_token, "Client-ID": os.environ.get('CLIENT-ID')}, timeout = 15) - r.raise_for_status() - info = r.json() - if info["data"] != []: self.channel_id = info["data"][0]["id"] - else: self.user_not_found = 1 - except requests.exceptions.RequestException as e: - self.getting_channel_id_error = 1 - print(f'\n{e}\n') - - def check_user(self): - try: - url = 'https://api.twitch.tv/helix/streams?user_id=' + self.channel_id - req = requests.get(url, headers = {"Authorization" : "Bearer " + self.oauth_token, "Client-ID": os.environ.get('CLIENT-ID')}, timeout = 30) - stream_data = req.json() - if len(stream_data['data']) == 1: - return True - else: - return False - except Exception as e: - print("ERROR checking user: ", e) - return False - - def toTZ(self, utc_str): - new_date = str(datetime.fromisoformat(utc_str.replace('Z', '+00:00')).astimezone(timezone(self.timezone))) - year = new_date[:4] - month = new_date[5:7] - day = new_date[8:10] - hour = new_date[11:13] - minute = new_date[14:16] - seconds = new_date[17:19] - date_formated = year + month + day + "_" + hour + "h" + minute + "m" + seconds + "s" - return date_formated - - def sendNotif(self, subject, content): - if self.notifications == 1: - sender = os.environ.get("SENDER") - receiver = os.environ.get("RECEIVER") - msg = MIMEMultipart() - msg['From'] = sender - msg['To'] = receiver - msg['Subject'] = self.username + " _ " + subject - body = "Current seccion is for " + self.username + "\n\n\n\n" + content - msg.attach(MIMEText((body), 'plain')) - server = smtplib.SMTP('smtp.gmail.com', 587) - server.starttls() - server.login(sender, os.environ.get("PASSWD")) - txt = msg.as_string() - server.sendmail(sender, receiver, txt) - server.quit() - - def loopcheck(self): - while True: - if self.check_user() is True: - live_date = datetime.now(timezone('UTC')) - live_date_min = live_date - timedelta(minutes=10) - live_date_plus = live_date + timedelta(minutes=10) - present_datetime = datetime.now(timezone(self.timezone)).strftime("%Y%m%d_%Hh%Mm%Ss") - raw_filename = present_datetime + ".ts" - live_filename = "LIVE_" + raw_filename - raw_vod_filename = "VOD_" + raw_filename - chat_json_filename = "CHAT_" + present_datetime + ".json" - chat_mp4_filename = "CHAT_" + present_datetime + ".mp4" - metadata_filename = "metadata_" + present_datetime + ".json" - recorded_filename = os.path.join(self.recorded_path, live_filename) - # start streamlink process - subprocess.call(["streamlink", '--http-header', 'Authorization=OAuth ' + os.environ.get('OAUTH-PRIVATE-TOKEN'), "--hls-segment-threads", str(self.hls_segments), "--hls-live-restart", "--twitch-disable-hosting", "twitch.tv/" + self.username, self.quality, "--retry-streams", str(self.refresh)] + self.debug_cmd + ["-o", recorded_filename]) - if(os.path.exists(recorded_filename) is True): - try: - vodurl = 'https://api.twitch.tv/helix/videos?user_id=' + str(self.channel_id) + '&type=archive' - vods = requests.get(vodurl, headers = {"Authorization" : "Bearer " + self.oauth_token, "Client-ID": os.environ.get('CLIENT-ID')}, timeout = 30) - vodsinfodic = json.loads(vods.text) - if vodsinfodic["data"][0] != []: - if live_date_min <= parser.parse(vodsinfodic["data"][0]["created_at"]) <= live_date_plus: - vod_id = vodsinfodic["data"][0]["id"] - created_at = vodsinfodic["data"][0]["created_at"] - created_at = self.toTZ(created_at) - raw_filename = created_at + ".ts" - live_filename = "LIVE_" + raw_filename - raw_vod_filename = "VOD_" + raw_filename - chat_json_filename = "CHAT_" + created_at + ".json" - chat_mp4_filename = "CHAT_" + created_at + ".mp4" - metadata_filename = "metadata_" + created_at + ".json" - try: - os.rename(recorded_filename,os.path.join(self.recorded_path, live_filename)) - recorded_filename = os.path.join(self.recorded_path, live_filename) - except Exception as e: - raw_filename = present_datetime + ".ts" - live_filename = "LIVE_" + raw_filename - raw_vod_filename = "VOD_" + raw_filename - chat_json_filename = "CHAT_" + present_datetime + ".json" - chat_mp4_filename = "CHAT_" + present_datetime + ".mp4" - metadata_filename = "metadata_" + present_datetime + ".json" - os.rename(recorded_filename,os.path.join(self.recorded_path, live_filename)) - recorded_filename = os.path.join(self.recorded_path, live_filename) - print('first exception as e\nAn error has occurred. VOD and chat will not be downloaded. Please check them manually.\n' + e) - self.sendNotif('ERROR - '+ present_datetime, 'An error has occurred. VOD and chat will not be downloaded. Please check them manually.\n ' + e) - if self.downloadMETADATA == 1: - self.sendNotif('Metadata - ' + created_at,'Downloading and saving metadata:\n' + json.dumps(vodsinfodic["data"][0], indent=4)) - with open(os.path.join(self.metadata_path, metadata_filename), 'w', encoding='utf-8') as f: - json.dump(vodsinfodic["data"][0], f, ensure_ascii=False, indent=4) - if self.downloadVOD == 1: - print('Downloading VOD: ' + vodsinfodic["data"][0]["title"]) - self.sendNotif('VOD - ' + created_at,'Downloading VOD: ' + vodsinfodic["data"][0]["title"]) - try: - subprocess.call(['streamlink', '--http-header', 'Authorization=OAuth ' + os.environ.get('OAUTH-PRIVATE-TOKEN'), "--hls-segment-threads", str(self.hls_segmentsVOD), "twitch.tv/videos/" + vod_id, self.quality] + self.debug_cmd + ["-o", os.path.join(self.recorded_path, raw_vod_filename)]) - except Exception as e: - print('Error', 'A ERROR has ocurred and the VOD will not be downloaded.\n' + e) - self.sendNotif('ERROR - ' + created_at, 'A ERROR has ocurred and the VOD will not be downloaded. \nerror:\n' + e) - if self.downloadCHAT == 1: - print('Downloading and rendering CHAT: ' + vodsinfodic["data"][0]["title"]) - self.sendNotif('CHAT - ' + created_at,'Downloading JSON and rendering chat logs from VOD:\n' + vodsinfodic["data"][0]["title"]) - try: - subprocess.call(["bash",str(pathlib.Path(__file__).parent.resolve())+"/bin/chat.sh", vod_id, os.path.join(self.chatJSON_path, chat_json_filename), os.path.join(self.chatMP4_path, chat_mp4_filename)]) - except Exception as e: - self.sendNotif('ERROR - ' + created_at, "A ERROR has ocurred and chat will need to be downloaded and rendered manually.\n" + e) - print("A ERROR has ocurred and chat will need to be downloaded and rendered manually\n" + e) - else: - print('A ERROR has ocurred, the latest VOD doesnt match with the livestream, the VOD is not published\nThe VOD and chat will not be downloaded and rendered.\nThe current livestream date: ' + present_datetime + '\nThe VOD date: ' + self.toTZ(vodsinfodic["data"][0]["created_at"])) - self.sendNotif('ERROR - ' + present_datetime, 'A ERROR has ocurred, the latest VOD doesnt match with the livestream, the VOD is not published\nThe VOD and chat will not be downloaded and rendered.\nThe current livestream date: ' + present_datetime + '\nThe VOD date: ' + self.toTZ(vodsinfodic["data"][0]["created_at"])) - else: - raw_filename = present_datetime + ".ts" - live_filename = "LIVE_" + raw_filename - raw_vod_filename = "VOD_" + raw_filename - chat_json_filename = "CHAT_" + present_datetime + ".json" - chat_mp4_filename = "CHAT_" + present_datetime + ".mp4" - metadata_filename = "metadata_" + present_datetime + ".json" - os.rename(recorded_filename,os.path.join(self.recorded_path, live_filename)) - recorded_filename = os.path.join(self.recorded_path, live_filename) - except Exception as e: - raw_filename = present_datetime + ".ts" - live_filename = "LIVE_" + raw_filename - raw_vod_filename = "VOD_" + raw_filename - chat_json_filename = "CHAT_" + present_datetime + ".json" - chat_mp4_filename = "CHAT_" + present_datetime + ".mp4" - metadata_filename = "metadata_" + present_datetime + ".json" - os.rename(recorded_filename,os.path.join(self.recorded_path, live_filename)) - recorded_filename = os.path.join(self.recorded_path, live_filename) - print('An error has occurred. VOD and chat will not be downloaded. Please check them manually.\n' + e) - self.sendNotif('ERROR - ' + present_datetime, 'An error has occurred. VOD and chat will not be downloaded. Please check them manually.\n' + e) - print("Recording stream is done. Fixing video file.") - self.sendNotif("STREAM DONE - " + present_datetime, "Recording stream is done. Fixing video file.") - if(os.path.exists(recorded_filename) is True): - file_mp4 = live_filename[:-2] + "mp4" - vod_filename = raw_vod_filename[:-2] + "mp4" - processed_filename = os.path.join(self.processed_path, file_mp4) - subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/ffmpeg', '-y', '-i', recorded_filename, '-analyzeduration', '2147483647', '-probesize', '2147483647', '-c:v', 'copy', '-c:a', 'copy', '-start_at_zero', '-copyts', processed_filename], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) - if(os.path.exists(os.path.join(self.recorded_path, raw_vod_filename)) is True): - subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/ffmpeg', '-y', '-i', os.path.join(self.recorded_path, raw_vod_filename), '-analyzeduration', '2147483647', '-probesize', '2147483647', '-c:v', 'copy', '-c:a', 'copy', '-start_at_zero', '-copyts', os.path.join(self.processed_path, vod_filename)], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) - else: - print("Skip fixing. File not found.") - print("Fixing is done.") - if self.uploadCloud == 1: - tree = subprocess.check_output(["tree", str(pathlib.Path(self.root_path).resolve())+"/"+self.username]).decode(sys.stdout.encoding).split("\n",1)[1] - print('Uploading the following files:\n' + tree) - self.sendNotif("UPLOADING - " + present_datetime, 'Uploading the following files: \n' + tree) - subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/upload.sh', str(pathlib.Path(self.root_path).resolve()),self.username]) - if self.deleteFiles == 1: - self.sendNotif("DELETING - " + present_datetime, "Deleting the files from current seccion.") - print(f'{Fore.RED}DELETING FILES{Style.RESET_ALL}') - print(f'{Fore.RED}Deleting ' + recorded_filename + f'{Style.RESET_ALL}') - os.remove(recorded_filename) - print(f'{Fore.RED}Deleting ' + processed_filename + f'{Style.RESET_ALL}') - os.remove(processed_filename) - if self.downloadVOD == 1: - if(os.path.exists(os.path.join(self.recorded_path, raw_vod_filename)) is True): - print(f'{Fore.RED}Deleting ' + os.path.join(self.recorded_path, raw_vod_filename) + f'{Style.RESET_ALL}') - os.remove(os.path.join(self.recorded_path, raw_vod_filename)) - if(os.path.exists(os.path.join(self.processed_path, vod_filename)) is True): - print(f'{Fore.RED}Deleting ' + os.path.join(self.processed_path, vod_filename) + f'{Style.RESET_ALL}') - os.remove(os.path.join(self.processed_path, vod_filename)) - if self.downloadCHAT == 1: - if(os.path.exists(os.path.join(self.chatJSON_path, chat_json_filename)) is True): - print(f'{Fore.RED}Deleting ' + os.path.join(self.chatJSON_path, chat_json_filename) + f'{Style.RESET_ALL}') - os.remove(os.path.join(self.chatJSON_path, chat_json_filename)) - if(os.path.exists(os.path.join(self.chatMP4_path, chat_mp4_filename)) is True): - print(f'{Fore.RED}Deleting ' + os.path.join(self.chatMP4_path, chat_mp4_filename) + f'{Style.RESET_ALL}') - os.remove(os.path.join(self.chatMP4_path, chat_mp4_filename)) - if self.downloadMETADATA == 1: - if(os.path.exists(os.path.join(self.metadata_path, metadata_filename)) is True): - print(f'{Fore.RED}Deleting ' + os.path.join(self.metadata_path, metadata_filename) + f'{Style.RESET_ALL}') - os.remove(os.path.join(self.metadata_path, metadata_filename)) - print('CURRENT SECCION HAVE FINISHED GOING BACK TO CHECKING') - self.sendNotif("SECCION DONE - " + present_datetime, 'CURRENT SECCION HAVE FINISHED GOING BACK TO CHECKING') - time.sleep(self.refresh) -def main(argv): - twitch_recorder = TwitchArchive() - usage_message = 'twitch-archive.py -u -q ' - try: - opts, args = getopt.getopt(argv,"u:q:",["username=","quality="]) - except getopt.GetoptError: - print (usage_message) - sys.exit(2) - for opt, arg in opts: - if opt == '-h': - print(usage_message) - sys.exit() - elif opt in ("-u", "--username"): - twitch_recorder.username = arg - elif opt in ("-q", "--quality"): - twitch_recorder.quality = arg - twitch_recorder.run() -if __name__ == "__main__": - main(sys.argv[1:]) diff --git a/twitch-archive.py b/twitch-archive.py index 8f020e8..df2d4ae 100644 --- a/twitch-archive.py +++ b/twitch-archive.py @@ -1,8 +1,7 @@ -import requests, os, time, json, sys, subprocess, getopt, smtplib, pathlib +import requests, os, time, json, sys, subprocess, getopt, smtplib, pathlib, glob from colorama import Fore, Style from datetime import datetime, timedelta from pytz import timezone -from dateutil import parser from dotenv import load_dotenv, find_dotenv from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText @@ -14,23 +13,22 @@ class TwitchArchive: self.quality = "best" # Qualities options: best/source high/720p medium/540p low/360p # global configuration self.root_path = r"archive" # Path where this script saves everything (livestream,VODs,chat,metadata) - self.timezone = "US/Eastern" # Timezones, you can see a list of the format timezone here: https://gist.github.com/heyalexej/8bf688fd67d7199be4a1682b3eec7568 - self.refresh = 5.0 # Time between checking (5.0 is recommended) - self.notifications = 0 # 0 - disable email notification of current seccion, 1 - enable email notification of current seccion + self.refresh = 5.0 # Time between checking (5.0 is recommended), avoid less than 1.0 + self.notifications = 1 # 0 - disable email notification of current seccion, 1 - enable email notification of current seccion self.downloadMETADATA = 1 # 0 - disable metadata downloading, 1 - enable metadata downloading self.downloadVOD = 1 # 0 - disable VOD downloading after stream finished, 1 - enable VOD downloading after stream finished (this option downloads the latest public vod) self.downloadCHAT = 1 # 0 - disable chat downloading and rendering, 1 - enable chat downloading and rendering - self.uploadCloud = 0 # 0 - disable upload to remote cloud, 1 - enable upload to remote cloud + self.uploadCloud = 1 # 0 - disable upload to remote cloud, 1 - enable upload to remote cloud self.deleteFiles = 0 # 0 - disable the deleting of files from current seccion after being uploaded to the cloud, 1 - enable the deleting files of files from current seccion after being uploaded to the cloud (BE CAREFUL WITH THIS OPTION) - self.streamlink_debug = 0 # 0 - disable streamlink debug display, 1 - enable streamlink debug display + self.cleanRaw = 1 # 0 - disable the deleting of raw (.ts) files, 1 - enable the deleteing of raw (.ts) files (if upload enable they will be deleted before) self.hls_segments = 3 # 1-10 for live stream, it's possible to use multiple threads to potentially increase the throughput. 2-3 is enough self.hls_segmentsVOD = 10 # 1-10 for downloading vod, it's possible to use multiple threads to potentially increase the throughput - def run(self): + def run(self): + self.terminal = self.get_OS() print('Twitch-Archive') print('Configuration:') print(f'Root path: {Fore.GREEN}' + str(pathlib.Path(self.root_path).resolve()) + f'{Style.RESET_ALL}') - print(f'Timezone: {Fore.GREEN}{self.timezone}{Style.RESET_ALL}') print(f'Refresh rate: {Fore.GREEN} {str(self.refresh)}{Style.RESET_ALL}') if self.notifications == 1: print(f'Email notifications: {Fore.GREEN}Enabled{Style.RESET_ALL}') else: print(f'Email notifications: {Fore.RED}Disabled{Style.RESET_ALL}') @@ -40,123 +38,77 @@ class TwitchArchive: else: print(f'VOD downloading: {Fore.RED}Disabled{Style.RESET_ALL}') if self.downloadCHAT == 1: print(f'Chat downloading {Fore.GREEN}Enabled{Style.RESET_ALL}') else: print(f'Chat downloading: {Fore.RED}Disabled{Style.RESET_ALL}') - if self.uploadCloud == 1: print(f'Upload to Google Drive: {Fore.GREEN}Enabled{Style.RESET_ALL}') - else: print(f'Upload to cloud service: {Fore.RED}Disabled{Style.RESET_ALL}') + if self.uploadCloud == 1: print(f'Upload to cloud storage: {Fore.GREEN}Enabled{Style.RESET_ALL}') + else: print(f'Upload to cloud storage: {Fore.RED}Disabled{Style.RESET_ALL}') if self.deleteFiles == 1: print(f'{Fore.RED}'+'\033[1m'+f'CAREFUL FILES ARE CONFIGURATED TO BE DELETED{Style.RESET_ALL}') else: print(f'{Fore.GREEN}'+'\033[1m'+f'Files will NOT be deleted{Style.RESET_ALL}') if self.uploadCloud == 0 and self.deleteFiles == 1: print(f'{Fore.RED}'+'\033[1m'+f'FILES WILL BE DELETED AND NO UPLOADED {Style.RESET_ALL}{Fore.GREEN}\n"CTRL + C"{Style.RESET_ALL}{Fore.RED}'+'\033[1m'+f' TO STOP AND CHANGED CONFIGURATION{Style.RESET_ALL}') self.oauth_token = self.get_oauth_token() - self.get_channel_id() - if self.streamlink_debug == 1: self.debug_cmd = "--loglevel trace".split() - else: self.debug_cmd = "".split() + self.channel_id = self.get_channel_id() - self.recorded_path = str(pathlib.Path(os.path.join(self.root_path,self.username,"video", "recorded")).absolute()) - self.processed_path = str(pathlib.Path(os.path.join(self.root_path, self.username, "video", "processed")).absolute()) + self.raw_path = str(pathlib.Path(os.path.join(self.root_path,self.username,"video", "raw")).absolute()) + self.video_path = str(pathlib.Path(os.path.join(self.root_path, self.username, "video")).absolute()) self.chatJSON_path = str(pathlib.Path(os.path.join(self.root_path, self.username, "chat", "json")).absolute()) - self.chatMP4_path = str(pathlib.Path(os.path.join(self.root_path, self.username, "chat", "rendered")).absolute()) + self.chatMP4_path = str(pathlib.Path(os.path.join(self.root_path, self.username, "chat")).absolute()) self.metadata_path = str(pathlib.Path(os.path.join(self.root_path, self.username, "metadata")).absolute()) - if(os.path.isdir(self.recorded_path) is False): os.makedirs(self.recorded_path) - if(os.path.isdir(self.processed_path) is False): os.makedirs(self.processed_path) + if(os.path.isdir(self.raw_path) is False): os.makedirs(self.raw_path) + if(os.path.isdir(self.video_path) is False): os.makedirs(self.video_path) if(os.path.isdir(self.chatJSON_path) is False): os.makedirs(self.chatJSON_path) if(os.path.isdir(self.chatMP4_path) is False): os.makedirs(self.chatMP4_path) if(os.path.isdir(self.metadata_path) is False): os.makedirs(self.metadata_path) - - try: - video_list = [f for f in os.listdir(self.recorded_path) if os.path.isfile(os.path.join(self.recorded_path, f))] - if(len(video_list) > 0): - print('Fixing previously recorded files.') - for f in video_list: - recorded_filename = os.path.join(self.recorded_path, f) - stream_dir_path = self.processed_path - if(os.path.isdir(stream_dir_path) is False): - os.makedirs(stream_dir_path) - print('Fixing ' + recorded_filename + '.') - try: - subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/ffmpeg.exe' , '-y', '-i', recorded_filename, '-analyzeduration', '2147483647', '-probesize', '2147483647', '-c:v', 'copy', '-c:a', 'copy', '-start_at_zero', '-copyts', os.path.join(stream_dir_path, f[:-2]+"mp4")], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) - except Exception as e: - print(e) - elif(os.path.exists(os.path.join(stream_dir_path, f)) is False): - print('Fixing ' + recorded_filename + '.') - try: - subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/ffmpeg.exe', '-y', '-i', recorded_filename, '-analyzeduration', '2147483647', '-probesize', '2147483647', '-c:v', 'copy', '-c:a', 'copy', '-start_at_zero', '-copyts', os.path.join(stream_dir_path, f[:-2]+"mp4")], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) - except Exception as e: - print(e) - except Exception as e: - print(e) + if not os.path.exists(os.path.join(self.root_path, ".log")): + with open(os.path.join(self.root_path, ".log"), 'w'): pass print(f"Checking for {Fore.GREEN}{self.username}{Style.RESET_ALL} every {Fore.GREEN}{self.refresh}{Style.RESET_ALL} seconds. Record with {Fore.GREEN}{self.quality}{Style.RESET_ALL} quality.") self.sendNotif("TWITCH ARCHIVE", f"Checking for {self.username} every {self.refresh} seconds. Record with {self.quality} quality.") self.loopcheck() + def get_OS(self): + if sys.platform.startswith('win32'): + return 'powershell.exe' + elif sys.platform.startswith('linux'): + return 'bash' + else: + print('OS no supported') + return + def get_oauth_token(self): try: - return requests.post(f"https://id.twitch.tv/oauth2/token?client_id={os.environ.get('CLIENT-ID')}&client_secret={os.environ.get('CLIENT-SECRET')}&grant_type=client_credentials").json()['access_token'] + return requests.post(f"https://id.twitch.tv/oauth2/token?client_id={os.getenv('CLIENT-ID')}&client_secret={os.getenv('CLIENT-SECRET')}&grant_type=client_credentials").json()['access_token'] except: return None def get_channel_id(self): - self.getting_channel_id_error = 0 - self.user_not_found = 0 - if self.oauth_token == None: - self.getting_channel_id_error = 1 - return - url = 'https://api.twitch.tv/helix/users?login=' + self.username try: - r = requests.get(url, headers = {"Authorization" : "Bearer " + self.oauth_token, "Client-ID": os.environ.get('CLIENT-ID')}, timeout = 15) + r = requests.get(f'https://api.twitch.tv/helix/users?login={self.username}', headers = {"Authorization" : "Bearer " + self.oauth_token, "Client-ID": os.getenv('CLIENT-ID')}, timeout = 15) r.raise_for_status() info = r.json() - if info["data"] != []: self.channel_id = info["data"][0]["id"] - else: self.user_not_found = 1 + if info["data"] != []: + return info["data"][0]["id"] + else: + return None except requests.exceptions.RequestException as e: - self.getting_channel_id_error = 1 print(f'\n{e}\n') - def check_user(self): - # 0: online, 1: not found, 2: error, 3: channel id error - info = None - if self.user_not_found != 1 and self.getting_channel_id_error != 1: - url = 'https://api.twitch.tv/helix/channels?broadcaster_id=' + str(self.channel_id) - status = 2 - try: - r = requests.get(url, headers = {"Authorization" : "Bearer " + self.oauth_token, "Client-ID": os.environ.get('CLIENT-ID')}, timeout = 15) - r.raise_for_status() - info = r.json() - status = 0 - except requests.exceptions.RequestException as e: - if e.response != None: - if e.response.status_code == 401: - print('\nRequest to Twitch returned an error %s, trying to get new oauth_token...'% (e.response.status_code)) - self.getting_channel_id_error = 1 - else: - print('\nRequest to Twitch returned an error %s, the response is:\n%s\n'% (e.response.status_code, e.response)) - else: - print(f'\n{e}\n') - elif self.user_not_found == 1: - status = 1 - else: - self.oauth_token = self.get_oauth_token() - self.get_channel_id() - status = 3 - - return status, info - - def toTZ(self, utc_str): - new_date = str(datetime.fromisoformat(utc_str.replace('Z', '+00:00')).astimezone(timezone(self.timezone))) - year = new_date[:4] - month = new_date[5:7] - day = new_date[8:10] - hour = new_date[11:13] - minute = new_date[14:16] - seconds = new_date[17:19] - date_formated = year + month + day + "_" + hour + "h" + minute + "m" + seconds + "s" - return date_formated + try: + url = 'https://api.twitch.tv/helix/streams?user_id=' + self.channel_id + live = requests.get(url, headers = {"Authorization" : "Bearer " + self.oauth_token, "Client-ID": os.environ.get('CLIENT-ID')}, timeout = 30) + stream_data = live.json() + if len(stream_data['data']) == 1: + self.live_info = stream_data['data'][0] + return True + else: + return False + except Exception as e: + print("ERROR checking user: ", e) + return False def sendNotif(self, subject, content): if self.notifications == 1: - sender = os.environ.get("SENDER") - receiver = os.environ.get("RECEIVER") + sender = os.getenv("SENDER") + receiver = os.getenv("RECEIVER") msg = MIMEMultipart() msg['From'] = sender msg['To'] = receiver @@ -165,174 +117,150 @@ class TwitchArchive: msg.attach(MIMEText((body), 'plain')) server = smtplib.SMTP('smtp.gmail.com', 587) server.starttls() - server.login(sender, os.environ.get("PASSWD")) + server.login(sender, os.getenv("PASSWD")) txt = msg.as_string() server.sendmail(sender, receiver, txt) server.quit() def loopcheck(self): while True: - status, info = self.check_user() - if status == 1: - print("Username not found. Invalid username or typo.") - time.sleep(self.refresh) - elif status == 2: - print(datetime.now(timezone(self.timezone)).strftime("%Y%m%d_%Hh%Mm%Ss")," ","Unexpected error. Try to check internet connection or client-id. Will try again in", self.refresh, "seconds.") - time.sleep(self.refresh) - elif status == 3: - print(datetime.now(timezone(self.timezone)).strftime("%Y%m%d_%Hh%Mm%Ss")," ","Error with channel id or oauth token. Try to check internet connection or client-id and client-secret. Will try again in", self.refresh, "seconds.") - time.sleep(self.refresh) - elif status == 0: - live_date = datetime.now(timezone('UTC')) - live_date_min = live_date - timedelta(minutes=5) - live_date_plus = live_date + timedelta(minutes=5) - present_datetime = datetime.now(timezone(self.timezone)).strftime("%Y%m%d_%Hh%Mm%Ss") - raw_filename = present_datetime + ".ts" - live_filename = "LIVE_" + raw_filename - raw_vod_filename = "VOD_" + raw_filename - chat_json_filename = "CHAT_" + present_datetime + ".json" - chat_mp4_filename = "CHAT_" + present_datetime + ".mp4" - metadata_filename = "metadata_" + present_datetime + ".json" - recorded_filename = os.path.join(self.recorded_path, live_filename) - # start streamlink process - subprocess.call(["streamlink", '--http-header', 'Authorization=OAuth ' + os.environ.get('OAUTH-PRIVATE-TOKEN'), "--hls-segment-threads", str(self.hls_segments), "--hls-live-restart", "--twitch-disable-hosting", "twitch.tv/" + self.username, self.quality, "--retry-streams", str(self.refresh)] + self.debug_cmd + ["-o", recorded_filename]) - if(os.path.exists(recorded_filename) is True): - status, info_tmp = self.check_user() - if info_tmp != None: - info = info_tmp - try: - vodurl = 'https://api.twitch.tv/helix/videos?user_id=' + str(self.channel_id) + '&type=archive' - vods = requests.get(vodurl, headers = {"Authorization" : "Bearer " + self.oauth_token, "Client-ID": os.environ.get('CLIENT-ID')}, timeout = 30) - vodsinfodic = json.loads(vods.text) - if vodsinfodic["data"][0] != []: - if live_date_min <= parser.parse(vodsinfodic["data"][0]["created_at"]) <= live_date_plus: - vod_id = vodsinfodic["data"][0]["id"] - created_at = vodsinfodic["data"][0]["created_at"] - created_at = self.toTZ(created_at) - raw_filename = created_at + ".ts" - live_filename = "LIVE_" + raw_filename - raw_vod_filename = "VOD_" + raw_filename - chat_json_filename = "CHAT_" + created_at + ".json" - chat_mp4_filename = "CHAT_" + created_at + ".mp4" - metadata_filename = "metadata_" + created_at + ".json" - try: - os.rename(recorded_filename,os.path.join(self.recorded_path, live_filename)) - recorded_filename = os.path.join(self.recorded_path, live_filename) - except Exception as e: - raw_filename = present_datetime + ".ts" - live_filename = "LIVE_" + raw_filename - raw_vod_filename = "VOD_" + raw_filename - chat_json_filename = "CHAT_" + present_datetime + ".json" - chat_mp4_filename = "CHAT_" + present_datetime + ".mp4" - metadata_filename = "metadata_" + present_datetime + ".json" - os.rename(recorded_filename,os.path.join(self.recorded_path, live_filename)) - recorded_filename = os.path.join(self.recorded_path, live_filename) - print('first exception as e\nAn error has occurred. VOD and chat will not be downloaded. Please check them manually.\n' + e) - self.sendNotif('ERROR - '+ present_datetime, 'An error has occurred. VOD and chat will not be downloaded. Please check them manually.\n ' + e) - if self.downloadMETADATA == 1: - self.sendNotif('Metadata - ' + created_at,'Downloading and saving metadata:\n' + json.dumps(vodsinfodic["data"][0], indent=4)) - with open(os.path.join(self.metadata_path, metadata_filename), 'w', encoding='utf-8') as f: - json.dump(vodsinfodic["data"][0], f, ensure_ascii=False, indent=4) - if self.downloadVOD == 1: - print('Downloading VOD: ' + vodsinfodic["data"][0]["title"]) - self.sendNotif('VOD - ' + created_at,'Downloading VOD: ' + vodsinfodic["data"][0]["title"]) - try: - subprocess.call(['streamlink', '--http-header', 'Authorization=OAuth ' + os.environ.get('OAUTH-PRIVATE-TOKEN'), "--hls-segment-threads", str(self.hls_segmentsVOD), "twitch.tv/videos/" + vod_id, self.quality] + self.debug_cmd + ["-o", os.path.join(self.recorded_path, raw_vod_filename)]) - except Exception as e: - print('Error', 'A ERROR has ocurred and the VOD will not be downloaded.\n' + e) - self.sendNotif('ERROR - ' + created_at, 'A ERROR has ocurred and the VOD will not be downloaded. \nerror:\n' + e) - if self.downloadCHAT == 1: - print('Downloading and rendering CHAT: ' + vodsinfodic["data"][0]["title"]) - self.sendNotif('CHAT - ' + created_at,'Downloading JSON and rendering chat logs from VOD:\n' + vodsinfodic["data"][0]["title"]) - try: - subprocess.call([str(pathlib.Path(__file__).parent.resolve())+"/bin/chat.bat", vod_id, os.path.join(self.chatJSON_path, chat_json_filename), os.path.join(self.chatMP4_path, chat_mp4_filename)]) - except Exception as e: - self.sendNotif('ERROR - ' + created_at, "A ERROR has ocurred and chat will need to be downloaded and rendered manually.\n" + e) - print("A ERROR has ocurred and chat will need to be downloaded and rendered manually\n" + e) - else: - print('A ERROR has ocurred, the latest VOD doesnt match with the livestream, the VOD is not published\nThe VOD and chat will not be downloaded and rendered.\nThe current livestream date: ' + present_datetime + '\nThe VOD date: ' + self.toTZ(vodsinfodic["data"][0]["created_at"])) - self.sendNotif('ERROR - ' + present_datetime, 'A ERROR has ocurred, the latest VOD doesnt match with the livestream, the VOD is not published\nThe VOD and chat will not be downloaded and rendered.\nThe current livestream date: ' + present_datetime + '\nThe VOD date: ' + self.toTZ(vodsinfodic["data"][0]["created_at"])) - else: - raw_filename = present_datetime + ".ts" - live_filename = "LIVE_" + raw_filename - raw_vod_filename = "VOD_" + raw_filename - chat_json_filename = "CHAT_" + present_datetime + ".json" - chat_mp4_filename = "CHAT_" + present_datetime + ".mp4" - metadata_filename = "metadata_" + present_datetime + ".json" - os.rename(recorded_filename,os.path.join(self.recorded_path, live_filename)) - recorded_filename = os.path.join(self.recorded_path, live_filename) - except Exception as e: - raw_filename = present_datetime + ".ts" - live_filename = "LIVE_" + raw_filename - raw_vod_filename = "VOD_" + raw_filename - chat_json_filename = "CHAT_" + present_datetime + ".json" - chat_mp4_filename = "CHAT_" + present_datetime + ".mp4" - metadata_filename = "metadata_" + present_datetime + ".json" - os.rename(recorded_filename,os.path.join(self.recorded_path, live_filename)) - recorded_filename = os.path.join(self.recorded_path, live_filename) - print('An error has occurred. VOD and chat will not be downloaded. Please check them manually.\n' + e) - self.sendNotif('ERROR - ' + present_datetime, 'An error has occurred. VOD and chat will not be downloaded. Please check them manually.\n' + e) - print("Recording stream is done. Fixing video file.") - self.sendNotif("STREAM DONE - " + present_datetime, "Recording stream is done. Fixing video file.") - if(os.path.exists(recorded_filename) is True): - file_mp4 = live_filename[:-2] + "mp4" - vod_filename = raw_vod_filename[:-2] + "mp4" - processed_filename = os.path.join(self.processed_path, file_mp4) - subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/ffmpeg.exe', '-y', '-i', recorded_filename, '-analyzeduration', '2147483647', '-probesize', '2147483647', '-c:v', 'copy', '-c:a', 'copy', '-start_at_zero', '-copyts', processed_filename], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) - if(os.path.exists(os.path.join(self.recorded_path, raw_vod_filename)) is True): - subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/ffmpeg.exe', '-y', '-i', os.path.join(self.recorded_path, raw_vod_filename), '-analyzeduration', '2147483647', '-probesize', '2147483647', '-c:v', 'copy', '-c:a', 'copy', '-start_at_zero', '-copyts', os.path.join(self.processed_path, vod_filename)], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) + if self.check_user() is True: + live_date = datetime.strptime(self.live_info["started_at"],'%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=timezone('UTC')).astimezone(tz=None).replace(tzinfo=None) + live_raw_filename = datetime.strftime(live_date,'%Y%m%d_%Hh%Mm%Ss') + + live_raw_path = os.path.join(self.raw_path, "LIVE_" + live_raw_filename + ".ts") + live_proc_path = os.path.join(self.video_path, "LIVE_" + live_raw_filename + ".mp4") + + with open(os.path.join(self.root_path, ".log")) as logs: + logs = logs.read() + log_id = self.live_info["started_at"] + " - " + self.live_info["title"] + if log_id in logs: + time.sleep(self.refresh) + + with open(os.path.join(self.root_path, ".log"), "r+") as logs: + log_id = self.live_info["started_at"] + " - " + self.username + " - " + self.live_info["title"] + for line in logs: + if log_id in line: + break + else: + logs.write(self.live_info["started_at"] + " - " + self.username + " - " + self.live_info["title"] + "\n") + + + self.sendNotif('Stream - ' + live_raw_filename, 'Streamer went live: ' + self.live_info["title"]) + subprocess.call([self.terminal,'streamlink', 'twitch.tv/'+ self.username, self.quality, '--http-header', '"Authorization=OAuth ' + os.getenv('OAUTH-PRIVATE-TOKEN') + '"', '--hls-segment-threads', str(self.hls_segments), '--hls-live-restart', '--retry-streams', str(self.refresh), '--output', live_raw_path]) + if(os.path.exists(live_raw_path) is True): + subprocess.call([self.terminal,str(pathlib.Path(__file__).parent.resolve())+'/bin/ffmpeg', '-y', '-i', live_raw_path, '-analyzeduration', '2147483647', '-probesize', '2147483647', '-c:v', 'copy', '-c:a', 'copy', '-start_at_zero', '-copyts', live_proc_path], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) else: print("Skip fixing. File not found.") - print("Fixing is done.") - if self.uploadCloud == 1: - tree = subprocess.run(["powershell.exe","tree", f"'{self.root_path}/{self.username}'", "/f"], capture_output=True, text=True).stdout.split("\n",2)[2] - print('Uploading the following files:\n' + tree) - self.sendNotif("UPLOADING - " + present_datetime, 'Uploading the following files: \n' + tree) - subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/upload.bat', str(pathlib.Path(self.root_path).resolve()),self.username]) - if self.deleteFiles == 1: - self.sendNotif("DELETING - " + present_datetime, "Deleting the files from current seccion.") - print(f'{Fore.RED}DELETING FILES{Style.RESET_ALL}') - print(f'{Fore.RED}Deleting ' + recorded_filename + f'{Style.RESET_ALL}') - os.remove(recorded_filename) - print(f'{Fore.RED}Deleting ' + processed_filename + f'{Style.RESET_ALL}') - os.remove(processed_filename) + try: + vodurl = f'https://api.twitch.tv/helix/videos?user_id={str(self.channel_id)}&period=day&type=archive' + vods = requests.get(vodurl, headers = {"Authorization" : "Bearer " + self.oauth_token, "Client-ID": os.getenv('CLIENT-ID')}, timeout = 30) + vodsinfo = json.loads(vods.text) + if vodsinfo["data"][0] != []: + vod_date = datetime.strptime(vodsinfo["data"][0]["created_at"],'%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=timezone('UTC')).astimezone(tz=None).replace(tzinfo=None) + vod_raw_filename = datetime.strftime(vod_date,'%Y%m%d_%Hh%Mm%Ss') + if self.live_info["id"] == vodsinfo["data"][0]["stream_id"]: + current_vod = vodsinfo["data"][0] + vod_raw_path = os.path.join(self.raw_path, "VOD_" + live_raw_filename + ".ts") + vod_proc_path = os.path.join(self.video_path, "VOD_" + live_raw_filename + ".mp4") + + if self.downloadMETADATA == 1: + self.sendNotif('Metadata - ' + live_raw_filename,'Downloading and saving metadata:\n' + json.dumps(current_vod, indent=4)) + with open(os.path.join(self.metadata_path, "METADA_" + live_raw_filename + ".json"), 'w', encoding='utf-8') as f: + json.dump(current_vod, f, ensure_ascii=False, indent=4) + + if self.downloadVOD == 1: + print('Downloading VOD: ' + current_vod["title"]) + self.sendNotif('VOD - ' + live_raw_filename,'Downloading VOD: ' + current_vod["title"]) + try: + subprocess.call([self.terminal,'streamlink', 'twitch.tv/videos/' + current_vod["id"], self.quality, '--http-header', '"Authorization=OAuth ' + os.getenv('OAUTH-PRIVATE-TOKEN') +'"', "--hls-segment-threads", str(self.hls_segmentsVOD), "-o", vod_raw_path]) + if(os.path.exists(vod_raw_path) is True): + subprocess.call([self.terminal,str(pathlib.Path(__file__).parent.resolve())+'/bin/ffmpeg', '-y', '-i', vod_raw_path, '-analyzeduration', '2147483647', '-probesize', '2147483647', '-c:v', 'copy', '-c:a', 'copy', '-start_at_zero', '-copyts', vod_proc_path], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) + else: + print("Skip fixing. File not found.") + except Exception as e: + print('Error', 'A ERROR has ocurred and the VOD will not be downloaded.\n') + self.sendNotif('ERROR - ' + live_raw_filename, 'A ERROR has ocurred and the VOD will not be downloaded.\n') + + if self.downloadCHAT == 1: + print('Downloading and rendering CHAT: ' + current_vod["title"]) + self.sendNotif('CHAT - ' + live_raw_filename,'Downloading JSON and rendering chat logs from VOD:\n' + current_vod["title"]) + try: + subprocess.call([self.terminal,str(pathlib.Path(__file__).parent.resolve())+"/bin/chat", current_vod["id"], os.path.join(self.chatJSON_path, "CHAT_" + live_raw_filename + ".json"), os.path.join(self.chatMP4_path, "CHAT_" + live_raw_filename + ".mp4")]) + except Exception as e: + self.sendNotif('ERROR - ' + live_raw_filename, "A ERROR has ocurred and chat will need to be downloaded and rendered manually.\n") + print("A ERROR has ocurred and chat will need to be downloaded and rendered manually\n") + else: + print('A ERROR has ocurred, the latest VOD doesnt match with the livestream, the VOD is not published\nThe VOD and chat will not be downloaded and rendered.\nThe current livestream date: ' + live_raw_filename + '\nThe VOD date: ' + vod_raw_filename) + self.sendNotif('ERROR - ' + live_raw_filename, 'A ERROR has ocurred, the latest VOD doesnt match with the livestream, the VOD is not published\nThe VOD and chat will not be downloaded and rendered.\nThe current livestream date: ' + live_raw_filename + '\nThe VOD date: ' + vod_raw_filename) + except Exception as e: + print('An error has occurred. VOD and chat will not be downloaded. Please check them manually.\n') + self.sendNotif('ERROR - ' + live_raw_filename, 'An error has occurred. VOD and chat will not be downloaded. Please check them manually.\n') + + if self.cleanRaw == 1: + print('Deleting raw files') + if(os.path.exists(live_raw_path) is True): os.remove(live_raw_path) if self.downloadVOD == 1: - if(os.path.exists(os.path.join(self.recorded_path, raw_vod_filename)) is True): - print(f'{Fore.RED}Deleting ' + os.path.join(self.recorded_path, raw_vod_filename) + f'{Style.RESET_ALL}') - os.remove(os.path.join(self.recorded_path, raw_vod_filename)) - if(os.path.exists(os.path.join(self.processed_path, vod_filename)) is True): - print(f'{Fore.RED}Deleting ' + os.path.join(self.processed_path, vod_filename) + f'{Style.RESET_ALL}') - os.remove(os.path.join(self.processed_path, vod_filename)) + if(os.path.exists(os.path.join(self.raw_path, "VOD_" + live_raw_filename + ".ts")) is True): + os.remove(os.path.join(self.raw_path, "VOD_" + live_raw_filename + ".ts")) + if self.uploadCloud == 1: + if self.terminal == 'powershell.exe': + tree = subprocess.run([self.terminal,"tree", f"'{self.root_path}/{self.username}'", "/f"], capture_output=True, text=True).stdout.split("\n",2)[2] + elif self.terminal == 'bash': + tree = subprocess.check_output([self.terminal,"tree", str(pathlib.Path(self.root_path).resolve())+"/"+self.username]).decode(sys.stdout.encoding) + print('Uploading the following files:\n' + tree) + self.sendNotif("UPLOADING - " + live_raw_filename, 'Uploading the following files: \n' + tree) + subprocess.call([self.terminal,str(pathlib.Path(__file__).parent.resolve())+'/bin/upload', str(pathlib.Path(self.root_path).resolve()),self.username]) + if self.deleteFiles == 1: + self.sendNotif("DELETING - " + live_raw_filename, "Deleting the files from current seccion.") + print(f'{Fore.RED}DELETING FILES{Style.RESET_ALL}') + if self.cleanRaw == 0: + print(f'{Fore.RED}Deleting ' + live_raw_path + f'{Style.RESET_ALL}') + os.remove(live_raw_path) + print(f'{Fore.RED}Deleting ' + live_proc_path + f'{Style.RESET_ALL}') + os.remove(live_proc_path) + if self.downloadVOD == 1: + if(os.path.exists(os.path.join(self.raw_path, "VOD_" + live_raw_filename + ".ts")) is True): + if self.cleanRaw == 0: + print(f'{Fore.RED}Deleting ' + os.path.join(self.raw_path, "VOD_" + live_raw_filename + ".ts") + f'{Style.RESET_ALL}') + os.remove(os.path.join(self.raw_path, "VOD_" + live_raw_filename + ".ts")) + if(os.path.exists(os.path.join(self.video_path, "VOD_" + live_raw_filename + ".mp4")) is True): + print(f'{Fore.RED}Deleting ' + os.path.join(self.video_path, "VOD_" + live_raw_filename + ".mp4") + f'{Style.RESET_ALL}') + os.remove(os.path.join(self.video_path, "VOD_" + live_raw_filename + ".mp4")) if self.downloadCHAT == 1: - if(os.path.exists(os.path.join(self.chatJSON_path, chat_json_filename)) is True): - print(f'{Fore.RED}Deleting ' + os.path.join(self.chatJSON_path, chat_json_filename) + f'{Style.RESET_ALL}') - os.remove(os.path.join(self.chatJSON_path, chat_json_filename)) - if(os.path.exists(os.path.join(self.chatMP4_path, chat_mp4_filename)) is True): - print(f'{Fore.RED}Deleting ' + os.path.join(self.chatMP4_path, chat_mp4_filename) + f'{Style.RESET_ALL}') - os.remove(os.path.join(self.chatMP4_path, chat_mp4_filename)) + if(os.path.exists(os.path.join(self.chatJSON_path, "CHAT_"+live_raw_filename + ".json")) is True): + print(f'{Fore.RED}Deleting ' + os.path.join(self.chatJSON_path, "CHAT_"+live_raw_filename + ".json") + f'{Style.RESET_ALL}') + os.remove(os.path.join(self.chatJSON_path, "CHAT_"+live_raw_filename + ".json")) + if(os.path.exists(os.path.join(self.chatMP4_path, "CHAT_"+live_raw_filename + ".mp4")) is True): + print(f'{Fore.RED}Deleting ' + os.path.join(self.chatMP4_path, "CHAT_"+live_raw_filename + ".mp4") + f'{Style.RESET_ALL}') + os.remove(os.path.join(self.chatMP4_path, "CHAT_"+live_raw_filename + ".mp4")) if self.downloadMETADATA == 1: - if(os.path.exists(os.path.join(self.metadata_path, metadata_filename)) is True): - print(f'{Fore.RED}Deleting ' + os.path.join(self.metadata_path, metadata_filename) + f'{Style.RESET_ALL}') - os.remove(os.path.join(self.metadata_path, metadata_filename)) + if(os.path.exists(os.path.join(self.metadata_path, "METADA_"+live_raw_filename+".json")) is True): + print(f'{Fore.RED}Deleting ' + os.path.join(self.metadata_path, "METADA_"+live_raw_filename+".json") + f'{Style.RESET_ALL}') + os.remove(os.path.join(self.metadata_path, "METADA_"+live_raw_filename+".json")) print('CURRENT SECCION HAVE FINISHED GOING BACK TO CHECKING') - self.sendNotif("SECCION DONE - " + present_datetime, 'CURRENT SECCION HAVE FINISHED GOING BACK TO CHECKING') + self.sendNotif("SECCION DONE - " + live_raw_filename, 'CURRENT SECCION HAVE FINISHED GOING BACK TO CHECKING') time.sleep(self.refresh) def main(argv): - twitch_recorder = TwitchArchive() - usage_message = 'twitch-archive.py -u -q ' + twitch_archive = TwitchArchive() + help_msg = 'Twitch-Archive\nPython script to record twitch live stream, download the VOD, metadata, chat and render it, and uploads them to any cloud storage.\n -h, --help Display this information\n -u, --username Twitch channel username\n -q, --quality best/source high/720p medium/480p worst/360p\n -v, --vod <1/0> Download vod\n -c, --chat <1/0> Download chat and render it\n -m, --metadata <1/0> Download metadata\n -r, --upload <1/0> Upload to cloud storage\n -d, --delete <1/0> Delete all files after upload (CAREFUL with this arg)\n -n, --notifications <1/0> Receive email notification of the proccess through gmail\n' try: - opts, args = getopt.getopt(argv,"u:q:",["username=","quality="]) + opts, args = getopt.getopt(argv,"h:u:q:v:c:m:r:d:n",["username=","quality=","vod=","chat=","metadata=","upload=","delete=","notifications="]) except getopt.GetoptError: - print (usage_message) + print (help_msg) sys.exit(2) for opt, arg in opts: - if opt == '-h': - print(usage_message) + if opt in ('-h', '--help'): + print(help_msg) sys.exit() - elif opt in ("-u", "--username"): - twitch_recorder.username = arg - elif opt in ("-q", "--quality"): - twitch_recorder.quality = arg - twitch_recorder.run() + elif opt in ("-u", "--username"): twitch_archive.username = arg + elif opt in ("-q", "--quality"): twitch_archive.quality = arg + elif opt in ("-v", "--vod"): twitch_archive.quality = int(arg) + elif opt in ("-c", "--chat"): twitch_archive.quality = int(arg) + elif opt in ("-m", "--metadata"): twitch_archive.quality = int(arg) + elif opt in ("-r", "--upload"): twitch_archive.quality = int(arg) + elif opt in ("-d", "--delete"): twitch_archive.quality = int(arg) + elif opt in ("-n", "--notifications"): twitch_archive.quality = int(arg) + twitch_archive.run() if __name__ == "__main__": main(sys.argv[1:]) \ No newline at end of file