TwitchDownloader/twitch-archive.py
2023-02-19 17:03:49 -05:00

308 lines
No EOL
24 KiB
Python

import requests, os, time, json, sys, subprocess, getopt, smtplib, pathlib, socket
from colorama import Fore, Style
from datetime import datetime, timedelta
from pytz import timezone
from dotenv import load_dotenv, find_dotenv
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
class TwitchArchive:
def __init__(self):
# user configuration
self.username = "dogzilar123" # Twitch streamer username
self.quality = "best" # Qualities options: best/source high/720p medium/540p low/360p audio_only
# global configuration
self.root_path = r"archive" # Path where this script saves everything (livestream,VODs,chat,metadata)
self.rclone_path = "remote:path" # Path to rclone remote storage
self.refresh = 5.0 # Time between checking (5.0 is recommended), avoid less than 1.0
self.streamlink_ttvlol = 0 # 0 - disable blocking ads with ttvlol, 1 - enable blocking ads with ttvlol, Uses this repo: https://github.com/2bc4/streamlink-ttvlol to block ads with ttvlol. Follows the steps on the repo to enable it.
self.notifications = 0 # 0 - disable email notification of current seccion, 1 - enable email notification of current seccion
self.downloadMETADATA = 1 # 0 - disable metadata downloading, 1 - enable metadata downloading
self.downloadVOD = 1 # 0 - disable VOD downloading after stream finished, 1 - enable VOD downloading after stream finished (this option downloads the latest public vod)
self.downloadCHAT = 1 # 0 - disable chat downloading and rendering, 1 - enable chat downloading and rendering
self.uploadCloud = 0 # 0 - disable upload to remote cloud, 1 - enable upload to remote cloud
self.deleteFiles = 0 # 0 - disable the deleting of files from current seccion after being uploaded to the cloud, 1 - enable the deleting files of files from current seccion after being uploaded to the cloud (BE CAREFUL WITH THIS OPTION)
self.onlyRaw = 0 # 0 - disable the converting of ts files to mp3/mp4, 1 - enable the converting of ts files to mp3/mp4 (only works for recording, the vod will still be downloaded to mp3/mp4)
self.cleanRaw = 0 # 0 - disable the deleting of raw (.ts) files, 1 - enable the deleteing of raw (.ts) files (if upload enable they will be deleted before)
self.hls_segments = 3 # 1-10 for live stream, it's possible to use multiple threads to potentially increase the throughput. 2-3 is enough
self.hls_segmentsVOD = 10 # 1-10 for downloading vod, it's possible to use multiple threads to potentially increase the throughput
def run(self):
self.os = self.get_OS()
if load_dotenv(find_dotenv()): load_dotenv(find_dotenv())
else:
print(f'{Fore.RED}\033[1mCREATE .env file with variables{Style.RESET_ALL}')
quit()
self.correct_user()
print('Twitch-Archive')
print('Configuration:')
print(f'Root path: {Fore.GREEN}' + str(pathlib.Path(self.root_path).resolve()) + f'{Style.RESET_ALL}')
print(f'Refresh rate: {Fore.GREEN} {str(self.refresh)}{Style.RESET_ALL}')
if self.notifications == 1: print(f'Email notifications: {Fore.GREEN}Enabled{Style.RESET_ALL}')
else: print(f'Email notifications: {Fore.RED}Disabled{Style.RESET_ALL}')
if self.downloadMETADATA == 1: print(f'Metada downloading {Fore.GREEN}Enabled{Style.RESET_ALL}')
else: print(f'Metada downloading: {Fore.RED}Disabled{Style.RESET_ALL}')
if self.downloadVOD == 1: print(f'VOD downloading {Fore.GREEN}Enabled{Style.RESET_ALL}')
else: print(f'VOD downloading: {Fore.RED}Disabled{Style.RESET_ALL}')
if self.downloadCHAT == 1: print(f'Chat downloading {Fore.GREEN}Enabled{Style.RESET_ALL}')
else: print(f'Chat downloading: {Fore.RED}Disabled{Style.RESET_ALL}')
if self.uploadCloud == 1: print(f'Upload to cloud storage: {Fore.GREEN}Enabled{Style.RESET_ALL}')
else: print(f'Upload to cloud storage: {Fore.RED}Disabled{Style.RESET_ALL}')
if self.deleteFiles == 1: print(f'{Fore.RED}'+'\033[1m'+f'CAREFUL FILES ARE CONFIGURATED TO BE DELETED{Style.RESET_ALL}')
else: print(f'{Fore.GREEN}'+'\033[1m'+f'Files will NOT be deleted{Style.RESET_ALL}')
if self.uploadCloud == 0 and self.deleteFiles == 1: print(f'{Fore.RED}'+'\033[1m'+f'FILES WILL BE DELETED AND NO UPLOADED {Style.RESET_ALL}{Fore.GREEN}\n"CTRL + C"{Style.RESET_ALL}{Fore.RED}'+'\033[1m'+f' TO STOP AND CHANGED CONFIGURATION{Style.RESET_ALL}')
self.raw_path = str(pathlib.Path(os.path.join(self.root_path,self.username,"video", "raw")).absolute())
self.video_path = str(pathlib.Path(os.path.join(self.root_path, self.username, "video")).absolute())
self.chatJSON_path = str(pathlib.Path(os.path.join(self.root_path, self.username, "chat", "json")).absolute())
self.chatMP4_path = str(pathlib.Path(os.path.join(self.root_path, self.username, "chat")).absolute())
self.metadata_path = str(pathlib.Path(os.path.join(self.root_path, self.username, "metadata")).absolute())
if(os.path.isdir(self.raw_path) is False): os.makedirs(self.raw_path)
if(os.path.isdir(self.video_path) is False): os.makedirs(self.video_path)
if(os.path.isdir(self.chatJSON_path) is False): os.makedirs(self.chatJSON_path)
if(os.path.isdir(self.chatMP4_path) is False): os.makedirs(self.chatMP4_path)
if(os.path.isdir(self.metadata_path) is False): os.makedirs(self.metadata_path)
if not os.path.exists(os.path.join(self.root_path, ".log")):
with open(os.path.join(self.root_path, ".log"), 'w'): pass
print(f"Checking for {Fore.GREEN}{self.username}{Style.RESET_ALL} every {Fore.GREEN}{self.refresh}{Style.RESET_ALL} seconds. Record with {Fore.GREEN}{self.quality}{Style.RESET_ALL} quality.")
self.sendNotif("TWITCH ARCHIVE", f"Checking for {self.username} every {self.refresh} seconds. Record with {self.quality} quality.")
self.loopcheck()
def get_OS(self):
if sys.platform.startswith('win32'):
return 'windows'
elif sys.platform.startswith('linux'):
return 'linux'
else:
print(f'{Fore.RED}\033[1mOS no supported{Style.RESET_ALL}')
quit()
def get_oauth_token(self):
try:
return requests.post(f"https://id.twitch.tv/oauth2/token?client_id={os.getenv('CLIENT-ID')}&client_secret={os.getenv('CLIENT-SECRET')}&grant_type=client_credentials").json()['access_token']
except:
print(f'{Fore.RED}\033[1mCheck your client-id and secret{Style.RESET_ALL}')
quit()
def correct_user(self):
try:
url = f'https://api.twitch.tv/helix/users?login={self.username}'
response = requests.get(url,headers = {"Authorization" : "Bearer " + self.get_oauth_token(), "Client-ID": os.getenv('CLIENT-ID')}, timeout = 15).json()
if response['data'] == []:
print(f'{Fore.RED}\033[1mUse a correct username{Style.RESET_ALL}')
quit()
except requests.exceptions.RequestException as e:
print(e)
def check_user(self):
query = 'query{user(login: "' + self.username + '") {stream{archiveVideo{id}title createdAt}}}'
try:
response = requests.post('https://gql.twitch.tv/gql',json={'query': query},headers={"Client-ID": "kimne78kx3ncx6brgo4mv6wki5h1ko"})
return json.loads(response.text)
except requests.exceptions.RequestException as e:
print(e)
quit()
def get_vod(self):
query = 'query {user(login: "' + self.username + '") {videos(first: 1) {edges {node {id title description recordedAt lengthSeconds animatedPreviewURL previewThumbnailURL(height: 1280, width: 720) thumbnailURLs(height: 1280, width: 720)}}}}}'
try:
response = requests.post('https://gql.twitch.tv/gql', json={'query': query}, headers={"Client-ID": "kimne78kx3ncx6brgo4mv6wki5h1ko"})
return json.loads(response.text)
except requests.exceptions.RequestException as e:
print(e)
def sendNotif(self, subject, content):
if self.notifications == 1:
try:
sender = os.getenv("SENDER")
receiver = os.getenv("RECEIVER")
msg = MIMEMultipart()
msg['From'] = sender
msg['To'] = receiver
msg['Subject'] = self.username + " _ " + subject
body = "Current seccion is for " + self.username + "\n\n\n\n" + content
msg.attach(MIMEText((body), 'plain'))
server = smtplib.SMTP('smtp.gmail.com', 587)
server.starttls()
server.login(sender, os.getenv("PASSWD"))
txt = msg.as_string()
server.sendmail(sender, receiver, txt)
server.quit()
except socket.error as e:
print(e)
def loopcheck(self):
while True:
is_live = self.check_user()['data']['user']['stream']
if is_live is not None:
is_live_ready = self.check_user()['data']['user']['stream']['title']
if is_live_ready is not None:
bin_path = str(pathlib.Path(__file__).parent.resolve())+"/bin"
live_date = datetime.strptime(is_live["createdAt"],'%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=timezone('UTC')).astimezone(tz=None).replace(tzinfo=None)
live_raw_filename = datetime.strftime(live_date,'%Y%m%d_%Hh%Mm%Ss')
live_raw_path = os.path.join(self.raw_path, "LIVE_" + live_raw_filename + ".ts")
live_proc_path = os.path.join(self.video_path, "LIVE_" + live_raw_filename + ".mp4")
with open(os.path.join(self.root_path, ".log"), encoding="utf-8") as logs:
logs = logs.read()
log_id = is_live["createdAt"] + " - " + self.username + " - " + is_live["title"]
if log_id in logs:
time.sleep(self.refresh)
with open(os.path.join(self.root_path, ".log"), "r+", encoding="utf-8") as logs:
log_id = is_live["createdAt"] + " - " + self.username + " - " + is_live["title"]
for line in logs:
if log_id in line:
time.sleep(self.refresh)
else:
logs.write(is_live["createdAt"] + " - " + self.username + " - " + is_live["title"] +"\n")
if self.streamlink_ttvlol == 1:ttvlol = ['--twitch-proxy-playlist=https://api.ttv.lol']
else: ttvlol = ''.split()
if self.quality == 'audio_only':
live_proc_path = os.path.join(self.video_path, "LIVE_" + live_raw_filename + ".mp3")
if (os.getenv("OAUTH-PRIVATE-TOKEN") != "" or os.getenv("OAUTH-PRIVATE-TOKEN") is not None or os.getenv("OAUTH-PRIVATE-TOKEN") != "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"):
auth = ['--twitch-api-header', 'Authorization=OAuth ' + os.getenv('OAUTH-PRIVATE-TOKEN')]
else: auth = ''.split()
self.sendNotif('Stream - ' + live_raw_filename, 'Streamer went live: ' + is_live["title"])
subprocess.call(['streamlink', 'twitch.tv/'+ self.username, self.quality, '--hls-segment-threads', str(self.hls_segments), '--hls-live-restart', '--retry-streams', str(self.refresh), '--twitch-disable-reruns', '-o', live_raw_path] + ttvlol + auth)
if(os.path.exists(live_raw_path) is True and self.onlyRaw == 0):
ffmpeg_settings = ['-y', '-i', live_raw_path, '-analyzeduration', '2147483647', '-probesize', '2147483647', '-c:v', 'copy', '-c:a', 'copy', '-start_at_zero', '-copyts', live_proc_path]
if self.quality == 'audio_only': ffmpeg_settings = ['-i', live_raw_path, '-vn', '-ar', '44100', '-ac', '2', '-b:a', '192k', live_proc_path]
if self.os == 'windows': subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/ffmpeg.exe']+ ffmpeg_settings, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
elif self.os == 'linux': subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/ffmpeg']+ ffmpeg_settings, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
else:
print("Skip fixing. File not found.")
current_vod = self.get_vod()['data']['user']['videos']['edges'][0]['node']
live_date_min = live_date - timedelta(minutes=1)
live_date_max = live_date + timedelta(minutes=1)
vod_date = datetime.strptime(current_vod["recordedAt"],'%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=timezone('UTC')).astimezone(tz=None).replace(tzinfo=None)
if live_date_min <= vod_date <= live_date_max:
vod_proc_path = os.path.join(self.video_path, "VOD_" + live_raw_filename + ".mp4")
chat_json_path = os.path.join(self.chatJSON_path, "CHAT_" + live_raw_filename + ".json")
chat_video_path = os.path.join(self.chatMP4_path, "CHAT_" + live_raw_filename + ".mp4")
if self.downloadMETADATA == 1:
self.sendNotif('Metadata - ' + live_raw_filename,'Downloading and saving metadata:\n' + json.dumps(current_vod, indent=4))
with open(os.path.join(self.metadata_path, "METADA_" + live_raw_filename + ".json"), 'w', encoding='utf-8') as f:
json.dump(current_vod, f, ensure_ascii=False, indent=4)
if self.quality == 'audio_only':
vod_proc_path = os.path.join(self.video_path, "VOD_" + live_raw_filename + ".mp3")
if self.downloadVOD == 1:
print('Downloading VOD: ' + current_vod["title"])
self.sendNotif('VOD - ' + live_raw_filename,'Downloading VOD: ' + current_vod["title"])
try:
if self.os == 'windows':subprocess.call([bin_path+"/TwitchDownloaderCLI.exe", 'videoDownload', '-u', str(current_vod["id"]), '-q', self.quality, "-t", str(self.hls_segmentsVOD), "--ffmpeg-path", bin_path +"/ffmpeg.exe", '--temp-path', bin_path+"/temp", "-o", vod_proc_path])
elif self.os == 'linux':subprocess.call([bin_path+"/TwitchDownloaderCLI", 'videoDownload', '-u', str(current_vod["id"]), '-q', self.quality, "-t", str(self.hls_segmentsVOD), "--ffmpeg-path", bin_path +"/ffmpeg.exe", '--temp-path', bin_path+"/temp", "-o", vod_proc_path])
except Exception as e:
print('Error', 'A ERROR has ocurred and the VOD will not be downloaded.\n')
self.sendNotif('ERROR - ' + live_raw_filename, 'A ERROR has ocurred and the VOD will not be downloaded.\n')
if self.downloadCHAT == 1:
print('Downloading and rendering CHAT: ' + current_vod["title"])
self.sendNotif('CHAT - ' + live_raw_filename,'Downloading JSON and rendering chat logs from VOD:\n' + current_vod["title"])
chat_settings = ["--background-color", "#FF111111", "-w", "500", "-h", "1080", "--outline", "true", "-f", "Arial", "--font-size", "22", "--update-rate", "1.0", "--offline", "--ffmpeg-path", f"{bin_path}/ffmpeg.exe", "--temp-path", f"{bin_path}/temp"]
try:
if self.os == 'windows':
subprocess.call([bin_path + '/TwitchDownloaderCLI.exe', 'chatdownload', '--id', current_vod["id"], '-o', chat_json_path, '-E'])
print('Rendering CHAT: ' + current_vod["title"])
subprocess.call([bin_path + '/TwitchDownloaderCLI.exe', 'chatrender', '-i', chat_json_path, '-o', chat_video_path] + chat_settings)
elif self.os == 'linux':
subprocess.call([bin_path + '/TwitchDownloaderCLI', 'chatdownload', '--id', current_vod["id"], '-o', chat_json_path, '-E'])
subprocess.call([bin_path + '/TwitchDownloaderCLI', 'chatrender', '-i', chat_json_path, '-o', chat_video_path] + chat_settings)
except Exception as e:
self.sendNotif('ERROR - ' + live_raw_filename, "A ERROR has ocurred and chat will need to be downloaded and rendered manually.\n")
print("A ERROR has ocurred and chat will need to be downloaded and rendered manually\n")
else:
print('not VOD associated with stream found')
if self.cleanRaw == 1:
print('Deleting raw files')
if(os.path.exists(live_raw_path) is True): os.remove(live_raw_path)
if self.uploadCloud == 1:
with open(str(pathlib.Path(__file__).parent.resolve())+"/bin/temp/upload.txt", "a") as myfile:
myfile.write("LIVE_" + live_raw_filename + ".ts\n"+"VOD_" + live_raw_filename + ".ts\n"+"LIVE_" + live_raw_filename + ".mp4\n"+"VOD_" + live_raw_filename + ".mp4\n"+"METADATA_" + live_raw_filename + ".json\n"+"CHAT_" + live_raw_filename + ".json\n"+"CHAT_" + live_raw_filename + ".mp4\n")
print('Uploading files')
self.sendNotif("UPLOADING - " + live_raw_filename, 'The files are being uploaded')
subprocess.call(['rclone', 'copy', str(pathlib.Path(self.root_path).resolve()), self.rclone_path, '--include-from', bin_path + '/temp/upload.txt'])
os.remove(str(pathlib.Path(__file__).parent.resolve())+"/bin/temp/upload.txt")
if self.deleteFiles == 1:
self.sendNotif("DELETING - " + live_raw_filename, "Deleting the files from current seccion.")
print(f'{Fore.RED}DELETING FILES{Style.RESET_ALL}')
if self.cleanRaw == 0:
print(f'{Fore.RED}Deleting ' + live_raw_path + f'{Style.RESET_ALL}')
os.remove(live_raw_path)
print(f'{Fore.RED}Deleting ' + live_proc_path + f'{Style.RESET_ALL}')
os.remove(live_proc_path)
if self.downloadVOD == 1:
if(os.path.exists(os.path.join(self.raw_path, "VOD_" + live_raw_filename + ".ts")) is True):
if self.cleanRaw == 0:
print(f'{Fore.RED}Deleting ' + os.path.join(self.raw_path, "VOD_" + live_raw_filename + ".ts") + f'{Style.RESET_ALL}')
os.remove(os.path.join(self.raw_path, "VOD_" + live_raw_filename + ".ts"))
if(os.path.exists(os.path.join(self.video_path, "VOD_" + live_raw_filename + ".mp4")) is True):
print(f'{Fore.RED}Deleting ' + os.path.join(self.video_path, "VOD_" + live_raw_filename + ".mp4") + f'{Style.RESET_ALL}')
os.remove(os.path.join(self.video_path, "VOD_" + live_raw_filename + ".mp4"))
if self.downloadCHAT == 1:
if(os.path.exists(os.path.join(self.chatJSON_path, "CHAT_"+live_raw_filename + ".json")) is True):
print(f'{Fore.RED}Deleting ' + os.path.join(self.chatJSON_path, "CHAT_"+live_raw_filename + ".json") + f'{Style.RESET_ALL}')
os.remove(os.path.join(self.chatJSON_path, "CHAT_"+live_raw_filename + ".json"))
if(os.path.exists(os.path.join(self.chatMP4_path, "CHAT_"+live_raw_filename + ".mp4")) is True):
print(f'{Fore.RED}Deleting ' + os.path.join(self.chatMP4_path, "CHAT_"+live_raw_filename + ".mp4") + f'{Style.RESET_ALL}')
os.remove(os.path.join(self.chatMP4_path, "CHAT_"+live_raw_filename + ".mp4"))
if self.downloadMETADATA == 1:
if(os.path.exists(os.path.join(self.metadata_path, "METADA_"+live_raw_filename+".json")) is True):
print(f'{Fore.RED}Deleting ' + os.path.join(self.metadata_path, "METADA_"+live_raw_filename+".json") + f'{Style.RESET_ALL}')
os.remove(os.path.join(self.metadata_path, "METADA_"+live_raw_filename+".json"))
print('CURRENT SECCION HAVE FINISHED GOING BACK TO CHECKING')
self.sendNotif("SECCION DONE - " + live_raw_filename, 'CURRENT SECCION HAVE FINISHED GOING BACK TO CHECKING')
time.sleep(self.refresh)
else: time.sleep(self.refresh)
else: time.sleep(self.refresh)
def main(argv):
twitch_archive = TwitchArchive()
help_msg = '''
Twitch-Archive
Python script to record twitch live stream, download the VOD, metadata, chat and render it, and uploads them to any cloud storage.
-h, --help Display this information
-u, --username <username> Twitch channel username
-q, --quality <quality> best/source high/720p medium/480p worst/360p
-a, --ttv-lol <1/0> Block ads with ttv-lol https://github.com/2bc4/streamlink-ttvlol
-v, --vod <1/0> Download vod
-c, --chat <1/0> Download chat and render it
-m, --metadata <1/0> Download metadata
-r, --upload <1/0> Upload to cloud storage
-d, --delete <1/0> Delete all files after upload (CAREFUL with this arg)
-n, --notifications <1/0> Receive email notification of the proccess through gmail
'''
try:
opts, args = getopt.getopt(argv,"h:u:q:v:c:m:r:d:n",["username=","quality=","vod=","chat=","metadata=","upload=","delete=","notifications="])
except getopt.GetoptError:
print (help_msg)
sys.exit(2)
for opt, arg in opts:
if opt in ('-h', '--help'):
print(help_msg)
sys.exit()
elif opt in ("-u", "--username"): twitch_archive.username = arg
elif opt in ("-q", "--quality"): twitch_archive.quality = arg
elif opt in ("-a", "--ttv-lol"): twitch_archive.streamlink_ttvlol = int(arg)
elif opt in ("-v", "--vod"): twitch_archive.downloadVOD = int(arg)
elif opt in ("-c", "--chat"): twitch_archive.downloadCHAT = int(arg)
elif opt in ("-m", "--metadata"): twitch_archive.downloadMETADATA = int(arg)
elif opt in ("-r", "--upload"): twitch_archive.uploadCloud = int(arg)
elif opt in ("-d", "--delete"): twitch_archive.deleteFiles = int(arg)
elif opt in ("-n", "--notifications"): twitch_archive.notifications = int(arg)
twitch_archive.run()
if __name__ == "__main__":
main(sys.argv[1:])