v.5.0
This commit is contained in:
Piero
parent
cc40ef1062
commit
4e861b6c43
5 changed files with 323 additions and 272 deletions
277
only-vod-chat.py
277
only-vod-chat.py
|
|
@ -1,6 +1,7 @@
|
|||
import requests, os, time, json, sys, subprocess, getopt, pathlib
|
||||
import requests, os, time, json, sys, subprocess, getopt, pathlib, locale
|
||||
from colorama import Fore, Style
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timedelta
|
||||
locale.setlocale(locale.LC_TIME, "es_ES")
|
||||
from pytz import timezone
|
||||
from dotenv import load_dotenv, find_dotenv
|
||||
|
||||
|
|
@ -13,10 +14,10 @@ class TwitchArchive:
|
|||
# global configuration
|
||||
self.root_path = r"archive" # Path where this script saves everything (livestream,VODs,chat,metadata)
|
||||
self.refresh = 60 # Time between checking (5.0 is recommended), avoid less than 1.0
|
||||
self.downloadVOD = 0 # 0 - disable VOD downloading after stream finished, 1 - enable VOD downloading after stream finished (this option downloads the latest public vod)
|
||||
self.downloadVOD = 1 # 0 - disable VOD downloading after stream finished, 1 - enable VOD downloading after stream finished (this option downloads the latest public vod)
|
||||
self.downloadCHAT = 1 # 0 - disable chat downloading and rendering, 1 - enable chat downloading and rendering
|
||||
self.uploadCloud = 1 # 0 - disable upload to remote cloud, 1 - enable upload to remote cloud
|
||||
self.deleteFiles = 1 # 0 - disable the deleting of files from current seccion after being uploaded to the cloud, 1 - enable the deleting files of files from current seccion after being uploaded to the cloud (BE CAREFUL WITH THIS OPTION)
|
||||
self.deleteFiles = 0 # 0 - disable the deleting of files from current seccion after being uploaded to the cloud, 1 - enable the deleting files of files from current seccion after being uploaded to the cloud (BE CAREFUL WITH THIS OPTION)
|
||||
self.hls_segmentsVOD = 10 # 1-10 for downloading vod, it's possible to use multiple threads to potentially increase the throughput
|
||||
|
||||
def run(self):
|
||||
|
|
@ -38,18 +39,18 @@ class TwitchArchive:
|
|||
self.oauth_token = self.get_oauth_token()
|
||||
self.channel_id = self.get_channel_id()
|
||||
|
||||
self.temp_path = str(pathlib.Path(os.path.join(self.root_path,self.username,"vod", "temp")).absolute())
|
||||
self.vod_path = str(pathlib.Path(os.path.join(self.root_path, self.username, "vod")).absolute())
|
||||
self.json_path = str(pathlib.Path(os.path.join(self.root_path, self.username, "chat", "json")).absolute())
|
||||
self.chat_path = str(pathlib.Path(os.path.join(self.root_path, self.username, "chat")).absolute())
|
||||
self.temp_path = str(pathlib.Path(os.path.join("VODS",self.root_path,self.username,"vod", "temp")).absolute())
|
||||
self.vod_path = str(pathlib.Path(os.path.join("VODS",self.root_path, self.username, "vod")).absolute())
|
||||
self.json_path = str(pathlib.Path(os.path.join("VODS",self.root_path, self.username, "chat", "json")).absolute())
|
||||
self.chat_path = str(pathlib.Path(os.path.join("VODS",self.root_path, self.username, "chat")).absolute())
|
||||
|
||||
if(os.path.isdir(self.temp_path) is False): os.makedirs(self.temp_path)
|
||||
if(os.path.isdir(self.vod_path) is False): os.makedirs(self.vod_path)
|
||||
if(os.path.isdir(self.json_path) is False): os.makedirs(self.json_path)
|
||||
if(os.path.isdir(self.chat_path) is False): os.makedirs(self.chat_path)
|
||||
|
||||
if not os.path.exists(os.path.join(self.root_path, ".log")):
|
||||
with open(os.path.join(self.root_path, ".log"), 'w'): pass
|
||||
if not os.path.exists(os.path.join('VODS',self.root_path, ".log")):
|
||||
with open(os.path.join('VODS',self.root_path, ".log"), 'w'): pass
|
||||
|
||||
print(f"Checking for {Fore.GREEN}{self.username}{Style.RESET_ALL} every {Fore.GREEN}{self.refresh}{Style.RESET_ALL} seconds to download VOD/CHAT")
|
||||
self.loopcheck()
|
||||
|
|
@ -63,123 +64,161 @@ class TwitchArchive:
|
|||
print('OS no supported')
|
||||
return
|
||||
|
||||
def get_oauth_token(self):
|
||||
try:
|
||||
return requests.post(f"https://id.twitch.tv/oauth2/token?client_id={os.getenv('CLIENT-ID')}&client_secret={os.getenv('CLIENT-SECRET')}&grant_type=client_credentials").json()['access_token']
|
||||
except:
|
||||
return None
|
||||
|
||||
def get_channel_id(self):
|
||||
try:
|
||||
r = requests.get(f'https://api.twitch.tv/helix/users?login={self.username}', headers = {"Authorization" : "Bearer " + self.oauth_token, "Client-ID": os.getenv('CLIENT-ID')}, timeout = 15)
|
||||
r.raise_for_status()
|
||||
info = r.json()
|
||||
if info["data"] != []:
|
||||
return info["data"][0]["id"]
|
||||
else:
|
||||
return None
|
||||
except requests.exceptions.RequestException as e:
|
||||
print(e)
|
||||
def check_user(self):
|
||||
try:
|
||||
url = 'https://api.twitch.tv/helix/streams?user_id=' + self.channel_id
|
||||
live = requests.get(url, headers = {"Authorization" : "Bearer " + self.oauth_token, "Client-ID": os.environ.get('CLIENT-ID')}, timeout = 30)
|
||||
stream_data = live.json()
|
||||
if len(stream_data['data']) == 1:
|
||||
self.live_info = stream_data['data'][0]
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
except Exception as e:
|
||||
print("API request ERROR")
|
||||
print(e)
|
||||
return False
|
||||
client_id = "kimne78kx3ncx6brgo4mv6wki5h1ko"
|
||||
query = '''
|
||||
query {
|
||||
user(login: "''' + self.username + '''") {
|
||||
stream {
|
||||
archiveVideo{
|
||||
id
|
||||
}
|
||||
title
|
||||
createdAt
|
||||
}
|
||||
}
|
||||
}
|
||||
'''
|
||||
url = 'https://gql.twitch.tv/gql'
|
||||
response = requests.post(url,json={'query': query},headers={"Client-ID": client_id})
|
||||
return json.loads(response.text)
|
||||
def get_vod(self):
|
||||
client_id = "kimne78kx3ncx6brgo4mv6wki5h1ko"
|
||||
query = '''
|
||||
query {
|
||||
user(login: "''' + self.username + '''") {
|
||||
videos(first: 1) {
|
||||
edges {
|
||||
node {
|
||||
id
|
||||
scope
|
||||
title
|
||||
description
|
||||
recordedAt
|
||||
lengthSeconds
|
||||
animatedPreviewURL
|
||||
previewThumbnailURL(height: 1280, width: 720)
|
||||
thumbnailURLs(height: 1280, width: 720)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
'''
|
||||
url = 'https://gql.twitch.tv/gql'
|
||||
response = requests.post(url, json={'query': query}, headers={"Client-ID": client_id})
|
||||
return json.loads(response.text)
|
||||
|
||||
def loopcheck(self):
|
||||
while True:
|
||||
if self.check_user() is True:
|
||||
live_temp_path = os.path.join(self.temp_path, "live_temp.ts")
|
||||
with open(os.path.join(self.root_path, ".log")) as logs:
|
||||
logs = logs.read()
|
||||
log_id = self.live_info["started_at"] + " - " + self.username + " - " + self.live_info["title"]
|
||||
if log_id in logs:
|
||||
time.sleep(self.refresh)
|
||||
with open(os.path.join(self.root_path, ".log"), "r+") as logs:
|
||||
log_id = self.live_info["started_at"] + " - " + self.username + " - " + self.live_info["title"]
|
||||
for line in logs:
|
||||
if log_id in line:
|
||||
break
|
||||
else:
|
||||
logs.write(self.live_info["started_at"] + " - " + self.username + " - " + self.live_info["title"] + "\n")
|
||||
is_live = self.check_user()['data']['user']['stream']
|
||||
if is_live is not None:
|
||||
is_live_ready = self.check_user()['data']['user']['stream']['title']
|
||||
if is_live_ready is not None:
|
||||
live_date = datetime.strptime(is_live["createdAt"],'%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=timezone('UTC')).astimezone(tz=None).replace(tzinfo=None)
|
||||
live_temp_path = os.path.join(self.temp_path, "live_temp.ts")
|
||||
|
||||
with open(os.path.join(self.root_path, ".log")) as logs:
|
||||
logs = logs.read()
|
||||
log_id = is_live["createdAt"] + " - " + self.username + " - " + is_live["title"]
|
||||
if log_id in logs:
|
||||
time.sleep(self.refresh)
|
||||
|
||||
subprocess.call(['streamlink', 'twitch.tv/'+ self.username, 'worst', '--hls-segment-threads', '3', '--retry-streams', str(self.refresh), '--twitch-disable-reruns', '-o', live_temp_path])
|
||||
try:
|
||||
vodurl = f'https://api.twitch.tv/helix/videos?user_id={str(self.channel_id)}&period=day&type=archive'
|
||||
vods = requests.get(vodurl, headers = {"Authorization" : "Bearer " + self.oauth_token, "Client-ID": os.getenv('CLIENT-ID')}, timeout = 30)
|
||||
vodsinfo = json.loads(vods.text)
|
||||
if vodsinfo["data"][0] != []:
|
||||
vod_date = datetime.strptime(vodsinfo["data"][0]["created_at"],'%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=timezone('UTC')).astimezone(tz=None).replace(tzinfo=None)
|
||||
vod_raw_filename = datetime.strftime(vod_date,'%Y%m%d_%Hh%Mm%Ss')
|
||||
if self.live_info["id"] == vodsinfo["data"][0]["stream_id"]:
|
||||
print('VOD AND CHAT AVAILABLE')
|
||||
current_vod = vodsinfo["data"][0]
|
||||
vod_raw_path = os.path.join(self.temp_path, "vod_temp.ts")
|
||||
vod_proc_path = os.path.join(self.vod_path, vod_raw_filename + ".mp4")
|
||||
chat_json_path = os.path.join(self.json_path, vod_raw_filename + ".json")
|
||||
chat_video_path = os.path.join(self.chat_path, vod_raw_filename + ".mp4")
|
||||
|
||||
if self.downloadVOD == 1:
|
||||
print('Downloading VOD: ' + current_vod["title"])
|
||||
try:
|
||||
subprocess.call(['streamlink', 'twitch.tv/videos/' + current_vod["id"], self.quality, "--hls-segment-threads", str(self.hls_segmentsVOD), "-o", vod_raw_path])
|
||||
if(os.path.exists(vod_raw_path) is True):
|
||||
if self.os == 'windows':subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/ffmpeg.exe', '-y', '-i', vod_raw_path, '-analyzeduration', '2147483647', '-probesize', '2147483647', '-c:v', 'copy', '-c:a', 'copy', '-start_at_zero', '-copyts', vod_proc_path], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
|
||||
elif self.os == 'linux':subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/ffmpeg', '-y', '-i', vod_raw_path, '-analyzeduration', '2147483647', '-probesize', '2147483647', '-c:v', 'copy', '-c:a', 'copy', '-start_at_zero', '-copyts', vod_proc_path], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
|
||||
else:
|
||||
print("Skip fixing. File not found.")
|
||||
except Exception as e:
|
||||
print('Error', 'A ERROR has ocurred and the VOD will not be downloaded.\n')
|
||||
if self.downloadCHAT == 1:
|
||||
print('Downloading and rendering CHAT: ' + current_vod["title"])
|
||||
try:
|
||||
if self.os == 'windows':subprocess.call([str(pathlib.Path(__file__).parent.resolve())+"/bin/chat.bat", current_vod["id"], chat_json_path, chat_video_path])
|
||||
elif self.os == 'linux':subprocess.call([str(pathlib.Path(__file__).parent.resolve())+"/bin/chat.sh", current_vod["id"], chat_json_path, chat_video_path])
|
||||
except Exception as e:
|
||||
print("A ERROR has ocurred and chat will need to be downloaded and rendered manually\n")
|
||||
|
||||
if self.uploadCloud == 1:
|
||||
if self.os == 'windows':
|
||||
tree = subprocess.run(['powershell.exe','tree', f'{self.root_path}/{self.username}', '/f'], capture_output=True, text=True).stdout.split("\n",2)[2]
|
||||
elif self.os == 'linux':
|
||||
tree = subprocess.check_output(['tree', str(pathlib.Path(self.root_path).resolve())+'/'+self.username]).decode(sys.stdout.encoding)
|
||||
print('Uploading the following files:\n' + tree)
|
||||
if self.os == 'windows':subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/upload.bat', str(pathlib.Path(self.root_path).resolve()),self.username])
|
||||
elif self.os == 'linux':subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/upload.sh', str(pathlib.Path(self.root_path).resolve()),self.username])
|
||||
|
||||
if self.deleteFiles == 1:
|
||||
print(f'{Fore.RED}DELETING FILES{Style.RESET_ALL}')
|
||||
if self.downloadVOD == 1:
|
||||
if(os.path.exists(vod_raw_path) is True):
|
||||
print(f'{Fore.RED}Deleting ' + vod_raw_path + f'{Style.RESET_ALL}')
|
||||
os.remove(vod_raw_path)
|
||||
if(os.path.exists(vod_proc_path) is True):
|
||||
print(f'{Fore.RED}Deleting ' + vod_proc_path + f'{Style.RESET_ALL}')
|
||||
os.remove(vod_proc_path)
|
||||
if self.downloadCHAT == 1:
|
||||
if(os.path.exists(chat_json_path) is True):
|
||||
print(f'{Fore.RED}Deleting ' + chat_json_path + f'{Style.RESET_ALL}')
|
||||
os.remove(chat_json_path)
|
||||
if(os.path.exists(chat_video_path) is True):
|
||||
print(f'{Fore.RED}Deleting ' + chat_video_path + f'{Style.RESET_ALL}')
|
||||
os.remove(chat_video_path)
|
||||
with open(os.path.join(self.root_path, ".log"), "r+") as logs:
|
||||
log_id = is_live["createdAt"] + " - " + self.username + " - " + is_live["title"]
|
||||
for line in logs:
|
||||
if log_id in line:
|
||||
break
|
||||
else:
|
||||
print('THE VOD/CHAT FOR CURRENT LIVESTREAM IS NOT AVAILABLE\nThe current livestream date: ' + self.live_info['started_at'] + '\nThe VOD date: ' + vodsinfo["data"][0]["created_at"])
|
||||
except Exception as e:
|
||||
print('API request error.')
|
||||
print(e)
|
||||
logs.write(is_live["createdAt"] + " - " + self.username + " - " + is_live["title"] +"\n")
|
||||
|
||||
print('CURRENT SECCION HAVE FINISHED GOING BACK TO CHECKING')
|
||||
time.sleep(self.refresh)
|
||||
subprocess.call(['streamlink', 'twitch.tv/'+ self.username, self.quality, '--twitch-api-header', 'Authorization=OAuth ' + os.getenv('OAUTH-PRIVATE-TOKEN'), '--hls-segment-threads', str(self.hls_segments), '--hls-live-restart', '--retry-streams', str(self.refresh), '--twitch-disable-reruns', '-o', live_temp_path])
|
||||
os.remove(live_temp_path)
|
||||
|
||||
current_vod = self.get_vod()['data']['user']['videos']['edges'][0]['node']
|
||||
vod_date = datetime.strptime(current_vod["recordedAt"],'%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=timezone('UTC')).astimezone(tz=None).replace(tzinfo=None)
|
||||
vod_raw_filename = datetime.strftime(vod_date,'%Y%m%d_%Hh%Mm%Ss')
|
||||
live_date_min = live_date - timedelta(minutes=1)
|
||||
live_date_max = live_date + timedelta(minutes=1)
|
||||
|
||||
if live_date_min <= vod_date <= live_date_max:
|
||||
print('VOD AND CHAT AVAILABLE')
|
||||
vod_raw_path = os.path.join(self.temp_path, "vod_temp.ts")
|
||||
vod_proc_path = os.path.join(self.vod_path, vod_raw_filename + ".mp4")
|
||||
chat_json_path = os.path.join(self.json_path, vod_raw_filename + ".json")
|
||||
chat_video_path = os.path.join(self.chat_path, vod_raw_filename + ".mp4")
|
||||
if self.username == 'KalathrasLolweapon':
|
||||
file_date = datetime.strptime(vod_raw_filename, '%Y%m%d_%Hh%Mm%Ss').date()
|
||||
|
||||
week_first = file_date - timedelta(days=file_date.weekday())
|
||||
week_last = week_first + timedelta(days=6)
|
||||
|
||||
vod_year = 'VOD - ' + str(file_date.year)
|
||||
vod_month = f'{file_date.month:02d} - ' + file_date.strftime("%B").upper()
|
||||
vod_week = file_date.strftime("%B").capitalize() + ' ' + str(week_first.day) + '-' + str(week_last.day)
|
||||
|
||||
chat_year = 'Chat - ' + str(file_date.year)
|
||||
chat_month = f'{file_date.month:02d} - ' + file_date.strftime("%B")
|
||||
|
||||
vod_path = str(pathlib.Path(os.path.join("VODS",vod_year,vod_month,vod_week)).absolute())
|
||||
chat_path = str(pathlib.Path(os.path.join("Chat",chat_year,chat_month)).absolute())
|
||||
if(os.path.isdir(vod_path) is False): os.makedirs(vod_path)
|
||||
if(os.path.isdir(chat_path) is False): os.makedirs(chat_path)
|
||||
chat_video_path = os.path.join(chat_path, vod_raw_filename + ".mp4")
|
||||
vod_proc_path = os.path.join(vod_path, vod_raw_filename + ".mp4")
|
||||
|
||||
if self.downloadVOD == 1:
|
||||
print('Downloading VOD: ' + current_vod["title"])
|
||||
try:
|
||||
subprocess.call(['streamlink', 'twitch.tv/videos/' + current_vod["id"], self.quality, "--hls-segment-threads", str(self.hls_segmentsVOD), "-o", vod_raw_path])
|
||||
if(os.path.exists(vod_raw_path) is True):
|
||||
if self.os == 'windows':subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/ffmpeg.exe', '-y', '-i', vod_raw_path, '-analyzeduration', '2147483647', '-probesize', '2147483647', '-c:v', 'copy', '-c:a', 'copy', '-start_at_zero', '-copyts', vod_proc_path], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
|
||||
elif self.os == 'linux':subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/ffmpeg', '-y', '-i', vod_raw_path, '-analyzeduration', '2147483647', '-probesize', '2147483647', '-c:v', 'copy', '-c:a', 'copy', '-start_at_zero', '-copyts', vod_proc_path], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
|
||||
os.remove(vod_raw_path)
|
||||
else:
|
||||
print("Skip fixing. File not found.")
|
||||
except Exception as e:
|
||||
print('Error', 'A ERROR has ocurred and the VOD will not be downloaded.\n')
|
||||
if self.downloadCHAT == 1:
|
||||
print('Downloading and rendering CHAT: ' + current_vod["title"])
|
||||
try:
|
||||
if self.os == 'windows':subprocess.call([str(pathlib.Path(__file__).parent.resolve())+"/bin/chat.bat", current_vod["id"], chat_json_path, chat_video_path])
|
||||
elif self.os == 'linux':subprocess.call([str(pathlib.Path(__file__).parent.resolve())+"/bin/chat.sh", current_vod["id"], chat_json_path, chat_video_path])
|
||||
except Exception as e:
|
||||
print("A ERROR has ocurred and chat will need to be downloaded and rendered manually\n")
|
||||
|
||||
if self.uploadCloud == 1:
|
||||
print('Uploading files:')
|
||||
if self.os == 'windows':
|
||||
if self.username == 'KalathrasLolweapon':
|
||||
subprocess.call(['rclone', 'copy', str(pathlib.Path(__file__).parent.resolve())+'/VODS', 'gd:VODS', '--progress'])
|
||||
subprocess.call(['rclone', 'copy', str(pathlib.Path(__file__).parent.resolve())+'/Chat', 'gd:Chat', '--progress'])
|
||||
else:subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/upload.bat', str(pathlib.Path(self.root_path).resolve()),self.username])
|
||||
elif self.os == 'linux':subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/upload.sh', str(pathlib.Path(self.root_path).resolve()),self.username])
|
||||
|
||||
if self.deleteFiles == 1:
|
||||
print(f'{Fore.RED}DELETING FILES{Style.RESET_ALL}')
|
||||
if self.downloadVOD == 1:
|
||||
if(os.path.exists(vod_raw_path) is True):
|
||||
print(f'{Fore.RED}Deleting ' + vod_raw_path + f'{Style.RESET_ALL}')
|
||||
os.remove(vod_raw_path)
|
||||
if(os.path.exists(vod_proc_path) is True):
|
||||
print(f'{Fore.RED}Deleting ' + vod_proc_path + f'{Style.RESET_ALL}')
|
||||
os.remove(vod_proc_path)
|
||||
if self.downloadCHAT == 1:
|
||||
if(os.path.exists(chat_json_path) is True):
|
||||
print(f'{Fore.RED}Deleting ' + chat_json_path + f'{Style.RESET_ALL}')
|
||||
os.remove(chat_json_path)
|
||||
if(os.path.exists(chat_video_path) is True):
|
||||
print(f'{Fore.RED}Deleting ' + chat_video_path + f'{Style.RESET_ALL}')
|
||||
os.remove(chat_video_path)
|
||||
else:
|
||||
print('THE VOD/CHAT FOR CURRENT LIVESTREAM IS NOT AVAILABLE\nThe current livestream date: ' + is_live["createdAt"] + '\nThe VOD date: ' + current_vod["recordedAt"])
|
||||
print('CURRENT SECCION HAVE FINISHED GOING BACK TO CHECKING')
|
||||
time.sleep(self.refresh)
|
||||
else: time.sleep(self.refresh)
|
||||
else: time.sleep(self.refresh)
|
||||
def main(argv):
|
||||
twitch_archive = TwitchArchive()
|
||||
help_msg = 'Twitch-Archive\nPython script to download the VOD and/or chat and render it, upload them to any cloud storage.\n -h, --help Display this information\n -u, --username <username> Twitch channel username\n -q, --quality <quality> best/source high/720p medium/480p worst/360p\n -v, --vod <1/0> Download vod\n -c, --chat <1/0> Download chat and render it\n -r, --upload <1/0> Upload to cloud storage\n -d, --delete <1/0> Delete all files after upload (CAREFUL with this arg)\n'
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue