clean up
This commit is contained in:
Piero
parent
4e861b6c43
commit
5d7fceef45
13 changed files with 217 additions and 199 deletions
|
|
@ -1,11 +1,10 @@
|
|||
import requests, os, time, json, sys, subprocess, getopt, smtplib, pathlib, glob
|
||||
import requests, os, time, json, sys, subprocess, getopt, smtplib, pathlib, socket
|
||||
from colorama import Fore, Style
|
||||
from datetime import datetime, timedelta
|
||||
from pytz import timezone
|
||||
from dotenv import load_dotenv, find_dotenv
|
||||
from email.mime.multipart import MIMEMultipart
|
||||
from email.mime.text import MIMEText
|
||||
load_dotenv(find_dotenv())
|
||||
class TwitchArchive:
|
||||
def __init__(self):
|
||||
# user configuration
|
||||
|
|
@ -13,19 +12,28 @@ class TwitchArchive:
|
|||
self.quality = "best" # Qualities options: best/source high/720p medium/540p low/360p
|
||||
# global configuration
|
||||
self.root_path = r"archive" # Path where this script saves everything (livestream,VODs,chat,metadata)
|
||||
self.rclone_path = "remote:path" # Path to rclone remote storage
|
||||
self.refresh = 5.0 # Time between checking (5.0 is recommended), avoid less than 1.0
|
||||
self.notifications = 1 # 0 - disable email notification of current seccion, 1 - enable email notification of current seccion
|
||||
self.notifications = 0 # 0 - disable email notification of current seccion, 1 - enable email notification of current seccion
|
||||
self.downloadMETADATA = 1 # 0 - disable metadata downloading, 1 - enable metadata downloading
|
||||
self.downloadVOD = 1 # 0 - disable VOD downloading after stream finished, 1 - enable VOD downloading after stream finished (this option downloads the latest public vod)
|
||||
self.downloadCHAT = 1 # 0 - disable chat downloading and rendering, 1 - enable chat downloading and rendering
|
||||
self.uploadCloud = 1 # 0 - disable upload to remote cloud, 1 - enable upload to remote cloud
|
||||
self.deleteFiles = 1 # 0 - disable the deleting of files from current seccion after being uploaded to the cloud, 1 - enable the deleting files of files from current seccion after being uploaded to the cloud (BE CAREFUL WITH THIS OPTION)
|
||||
self.deleteFiles = 0 # 0 - disable the deleting of files from current seccion after being uploaded to the cloud, 1 - enable the deleting files of files from current seccion after being uploaded to the cloud (BE CAREFUL WITH THIS OPTION)
|
||||
self.cleanRaw = 1 # 0 - disable the deleting of raw (.ts) files, 1 - enable the deleteing of raw (.ts) files (if upload enable they will be deleted before)
|
||||
self.hls_segments = 3 # 1-10 for live stream, it's possible to use multiple threads to potentially increase the throughput. 2-3 is enough
|
||||
self.hls_segmentsVOD = 10 # 1-10 for downloading vod, it's possible to use multiple threads to potentially increase the throughput
|
||||
|
||||
def run(self):
|
||||
self.os = self.get_OS()
|
||||
|
||||
if load_dotenv(find_dotenv()): load_dotenv(find_dotenv())
|
||||
else:
|
||||
print(f'{Fore.RED}\033[1mCREATE .env file with variables{Style.RESET_ALL}')
|
||||
quit()
|
||||
|
||||
self.correct_user()
|
||||
|
||||
print('Twitch-Archive')
|
||||
print('Configuration:')
|
||||
print(f'Root path: {Fore.GREEN}' + str(pathlib.Path(self.root_path).resolve()) + f'{Style.RESET_ALL}')
|
||||
|
|
@ -55,7 +63,7 @@ class TwitchArchive:
|
|||
if(os.path.isdir(self.chatJSON_path) is False): os.makedirs(self.chatJSON_path)
|
||||
if(os.path.isdir(self.chatMP4_path) is False): os.makedirs(self.chatMP4_path)
|
||||
if(os.path.isdir(self.metadata_path) is False): os.makedirs(self.metadata_path)
|
||||
if not os.path.exists(os.path.join(self.root_path, ".log")):
|
||||
if not os.path.exists(os.path.join(self.root_path, ".log")):
|
||||
with open(os.path.join(self.root_path, ".log"), 'w'): pass
|
||||
|
||||
print(f"Checking for {Fore.GREEN}{self.username}{Style.RESET_ALL} every {Fore.GREEN}{self.refresh}{Style.RESET_ALL} seconds. Record with {Fore.GREEN}{self.quality}{Style.RESET_ALL} quality.")
|
||||
|
|
@ -68,72 +76,62 @@ class TwitchArchive:
|
|||
elif sys.platform.startswith('linux'):
|
||||
return 'linux'
|
||||
else:
|
||||
print('OS no supported')
|
||||
return
|
||||
print(f'{Fore.RED}\033[1mOS no supported{Style.RESET_ALL}')
|
||||
quit()
|
||||
|
||||
def get_oauth_token(self):
|
||||
try:
|
||||
return requests.post(f"https://id.twitch.tv/oauth2/token?client_id={os.getenv('CLIENT-ID')}&client_secret={os.getenv('CLIENT-SECRET')}&grant_type=client_credentials").json()['access_token']
|
||||
except:
|
||||
print(f'{Fore.RED}\033[1mCheck your client-id and secret{Style.RESET_ALL}')
|
||||
quit()
|
||||
|
||||
def correct_user(self):
|
||||
try:
|
||||
url = f'https://api.twitch.tv/helix/users?login={self.username}'
|
||||
response = requests.get(url,headers = {"Authorization" : "Bearer " + self.get_oauth_token(), "Client-ID": os.getenv('CLIENT-ID')}, timeout = 15).json()
|
||||
if response['data'] == []:
|
||||
print(f'{Fore.RED}\033[1mUse a correct username{Style.RESET_ALL}')
|
||||
quit()
|
||||
except requests.exceptions.RequestException as e:
|
||||
print(e)
|
||||
|
||||
def check_user(self):
|
||||
client_id = "kimne78kx3ncx6brgo4mv6wki5h1ko"
|
||||
query = '''
|
||||
query {
|
||||
user(login: "''' + self.username + '''") {
|
||||
stream {
|
||||
archiveVideo{
|
||||
id
|
||||
}
|
||||
title
|
||||
createdAt
|
||||
}
|
||||
}
|
||||
}
|
||||
'''
|
||||
url = 'https://gql.twitch.tv/gql'
|
||||
response = requests.post(url,json={'query': query},headers={"Client-ID": client_id})
|
||||
return json.loads(response.text)
|
||||
|
||||
query = 'query{user(login: "' + self.username + '") {stream{archiveVideo{id}title createdAt}}}'
|
||||
try:
|
||||
response = requests.post('https://gql.twitch.tv/gql',json={'query': query},headers={"Client-ID": "kimne78kx3ncx6brgo4mv6wki5h1ko"})
|
||||
return json.loads(response.text)
|
||||
except requests.exceptions.RequestException as e:
|
||||
print(e)
|
||||
quit()
|
||||
|
||||
def get_vod(self):
|
||||
client_id = "kimne78kx3ncx6brgo4mv6wki5h1ko"
|
||||
query = '''
|
||||
query {
|
||||
user(login: "''' + self.username + '''") {
|
||||
videos(first: 1) {
|
||||
edges {
|
||||
node {
|
||||
id
|
||||
scope
|
||||
title
|
||||
description
|
||||
recordedAt
|
||||
lengthSeconds
|
||||
animatedPreviewURL
|
||||
previewThumbnailURL(height: 1280, width: 720)
|
||||
thumbnailURLs(height: 1280, width: 720)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
'''
|
||||
url = 'https://gql.twitch.tv/gql'
|
||||
response = requests.post(url, json={'query': query}, headers={"Client-ID": client_id})
|
||||
return json.loads(response.text)
|
||||
query = 'query {user(login: "' + self.username + '") {videos(first: 1) {edges {node {id title description recordedAt lengthSeconds animatedPreviewURL previewThumbnailURL(height: 1280, width: 720) thumbnailURLs(height: 1280, width: 720)}}}}}'
|
||||
try:
|
||||
response = requests.post('https://gql.twitch.tv/gql', json={'query': query}, headers={"Client-ID": "kimne78kx3ncx6brgo4mv6wki5h1ko"})
|
||||
return json.loads(response.text)
|
||||
except requests.exceptions.RequestException as e:
|
||||
print(e)
|
||||
|
||||
def sendNotif(self, subject, content):
|
||||
if self.notifications == 1:
|
||||
sender = os.getenv("SENDER")
|
||||
receiver = os.getenv("RECEIVER")
|
||||
msg = MIMEMultipart()
|
||||
msg['From'] = sender
|
||||
msg['To'] = receiver
|
||||
msg['Subject'] = self.username + " _ " + subject
|
||||
body = "Current seccion is for " + self.username + "\n\n\n\n" + content
|
||||
msg.attach(MIMEText((body), 'plain'))
|
||||
server = smtplib.SMTP('smtp.gmail.com', 587)
|
||||
server.starttls()
|
||||
server.login(sender, os.getenv("PASSWD"))
|
||||
txt = msg.as_string()
|
||||
server.sendmail(sender, receiver, txt)
|
||||
server.quit()
|
||||
try:
|
||||
sender = os.getenv("SENDER")
|
||||
receiver = os.getenv("RECEIVER")
|
||||
msg = MIMEMultipart()
|
||||
msg['From'] = sender
|
||||
msg['To'] = receiver
|
||||
msg['Subject'] = self.username + " _ " + subject
|
||||
body = "Current seccion is for " + self.username + "\n\n\n\n" + content
|
||||
msg.attach(MIMEText((body), 'plain'))
|
||||
server = smtplib.SMTP('smtp.gmail.com', 587)
|
||||
server.starttls()
|
||||
server.login(sender, os.getenv("PASSWD"))
|
||||
txt = msg.as_string()
|
||||
server.sendmail(sender, receiver, txt)
|
||||
server.quit()
|
||||
except socket.error as e:
|
||||
print(e)
|
||||
|
||||
def loopcheck(self):
|
||||
while True:
|
||||
|
|
@ -141,19 +139,19 @@ class TwitchArchive:
|
|||
if is_live is not None:
|
||||
is_live_ready = self.check_user()['data']['user']['stream']['title']
|
||||
if is_live_ready is not None:
|
||||
bin_path = str(pathlib.Path(__file__).parent.resolve())+"/bin"
|
||||
live_date = datetime.strptime(is_live["createdAt"],'%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=timezone('UTC')).astimezone(tz=None).replace(tzinfo=None)
|
||||
live_raw_filename = datetime.strftime(live_date,'%Y%m%d_%Hh%Mm%Ss')
|
||||
|
||||
live_raw_filename = datetime.strftime(live_date,'%Y%m%d_%Hh%Mm%Ss')
|
||||
live_raw_path = os.path.join(self.raw_path, "LIVE_" + live_raw_filename + ".ts")
|
||||
live_proc_path = os.path.join(self.video_path, "LIVE_" + live_raw_filename + ".mp4")
|
||||
|
||||
with open(os.path.join(self.root_path, ".log")) as logs:
|
||||
with open(os.path.join(self.root_path, ".log"), encoding="utf-8") as logs:
|
||||
logs = logs.read()
|
||||
log_id = is_live["createdAt"] + " - " + self.username + " - " + is_live["title"]
|
||||
if log_id in logs:
|
||||
time.sleep(self.refresh)
|
||||
|
||||
with open(os.path.join(self.root_path, ".log"), "r+") as logs:
|
||||
with open(os.path.join(self.root_path, ".log"), "r+", encoding="utf-8") as logs:
|
||||
log_id = is_live["createdAt"] + " - " + self.username + " - " + is_live["title"]
|
||||
for line in logs:
|
||||
if log_id in line:
|
||||
|
|
@ -176,9 +174,10 @@ class TwitchArchive:
|
|||
vod_date = datetime.strptime(current_vod["recordedAt"],'%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=timezone('UTC')).astimezone(tz=None).replace(tzinfo=None)
|
||||
|
||||
if live_date_min <= vod_date <= live_date_max:
|
||||
vod_raw_path = os.path.join(self.raw_path, "VOD_" + live_raw_filename + ".ts")
|
||||
vod_proc_path = os.path.join(self.video_path, "VOD_" + live_raw_filename + ".mp4")
|
||||
|
||||
vod_proc_path = os.path.join(self.video_path, live_raw_filename + ".mp4")
|
||||
chat_json_path = os.path.join(self.chatJSON_path, live_raw_filename + ".json")
|
||||
chat_video_path = os.path.join(self.chatMP4_path, live_raw_filename + ".mp4")
|
||||
if self.downloadMETADATA == 1:
|
||||
self.sendNotif('Metadata - ' + live_raw_filename,'Downloading and saving metadata:\n' + json.dumps(current_vod, indent=4))
|
||||
with open(os.path.join(self.metadata_path, "METADA_" + live_raw_filename + ".json"), 'w', encoding='utf-8') as f:
|
||||
|
|
@ -188,12 +187,9 @@ class TwitchArchive:
|
|||
print('Downloading VOD: ' + current_vod["title"])
|
||||
self.sendNotif('VOD - ' + live_raw_filename,'Downloading VOD: ' + current_vod["title"])
|
||||
try:
|
||||
subprocess.call(['streamlink', 'twitch.tv/videos/' + str(current_vod["id"]), self.quality, '--twitch-api-header', 'Authorization=OAuth ' + os.getenv('OAUTH-PRIVATE-TOKEN'), "--hls-segment-threads", str(self.hls_segmentsVOD), "-o", vod_raw_path])
|
||||
if(os.path.exists(vod_raw_path) is True):
|
||||
if self.os == 'windows':subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/ffmpeg.exe', '-y', '-i', vod_raw_path, '-analyzeduration', '2147483647', '-probesize', '2147483647', '-c:v', 'copy', '-c:a', 'copy', '-start_at_zero', '-copyts', vod_proc_path], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
|
||||
elif self.os == 'linux':subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/ffmpeg', '-y', '-i', vod_raw_path, '-analyzeduration', '2147483647', '-probesize', '2147483647', '-c:v', 'copy', '-c:a', 'copy', '-start_at_zero', '-copyts', vod_proc_path], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
|
||||
else:
|
||||
print("Skip fixing. File not found.")
|
||||
if self.os == 'windows':subprocess.call([bin_path+"/TwitchDownloaderCLI.exe", 'videoDownload', '-u', str(current_vod["id"]), '-q', self.quality, "-t", str(self.hls_segmentsVOD), "--ffmpeg-path", bin_path +"/ffmpeg.exe", '--temp-path', bin_path+"/temp", "-o", vod_proc_path])
|
||||
elif self.os == 'linux':subprocess.call([bin_path+"/TwitchDownloaderCLI", 'videoDownload', '-u', str(current_vod["id"]), '-q', self.quality, "-t", str(self.hls_segmentsVOD), "--ffmpeg-path", bin_path +"/ffmpeg.exe", '--temp-path', bin_path+"/temp", "-o", vod_proc_path])
|
||||
|
||||
except Exception as e:
|
||||
print('Error', 'A ERROR has ocurred and the VOD will not be downloaded.\n')
|
||||
self.sendNotif('ERROR - ' + live_raw_filename, 'A ERROR has ocurred and the VOD will not be downloaded.\n')
|
||||
|
|
@ -201,9 +197,15 @@ class TwitchArchive:
|
|||
if self.downloadCHAT == 1:
|
||||
print('Downloading and rendering CHAT: ' + current_vod["title"])
|
||||
self.sendNotif('CHAT - ' + live_raw_filename,'Downloading JSON and rendering chat logs from VOD:\n' + current_vod["title"])
|
||||
chat_settings = ["--background-color", "#FF111111", "-w", "500", "-h", "1080", "--outline", "true", "-f", "Arial", "--font-size", "22", "--update-rate", "1.0", "--offline", "--ffmpeg-path", f"{bin_path}/ffmpeg.exe", "--temp-path", f"{bin_path}/temp"]
|
||||
try:
|
||||
if self.os == 'windows':subprocess.call([str(pathlib.Path(__file__).parent.resolve())+"/bin/chat.bat", str(current_vod["id"]), os.path.join(self.chatJSON_path, "CHAT_" + live_raw_filename + ".json"), os.path.join(self.chatMP4_path, "CHAT_" + live_raw_filename + ".mp4")])
|
||||
elif self.os == 'linux':subprocess.call([str(pathlib.Path(__file__).parent.resolve())+"/bin/chat.sh", str(current_vod["id"]), os.path.join(self.chatJSON_path, "CHAT_" + live_raw_filename + ".json"), os.path.join(self.chatMP4_path, "CHAT_" + live_raw_filename + ".mp4")])
|
||||
if self.os == 'windows':
|
||||
subprocess.call([bin_path + '/TwitchDownloaderCLI.exe', 'chatdownload', '--id', current_vod["id"], '-o', chat_json_path, '-E'])
|
||||
print('Rendering CHAT: ' + current_vod["title"])
|
||||
subprocess.call([bin_path + '/TwitchDownloaderCLI.exe', 'chatrender', '-i', chat_json_path, '-o', chat_video_path] + chat_settings)
|
||||
elif self.os == 'linux':
|
||||
subprocess.call([bin_path + '/TwitchDownloaderCLI', 'chatdownload', '--id', current_vod["id"], '-o', chat_json_path, '-E'])
|
||||
subprocess.call([bin_path + '/TwitchDownloaderCLI', 'chatrender', '-i', chat_json_path, '-o', chat_video_path] + chat_settings)
|
||||
except Exception as e:
|
||||
self.sendNotif('ERROR - ' + live_raw_filename, "A ERROR has ocurred and chat will need to be downloaded and rendered manually.\n")
|
||||
print("A ERROR has ocurred and chat will need to be downloaded and rendered manually\n")
|
||||
|
|
@ -212,16 +214,12 @@ class TwitchArchive:
|
|||
if self.cleanRaw == 1:
|
||||
print('Deleting raw files')
|
||||
if(os.path.exists(live_raw_path) is True): os.remove(live_raw_path)
|
||||
if self.downloadVOD == 1:
|
||||
if(os.path.exists(os.path.join(self.raw_path, "VOD_" + live_raw_filename + ".ts")) is True):
|
||||
os.remove(os.path.join(self.raw_path, "VOD_" + live_raw_filename + ".ts"))
|
||||
if self.uploadCloud == 1:
|
||||
with open(str(pathlib.Path(__file__).parent.resolve())+"/bin/temp/upload.txt", "a") as myfile:
|
||||
myfile.write("LIVE_" + live_raw_filename + ".ts\n"+"VOD_" + live_raw_filename + ".ts\n"+"LIVE_" + live_raw_filename + ".mp4\n"+"VOD_" + live_raw_filename + ".mp4\n"+"METADATA_" + live_raw_filename + ".json\n"+"CHAT_" + live_raw_filename + ".json\n"+"CHAT_" + live_raw_filename + ".mp4\n")
|
||||
print('Uploading files')
|
||||
self.sendNotif("UPLOADING - " + live_raw_filename, 'The files are being uploaded')
|
||||
if self.os == 'windows':subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/upload.bat', str(pathlib.Path(self.root_path).resolve()),self.username])
|
||||
elif self.os == 'linux':subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/upload.sh', str(pathlib.Path(self.root_path).resolve()),self.username])
|
||||
subprocess.call(['rclone', 'copy', str(pathlib.Path(self.root_path).resolve()), self.rclone_path, '--include-from', bin_path + '/temp/upload.txt'])
|
||||
os.remove(str(pathlib.Path(__file__).parent.resolve())+"/bin/temp/upload.txt")
|
||||
if self.deleteFiles == 1:
|
||||
self.sendNotif("DELETING - " + live_raw_filename, "Deleting the files from current seccion.")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue