maddo/TwitchDownloader
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

v.4.0

Browse source
This commit is contained in:
Piero 2022-12-09 06:10:02 -05:00
commit c9b5de2d22
7 changed files with 176 additions and 560 deletions
Download patch file Download diff file Expand all files Collapse all files

4
.gitignore vendored
View file

@ -1,2 +1,4 @@
# Environments
.env
.env
# Tests
test.py

8
README.md
View file

@ -1,14 +1,14 @@
# Twitch Archive
Inspired by https://github.com/EnterGin/Auto-Stream-Recording-Twitch
Python script to check, download live stream, VOD, chat and upload them to any cloud service supported by rclone.
Python script to check, download live stream, VOD, chat and upload them to any cloud storage supported by rclone.
## Requirements
- [Python 3](https://www.python.org/downloads/)
- [Streamlink](https://github.com/streamlink/streamlink)
## Getting started
1. Install Python 3
2. Install Streamlink
3. If you want to upload to any cloud service using rclone, [configure rclone](https://rclone.org/docs/#configure) (Doesnt need to download, the `rclone.exe` is avalible in [bin/rclone.exe](https://github.com/piero0920/Twitch-Archive/blob/main/bin/rclone.exe)).
1. Install Python 3.x
2. Install Streamlink 5.1.x
3. If you want to upload to any cloud storage using rclone, [configure rclone](https://rclone.org/docs/#configure).
4. `git clone https://github.com/piero0920/Twitch-Archive.git`
5. `cd Twitch-Archive`
6. `pip install -r requirements.txt`

2
bin/upload.bat
View file

@ -5,4 +5,4 @@ if "%root_path:~-1%" == "\" set "root_path_1=%root_path:~0,-1%"
for %%f in ("%root_path_1%") do set "root_path_name=%%~nxf"
FOR /F "eol=# tokens=*" %%i in (%~dp0\..\.env) do SET %%i
CD %~dp0
rclone.exe copy ../%root_path%/%user% %remote%/%root_path_name%/%user% --progress
rclone.exe copy %root_path%/%user% %remote%/%root_path_name%/%user% --progress

2
extra.md
View file

@ -30,7 +30,7 @@ This option will Download the latest public VOD, if the streamer hasn't publishe
Using a simple api request downloads the `.json` metadata of the latest VOD and saves it to:
`/root_path/streamer_username/metadata/metada_yyyymmdd_hhmmss.json`
### Upload to the cloud
Using [rclone](https://rclone.org/) after everything being downloaded and rendered , it will upload every file from the `root_path/streamer` folder to any cloud service supported by rclone such as [Google Drive, Mega, One Drive, etc.](https://rclone.org/overview/#features)
Using [rclone](https://rclone.org/) after everything being downloaded and rendered , it will upload every file from the `root_path/streamer` folder to any cloud storage supported by rclone such as [Google Drive, Mega, One Drive, etc.](https://rclone.org/overview/#features)
The destination path to where it will be uploaded it has to be stated in the `.env` file.
Example:
```env

5
requirements.txt
View file

@ -1,5 +0,0 @@
colorama==0.4.6
python-dotenv==0.21.0
python_dateutil==2.8.2
pytz==2022.6
requests==2.28.1

309
twitch-archive
View file

@ -1,309 +0,0 @@
import requests, os, time, json, sys, subprocess, getopt, smtplib, pathlib
from colorama import Fore, Style
from datetime import datetime, timedelta
from pytz import timezone
from dateutil import parser
from dotenv import load_dotenv, find_dotenv
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
load_dotenv(find_dotenv())
class TwitchArchive:
def __init__(self):
# user configuration
self.username = "KalathrasLolweapon" # Twitch streamer username
self.quality = "best" # Qualities options: best/source high/720p medium/540p low/360p
# global configuration
self.root_path = r"archive" # Path where this script saves everything (livestream,VODs,chat,metadata)
self.timezone = "US/Eastern" # Timezones, you can see a list of the format timezone here: https://gist.github.com/heyalexej/8bf688fd67d7199be4a1682b3eec7568
self.refresh = 5.0 # Time between checking (5.0 is recommended)
self.notifications = 0 # 0 - disable email notification of current seccion, 1 - enable email notification of current seccion
self.downloadMETADATA = 1 # 0 - disable metadata downloading, 1 - enable metadata downloading
self.downloadVOD = 1 # 0 - disable VOD downloading after stream finished, 1 - enable VOD downloading after stream finished (this option downloads the latest public vod)
self.downloadCHAT = 1 # 0 - disable chat downloading and rendering, 1 - enable chat downloading and rendering
self.uploadCloud = 0 # 0 - disable upload to remote cloud, 1 - enable upload to remote cloud
self.deleteFiles = 0 # 0 - disable the deleting of files from current seccion after being uploaded to the cloud, 1 - enable the deleting files of files from current seccion after being uploaded to the cloud (BE CAREFUL WITH THIS OPTION)
self.streamlink_debug = 0 # 0 - disable streamlink debug display, 1 - enable streamlink debug display
self.hls_segments = 3 # 1-10 for live stream, it's possible to use multiple threads to potentially increase the throughput. 2-3 is enough
self.hls_segmentsVOD = 10 # 1-10 for downloading vod, it's possible to use multiple threads to potentially increase the throughput
def run(self):
print('Twitch-Archive')
print('Configuration:')
print(f'Root path: {Fore.GREEN}' + str(pathlib.Path(self.root_path).resolve()) + f'{Style.RESET_ALL}')
print(f'Timezone: {Fore.GREEN}{self.timezone}{Style.RESET_ALL}')
print(f'Refresh rate: {Fore.GREEN} {str(self.refresh)}{Style.RESET_ALL}')
if self.notifications == 1: print(f'Email notifications: {Fore.GREEN}Enabled{Style.RESET_ALL}')
else: print(f'Email notifications: {Fore.RED}Disabled{Style.RESET_ALL}')
if self.downloadMETADATA == 1: print(f'Metada downloading {Fore.GREEN}Enabled{Style.RESET_ALL}')
else: print(f'Metada downloading: {Fore.RED}Disabled{Style.RESET_ALL}')
if self.downloadVOD == 1: print(f'VOD downloading {Fore.GREEN}Enabled{Style.RESET_ALL}')
else: print(f'VOD downloading: {Fore.RED}Disabled{Style.RESET_ALL}')
if self.downloadCHAT == 1: print(f'Chat downloading {Fore.GREEN}Enabled{Style.RESET_ALL}')
else: print(f'Chat downloading: {Fore.RED}Disabled{Style.RESET_ALL}')
if self.uploadCloud == 1: print(f'Upload to Google Drive: {Fore.GREEN}Enabled{Style.RESET_ALL}')
else: print(f'Upload to cloud service: {Fore.RED}Disabled{Style.RESET_ALL}')
if self.deleteFiles == 1: print(f'{Fore.RED}'+'\033[1m'+f'CAREFUL FILES ARE CONFIGURATED TO BE DELETED{Style.RESET_ALL}')
else: print(f'{Fore.GREEN}'+'\033[1m'+f'Files will NOT be deleted{Style.RESET_ALL}')
if self.uploadCloud == 0 and self.deleteFiles == 1: print(f'{Fore.RED}'+'\033[1m'+f'FILES WILL BE DELETED AND NO UPLOADED {Style.RESET_ALL}{Fore.GREEN}\n"CTRL + C"{Style.RESET_ALL}{Fore.RED}'+'\033[1m'+f' TO STOP AND CHANGED CONFIGURATION{Style.RESET_ALL}')
self.oauth_token = self.get_oauth_token()
self.get_channel_id()
if self.streamlink_debug == 1: self.debug_cmd = "--loglevel trace".split()
else: self.debug_cmd = "".split()
self.recorded_path = str(pathlib.Path(os.path.join(self.root_path,self.username,"video", "recorded")).absolute())
self.processed_path = str(pathlib.Path(os.path.join(self.root_path, self.username, "video", "processed")).absolute())
self.chatJSON_path = str(pathlib.Path(os.path.join(self.root_path, self.username, "chat", "json")).absolute())
self.chatMP4_path = str(pathlib.Path(os.path.join(self.root_path, self.username, "chat", "rendered")).absolute())
self.metadata_path = str(pathlib.Path(os.path.join(self.root_path, self.username, "metadata")).absolute())
if(os.path.isdir(self.recorded_path) is False): os.makedirs(self.recorded_path)
if(os.path.isdir(self.processed_path) is False): os.makedirs(self.processed_path)
if(os.path.isdir(self.chatJSON_path) is False): os.makedirs(self.chatJSON_path)
if(os.path.isdir(self.chatMP4_path) is False): os.makedirs(self.chatMP4_path)
if(os.path.isdir(self.metadata_path) is False): os.makedirs(self.metadata_path)
try:
video_list = [f for f in os.listdir(self.recorded_path) if os.path.isfile(os.path.join(self.recorded_path, f))]
if(len(video_list) > 0):
print('Fixing previously recorded files.')
for f in video_list:
recorded_filename = os.path.join(self.recorded_path, f)
stream_dir_path = self.processed_path
if(os.path.isdir(stream_dir_path) is False):
os.makedirs(stream_dir_path)
print('Fixing ' + recorded_filename + '.')
try:
subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/ffmpeg' , '-y', '-i', recorded_filename, '-analyzeduration', '2147483647', '-probesize', '2147483647', '-c:v', 'copy', '-c:a', 'copy', '-start_at_zero', '-copyts', os.path.join(stream_dir_path, f[:-2]+"mp4")], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
except Exception as e:
print(e)
elif(os.path.exists(os.path.join(stream_dir_path, f)) is False):
print('Fixing ' + recorded_filename + '.')
try:
subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/ffmpeg', '-y', '-i', recorded_filename, '-analyzeduration', '2147483647', '-probesize', '2147483647', '-c:v', 'copy', '-c:a', 'copy', '-start_at_zero', '-copyts', os.path.join(stream_dir_path, f[:-2]+"mp4")], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
except Exception as e:
print(e)
except Exception as e:
print(e)
print(f"Checking for {Fore.GREEN}{self.username}{Style.RESET_ALL} every {Fore.GREEN}{self.refresh}{Style.RESET_ALL} seconds. Record with {Fore.GREEN}{self.quality}{Style.RESET_ALL} quality.")
self.sendNotif("TWITCH ARCHIVE", f"Checking for {self.username} every {self.refresh} seconds. Record with {self.quality} quality.")
self.loopcheck()
def get_oauth_token(self):
try:
return requests.post(f"https://id.twitch.tv/oauth2/token?client_id={os.environ.get('CLIENT-ID')}&client_secret={os.environ.get('CLIENT-SECRET')}&grant_type=client_credentials").json()['access_token']
except:
return None
def get_channel_id(self):
self.getting_channel_id_error = 0
self.user_not_found = 0
if self.oauth_token == None:
self.getting_channel_id_error = 1
return
url = 'https://api.twitch.tv/helix/users?login=' + self.username
try:
r = requests.get(url, headers = {"Authorization" : "Bearer " + self.oauth_token, "Client-ID": os.environ.get('CLIENT-ID')}, timeout = 15)
r.raise_for_status()
info = r.json()
if info["data"] != []: self.channel_id = info["data"][0]["id"]
else: self.user_not_found = 1
except requests.exceptions.RequestException as e:
self.getting_channel_id_error = 1
print(f'\n{e}\n')
def check_user(self):
try:
url = 'https://api.twitch.tv/helix/streams?user_id=' + self.channel_id
req = requests.get(url, headers = {"Authorization" : "Bearer " + self.oauth_token, "Client-ID": os.environ.get('CLIENT-ID')}, timeout = 30)
stream_data = req.json()
if len(stream_data['data']) == 1:
return True
else:
return False
except Exception as e:
print("ERROR checking user: ", e)
return False
def toTZ(self, utc_str):
new_date = str(datetime.fromisoformat(utc_str.replace('Z', '+00:00')).astimezone(timezone(self.timezone)))
year = new_date[:4]
month = new_date[5:7]
day = new_date[8:10]
hour = new_date[11:13]
minute = new_date[14:16]
seconds = new_date[17:19]
date_formated = year + month + day + "_" + hour + "h" + minute + "m" + seconds + "s"
return date_formated
def sendNotif(self, subject, content):
if self.notifications == 1:
sender = os.environ.get("SENDER")
receiver = os.environ.get("RECEIVER")
msg = MIMEMultipart()
msg['From'] = sender
msg['To'] = receiver
msg['Subject'] = self.username + " _ " + subject
body = "Current seccion is for " + self.username + "\n\n\n\n" + content
msg.attach(MIMEText((body), 'plain'))
server = smtplib.SMTP('smtp.gmail.com', 587)
server.starttls()
server.login(sender, os.environ.get("PASSWD"))
txt = msg.as_string()
server.sendmail(sender, receiver, txt)
server.quit()
def loopcheck(self):
while True:
if self.check_user() is True:
live_date = datetime.now(timezone('UTC'))
live_date_min = live_date - timedelta(minutes=10)
live_date_plus = live_date + timedelta(minutes=10)
present_datetime = datetime.now(timezone(self.timezone)).strftime("%Y%m%d_%Hh%Mm%Ss")
raw_filename = present_datetime + ".ts"
live_filename = "LIVE_" + raw_filename
raw_vod_filename = "VOD_" + raw_filename
chat_json_filename = "CHAT_" + present_datetime + ".json"
chat_mp4_filename = "CHAT_" + present_datetime + ".mp4"
metadata_filename = "metadata_" + present_datetime + ".json"
recorded_filename = os.path.join(self.recorded_path, live_filename)
# start streamlink process
subprocess.call(["streamlink", '--http-header', 'Authorization=OAuth ' + os.environ.get('OAUTH-PRIVATE-TOKEN'), "--hls-segment-threads", str(self.hls_segments), "--hls-live-restart", "--twitch-disable-hosting", "twitch.tv/" + self.username, self.quality, "--retry-streams", str(self.refresh)] + self.debug_cmd + ["-o", recorded_filename])
if(os.path.exists(recorded_filename) is True):
try:
vodurl = 'https://api.twitch.tv/helix/videos?user_id=' + str(self.channel_id) + '&type=archive'
vods = requests.get(vodurl, headers = {"Authorization" : "Bearer " + self.oauth_token, "Client-ID": os.environ.get('CLIENT-ID')}, timeout = 30)
vodsinfodic = json.loads(vods.text)
if vodsinfodic["data"][0] != []:
if live_date_min <= parser.parse(vodsinfodic["data"][0]["created_at"]) <= live_date_plus:
vod_id = vodsinfodic["data"][0]["id"]
created_at = vodsinfodic["data"][0]["created_at"]
created_at = self.toTZ(created_at)
raw_filename = created_at + ".ts"
live_filename = "LIVE_" + raw_filename
raw_vod_filename = "VOD_" + raw_filename
chat_json_filename = "CHAT_" + created_at + ".json"
chat_mp4_filename = "CHAT_" + created_at + ".mp4"
metadata_filename = "metadata_" + created_at + ".json"
try:
os.rename(recorded_filename,os.path.join(self.recorded_path, live_filename))
recorded_filename = os.path.join(self.recorded_path, live_filename)
except Exception as e:
raw_filename = present_datetime + ".ts"
live_filename = "LIVE_" + raw_filename
raw_vod_filename = "VOD_" + raw_filename
chat_json_filename = "CHAT_" + present_datetime + ".json"
chat_mp4_filename = "CHAT_" + present_datetime + ".mp4"
metadata_filename = "metadata_" + present_datetime + ".json"
os.rename(recorded_filename,os.path.join(self.recorded_path, live_filename))
recorded_filename = os.path.join(self.recorded_path, live_filename)
print('first exception as e\nAn error has occurred. VOD and chat will not be downloaded. Please check them manually.\n' + e)
self.sendNotif('ERROR - '+ present_datetime, 'An error has occurred. VOD and chat will not be downloaded. Please check them manually.\n ' + e)
if self.downloadMETADATA == 1:
self.sendNotif('Metadata - ' + created_at,'Downloading and saving metadata:\n' + json.dumps(vodsinfodic["data"][0], indent=4))
with open(os.path.join(self.metadata_path, metadata_filename), 'w', encoding='utf-8') as f:
json.dump(vodsinfodic["data"][0], f, ensure_ascii=False, indent=4)
if self.downloadVOD == 1:
print('Downloading VOD: ' + vodsinfodic["data"][0]["title"])
self.sendNotif('VOD - ' + created_at,'Downloading VOD: ' + vodsinfodic["data"][0]["title"])
try:
subprocess.call(['streamlink', '--http-header', 'Authorization=OAuth ' + os.environ.get('OAUTH-PRIVATE-TOKEN'), "--hls-segment-threads", str(self.hls_segmentsVOD), "twitch.tv/videos/" + vod_id, self.quality] + self.debug_cmd + ["-o", os.path.join(self.recorded_path, raw_vod_filename)])
except Exception as e:
print('Error', 'A ERROR has ocurred and the VOD will not be downloaded.\n' + e)
self.sendNotif('ERROR - ' + created_at, 'A ERROR has ocurred and the VOD will not be downloaded. \nerror:\n' + e)
if self.downloadCHAT == 1:
print('Downloading and rendering CHAT: ' + vodsinfodic["data"][0]["title"])
self.sendNotif('CHAT - ' + created_at,'Downloading JSON and rendering chat logs from VOD:\n' + vodsinfodic["data"][0]["title"])
try:
subprocess.call(["bash",str(pathlib.Path(__file__).parent.resolve())+"/bin/chat.sh", vod_id, os.path.join(self.chatJSON_path, chat_json_filename), os.path.join(self.chatMP4_path, chat_mp4_filename)])
except Exception as e:
self.sendNotif('ERROR - ' + created_at, "A ERROR has ocurred and chat will need to be downloaded and rendered manually.\n" + e)
print("A ERROR has ocurred and chat will need to be downloaded and rendered manually\n" + e)
else:
print('A ERROR has ocurred, the latest VOD doesnt match with the livestream, the VOD is not published\nThe VOD and chat will not be downloaded and rendered.\nThe current livestream date: ' + present_datetime + '\nThe VOD date: ' + self.toTZ(vodsinfodic["data"][0]["created_at"]))
self.sendNotif('ERROR - ' + present_datetime, 'A ERROR has ocurred, the latest VOD doesnt match with the livestream, the VOD is not published\nThe VOD and chat will not be downloaded and rendered.\nThe current livestream date: ' + present_datetime + '\nThe VOD date: ' + self.toTZ(vodsinfodic["data"][0]["created_at"]))
else:
raw_filename = present_datetime + ".ts"
live_filename = "LIVE_" + raw_filename
raw_vod_filename = "VOD_" + raw_filename
chat_json_filename = "CHAT_" + present_datetime + ".json"
chat_mp4_filename = "CHAT_" + present_datetime + ".mp4"
metadata_filename = "metadata_" + present_datetime + ".json"
os.rename(recorded_filename,os.path.join(self.recorded_path, live_filename))
recorded_filename = os.path.join(self.recorded_path, live_filename)
except Exception as e:
raw_filename = present_datetime + ".ts"
live_filename = "LIVE_" + raw_filename
raw_vod_filename = "VOD_" + raw_filename
chat_json_filename = "CHAT_" + present_datetime + ".json"
chat_mp4_filename = "CHAT_" + present_datetime + ".mp4"
metadata_filename = "metadata_" + present_datetime + ".json"
os.rename(recorded_filename,os.path.join(self.recorded_path, live_filename))
recorded_filename = os.path.join(self.recorded_path, live_filename)
print('An error has occurred. VOD and chat will not be downloaded. Please check them manually.\n' + e)
self.sendNotif('ERROR - ' + present_datetime, 'An error has occurred. VOD and chat will not be downloaded. Please check them manually.\n' + e)
print("Recording stream is done. Fixing video file.")
self.sendNotif("STREAM DONE - " + present_datetime, "Recording stream is done. Fixing video file.")
if(os.path.exists(recorded_filename) is True):
file_mp4 = live_filename[:-2] + "mp4"
vod_filename = raw_vod_filename[:-2] + "mp4"
processed_filename = os.path.join(self.processed_path, file_mp4)
subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/ffmpeg', '-y', '-i', recorded_filename, '-analyzeduration', '2147483647', '-probesize', '2147483647', '-c:v', 'copy', '-c:a', 'copy', '-start_at_zero', '-copyts', processed_filename], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
if(os.path.exists(os.path.join(self.recorded_path, raw_vod_filename)) is True):
subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/ffmpeg', '-y', '-i', os.path.join(self.recorded_path, raw_vod_filename), '-analyzeduration', '2147483647', '-probesize', '2147483647', '-c:v', 'copy', '-c:a', 'copy', '-start_at_zero', '-copyts', os.path.join(self.processed_path, vod_filename)], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
else:
print("Skip fixing. File not found.")
print("Fixing is done.")
if self.uploadCloud == 1:
tree = subprocess.check_output(["tree", str(pathlib.Path(self.root_path).resolve())+"/"+self.username]).decode(sys.stdout.encoding).split("\n",1)[1]
print('Uploading the following files:\n' + tree)
self.sendNotif("UPLOADING - " + present_datetime, 'Uploading the following files: \n' + tree)
subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/upload.sh', str(pathlib.Path(self.root_path).resolve()),self.username])
if self.deleteFiles == 1:
self.sendNotif("DELETING - " + present_datetime, "Deleting the files from current seccion.")
print(f'{Fore.RED}DELETING FILES{Style.RESET_ALL}')
print(f'{Fore.RED}Deleting ' + recorded_filename + f'{Style.RESET_ALL}')
os.remove(recorded_filename)
print(f'{Fore.RED}Deleting ' + processed_filename + f'{Style.RESET_ALL}')
os.remove(processed_filename)
if self.downloadVOD == 1:
if(os.path.exists(os.path.join(self.recorded_path, raw_vod_filename)) is True):
print(f'{Fore.RED}Deleting ' + os.path.join(self.recorded_path, raw_vod_filename) + f'{Style.RESET_ALL}')
os.remove(os.path.join(self.recorded_path, raw_vod_filename))
if(os.path.exists(os.path.join(self.processed_path, vod_filename)) is True):
print(f'{Fore.RED}Deleting ' + os.path.join(self.processed_path, vod_filename) + f'{Style.RESET_ALL}')
os.remove(os.path.join(self.processed_path, vod_filename))
if self.downloadCHAT == 1:
if(os.path.exists(os.path.join(self.chatJSON_path, chat_json_filename)) is True):
print(f'{Fore.RED}Deleting ' + os.path.join(self.chatJSON_path, chat_json_filename) + f'{Style.RESET_ALL}')
os.remove(os.path.join(self.chatJSON_path, chat_json_filename))
if(os.path.exists(os.path.join(self.chatMP4_path, chat_mp4_filename)) is True):
print(f'{Fore.RED}Deleting ' + os.path.join(self.chatMP4_path, chat_mp4_filename) + f'{Style.RESET_ALL}')
os.remove(os.path.join(self.chatMP4_path, chat_mp4_filename))
if self.downloadMETADATA == 1:
if(os.path.exists(os.path.join(self.metadata_path, metadata_filename)) is True):
print(f'{Fore.RED}Deleting ' + os.path.join(self.metadata_path, metadata_filename) + f'{Style.RESET_ALL}')
os.remove(os.path.join(self.metadata_path, metadata_filename))
print('CURRENT SECCION HAVE FINISHED GOING BACK TO CHECKING')
self.sendNotif("SECCION DONE - " + present_datetime, 'CURRENT SECCION HAVE FINISHED GOING BACK TO CHECKING')
time.sleep(self.refresh)
def main(argv):
twitch_recorder = TwitchArchive()
usage_message = 'twitch-archive.py -u <username> -q <quality>'
try:
opts, args = getopt.getopt(argv,"u:q:",["username=","quality="])
except getopt.GetoptError:
print (usage_message)
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print(usage_message)
sys.exit()
elif opt in ("-u", "--username"):
twitch_recorder.username = arg
elif opt in ("-q", "--quality"):
twitch_recorder.quality = arg
twitch_recorder.run()
if __name__ == "__main__":
main(sys.argv[1:])

416
twitch-archive.py
View file

@ -1,8 +1,7 @@
import requests, os, time, json, sys, subprocess, getopt, smtplib, pathlib
import requests, os, time, json, sys, subprocess, getopt, smtplib, pathlib, glob
from colorama import Fore, Style
from datetime import datetime, timedelta
from pytz import timezone
from dateutil import parser
from dotenv import load_dotenv, find_dotenv
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
@ -14,23 +13,22 @@ class TwitchArchive:
self.quality = "best" # Qualities options: best/source high/720p medium/540p low/360p
# global configuration
self.root_path = r"archive" # Path where this script saves everything (livestream,VODs,chat,metadata)
self.timezone = "US/Eastern" # Timezones, you can see a list of the format timezone here: https://gist.github.com/heyalexej/8bf688fd67d7199be4a1682b3eec7568
self.refresh = 5.0 # Time between checking (5.0 is recommended)
self.notifications = 0 # 0 - disable email notification of current seccion, 1 - enable email notification of current seccion
self.refresh = 5.0 # Time between checking (5.0 is recommended), avoid less than 1.0
self.notifications = 1 # 0 - disable email notification of current seccion, 1 - enable email notification of current seccion
self.downloadMETADATA = 1 # 0 - disable metadata downloading, 1 - enable metadata downloading
self.downloadVOD = 1 # 0 - disable VOD downloading after stream finished, 1 - enable VOD downloading after stream finished (this option downloads the latest public vod)
self.downloadCHAT = 1 # 0 - disable chat downloading and rendering, 1 - enable chat downloading and rendering
self.uploadCloud = 0 # 0 - disable upload to remote cloud, 1 - enable upload to remote cloud
self.uploadCloud = 1 # 0 - disable upload to remote cloud, 1 - enable upload to remote cloud
self.deleteFiles = 0 # 0 - disable the deleting of files from current seccion after being uploaded to the cloud, 1 - enable the deleting files of files from current seccion after being uploaded to the cloud (BE CAREFUL WITH THIS OPTION)
self.streamlink_debug = 0 # 0 - disable streamlink debug display, 1 - enable streamlink debug display
self.cleanRaw = 1 # 0 - disable the deleting of raw (.ts) files, 1 - enable the deleteing of raw (.ts) files (if upload enable they will be deleted before)
self.hls_segments = 3 # 1-10 for live stream, it's possible to use multiple threads to potentially increase the throughput. 2-3 is enough
self.hls_segmentsVOD = 10 # 1-10 for downloading vod, it's possible to use multiple threads to potentially increase the throughput
def run(self):
def run(self):
self.terminal = self.get_OS()
print('Twitch-Archive')
print('Configuration:')
print(f'Root path: {Fore.GREEN}' + str(pathlib.Path(self.root_path).resolve()) + f'{Style.RESET_ALL}')
print(f'Timezone: {Fore.GREEN}{self.timezone}{Style.RESET_ALL}')
print(f'Refresh rate: {Fore.GREEN} {str(self.refresh)}{Style.RESET_ALL}')
if self.notifications == 1: print(f'Email notifications: {Fore.GREEN}Enabled{Style.RESET_ALL}')
else: print(f'Email notifications: {Fore.RED}Disabled{Style.RESET_ALL}')
@ -40,123 +38,77 @@ class TwitchArchive:
else: print(f'VOD downloading: {Fore.RED}Disabled{Style.RESET_ALL}')
if self.downloadCHAT == 1: print(f'Chat downloading {Fore.GREEN}Enabled{Style.RESET_ALL}')
else: print(f'Chat downloading: {Fore.RED}Disabled{Style.RESET_ALL}')
if self.uploadCloud == 1: print(f'Upload to Google Drive: {Fore.GREEN}Enabled{Style.RESET_ALL}')
else: print(f'Upload to cloud service: {Fore.RED}Disabled{Style.RESET_ALL}')
if self.uploadCloud == 1: print(f'Upload to cloud storage: {Fore.GREEN}Enabled{Style.RESET_ALL}')
else: print(f'Upload to cloud storage: {Fore.RED}Disabled{Style.RESET_ALL}')
if self.deleteFiles == 1: print(f'{Fore.RED}'+'\033[1m'+f'CAREFUL FILES ARE CONFIGURATED TO BE DELETED{Style.RESET_ALL}')
else: print(f'{Fore.GREEN}'+'\033[1m'+f'Files will NOT be deleted{Style.RESET_ALL}')
if self.uploadCloud == 0 and self.deleteFiles == 1: print(f'{Fore.RED}'+'\033[1m'+f'FILES WILL BE DELETED AND NO UPLOADED {Style.RESET_ALL}{Fore.GREEN}\n"CTRL + C"{Style.RESET_ALL}{Fore.RED}'+'\033[1m'+f' TO STOP AND CHANGED CONFIGURATION{Style.RESET_ALL}')
self.oauth_token = self.get_oauth_token()
self.get_channel_id()
if self.streamlink_debug == 1: self.debug_cmd = "--loglevel trace".split()
else: self.debug_cmd = "".split()
self.channel_id = self.get_channel_id()
self.recorded_path = str(pathlib.Path(os.path.join(self.root_path,self.username,"video", "recorded")).absolute())
self.processed_path = str(pathlib.Path(os.path.join(self.root_path, self.username, "video", "processed")).absolute())
self.raw_path = str(pathlib.Path(os.path.join(self.root_path,self.username,"video", "raw")).absolute())
self.video_path = str(pathlib.Path(os.path.join(self.root_path, self.username, "video")).absolute())
self.chatJSON_path = str(pathlib.Path(os.path.join(self.root_path, self.username, "chat", "json")).absolute())
self.chatMP4_path = str(pathlib.Path(os.path.join(self.root_path, self.username, "chat", "rendered")).absolute())
self.chatMP4_path = str(pathlib.Path(os.path.join(self.root_path, self.username, "chat")).absolute())
self.metadata_path = str(pathlib.Path(os.path.join(self.root_path, self.username, "metadata")).absolute())
if(os.path.isdir(self.recorded_path) is False): os.makedirs(self.recorded_path)
if(os.path.isdir(self.processed_path) is False): os.makedirs(self.processed_path)
if(os.path.isdir(self.raw_path) is False): os.makedirs(self.raw_path)
if(os.path.isdir(self.video_path) is False): os.makedirs(self.video_path)
if(os.path.isdir(self.chatJSON_path) is False): os.makedirs(self.chatJSON_path)
if(os.path.isdir(self.chatMP4_path) is False): os.makedirs(self.chatMP4_path)
if(os.path.isdir(self.metadata_path) is False): os.makedirs(self.metadata_path)
try:
video_list = [f for f in os.listdir(self.recorded_path) if os.path.isfile(os.path.join(self.recorded_path, f))]
if(len(video_list) > 0):
print('Fixing previously recorded files.')
for f in video_list:
recorded_filename = os.path.join(self.recorded_path, f)
stream_dir_path = self.processed_path
if(os.path.isdir(stream_dir_path) is False):
os.makedirs(stream_dir_path)
print('Fixing ' + recorded_filename + '.')
try:
subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/ffmpeg.exe' , '-y', '-i', recorded_filename, '-analyzeduration', '2147483647', '-probesize', '2147483647', '-c:v', 'copy', '-c:a', 'copy', '-start_at_zero', '-copyts', os.path.join(stream_dir_path, f[:-2]+"mp4")], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
except Exception as e:
print(e)
elif(os.path.exists(os.path.join(stream_dir_path, f)) is False):
print('Fixing ' + recorded_filename + '.')
try:
subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/ffmpeg.exe', '-y', '-i', recorded_filename, '-analyzeduration', '2147483647', '-probesize', '2147483647', '-c:v', 'copy', '-c:a', 'copy', '-start_at_zero', '-copyts', os.path.join(stream_dir_path, f[:-2]+"mp4")], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
except Exception as e:
print(e)
except Exception as e:
print(e)
if not os.path.exists(os.path.join(self.root_path, ".log")):
with open(os.path.join(self.root_path, ".log"), 'w'): pass
print(f"Checking for {Fore.GREEN}{self.username}{Style.RESET_ALL} every {Fore.GREEN}{self.refresh}{Style.RESET_ALL} seconds. Record with {Fore.GREEN}{self.quality}{Style.RESET_ALL} quality.")
self.sendNotif("TWITCH ARCHIVE", f"Checking for {self.username} every {self.refresh} seconds. Record with {self.quality} quality.")
self.loopcheck()
def get_OS(self):
if sys.platform.startswith('win32'):
return 'powershell.exe'
elif sys.platform.startswith('linux'):
return 'bash'
else:
print('OS no supported')
return
def get_oauth_token(self):
try:
return requests.post(f"https://id.twitch.tv/oauth2/token?client_id={os.environ.get('CLIENT-ID')}&client_secret={os.environ.get('CLIENT-SECRET')}&grant_type=client_credentials").json()['access_token']
return requests.post(f"https://id.twitch.tv/oauth2/token?client_id={os.getenv('CLIENT-ID')}&client_secret={os.getenv('CLIENT-SECRET')}&grant_type=client_credentials").json()['access_token']
except:
return None
def get_channel_id(self):
self.getting_channel_id_error = 0
self.user_not_found = 0
if self.oauth_token == None:
self.getting_channel_id_error = 1
return
url = 'https://api.twitch.tv/helix/users?login=' + self.username
try:
r = requests.get(url, headers = {"Authorization" : "Bearer " + self.oauth_token, "Client-ID": os.environ.get('CLIENT-ID')}, timeout = 15)
r = requests.get(f'https://api.twitch.tv/helix/users?login={self.username}', headers = {"Authorization" : "Bearer " + self.oauth_token, "Client-ID": os.getenv('CLIENT-ID')}, timeout = 15)
r.raise_for_status()
info = r.json()
if info["data"] != []: self.channel_id = info["data"][0]["id"]
else: self.user_not_found = 1
if info["data"] != []:
return info["data"][0]["id"]
else:
return None
except requests.exceptions.RequestException as e:
self.getting_channel_id_error = 1
print(f'\n{e}\n')
def check_user(self):
# 0: online, 1: not found, 2: error, 3: channel id error
info = None
if self.user_not_found != 1 and self.getting_channel_id_error != 1:
url = 'https://api.twitch.tv/helix/channels?broadcaster_id=' + str(self.channel_id)
status = 2
try:
r = requests.get(url, headers = {"Authorization" : "Bearer " + self.oauth_token, "Client-ID": os.environ.get('CLIENT-ID')}, timeout = 15)
r.raise_for_status()
info = r.json()
status = 0
except requests.exceptions.RequestException as e:
if e.response != None:
if e.response.status_code == 401:
print('\nRequest to Twitch returned an error %s, trying to get new oauth_token...'% (e.response.status_code))
self.getting_channel_id_error = 1
else:
print('\nRequest to Twitch returned an error %s, the response is:\n%s\n'% (e.response.status_code, e.response))
else:
print(f'\n{e}\n')
elif self.user_not_found == 1:
status = 1
else:
self.oauth_token = self.get_oauth_token()
self.get_channel_id()
status = 3
return status, info
def toTZ(self, utc_str):
new_date = str(datetime.fromisoformat(utc_str.replace('Z', '+00:00')).astimezone(timezone(self.timezone)))
year = new_date[:4]
month = new_date[5:7]
day = new_date[8:10]
hour = new_date[11:13]
minute = new_date[14:16]
seconds = new_date[17:19]
date_formated = year + month + day + "_" + hour + "h" + minute + "m" + seconds + "s"
return date_formated
try:
url = 'https://api.twitch.tv/helix/streams?user_id=' + self.channel_id
live = requests.get(url, headers = {"Authorization" : "Bearer " + self.oauth_token, "Client-ID": os.environ.get('CLIENT-ID')}, timeout = 30)
stream_data = live.json()
if len(stream_data['data']) == 1:
self.live_info = stream_data['data'][0]
return True
else:
return False
except Exception as e:
print("ERROR checking user: ", e)
return False
def sendNotif(self, subject, content):
if self.notifications == 1:
sender = os.environ.get("SENDER")
receiver = os.environ.get("RECEIVER")
sender = os.getenv("SENDER")
receiver = os.getenv("RECEIVER")
msg = MIMEMultipart()
msg['From'] = sender
msg['To'] = receiver
@ -165,174 +117,150 @@ class TwitchArchive:
msg.attach(MIMEText((body), 'plain'))
server = smtplib.SMTP('smtp.gmail.com', 587)
server.starttls()
server.login(sender, os.environ.get("PASSWD"))
server.login(sender, os.getenv("PASSWD"))
txt = msg.as_string()
server.sendmail(sender, receiver, txt)
server.quit()
def loopcheck(self):
while True:
status, info = self.check_user()
if status == 1:
print("Username not found. Invalid username or typo.")
time.sleep(self.refresh)
elif status == 2:
print(datetime.now(timezone(self.timezone)).strftime("%Y%m%d_%Hh%Mm%Ss")," ","Unexpected error. Try to check internet connection or client-id. Will try again in", self.refresh, "seconds.")
time.sleep(self.refresh)
elif status == 3:
print(datetime.now(timezone(self.timezone)).strftime("%Y%m%d_%Hh%Mm%Ss")," ","Error with channel id or oauth token. Try to check internet connection or client-id and client-secret. Will try again in", self.refresh, "seconds.")
time.sleep(self.refresh)
elif status == 0:
live_date = datetime.now(timezone('UTC'))
live_date_min = live_date - timedelta(minutes=5)
live_date_plus = live_date + timedelta(minutes=5)
present_datetime = datetime.now(timezone(self.timezone)).strftime("%Y%m%d_%Hh%Mm%Ss")
raw_filename = present_datetime + ".ts"
live_filename = "LIVE_" + raw_filename
raw_vod_filename = "VOD_" + raw_filename
chat_json_filename = "CHAT_" + present_datetime + ".json"
chat_mp4_filename = "CHAT_" + present_datetime + ".mp4"
metadata_filename = "metadata_" + present_datetime + ".json"
recorded_filename = os.path.join(self.recorded_path, live_filename)
# start streamlink process
subprocess.call(["streamlink", '--http-header', 'Authorization=OAuth ' + os.environ.get('OAUTH-PRIVATE-TOKEN'), "--hls-segment-threads", str(self.hls_segments), "--hls-live-restart", "--twitch-disable-hosting", "twitch.tv/" + self.username, self.quality, "--retry-streams", str(self.refresh)] + self.debug_cmd + ["-o", recorded_filename])
if(os.path.exists(recorded_filename) is True):
status, info_tmp = self.check_user()
if info_tmp != None:
info = info_tmp
try:
vodurl = 'https://api.twitch.tv/helix/videos?user_id=' + str(self.channel_id) + '&type=archive'
vods = requests.get(vodurl, headers = {"Authorization" : "Bearer " + self.oauth_token, "Client-ID": os.environ.get('CLIENT-ID')}, timeout = 30)
vodsinfodic = json.loads(vods.text)
if vodsinfodic["data"][0] != []:
if live_date_min <= parser.parse(vodsinfodic["data"][0]["created_at"]) <= live_date_plus:
vod_id = vodsinfodic["data"][0]["id"]
created_at = vodsinfodic["data"][0]["created_at"]
created_at = self.toTZ(created_at)
raw_filename = created_at + ".ts"
live_filename = "LIVE_" + raw_filename
raw_vod_filename = "VOD_" + raw_filename
chat_json_filename = "CHAT_" + created_at + ".json"
chat_mp4_filename = "CHAT_" + created_at + ".mp4"
metadata_filename = "metadata_" + created_at + ".json"
try:
os.rename(recorded_filename,os.path.join(self.recorded_path, live_filename))
recorded_filename = os.path.join(self.recorded_path, live_filename)
except Exception as e:
raw_filename = present_datetime + ".ts"
live_filename = "LIVE_" + raw_filename
raw_vod_filename = "VOD_" + raw_filename
chat_json_filename = "CHAT_" + present_datetime + ".json"
chat_mp4_filename = "CHAT_" + present_datetime + ".mp4"
metadata_filename = "metadata_" + present_datetime + ".json"
os.rename(recorded_filename,os.path.join(self.recorded_path, live_filename))
recorded_filename = os.path.join(self.recorded_path, live_filename)
print('first exception as e\nAn error has occurred. VOD and chat will not be downloaded. Please check them manually.\n' + e)
self.sendNotif('ERROR - '+ present_datetime, 'An error has occurred. VOD and chat will not be downloaded. Please check them manually.\n ' + e)
if self.downloadMETADATA == 1:
self.sendNotif('Metadata - ' + created_at,'Downloading and saving metadata:\n' + json.dumps(vodsinfodic["data"][0], indent=4))
with open(os.path.join(self.metadata_path, metadata_filename), 'w', encoding='utf-8') as f:
json.dump(vodsinfodic["data"][0], f, ensure_ascii=False, indent=4)
if self.downloadVOD == 1:
print('Downloading VOD: ' + vodsinfodic["data"][0]["title"])
self.sendNotif('VOD - ' + created_at,'Downloading VOD: ' + vodsinfodic["data"][0]["title"])
try:
subprocess.call(['streamlink', '--http-header', 'Authorization=OAuth ' + os.environ.get('OAUTH-PRIVATE-TOKEN'), "--hls-segment-threads", str(self.hls_segmentsVOD), "twitch.tv/videos/" + vod_id, self.quality] + self.debug_cmd + ["-o", os.path.join(self.recorded_path, raw_vod_filename)])
except Exception as e:
print('Error', 'A ERROR has ocurred and the VOD will not be downloaded.\n' + e)
self.sendNotif('ERROR - ' + created_at, 'A ERROR has ocurred and the VOD will not be downloaded. \nerror:\n' + e)
if self.downloadCHAT == 1:
print('Downloading and rendering CHAT: ' + vodsinfodic["data"][0]["title"])
self.sendNotif('CHAT - ' + created_at,'Downloading JSON and rendering chat logs from VOD:\n' + vodsinfodic["data"][0]["title"])
try:
subprocess.call([str(pathlib.Path(__file__).parent.resolve())+"/bin/chat.bat", vod_id, os.path.join(self.chatJSON_path, chat_json_filename), os.path.join(self.chatMP4_path, chat_mp4_filename)])
except Exception as e:
self.sendNotif('ERROR - ' + created_at, "A ERROR has ocurred and chat will need to be downloaded and rendered manually.\n" + e)
print("A ERROR has ocurred and chat will need to be downloaded and rendered manually\n" + e)
else:
print('A ERROR has ocurred, the latest VOD doesnt match with the livestream, the VOD is not published\nThe VOD and chat will not be downloaded and rendered.\nThe current livestream date: ' + present_datetime + '\nThe VOD date: ' + self.toTZ(vodsinfodic["data"][0]["created_at"]))
self.sendNotif('ERROR - ' + present_datetime, 'A ERROR has ocurred, the latest VOD doesnt match with the livestream, the VOD is not published\nThe VOD and chat will not be downloaded and rendered.\nThe current livestream date: ' + present_datetime + '\nThe VOD date: ' + self.toTZ(vodsinfodic["data"][0]["created_at"]))
else:
raw_filename = present_datetime + ".ts"
live_filename = "LIVE_" + raw_filename
raw_vod_filename = "VOD_" + raw_filename
chat_json_filename = "CHAT_" + present_datetime + ".json"
chat_mp4_filename = "CHAT_" + present_datetime + ".mp4"
metadata_filename = "metadata_" + present_datetime + ".json"
os.rename(recorded_filename,os.path.join(self.recorded_path, live_filename))
recorded_filename = os.path.join(self.recorded_path, live_filename)
except Exception as e:
raw_filename = present_datetime + ".ts"
live_filename = "LIVE_" + raw_filename
raw_vod_filename = "VOD_" + raw_filename
chat_json_filename = "CHAT_" + present_datetime + ".json"
chat_mp4_filename = "CHAT_" + present_datetime + ".mp4"
metadata_filename = "metadata_" + present_datetime + ".json"
os.rename(recorded_filename,os.path.join(self.recorded_path, live_filename))
recorded_filename = os.path.join(self.recorded_path, live_filename)
print('An error has occurred. VOD and chat will not be downloaded. Please check them manually.\n' + e)
self.sendNotif('ERROR - ' + present_datetime, 'An error has occurred. VOD and chat will not be downloaded. Please check them manually.\n' + e)
print("Recording stream is done. Fixing video file.")
self.sendNotif("STREAM DONE - " + present_datetime, "Recording stream is done. Fixing video file.")
if(os.path.exists(recorded_filename) is True):
file_mp4 = live_filename[:-2] + "mp4"
vod_filename = raw_vod_filename[:-2] + "mp4"
processed_filename = os.path.join(self.processed_path, file_mp4)
subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/ffmpeg.exe', '-y', '-i', recorded_filename, '-analyzeduration', '2147483647', '-probesize', '2147483647', '-c:v', 'copy', '-c:a', 'copy', '-start_at_zero', '-copyts', processed_filename], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
if(os.path.exists(os.path.join(self.recorded_path, raw_vod_filename)) is True):
subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/ffmpeg.exe', '-y', '-i', os.path.join(self.recorded_path, raw_vod_filename), '-analyzeduration', '2147483647', '-probesize', '2147483647', '-c:v', 'copy', '-c:a', 'copy', '-start_at_zero', '-copyts', os.path.join(self.processed_path, vod_filename)], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
if self.check_user() is True:
live_date = datetime.strptime(self.live_info["started_at"],'%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=timezone('UTC')).astimezone(tz=None).replace(tzinfo=None)
live_raw_filename = datetime.strftime(live_date,'%Y%m%d_%Hh%Mm%Ss')
live_raw_path = os.path.join(self.raw_path, "LIVE_" + live_raw_filename + ".ts")
live_proc_path = os.path.join(self.video_path, "LIVE_" + live_raw_filename + ".mp4")
with open(os.path.join(self.root_path, ".log")) as logs:
logs = logs.read()
log_id = self.live_info["started_at"] + " - " + self.live_info["title"]
if log_id in logs:
time.sleep(self.refresh)
with open(os.path.join(self.root_path, ".log"), "r+") as logs:
log_id = self.live_info["started_at"] + " - " + self.username + " - " + self.live_info["title"]
for line in logs:
if log_id in line:
break
else:
logs.write(self.live_info["started_at"] + " - " + self.username + " - " + self.live_info["title"] + "\n")
self.sendNotif('Stream - ' + live_raw_filename, 'Streamer went live: ' + self.live_info["title"])
subprocess.call([self.terminal,'streamlink', 'twitch.tv/'+ self.username, self.quality, '--http-header', '"Authorization=OAuth ' + os.getenv('OAUTH-PRIVATE-TOKEN') + '"', '--hls-segment-threads', str(self.hls_segments), '--hls-live-restart', '--retry-streams', str(self.refresh), '--output', live_raw_path])
if(os.path.exists(live_raw_path) is True):
subprocess.call([self.terminal,str(pathlib.Path(__file__).parent.resolve())+'/bin/ffmpeg', '-y', '-i', live_raw_path, '-analyzeduration', '2147483647', '-probesize', '2147483647', '-c:v', 'copy', '-c:a', 'copy', '-start_at_zero', '-copyts', live_proc_path], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
else:
print("Skip fixing. File not found.")
print("Fixing is done.")
if self.uploadCloud == 1:
tree = subprocess.run(["powershell.exe","tree", f"'{self.root_path}/{self.username}'", "/f"], capture_output=True, text=True).stdout.split("\n",2)[2]
print('Uploading the following files:\n' + tree)
self.sendNotif("UPLOADING - " + present_datetime, 'Uploading the following files: \n' + tree)
subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/upload.bat', str(pathlib.Path(self.root_path).resolve()),self.username])
if self.deleteFiles == 1:
self.sendNotif("DELETING - " + present_datetime, "Deleting the files from current seccion.")
print(f'{Fore.RED}DELETING FILES{Style.RESET_ALL}')
print(f'{Fore.RED}Deleting ' + recorded_filename + f'{Style.RESET_ALL}')
os.remove(recorded_filename)
print(f'{Fore.RED}Deleting ' + processed_filename + f'{Style.RESET_ALL}')
os.remove(processed_filename)
try:
vodurl = f'https://api.twitch.tv/helix/videos?user_id={str(self.channel_id)}&period=day&type=archive'
vods = requests.get(vodurl, headers = {"Authorization" : "Bearer " + self.oauth_token, "Client-ID": os.getenv('CLIENT-ID')}, timeout = 30)
vodsinfo = json.loads(vods.text)
if vodsinfo["data"][0] != []:
vod_date = datetime.strptime(vodsinfo["data"][0]["created_at"],'%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=timezone('UTC')).astimezone(tz=None).replace(tzinfo=None)
vod_raw_filename = datetime.strftime(vod_date,'%Y%m%d_%Hh%Mm%Ss')
if self.live_info["id"] == vodsinfo["data"][0]["stream_id"]:
current_vod = vodsinfo["data"][0]
vod_raw_path = os.path.join(self.raw_path, "VOD_" + live_raw_filename + ".ts")
vod_proc_path = os.path.join(self.video_path, "VOD_" + live_raw_filename + ".mp4")
if self.downloadMETADATA == 1:
self.sendNotif('Metadata - ' + live_raw_filename,'Downloading and saving metadata:\n' + json.dumps(current_vod, indent=4))
with open(os.path.join(self.metadata_path, "METADA_" + live_raw_filename + ".json"), 'w', encoding='utf-8') as f:
json.dump(current_vod, f, ensure_ascii=False, indent=4)
if self.downloadVOD == 1:
print('Downloading VOD: ' + current_vod["title"])
self.sendNotif('VOD - ' + live_raw_filename,'Downloading VOD: ' + current_vod["title"])
try:
subprocess.call([self.terminal,'streamlink', 'twitch.tv/videos/' + current_vod["id"], self.quality, '--http-header', '"Authorization=OAuth ' + os.getenv('OAUTH-PRIVATE-TOKEN') +'"', "--hls-segment-threads", str(self.hls_segmentsVOD), "-o", vod_raw_path])
if(os.path.exists(vod_raw_path) is True):
subprocess.call([self.terminal,str(pathlib.Path(__file__).parent.resolve())+'/bin/ffmpeg', '-y', '-i', vod_raw_path, '-analyzeduration', '2147483647', '-probesize', '2147483647', '-c:v', 'copy', '-c:a', 'copy', '-start_at_zero', '-copyts', vod_proc_path], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
else:
print("Skip fixing. File not found.")
except Exception as e:
print('Error', 'A ERROR has ocurred and the VOD will not be downloaded.\n')
self.sendNotif('ERROR - ' + live_raw_filename, 'A ERROR has ocurred and the VOD will not be downloaded.\n')
if self.downloadCHAT == 1:
print('Downloading and rendering CHAT: ' + current_vod["title"])
self.sendNotif('CHAT - ' + live_raw_filename,'Downloading JSON and rendering chat logs from VOD:\n' + current_vod["title"])
try:
subprocess.call([self.terminal,str(pathlib.Path(__file__).parent.resolve())+"/bin/chat", current_vod["id"], os.path.join(self.chatJSON_path, "CHAT_" + live_raw_filename + ".json"), os.path.join(self.chatMP4_path, "CHAT_" + live_raw_filename + ".mp4")])
except Exception as e:
self.sendNotif('ERROR - ' + live_raw_filename, "A ERROR has ocurred and chat will need to be downloaded and rendered manually.\n")
print("A ERROR has ocurred and chat will need to be downloaded and rendered manually\n")
else:
print('A ERROR has ocurred, the latest VOD doesnt match with the livestream, the VOD is not published\nThe VOD and chat will not be downloaded and rendered.\nThe current livestream date: ' + live_raw_filename + '\nThe VOD date: ' + vod_raw_filename)
self.sendNotif('ERROR - ' + live_raw_filename, 'A ERROR has ocurred, the latest VOD doesnt match with the livestream, the VOD is not published\nThe VOD and chat will not be downloaded and rendered.\nThe current livestream date: ' + live_raw_filename + '\nThe VOD date: ' + vod_raw_filename)
except Exception as e:
print('An error has occurred. VOD and chat will not be downloaded. Please check them manually.\n')
self.sendNotif('ERROR - ' + live_raw_filename, 'An error has occurred. VOD and chat will not be downloaded. Please check them manually.\n')
if self.cleanRaw == 1:
print('Deleting raw files')
if(os.path.exists(live_raw_path) is True): os.remove(live_raw_path)
if self.downloadVOD == 1:
if(os.path.exists(os.path.join(self.recorded_path, raw_vod_filename)) is True):
print(f'{Fore.RED}Deleting ' + os.path.join(self.recorded_path, raw_vod_filename) + f'{Style.RESET_ALL}')
os.remove(os.path.join(self.recorded_path, raw_vod_filename))
if(os.path.exists(os.path.join(self.processed_path, vod_filename)) is True):
print(f'{Fore.RED}Deleting ' + os.path.join(self.processed_path, vod_filename) + f'{Style.RESET_ALL}')
os.remove(os.path.join(self.processed_path, vod_filename))
if(os.path.exists(os.path.join(self.raw_path, "VOD_" + live_raw_filename + ".ts")) is True):
os.remove(os.path.join(self.raw_path, "VOD_" + live_raw_filename + ".ts"))
if self.uploadCloud == 1:
if self.terminal == 'powershell.exe':
tree = subprocess.run([self.terminal,"tree", f"'{self.root_path}/{self.username}'", "/f"], capture_output=True, text=True).stdout.split("\n",2)[2]
elif self.terminal == 'bash':
tree = subprocess.check_output([self.terminal,"tree", str(pathlib.Path(self.root_path).resolve())+"/"+self.username]).decode(sys.stdout.encoding)
print('Uploading the following files:\n' + tree)
self.sendNotif("UPLOADING - " + live_raw_filename, 'Uploading the following files: \n' + tree)
subprocess.call([self.terminal,str(pathlib.Path(__file__).parent.resolve())+'/bin/upload', str(pathlib.Path(self.root_path).resolve()),self.username])
if self.deleteFiles == 1:
self.sendNotif("DELETING - " + live_raw_filename, "Deleting the files from current seccion.")
print(f'{Fore.RED}DELETING FILES{Style.RESET_ALL}')
if self.cleanRaw == 0:
print(f'{Fore.RED}Deleting ' + live_raw_path + f'{Style.RESET_ALL}')
os.remove(live_raw_path)
print(f'{Fore.RED}Deleting ' + live_proc_path + f'{Style.RESET_ALL}')
os.remove(live_proc_path)
if self.downloadVOD == 1:
if(os.path.exists(os.path.join(self.raw_path, "VOD_" + live_raw_filename + ".ts")) is True):
if self.cleanRaw == 0:
print(f'{Fore.RED}Deleting ' + os.path.join(self.raw_path, "VOD_" + live_raw_filename + ".ts") + f'{Style.RESET_ALL}')
os.remove(os.path.join(self.raw_path, "VOD_" + live_raw_filename + ".ts"))
if(os.path.exists(os.path.join(self.video_path, "VOD_" + live_raw_filename + ".mp4")) is True):
print(f'{Fore.RED}Deleting ' + os.path.join(self.video_path, "VOD_" + live_raw_filename + ".mp4") + f'{Style.RESET_ALL}')
os.remove(os.path.join(self.video_path, "VOD_" + live_raw_filename + ".mp4"))
if self.downloadCHAT == 1:
if(os.path.exists(os.path.join(self.chatJSON_path, chat_json_filename)) is True):
print(f'{Fore.RED}Deleting ' + os.path.join(self.chatJSON_path, chat_json_filename) + f'{Style.RESET_ALL}')
os.remove(os.path.join(self.chatJSON_path, chat_json_filename))
if(os.path.exists(os.path.join(self.chatMP4_path, chat_mp4_filename)) is True):
print(f'{Fore.RED}Deleting ' + os.path.join(self.chatMP4_path, chat_mp4_filename) + f'{Style.RESET_ALL}')
os.remove(os.path.join(self.chatMP4_path, chat_mp4_filename))
if(os.path.exists(os.path.join(self.chatJSON_path, "CHAT_"+live_raw_filename + ".json")) is True):
print(f'{Fore.RED}Deleting ' + os.path.join(self.chatJSON_path, "CHAT_"+live_raw_filename + ".json") + f'{Style.RESET_ALL}')
os.remove(os.path.join(self.chatJSON_path, "CHAT_"+live_raw_filename + ".json"))
if(os.path.exists(os.path.join(self.chatMP4_path, "CHAT_"+live_raw_filename + ".mp4")) is True):
print(f'{Fore.RED}Deleting ' + os.path.join(self.chatMP4_path, "CHAT_"+live_raw_filename + ".mp4") + f'{Style.RESET_ALL}')
os.remove(os.path.join(self.chatMP4_path, "CHAT_"+live_raw_filename + ".mp4"))
if self.downloadMETADATA == 1:
if(os.path.exists(os.path.join(self.metadata_path, metadata_filename)) is True):
print(f'{Fore.RED}Deleting ' + os.path.join(self.metadata_path, metadata_filename) + f'{Style.RESET_ALL}')
os.remove(os.path.join(self.metadata_path, metadata_filename))
if(os.path.exists(os.path.join(self.metadata_path, "METADA_"+live_raw_filename+".json")) is True):
print(f'{Fore.RED}Deleting ' + os.path.join(self.metadata_path, "METADA_"+live_raw_filename+".json") + f'{Style.RESET_ALL}')
os.remove(os.path.join(self.metadata_path, "METADA_"+live_raw_filename+".json"))
print('CURRENT SECCION HAVE FINISHED GOING BACK TO CHECKING')
self.sendNotif("SECCION DONE - " + present_datetime, 'CURRENT SECCION HAVE FINISHED GOING BACK TO CHECKING')
self.sendNotif("SECCION DONE - " + live_raw_filename, 'CURRENT SECCION HAVE FINISHED GOING BACK TO CHECKING')
time.sleep(self.refresh)
def main(argv):
twitch_recorder = TwitchArchive()
usage_message = 'twitch-archive.py -u <username> -q <quality>'
twitch_archive = TwitchArchive()
help_msg = 'Twitch-Archive\nPython script to record twitch live stream, download the VOD, metadata, chat and render it, and uploads them to any cloud storage.\n -h, --help Display this information\n -u, --username <username> Twitch channel username\n -q, --quality <quality> best/source high/720p medium/480p worst/360p\n -v, --vod <1/0> Download vod\n -c, --chat <1/0> Download chat and render it\n -m, --metadata <1/0> Download metadata\n -r, --upload <1/0> Upload to cloud storage\n -d, --delete <1/0> Delete all files after upload (CAREFUL with this arg)\n -n, --notifications <1/0> Receive email notification of the proccess through gmail\n'
try:
opts, args = getopt.getopt(argv,"u:q:",["username=","quality="])
opts, args = getopt.getopt(argv,"h:u:q:v:c:m:r:d:n",["username=","quality=","vod=","chat=","metadata=","upload=","delete=","notifications="])
except getopt.GetoptError:
print (usage_message)
print (help_msg)
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print(usage_message)
if opt in ('-h', '--help'):
print(help_msg)
sys.exit()
elif opt in ("-u", "--username"):
twitch_recorder.username = arg
elif opt in ("-q", "--quality"):
twitch_recorder.quality = arg
twitch_recorder.run()
elif opt in ("-u", "--username"): twitch_archive.username = arg
elif opt in ("-q", "--quality"): twitch_archive.quality = arg
elif opt in ("-v", "--vod"): twitch_archive.quality = int(arg)
elif opt in ("-c", "--chat"): twitch_archive.quality = int(arg)
elif opt in ("-m", "--metadata"): twitch_archive.quality = int(arg)
elif opt in ("-r", "--upload"): twitch_archive.quality = int(arg)
elif opt in ("-d", "--delete"): twitch_archive.quality = int(arg)
elif opt in ("-n", "--notifications"): twitch_archive.quality = int(arg)
twitch_archive.run()
if __name__ == "__main__":
main(sys.argv[1:])