maddo/TwitchDownloader
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

clean up

Browse source
This commit is contained in:
Piero 2022-12-15 13:14:44 -05:00
commit 5d7fceef45
13 changed files with 217 additions and 199 deletions
Download patch file Download diff file Expand all files Collapse all files

4
.gitignore vendored
View file

@ -1,4 +1,6 @@
# Environments
.env
# Tests
test.py
test.py
a.py
subprocess_time.py

3
README.md
View file

@ -20,6 +20,9 @@ OAUTH-PRIVATE-TOKEN=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx # optional to record without
```
8. if you want to enable/disable more available options, edit `twitch-archive.py`
9. run `Python twitch-archive.py` or for multiple streamers `Python twitch-archive.py -u streamer`
[...](https://github.com/piero0920/Twitch-Archive/blob/main/extra.md)
<!---
## Features
- Auto records the live stream | [Streamlink](https://streamlink.github.io/)

BIN
bin/TwitchDownloaderCLI (Stored with Git LFS)
View file

Binary file not shown.

BIN
bin/TwitchDownloaderCLI.exe (Stored with Git LFS)
View file

Binary file not shown.

7
bin/chat.bat
View file

@ -1,7 +0,0 @@
@echo off
set vodid=%1
set json=%2
set mp4=%3
CD %~dp0
TwitchDownloaderCLI.exe chatdownload --id %vodid% -o %json% -E
TwitchDownloaderCLI.exe chatrender -i %json% -o %mp4% --background-color #FF111111 -w 500 -h 1080 --outline true -f Arial --font-size 22 --update-rate 1.0 --offline --ffmpeg-path ./ffmpeg.exe --temp-path ./temp

3
bin/chat.sh
View file

@ -1,3 +0,0 @@
#!/bin/sh
$(dirname "$0")/TwitchDownloaderCLI -m ChatDownload --id $1 -o $2 --embed-emotes
$(dirname "$0")/TwitchDownloaderCLI -m ChatRender -i $2 -o $3 --background-color "#FF111111" -w 500 -h 1080 --outline true -f Arial --font-size 22 --update-rate 1.0 --ffmpeg-path $(dirname "$0")/ffmpeg --temp-path $(dirname "$0")/temp

BIN
bin/rclone.exe
View file

Binary file not shown.

8
bin/upload.bat
View file

@ -1,8 +0,0 @@
@echo off
set root_path=%1
set user=%2
if "%root_path:~-1%" == "\" set "root_path_1=%root_path:~0,-1%"
for %%f in ("%root_path_1%") do set "root_path_name=%%~nxf"
FOR /F "eol=# tokens=*" %%i in (%~dp0\..\.env) do SET %%i
CD %~dp0
rclone.exe copy %root_path%/%user% %remote%/%root_path_name%/%user% --progress --include-from %~dp0/temp/upload.txt

2
bin/upload.sh
View file

@ -1,2 +0,0 @@
#!/bin/sh
rclone copy $1/$2 gd:VODS/$(basename $1)/$2 --progress --include-from $(dirname "$0")/temp/upload.txt

4
bin/upload_files.txt
View file

@ -1,4 +0,0 @@
C:\Users\piero\Documents\GitHub\Twitch-Archive\archive\piero_fn\chat\CHAT_20221209_12h32m34s.mp4
C:\Users\piero\Documents\GitHub\Twitch-Archive\archive\piero_fn\metadata\METADA_20221209_12h32m34s.json
C:\Users\piero\Documents\GitHub\Twitch-Archive\archive\piero_fn\video\LIVE_20221209_12h26m03s.mp4
C:\Users\piero\Documents\GitHub\Twitch-Archive\archive\piero_fn\video\VOD_20221209_12h32m34s.mp4

1
extra.md
View file

@ -1,5 +1,4 @@
## Linux
- use [twitch-archive](https://github.com/piero0920/Twitch-Archive/blob/main/twitch-archive)
- install and configure rclone
```
cd bin

212
only-vod-chat.py
View file

@ -1,11 +1,10 @@
import requests, os, time, json, sys, subprocess, getopt, pathlib, locale
import requests, os, time, json, sys, subprocess, getopt, pathlib, locale, re
from colorama import Fore, Style
from datetime import datetime, timedelta
locale.setlocale(locale.LC_TIME, "es_ES")
from pytz import timezone
from dotenv import load_dotenv, find_dotenv
load_dotenv(find_dotenv())
class TwitchArchive:
def __init__(self):
# user configuration
@ -16,12 +15,23 @@ class TwitchArchive:
self.refresh = 60 # Time between checking (5.0 is recommended), avoid less than 1.0
self.downloadVOD = 1 # 0 - disable VOD downloading after stream finished, 1 - enable VOD downloading after stream finished (this option downloads the latest public vod)
self.downloadCHAT = 1 # 0 - disable chat downloading and rendering, 1 - enable chat downloading and rendering
self.downloadClips = 1
self.downloadMuted = 1
self.downloadChatHTML = 1
self.uploadCloud = 1 # 0 - disable upload to remote cloud, 1 - enable upload to remote cloud
self.deleteFiles = 0 # 0 - disable the deleting of files from current seccion after being uploaded to the cloud, 1 - enable the deleting files of files from current seccion after being uploaded to the cloud (BE CAREFUL WITH THIS OPTION)
self.hls_segmentsVOD = 10 # 1-10 for downloading vod, it's possible to use multiple threads to potentially increase the throughput
def run(self):
self.os = self.get_OS()
if load_dotenv(find_dotenv()): load_dotenv(find_dotenv())
else:
print(f'{Fore.RED}\033[1mCREATE .env file with variables{Style.RESET_ALL}')
quit()
self.correct_user()
print('Twitch-Archive --- ONLY VOD/CHAT')
print('Configuration:')
print(f'Root path: {Fore.GREEN}' + str(pathlib.Path(self.root_path).resolve()) + f'{Style.RESET_ALL}')
@ -36,21 +46,8 @@ class TwitchArchive:
else: print(f'{Fore.GREEN}'+'\033[1m'+f'Files will NOT be deleted{Style.RESET_ALL}')
if self.uploadCloud == 0 and self.deleteFiles == 1: print(f'{Fore.RED}'+'\033[1m'+f'FILES WILL BE DELETED AND NO UPLOADED {Style.RESET_ALL}{Fore.GREEN}\n"CTRL + C"{Style.RESET_ALL}{Fore.RED}'+'\033[1m'+f' TO STOP AND CHANGED CONFIGURATION{Style.RESET_ALL}')
self.oauth_token = self.get_oauth_token()
self.channel_id = self.get_channel_id()
self.temp_path = str(pathlib.Path(os.path.join("VODS",self.root_path,self.username,"vod", "temp")).absolute())
self.vod_path = str(pathlib.Path(os.path.join("VODS",self.root_path, self.username, "vod")).absolute())
self.json_path = str(pathlib.Path(os.path.join("VODS",self.root_path, self.username, "chat", "json")).absolute())
self.chat_path = str(pathlib.Path(os.path.join("VODS",self.root_path, self.username, "chat")).absolute())
if(os.path.isdir(self.temp_path) is False): os.makedirs(self.temp_path)
if(os.path.isdir(self.vod_path) is False): os.makedirs(self.vod_path)
if(os.path.isdir(self.json_path) is False): os.makedirs(self.json_path)
if(os.path.isdir(self.chat_path) is False): os.makedirs(self.chat_path)
if not os.path.exists(os.path.join('VODS',self.root_path, ".log")):
with open(os.path.join('VODS',self.root_path, ".log"), 'w'): pass
if not os.path.exists(os.path.join(str(pathlib.Path(__file__).parent.resolve())+'/bin/temp/', ".log")):
with open(os.path.join(str(pathlib.Path(__file__).parent.resolve())+'/bin/temp/', ".log"), 'w'): pass
print(f"Checking for {Fore.GREEN}{self.username}{Style.RESET_ALL} every {Fore.GREEN}{self.refresh}{Style.RESET_ALL} seconds to download VOD/CHAT")
self.loopcheck()
@ -64,68 +61,56 @@ class TwitchArchive:
print('OS no supported')
return
def get_oauth_token(self):
try:
return requests.post(f"https://id.twitch.tv/oauth2/token?client_id={os.getenv('CLIENT-ID')}&client_secret={os.getenv('CLIENT-SECRET')}&grant_type=client_credentials").json()['access_token']
except:
print(f'{Fore.RED}\033[1mCheck your client-id and secret{Style.RESET_ALL}')
quit()
def correct_user(self):
try:
url = f'https://api.twitch.tv/helix/users?login={self.username}'
response = requests.get(url,headers = {"Authorization" : "Bearer " + self.get_oauth_token(), "Client-ID": os.getenv('CLIENT-ID')}, timeout = 15).json()
if response['data'] == []:
print(f'{Fore.RED}\033[1mUse a correct username{Style.RESET_ALL}')
quit()
except requests.exceptions.RequestException as e:
print(e)
def check_user(self):
client_id = "kimne78kx3ncx6brgo4mv6wki5h1ko"
query = '''
query {
user(login: "''' + self.username + '''") {
stream {
archiveVideo{
id
}
title
createdAt
}
}
}
'''
url = 'https://gql.twitch.tv/gql'
response = requests.post(url,json={'query': query},headers={"Client-ID": client_id})
return json.loads(response.text)
query = 'query{user(login: "' + self.username + '") {stream{archiveVideo{id}title createdAt}}}'
try:
response = requests.post('https://gql.twitch.tv/gql',json={'query': query},headers={"Client-ID": "kimne78kx3ncx6brgo4mv6wki5h1ko"})
return json.loads(response.text)
except requests.exceptions.RequestException as e:
print(e)
quit()
def get_vod(self):
client_id = "kimne78kx3ncx6brgo4mv6wki5h1ko"
query = '''
query {
user(login: "''' + self.username + '''") {
videos(first: 1) {
edges {
node {
id
scope
title
description
recordedAt
lengthSeconds
animatedPreviewURL
previewThumbnailURL(height: 1280, width: 720)
thumbnailURLs(height: 1280, width: 720)
}
}
}
}
}
'''
url = 'https://gql.twitch.tv/gql'
response = requests.post(url, json={'query': query}, headers={"Client-ID": client_id})
return json.loads(response.text)
query = 'query {user(login: "' + self.username + '"){videos(first: 1){edges {node {id title recordedAt lengthSeconds tags muteInfo{ mutedSegmentConnection{ nodes{ duration offset }}} topClips(first: 10) { edges{ node{ id slug viewCount title createdAt curator { displayName } durationSeconds url thumbnailURL(width: 480, height: 272)}}}}}}}}'
try:
response = requests.post('https://gql.twitch.tv/gql', json={'query': query}, headers={"Client-ID": "kimne78kx3ncx6brgo4mv6wki5h1ko"})
return json.loads(response.text)
except requests.exceptions.RequestException as e:
print(e)
def loopcheck(self):
while True:
is_live = self.check_user()['data']['user']['stream']
if is_live is not None:
is_live_ready = self.check_user()['data']['user']['stream']['title']
is_live_ready = self.check_user()['data']['user']['stream']['archiveVideo']
if is_live_ready is not None:
live_date = datetime.strptime(is_live["createdAt"],'%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=timezone('UTC')).astimezone(tz=None).replace(tzinfo=None)
live_temp_path = os.path.join(self.temp_path, "live_temp.ts")
bin_path = str(pathlib.Path(__file__).parent.resolve())+"/bin"
live_temp_path = os.path.join(bin_path+'/temp/', "live_temp.ts")
with open(os.path.join(self.root_path, ".log")) as logs:
with open(os.path.join(bin_path+'/temp/', ".log")) as logs:
logs = logs.read()
log_id = is_live["createdAt"] + " - " + self.username + " - " + is_live["title"]
if log_id in logs:
time.sleep(self.refresh)
with open(os.path.join(self.root_path, ".log"), "r+") as logs:
with open(os.path.join(bin_path+'/temp/', ".log"), "r+") as logs:
log_id = is_live["createdAt"] + " - " + self.username + " - " + is_live["title"]
for line in logs:
if log_id in line:
@ -133,21 +118,37 @@ class TwitchArchive:
else:
logs.write(is_live["createdAt"] + " - " + self.username + " - " + is_live["title"] +"\n")
subprocess.call(['streamlink', 'twitch.tv/'+ self.username, self.quality, '--twitch-api-header', 'Authorization=OAuth ' + os.getenv('OAUTH-PRIVATE-TOKEN'), '--hls-segment-threads', str(self.hls_segments), '--hls-live-restart', '--retry-streams', str(self.refresh), '--twitch-disable-reruns', '-o', live_temp_path])
subprocess.call(['streamlink', 'twitch.tv/'+ self.username, self.quality, '--twitch-api-header', 'Authorization=OAuth ' + os.getenv('OAUTH-PRIVATE-TOKEN'), '--hls-live-restart', '--retry-streams', str(self.refresh), '--twitch-disable-reruns', '-o', live_temp_path])
os.remove(live_temp_path)
current_vod = self.get_vod()['data']['user']['videos']['edges'][0]['node']
vod_date = datetime.strptime(current_vod["recordedAt"],'%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=timezone('UTC')).astimezone(tz=None).replace(tzinfo=None)
vod_raw_filename = datetime.strftime(vod_date,'%Y%m%d_%Hh%Mm%Ss')
live_date_min = live_date - timedelta(minutes=1)
live_date_max = live_date + timedelta(minutes=1)
if live_date_min <= vod_date <= live_date_max:
print('VOD AND CHAT AVAILABLE')
vod_raw_path = os.path.join(self.temp_path, "vod_temp.ts")
if is_live_ready['id'] == current_vod['id']:
print('VOD AND CHAT AVAILABLE')
self.vod_path = str(pathlib.Path(os.path.join("VODS",self.root_path, self.username, vod_raw_filename ,"vod")).absolute())
self.clips_path = str(pathlib.Path(os.path.join("VODS",self.root_path, self.username, vod_raw_filename , "clips")).absolute())
self.muted_path = str(pathlib.Path(os.path.join("VODS",self.root_path, self.username, vod_raw_filename , "muted")).absolute())
self.json_path = str(pathlib.Path(os.path.join("VODS",self.root_path, self.username, vod_raw_filename ,"chat", "json")).absolute())
self.chat_path = str(pathlib.Path(os.path.join("VODS",self.root_path, self.username, vod_raw_filename , "chat")).absolute())
self.html_path = str(pathlib.Path(os.path.join("VODS",self.root_path, self.username, vod_raw_filename ,"chat", "html")).absolute())
if(os.path.isdir(self.vod_path) is False): os.makedirs(self.vod_path)
if(os.path.isdir(self.clips_path) is False): os.makedirs(self.clips_path)
if(os.path.isdir(self.muted_path) is False): os.makedirs(self.muted_path)
if(os.path.isdir(self.json_path) is False): os.makedirs(self.json_path)
if(os.path.isdir(self.chat_path) is False): os.makedirs(self.chat_path)
if(os.path.isdir(self.html_path) is False): os.makedirs(self.html_path)
vod_raw_path = os.path.join(bin_path+'/temp', "vod_temp.ts")
vod_proc_path = os.path.join(self.vod_path, vod_raw_filename + ".mp4")
chat_json_path = os.path.join(self.json_path, vod_raw_filename + ".json")
chat_video_path = os.path.join(self.chat_path, vod_raw_filename + ".mp4")
with open(os.path.join(str(pathlib.Path(os.path.join("VODS",self.root_path, self.username, vod_raw_filename)).absolute()), 'metadata.json'), 'w', encoding='utf-8') as f:
json.dump(current_vod, f, ensure_ascii=False, indent=4)
if self.username == 'KalathrasLolweapon':
file_date = datetime.strptime(vod_raw_filename, '%Y%m%d_%Hh%Mm%Ss').date()
@ -165,37 +166,76 @@ class TwitchArchive:
chat_path = str(pathlib.Path(os.path.join("Chat",chat_year,chat_month)).absolute())
if(os.path.isdir(vod_path) is False): os.makedirs(vod_path)
if(os.path.isdir(chat_path) is False): os.makedirs(chat_path)
clean_vod_title = re.sub(r'[/\\:*?"<>|]', '_', current_vod['title'])
if len(clean_vod_title) > 202:
dif = len(clean_vod_title) - 202
clean_vod_title[:-dif]
chat_video_path = os.path.join(chat_path, vod_raw_filename + ".mp4")
vod_proc_path = os.path.join(vod_path, vod_raw_filename + ".mp4")
vod_proc_path = os.path.join(vod_path, vod_raw_filename + '_'+ clean_vod_title + ".mp4")
if self.downloadVOD == 1:
print('Downloading VOD: ' + current_vod["title"])
try:
subprocess.call(['streamlink', 'twitch.tv/videos/' + current_vod["id"], self.quality, "--hls-segment-threads", str(self.hls_segmentsVOD), "-o", vod_raw_path])
if(os.path.exists(vod_raw_path) is True):
if self.os == 'windows':subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/ffmpeg.exe', '-y', '-i', vod_raw_path, '-analyzeduration', '2147483647', '-probesize', '2147483647', '-c:v', 'copy', '-c:a', 'copy', '-start_at_zero', '-copyts', vod_proc_path], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
elif self.os == 'linux':subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/ffmpeg', '-y', '-i', vod_raw_path, '-analyzeduration', '2147483647', '-probesize', '2147483647', '-c:v', 'copy', '-c:a', 'copy', '-start_at_zero', '-copyts', vod_proc_path], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
os.remove(vod_raw_path)
else:
print("Skip fixing. File not found.")
subprocess.call([bin_path+"/TwitchDownloaderCLI.exe", 'videoDownload', '-u', str(current_vod["id"]), '-q', self.quality, "-t", str(self.hls_segmentsVOD), "--ffmpeg-path", bin_path +"/ffmpeg.exe", '--temp-path', bin_path+"/temp", "-o", vod_proc_path])
except Exception as e:
print('Error', 'A ERROR has ocurred and the VOD will not be downloaded.\n')
if self.downloadMuted == 1:
if current_vod['muteInfo']['mutedSegmentConnection'] is not None:
mutedSegments = current_vod['muteInfo']['mutedSegmentConnection']['nodes']
print('Downloading ' + len(mutedSegments) + ' muted segmes')
for muted in mutedSegments:
beg = time.strftime('%Hh%Mm%Ss', time.gmtime(muted['offset']))
end = time.strftime('%Hh%Mm%Ss', time.gmtime(muted['offset'] + muted['duration']))
muted_filename_path = os.path.join(self.muted_path, beg + '-' + end + ".mp4")
subprocess.call([bin_path+"/TwitchDownloaderCLI.exe", 'videoDownload', '-u', current_vod['id'], '-q', self.quality, '-b', str(muted['offset']), '-e', str(muted['offset'] + muted['duration']), "-t", str(self.hls_segmentsVOD), "--ffmpeg-path", bin_path+"/ffmpeg.exe", '--temp-path', bin_path+"/temp" '-o', muted_filename_path])
else:
print('The VOD has no muted segments')
if self.downloadClips == 1:
topClips = current_vod['topClips']['edges']
if topClips != []:
print('Downloading Clips')
for clips in topClips:
clip = clips['node']
clean_title = re.sub(r'[/\\:*?"<>|]', '_', clip['title'])
clip_filename_path = os.path.join(self.clips_path, clean_title + '_'+ clip['id'] +".mp4")
subprocess.call([bin_path+"/TwitchDownloaderCLI.exe", 'clipDownload', '-u', clip['slug'], '-q', self.quality, '-o', clip_filename_path])
else:
print('No Clips has being made during the stream')
if self.downloadCHAT == 1:
print('Downloading and rendering CHAT: ' + current_vod["title"])
print('Downloading CHAT: ' + current_vod["title"])
chat_settings = ["--background-color", "#FF111111", "-w", "500", "-h", "1080", "--outline", "true", "-f", "Arial", "--font-size", "22", "--update-rate", "1.0", "--offline", "--ffmpeg-path", f"{bin_path}/ffmpeg.exe", "--temp-path", f"{bin_path}/temp"]
try:
if self.os == 'windows':subprocess.call([str(pathlib.Path(__file__).parent.resolve())+"/bin/chat.bat", current_vod["id"], chat_json_path, chat_video_path])
elif self.os == 'linux':subprocess.call([str(pathlib.Path(__file__).parent.resolve())+"/bin/chat.sh", current_vod["id"], chat_json_path, chat_video_path])
if self.os == 'windows':
subprocess.call([bin_path + '/TwitchDownloaderCLI.exe', 'chatdownload', '--id', current_vod["id"], '-o', chat_json_path, '-E'])
print('Rendering CHAT: ' + current_vod["title"])
subprocess.call([bin_path + '/TwitchDownloaderCLI.exe', 'chatrender', '-i', chat_json_path, '-o', chat_video_path] + chat_settings)
elif self.os == 'linux':
subprocess.call([bin_path + '/TwitchDownloaderCLI', 'chatdownload', '--id', current_vod["id"], '-o', chat_json_path, '-E'])
subprocess.call([bin_path + '/TwitchDownloaderCLI', 'chatrender', '-i', chat_json_path, '-o', chat_video_path] + chat_settings)
except Exception as e:
print("A ERROR has ocurred and chat will need to be downloaded and rendered manually\n")
if self.downloadChatHTML == 1:
print('Downloading chat to html format')
chat_html_path = os.path.join(self.html_path, vod_raw_filename + ".html")
try:
if self.os == 'windows': subprocess.call([bin_path+"/TwitchDownloaderCLI.exe", "chatupdate", "-i", chat_json_path, "-o", chat_html_path, "-E", "--temp-path", f"{bin_path}/temp"])
elif self.os == 'linux': subprocess.call([bin_path+"/TwitchDownloaderCLI", "chatupdate", "-i", chat_json_path, "-o", chat_html_path, "-E", "--temp-path", f"{bin_path}/temp"])
if self.username == 'KalathrasLolweapon':
print('Uploading html chat to b2 bucket')
subprocess.call(['rclone', 'copy', chat_html_path, 'b2:kala-help/chat_html', '--progress'])
except Exception as e:
print('A ERROR has ocurred and chat will need to be updated to html manually')
if self.uploadCloud == 1:
print('Uploading files:')
if self.os == 'windows':
if self.username == 'KalathrasLolweapon':
subprocess.call(['rclone', 'copy', str(pathlib.Path(__file__).parent.resolve())+'/VODS', 'gd:VODS', '--progress'])
subprocess.call(['rclone', 'copy', str(pathlib.Path(__file__).parent.resolve())+'/Chat', 'gd:Chat', '--progress'])
else:subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/upload.bat', str(pathlib.Path(self.root_path).resolve()),self.username])
elif self.os == 'linux':subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/upload.sh', str(pathlib.Path(self.root_path).resolve()),self.username])
subprocess.call(['rclone', 'copy', str(pathlib.Path(__file__).parent.resolve())+'/VODS', 'GD:VODS', '--progress'])
subprocess.call(['rclone', 'copy', str(pathlib.Path(__file__).parent.resolve())+'/Chat', 'GD:Chat', '--progress'])
else:subprocess.call(['rclone', 'copy', str(pathlib.Path(__file__).parent.resolve())+'/VODS', 'GD:VODS', '--progress'])
elif self.os == 'linux':subprocess.call([bin_path+'/upload.sh', str(pathlib.Path(self.root_path).resolve()),self.username])
if self.deleteFiles == 1:
print(f'{Fore.RED}DELETING FILES{Style.RESET_ALL}')

164
twitch-archive.py
View file

@ -1,11 +1,10 @@
import requests, os, time, json, sys, subprocess, getopt, smtplib, pathlib, glob
import requests, os, time, json, sys, subprocess, getopt, smtplib, pathlib, socket
from colorama import Fore, Style
from datetime import datetime, timedelta
from pytz import timezone
from dotenv import load_dotenv, find_dotenv
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
load_dotenv(find_dotenv())
class TwitchArchive:
def __init__(self):
# user configuration
@ -13,19 +12,28 @@ class TwitchArchive:
self.quality = "best" # Qualities options: best/source high/720p medium/540p low/360p
# global configuration
self.root_path = r"archive" # Path where this script saves everything (livestream,VODs,chat,metadata)
self.rclone_path = "remote:path" # Path to rclone remote storage
self.refresh = 5.0 # Time between checking (5.0 is recommended), avoid less than 1.0
self.notifications = 1 # 0 - disable email notification of current seccion, 1 - enable email notification of current seccion
self.notifications = 0 # 0 - disable email notification of current seccion, 1 - enable email notification of current seccion
self.downloadMETADATA = 1 # 0 - disable metadata downloading, 1 - enable metadata downloading
self.downloadVOD = 1 # 0 - disable VOD downloading after stream finished, 1 - enable VOD downloading after stream finished (this option downloads the latest public vod)
self.downloadCHAT = 1 # 0 - disable chat downloading and rendering, 1 - enable chat downloading and rendering
self.uploadCloud = 1 # 0 - disable upload to remote cloud, 1 - enable upload to remote cloud
self.deleteFiles = 1 # 0 - disable the deleting of files from current seccion after being uploaded to the cloud, 1 - enable the deleting files of files from current seccion after being uploaded to the cloud (BE CAREFUL WITH THIS OPTION)
self.deleteFiles = 0 # 0 - disable the deleting of files from current seccion after being uploaded to the cloud, 1 - enable the deleting files of files from current seccion after being uploaded to the cloud (BE CAREFUL WITH THIS OPTION)
self.cleanRaw = 1 # 0 - disable the deleting of raw (.ts) files, 1 - enable the deleteing of raw (.ts) files (if upload enable they will be deleted before)
self.hls_segments = 3 # 1-10 for live stream, it's possible to use multiple threads to potentially increase the throughput. 2-3 is enough
self.hls_segmentsVOD = 10 # 1-10 for downloading vod, it's possible to use multiple threads to potentially increase the throughput
def run(self):
self.os = self.get_OS()
if load_dotenv(find_dotenv()): load_dotenv(find_dotenv())
else:
print(f'{Fore.RED}\033[1mCREATE .env file with variables{Style.RESET_ALL}')
quit()
self.correct_user()
print('Twitch-Archive')
print('Configuration:')
print(f'Root path: {Fore.GREEN}' + str(pathlib.Path(self.root_path).resolve()) + f'{Style.RESET_ALL}')
@ -55,7 +63,7 @@ class TwitchArchive:
if(os.path.isdir(self.chatJSON_path) is False): os.makedirs(self.chatJSON_path)
if(os.path.isdir(self.chatMP4_path) is False): os.makedirs(self.chatMP4_path)
if(os.path.isdir(self.metadata_path) is False): os.makedirs(self.metadata_path)
if not os.path.exists(os.path.join(self.root_path, ".log")):
if not os.path.exists(os.path.join(self.root_path, ".log")):
with open(os.path.join(self.root_path, ".log"), 'w'): pass
print(f"Checking for {Fore.GREEN}{self.username}{Style.RESET_ALL} every {Fore.GREEN}{self.refresh}{Style.RESET_ALL} seconds. Record with {Fore.GREEN}{self.quality}{Style.RESET_ALL} quality.")
@ -68,72 +76,62 @@ class TwitchArchive:
elif sys.platform.startswith('linux'):
return 'linux'
else:
print('OS no supported')
return
print(f'{Fore.RED}\033[1mOS no supported{Style.RESET_ALL}')
quit()
def get_oauth_token(self):
try:
return requests.post(f"https://id.twitch.tv/oauth2/token?client_id={os.getenv('CLIENT-ID')}&client_secret={os.getenv('CLIENT-SECRET')}&grant_type=client_credentials").json()['access_token']
except:
print(f'{Fore.RED}\033[1mCheck your client-id and secret{Style.RESET_ALL}')
quit()
def correct_user(self):
try:
url = f'https://api.twitch.tv/helix/users?login={self.username}'
response = requests.get(url,headers = {"Authorization" : "Bearer " + self.get_oauth_token(), "Client-ID": os.getenv('CLIENT-ID')}, timeout = 15).json()
if response['data'] == []:
print(f'{Fore.RED}\033[1mUse a correct username{Style.RESET_ALL}')
quit()
except requests.exceptions.RequestException as e:
print(e)
def check_user(self):
client_id = "kimne78kx3ncx6brgo4mv6wki5h1ko"
query = '''
query {
user(login: "''' + self.username + '''") {
stream {
archiveVideo{
id
}
title
createdAt
}
}
}
'''
url = 'https://gql.twitch.tv/gql'
response = requests.post(url,json={'query': query},headers={"Client-ID": client_id})
return json.loads(response.text)
query = 'query{user(login: "' + self.username + '") {stream{archiveVideo{id}title createdAt}}}'
try:
response = requests.post('https://gql.twitch.tv/gql',json={'query': query},headers={"Client-ID": "kimne78kx3ncx6brgo4mv6wki5h1ko"})
return json.loads(response.text)
except requests.exceptions.RequestException as e:
print(e)
quit()
def get_vod(self):
client_id = "kimne78kx3ncx6brgo4mv6wki5h1ko"
query = '''
query {
user(login: "''' + self.username + '''") {
videos(first: 1) {
edges {
node {
id
scope
title
description
recordedAt
lengthSeconds
animatedPreviewURL
previewThumbnailURL(height: 1280, width: 720)
thumbnailURLs(height: 1280, width: 720)
}
}
}
}
}
'''
url = 'https://gql.twitch.tv/gql'
response = requests.post(url, json={'query': query}, headers={"Client-ID": client_id})
return json.loads(response.text)
query = 'query {user(login: "' + self.username + '") {videos(first: 1) {edges {node {id title description recordedAt lengthSeconds animatedPreviewURL previewThumbnailURL(height: 1280, width: 720) thumbnailURLs(height: 1280, width: 720)}}}}}'
try:
response = requests.post('https://gql.twitch.tv/gql', json={'query': query}, headers={"Client-ID": "kimne78kx3ncx6brgo4mv6wki5h1ko"})
return json.loads(response.text)
except requests.exceptions.RequestException as e:
print(e)
def sendNotif(self, subject, content):
if self.notifications == 1:
sender = os.getenv("SENDER")
receiver = os.getenv("RECEIVER")
msg = MIMEMultipart()
msg['From'] = sender
msg['To'] = receiver
msg['Subject'] = self.username + " _ " + subject
body = "Current seccion is for " + self.username + "\n\n\n\n" + content
msg.attach(MIMEText((body), 'plain'))
server = smtplib.SMTP('smtp.gmail.com', 587)
server.starttls()
server.login(sender, os.getenv("PASSWD"))
txt = msg.as_string()
server.sendmail(sender, receiver, txt)
server.quit()
try:
sender = os.getenv("SENDER")
receiver = os.getenv("RECEIVER")
msg = MIMEMultipart()
msg['From'] = sender
msg['To'] = receiver
msg['Subject'] = self.username + " _ " + subject
body = "Current seccion is for " + self.username + "\n\n\n\n" + content
msg.attach(MIMEText((body), 'plain'))
server = smtplib.SMTP('smtp.gmail.com', 587)
server.starttls()
server.login(sender, os.getenv("PASSWD"))
txt = msg.as_string()
server.sendmail(sender, receiver, txt)
server.quit()
except socket.error as e:
print(e)
def loopcheck(self):
while True:
@ -141,19 +139,19 @@ class TwitchArchive:
if is_live is not None:
is_live_ready = self.check_user()['data']['user']['stream']['title']
if is_live_ready is not None:
bin_path = str(pathlib.Path(__file__).parent.resolve())+"/bin"
live_date = datetime.strptime(is_live["createdAt"],'%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=timezone('UTC')).astimezone(tz=None).replace(tzinfo=None)
live_raw_filename = datetime.strftime(live_date,'%Y%m%d_%Hh%Mm%Ss')
live_raw_filename = datetime.strftime(live_date,'%Y%m%d_%Hh%Mm%Ss')
live_raw_path = os.path.join(self.raw_path, "LIVE_" + live_raw_filename + ".ts")
live_proc_path = os.path.join(self.video_path, "LIVE_" + live_raw_filename + ".mp4")
with open(os.path.join(self.root_path, ".log")) as logs:
with open(os.path.join(self.root_path, ".log"), encoding="utf-8") as logs:
logs = logs.read()
log_id = is_live["createdAt"] + " - " + self.username + " - " + is_live["title"]
if log_id in logs:
time.sleep(self.refresh)
with open(os.path.join(self.root_path, ".log"), "r+") as logs:
with open(os.path.join(self.root_path, ".log"), "r+", encoding="utf-8") as logs:
log_id = is_live["createdAt"] + " - " + self.username + " - " + is_live["title"]
for line in logs:
if log_id in line:
@ -176,9 +174,10 @@ class TwitchArchive:
vod_date = datetime.strptime(current_vod["recordedAt"],'%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=timezone('UTC')).astimezone(tz=None).replace(tzinfo=None)
if live_date_min <= vod_date <= live_date_max:
vod_raw_path = os.path.join(self.raw_path, "VOD_" + live_raw_filename + ".ts")
vod_proc_path = os.path.join(self.video_path, "VOD_" + live_raw_filename + ".mp4")
vod_proc_path = os.path.join(self.video_path, live_raw_filename + ".mp4")
chat_json_path = os.path.join(self.chatJSON_path, live_raw_filename + ".json")
chat_video_path = os.path.join(self.chatMP4_path, live_raw_filename + ".mp4")
if self.downloadMETADATA == 1:
self.sendNotif('Metadata - ' + live_raw_filename,'Downloading and saving metadata:\n' + json.dumps(current_vod, indent=4))
with open(os.path.join(self.metadata_path, "METADA_" + live_raw_filename + ".json"), 'w', encoding='utf-8') as f:
@ -188,12 +187,9 @@ class TwitchArchive:
print('Downloading VOD: ' + current_vod["title"])
self.sendNotif('VOD - ' + live_raw_filename,'Downloading VOD: ' + current_vod["title"])
try:
subprocess.call(['streamlink', 'twitch.tv/videos/' + str(current_vod["id"]), self.quality, '--twitch-api-header', 'Authorization=OAuth ' + os.getenv('OAUTH-PRIVATE-TOKEN'), "--hls-segment-threads", str(self.hls_segmentsVOD), "-o", vod_raw_path])
if(os.path.exists(vod_raw_path) is True):
if self.os == 'windows':subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/ffmpeg.exe', '-y', '-i', vod_raw_path, '-analyzeduration', '2147483647', '-probesize', '2147483647', '-c:v', 'copy', '-c:a', 'copy', '-start_at_zero', '-copyts', vod_proc_path], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
elif self.os == 'linux':subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/ffmpeg', '-y', '-i', vod_raw_path, '-analyzeduration', '2147483647', '-probesize', '2147483647', '-c:v', 'copy', '-c:a', 'copy', '-start_at_zero', '-copyts', vod_proc_path], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
else:
print("Skip fixing. File not found.")
if self.os == 'windows':subprocess.call([bin_path+"/TwitchDownloaderCLI.exe", 'videoDownload', '-u', str(current_vod["id"]), '-q', self.quality, "-t", str(self.hls_segmentsVOD), "--ffmpeg-path", bin_path +"/ffmpeg.exe", '--temp-path', bin_path+"/temp", "-o", vod_proc_path])
elif self.os == 'linux':subprocess.call([bin_path+"/TwitchDownloaderCLI", 'videoDownload', '-u', str(current_vod["id"]), '-q', self.quality, "-t", str(self.hls_segmentsVOD), "--ffmpeg-path", bin_path +"/ffmpeg.exe", '--temp-path', bin_path+"/temp", "-o", vod_proc_path])
except Exception as e:
print('Error', 'A ERROR has ocurred and the VOD will not be downloaded.\n')
self.sendNotif('ERROR - ' + live_raw_filename, 'A ERROR has ocurred and the VOD will not be downloaded.\n')
@ -201,9 +197,15 @@ class TwitchArchive:
if self.downloadCHAT == 1:
print('Downloading and rendering CHAT: ' + current_vod["title"])
self.sendNotif('CHAT - ' + live_raw_filename,'Downloading JSON and rendering chat logs from VOD:\n' + current_vod["title"])
chat_settings = ["--background-color", "#FF111111", "-w", "500", "-h", "1080", "--outline", "true", "-f", "Arial", "--font-size", "22", "--update-rate", "1.0", "--offline", "--ffmpeg-path", f"{bin_path}/ffmpeg.exe", "--temp-path", f"{bin_path}/temp"]
try:
if self.os == 'windows':subprocess.call([str(pathlib.Path(__file__).parent.resolve())+"/bin/chat.bat", str(current_vod["id"]), os.path.join(self.chatJSON_path, "CHAT_" + live_raw_filename + ".json"), os.path.join(self.chatMP4_path, "CHAT_" + live_raw_filename + ".mp4")])
elif self.os == 'linux':subprocess.call([str(pathlib.Path(__file__).parent.resolve())+"/bin/chat.sh", str(current_vod["id"]), os.path.join(self.chatJSON_path, "CHAT_" + live_raw_filename + ".json"), os.path.join(self.chatMP4_path, "CHAT_" + live_raw_filename + ".mp4")])
if self.os == 'windows':
subprocess.call([bin_path + '/TwitchDownloaderCLI.exe', 'chatdownload', '--id', current_vod["id"], '-o', chat_json_path, '-E'])
print('Rendering CHAT: ' + current_vod["title"])
subprocess.call([bin_path + '/TwitchDownloaderCLI.exe', 'chatrender', '-i', chat_json_path, '-o', chat_video_path] + chat_settings)
elif self.os == 'linux':
subprocess.call([bin_path + '/TwitchDownloaderCLI', 'chatdownload', '--id', current_vod["id"], '-o', chat_json_path, '-E'])
subprocess.call([bin_path + '/TwitchDownloaderCLI', 'chatrender', '-i', chat_json_path, '-o', chat_video_path] + chat_settings)
except Exception as e:
self.sendNotif('ERROR - ' + live_raw_filename, "A ERROR has ocurred and chat will need to be downloaded and rendered manually.\n")
print("A ERROR has ocurred and chat will need to be downloaded and rendered manually\n")
@ -212,16 +214,12 @@ class TwitchArchive:
if self.cleanRaw == 1:
print('Deleting raw files')
if(os.path.exists(live_raw_path) is True): os.remove(live_raw_path)
if self.downloadVOD == 1:
if(os.path.exists(os.path.join(self.raw_path, "VOD_" + live_raw_filename + ".ts")) is True):
os.remove(os.path.join(self.raw_path, "VOD_" + live_raw_filename + ".ts"))
if self.uploadCloud == 1:
with open(str(pathlib.Path(__file__).parent.resolve())+"/bin/temp/upload.txt", "a") as myfile:
myfile.write("LIVE_" + live_raw_filename + ".ts\n"+"VOD_" + live_raw_filename + ".ts\n"+"LIVE_" + live_raw_filename + ".mp4\n"+"VOD_" + live_raw_filename + ".mp4\n"+"METADATA_" + live_raw_filename + ".json\n"+"CHAT_" + live_raw_filename + ".json\n"+"CHAT_" + live_raw_filename + ".mp4\n")
print('Uploading files')
self.sendNotif("UPLOADING - " + live_raw_filename, 'The files are being uploaded')
if self.os == 'windows':subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/upload.bat', str(pathlib.Path(self.root_path).resolve()),self.username])
elif self.os == 'linux':subprocess.call([str(pathlib.Path(__file__).parent.resolve())+'/bin/upload.sh', str(pathlib.Path(self.root_path).resolve()),self.username])
subprocess.call(['rclone', 'copy', str(pathlib.Path(self.root_path).resolve()), self.rclone_path, '--include-from', bin_path + '/temp/upload.txt'])
os.remove(str(pathlib.Path(__file__).parent.resolve())+"/bin/temp/upload.txt")
if self.deleteFiles == 1:
self.sendNotif("DELETING - " + live_raw_filename, "Deleting the files from current seccion.")