Refactor downloader and file manager for improved rclone integration and add healthcheck and smoke test options

- Renamed download flags in ContentDownloader for clarity.
- Enhanced FileManager with methods to build upload paths and verify existing files for rclone uploads.
- Updated StreamProcessor to return success status for stream processing.
- Added rclone smoke test and healthcheck functions to validate configuration and tool availability.
- Improved environment variable handling with a utility function.
- Updated TwitchArchive to incorporate new rclone verification and processing logic.
- Added unit tests for new functionality and refactored existing tests for clarity and coverage.

Co-authored-by: Copilot <copilot@github.com>
This commit is contained in:
MaddoScientisto 2026-04-25 11:54:03 +02:00
commit f97e0200d6
23 changed files with 1013 additions and 289 deletions

18
.dockerignore Normal file
View file

@ -0,0 +1,18 @@
.git
.github
.forgejo
.vs
.vscode
venv*/
__pycache__/
*.pyc
archive/
dotnet/
tests/
.pytest_cache/
bin/temp/
bin/ffmpeg
bin/ffmpeg.exe
bin/ffprobe
bin/TwitchDownloaderCLI
bin/TwitchDownloaderCLI.exe

16
.env.development Normal file
View file

@ -0,0 +1,16 @@
TWITCH_ARCHIVE_DEV_IMAGE=twitch-archive-local
TWITCH_ARCHIVE_CONTAINER_NAME=twitch-archive-dev
TWITCH_ARCHIVE_APP_ENV_FILE=./.env.development
TWITCH_ARCHIVE_ARCHIVE_BIND=./archive
TWITCH_ARCHIVE_CONFIG_BIND=./config
TWITCH_ARCHIVE_ARGS=-u vinesauce --verbose
TWITCH_ARCHIVE_HEALTHCHECK_STREAMER=vinesauce
TWITCH_ARCHIVE_RCLONE_CONFIG=/app/config/rclone.conf
PYTHONUNBUFFERED=1
TZ=UTC
CLIENT-ID=vdyevjvllziylzwsm3y925p79pwtua
CLIENT-SECRET=y906xadsmf22q54suuzsmfnfav3jc7
OAUTH-PRIVATE-TOKEN=ll4kvlmxuajfgi9lgi5d8mkeglsyvm
SENDER=
RECEIVER=
PASSWD=

16
.env.production Normal file
View file

@ -0,0 +1,16 @@
TWITCH_ARCHIVE_IMAGE=forgejo.maddoscientisto.net/maddo/twitch-archive:latest
TWITCH_ARCHIVE_CONTAINER_NAME=twitch-archive
TWITCH_ARCHIVE_APP_ENV_FILE=./.env.production
TWITCH_ARCHIVE_ARCHIVE_BIND=./archive
TWITCH_ARCHIVE_CONFIG_BIND=./config
TWITCH_ARCHIVE_ARGS=-u vinesauce
TWITCH_ARCHIVE_HEALTHCHECK_STREAMER=vinesauce
TWITCH_ARCHIVE_RCLONE_CONFIG=/app/config/rclone.conf
PYTHONUNBUFFERED=1
TZ=UTC
CLIENT-ID=vdyevjvllziylzwsm3y925p79pwtua
CLIENT-SECRET=y906xadsmf22q54suuzsmfnfav3jc7
OAUTH-PRIVATE-TOKEN=ll4kvlmxuajfgi9lgi5d8mkeglsyvm
SENDER=
RECEIVER=
PASSWD=

View file

@ -0,0 +1,127 @@
name: Publish Twitch Archive Container
on:
push:
branches:
- master
- main
paths:
- docker/python.Dockerfile
- docker/entrypoint.sh
- docker-compose.yml
- docker-compose.override.yml
- requirements.txt
- twitch-archive.py
- run_chat_only.py
- modules/**
- .forgejo/workflows/publish-python-container.yml
workflow_dispatch:
env:
REGISTRY: ${{ vars.FORGEJO_REGISTRY }}
IMAGE_NAMESPACE: ${{ vars.IMAGE_NAMESPACE }}
IMAGE_NAME: ${{ vars.IMAGE_NAME != '' && vars.IMAGE_NAME || 'twitch-archive' }}
BUILD_CONTEXT: .
DOCKERFILE_PATH: docker/python.Dockerfile
jobs:
publish:
runs-on: docker
env:
DOCKER_HOST: ${{ vars.DOCKER_HOST != '' && vars.DOCKER_HOST || 'tcp://172.17.0.1:2375' }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Validate workflow variables
run: |
set -eu
if [ -z "${REGISTRY}" ]; then echo "vars.FORGEJO_REGISTRY is required"; exit 1; fi
if [ -z "${IMAGE_NAMESPACE}" ]; then echo "vars.IMAGE_NAMESPACE is required"; exit 1; fi
if [ ! -f "${DOCKERFILE_PATH}" ]; then echo "Dockerfile not found at ${DOCKERFILE_PATH}"; exit 1; fi
if [ ! -f "requirements.txt" ]; then echo "requirements.txt is missing"; exit 1; fi
- name: Ensure Docker CLI exists
run: |
set -eu
if command -v docker >/dev/null 2>&1; then
docker --version
exit 0
fi
ARCH="$(uname -m)"
case "${ARCH}" in
x86_64) DOCKER_ARCH="x86_64" ;;
aarch64|arm64) DOCKER_ARCH="aarch64" ;;
*) echo "Unsupported architecture for Docker CLI bootstrap: ${ARCH}"; exit 1 ;;
esac
DOCKER_CLI_VERSION="27.5.1"
curl -fsSL "https://download.docker.com/linux/static/stable/${DOCKER_ARCH}/docker-${DOCKER_CLI_VERSION}.tgz" -o docker.tgz
tar -xzf docker.tgz
mkdir -p "${HOME}/.local/bin"
mv docker/docker "${HOME}/.local/bin/docker"
chmod +x "${HOME}/.local/bin/docker"
echo "${HOME}/.local/bin" >> "${FORGEJO_PATH}"
"${HOME}/.local/bin/docker" --version
- name: Ensure Docker Buildx exists
run: |
set -eu
if docker buildx version >/dev/null 2>&1; then
docker buildx version
exit 0
fi
ARCH="$(uname -m)"
case "${ARCH}" in
x86_64) BUILDX_ARCH="amd64" ;;
aarch64|arm64) BUILDX_ARCH="arm64" ;;
*) echo "Unsupported architecture for Docker Buildx bootstrap: ${ARCH}"; exit 1 ;;
esac
BUILDX_VERSION="v0.21.1"
mkdir -p "${HOME}/.docker/cli-plugins"
curl -fsSL "https://github.com/docker/buildx/releases/download/${BUILDX_VERSION}/buildx-${BUILDX_VERSION}.linux-${BUILDX_ARCH}" -o "${HOME}/.docker/cli-plugins/docker-buildx"
chmod +x "${HOME}/.docker/cli-plugins/docker-buildx"
docker buildx version
- name: Check Docker daemon connectivity
run: |
set -eu
echo "Using DOCKER_HOST=${DOCKER_HOST}"
docker version
docker info >/dev/null
- name: Create Buildx builder
run: |
set -eu
docker buildx rm forgejo-builder >/dev/null 2>&1 || true
docker buildx create --name forgejo-builder --driver docker-container --use
docker buildx inspect --bootstrap
- name: Validate registry secrets
run: |
set -eu
if [ -z "${{ secrets.FORGEJO_REGISTRY_USERNAME }}" ]; then echo "secrets.FORGEJO_REGISTRY_USERNAME is required"; exit 1; fi
if [ -z "${{ secrets.FORGEJO_REGISTRY_TOKEN }}" ]; then echo "secrets.FORGEJO_REGISTRY_TOKEN is required"; exit 1; fi
- name: Login to registry
run: |
set -eu
echo "${{ secrets.FORGEJO_REGISTRY_TOKEN }}" | docker login "${REGISTRY}" -u "${{ secrets.FORGEJO_REGISTRY_USERNAME }}" --password-stdin
- name: Build and push image
run: |
set -eu
IMAGE_REF="${REGISTRY}/${IMAGE_NAMESPACE}/${IMAGE_NAME}"
SHA_TAG="${IMAGE_REF}:sha-${FORGEJO_SHA}"
BRANCH_TAG="${IMAGE_REF}:${FORGEJO_REF_NAME}"
docker buildx build \
--file "${DOCKERFILE_PATH}" \
--tag "${SHA_TAG}" \
--tag "${BRANCH_TAG}" \
--tag "${IMAGE_REF}:latest" \
--push \
"${BUILD_CONTEXT}"

2
.gitignore vendored
View file

@ -4,6 +4,8 @@ config/global.json
# Streamer-specific configurations (personal settings) # Streamer-specific configurations (personal settings)
config/streamers/*.json config/streamers/*.json
config/rclone.conf
config/rclone.conf.*
# Python cache # Python cache
__pycache__/ __pycache__/

View file

@ -26,6 +26,73 @@ Notes:
Python script to check, download live stream, VOD, chat and upload them to any cloud storage supported by rclone. Python script to check, download live stream, VOD, chat and upload them to any cloud storage supported by rclone.
## Docker
This repository now includes a Python-only container setup for the archiver. The dotnet subapp is not part of this container flow.
Files:
- `docker/python.Dockerfile`: production image for the Python archiver
- `docker-compose.yml`: deployment-oriented compose file
- `docker-compose.override.yml`: local development and testing override
- `.env.production`: production container and app environment template
- `.env.development`: development container and app environment template
- `dockerstart.bat`: Windows helper to run the container like the old batch launcher
### Container layout
- Mount your external archive folder to `/app/archive`
- Mount your external config folder to `/app/config`
- Put your `rclone.conf` file at `/app/config/rclone.conf` on the mounted host path
- The container exports `RCLONE_CONFIG=/app/config/rclone.conf`, so rclone will use that file automatically
### Production deployment
1. Edit `.env.production` with your image name, Twitch credentials, bind paths, and default arguments.
2. Place your streamer JSON files and `rclone.conf` in the mounted config folder.
3. Start the container:
```powershell
docker compose --env-file .env.production up -d
```
4. Follow logs:
```powershell
docker compose --env-file .env.production logs -f twitch-archive
```
### Development and local testing
The override compose file builds the image locally and mounts the repository for faster iteration.
Start it with:
```powershell
docker compose --env-file .env.development -f docker-compose.yml -f docker-compose.override.yml up --build
```
Run a one-off manual test for another streamer:
```powershell
docker compose --env-file .env.development -f docker-compose.yml -f docker-compose.override.yml run --rm twitch-archive python twitch-archive.py -u hackerling --verbose
```
Or use the Windows helper:
```powershell
.\dockerstart.bat vinesauce --verbose
```
That batch launcher mirrors the old pattern and expands to a compose `run` command, so you can test any streamer manually.
### Healthcheck and smoke tests
- Container healthcheck command: `python twitch-archive.py --healthcheck -u vinesauce`
- Rclone smoke test command: `python twitch-archive.py -u vinesauce --rclone-smoke-test`
The healthcheck verifies config loading plus `streamlink`, `ffmpeg`, `TwitchDownloaderCLI`, and `rclone` availability. The smoke test writes a tiny file, uploads it with the configured rclone remote, and prints the live rclone output into the container logs.
## ⚡ FFmpeg 8.0 Enhanced ## ⚡ FFmpeg 8.0 Enhanced
Now with FFmpeg 8.0+ support featuring hardware acceleration and performance improvements! Now with FFmpeg 8.0+ support featuring hardware acceleration and performance improvements!
- **5-10x faster encoding** with NVIDIA, Intel, or AMD GPUs - **5-10x faster encoding** with NVIDIA, Intel, or AMD GPUs

View file

@ -0,0 +1,17 @@
services:
twitch-archive:
build:
context: .
dockerfile: docker/python.Dockerfile
image: ${TWITCH_ARCHIVE_DEV_IMAGE:-twitch-archive-local}
restart: "no"
env_file:
- ${TWITCH_ARCHIVE_APP_ENV_FILE:-./.env.development}
command:
- sh
- -lc
- python twitch-archive.py ${TWITCH_ARCHIVE_ARGS:--u vinesauce --verbose}
volumes:
- .:/app
- ${TWITCH_ARCHIVE_ARCHIVE_BIND:-./archive}:/app/archive
- ${TWITCH_ARCHIVE_CONFIG_BIND:-./config}:/app/config

28
docker-compose.yml Normal file
View file

@ -0,0 +1,28 @@
services:
twitch-archive:
image: ${TWITCH_ARCHIVE_IMAGE:-forgejo.maddoscientisto.net/maddo/twitch-archive:latest}
container_name: ${TWITCH_ARCHIVE_CONTAINER_NAME:-twitch-archive}
restart: unless-stopped
init: true
env_file:
- ${TWITCH_ARCHIVE_APP_ENV_FILE:-./.env.production}
environment:
PYTHONUNBUFFERED: ${PYTHONUNBUFFERED:-1}
TZ: ${TZ:-UTC}
RCLONE_CONFIG: ${TWITCH_ARCHIVE_RCLONE_CONFIG:-/app/config/rclone.conf}
TWITCH_ARCHIVE_HEALTHCHECK_STREAMER: ${TWITCH_ARCHIVE_HEALTHCHECK_STREAMER:-vinesauce}
command:
- sh
- -lc
- python twitch-archive.py ${TWITCH_ARCHIVE_ARGS:--u vinesauce}
volumes:
- ${TWITCH_ARCHIVE_ARCHIVE_BIND:-./archive}:/app/archive
- ${TWITCH_ARCHIVE_CONFIG_BIND:-./config}:/app/config
healthcheck:
test:
- CMD-SHELL
- python twitch-archive.py --healthcheck -u ${TWITCH_ARCHIVE_HEALTHCHECK_STREAMER:-vinesauce}
interval: 30s
timeout: 10s
retries: 3
start_period: 30s

6
docker/entrypoint.sh Normal file
View file

@ -0,0 +1,6 @@
#!/bin/sh
set -eu
mkdir -p /app/archive /app/config /app/bin/temp
exec "$@"

49
docker/python.Dockerfile Normal file
View file

@ -0,0 +1,49 @@
FROM python:3.12-slim
ARG TWITCH_DOWNLOADER_VERSION=1.56.4
ARG TARGETARCH
ENV PYTHONUNBUFFERED=1 \
PIP_NO_CACHE_DIR=1 \
RCLONE_CONFIG=/app/config/rclone.conf
WORKDIR /app
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
ca-certificates \
curl \
ffmpeg \
libicu-dev \
rclone \
unzip \
&& rm -rf /var/lib/apt/lists/*
RUN set -eux; \
case "${TARGETARCH}" in \
amd64) twitch_downloader_arch='Linux-x64' ;; \
arm64) twitch_downloader_arch='LinuxArm64' ;; \
arm) twitch_downloader_arch='LinuxArm' ;; \
*) echo "Unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; \
esac; \
curl -fsSL -o /tmp/TwitchDownloaderCLI.zip "https://github.com/lay295/TwitchDownloader/releases/download/${TWITCH_DOWNLOADER_VERSION}/TwitchDownloaderCLI-${TWITCH_DOWNLOADER_VERSION}-${twitch_downloader_arch}.zip"; \
mkdir -p /tmp/twitchdownloader; \
unzip -j /tmp/TwitchDownloaderCLI.zip TwitchDownloaderCLI -d /tmp/twitchdownloader; \
install -m 0755 /tmp/twitchdownloader/TwitchDownloaderCLI /usr/local/bin/TwitchDownloaderCLI; \
rm -rf /tmp/TwitchDownloaderCLI.zip /tmp/twitchdownloader
COPY requirements.txt ./requirements.txt
RUN python -m pip install --upgrade pip \
&& python -m pip install -r requirements.txt
COPY . /app
RUN mkdir -p /app/archive /app/config /app/bin/temp
COPY docker/entrypoint.sh /usr/local/bin/twitch-archive-entrypoint
RUN chmod +x /usr/local/bin/twitch-archive-entrypoint
ENTRYPOINT ["twitch-archive-entrypoint"]
CMD ["python", "twitch-archive.py", "-u", "vinesauce"]

4
dockerrebuild.bat Normal file
View file

@ -0,0 +1,4 @@
@echo off
setlocal
docker compose --env-file .env.development -f docker-compose.yml -f docker-compose.override.yml build --no-cache --pull twitch-archive

20
dockerstart.bat Normal file
View file

@ -0,0 +1,20 @@
@echo off
setlocal
if "%~1"=="" (
echo Usage: .\dockerstart.bat streamer [additional args]
exit /b 1
)
set STREAMER=%~1
shift
set EXTRA_ARGS=
:collect_args
if "%~1"=="" goto run_compose
set EXTRA_ARGS=%EXTRA_ARGS% %~1
shift
goto collect_args
:run_compose
docker compose --env-file .env.development -f docker-compose.yml -f docker-compose.override.yml run --rm twitch-archive python twitch-archive.py -u %STREAMER%%EXTRA_ARGS%

View file

@ -17,49 +17,39 @@
</div> </div>
<div class="card"> <div class="card">
<label title="Set a quality override for this streamer">Quality</label> <label title="Quality for this streamer">Quality</label>
<InputCheckbox Value="overrideQuality" ValueChanged="@( (bool v) => overrideQuality = v )" ValueExpression="() => overrideQuality" title="Override default quality" /> Override <InputText @bind-Value="model.Quality" placeholder="@(global?.DefaultQuality ?? "")" title="Enter a quality string (e.g., best, 720p)" />
<InputText @bind="model.Quality" disabled="@(!overrideQuality)" placeholder="@(global?.DefaultQuality ?? "")" title="Enter a quality string (e.g., best, 720p)" />
</div> </div>
<div> <div>
<label title="Toggle cloud upload for this streamer">Upload to Cloud</label> <label title="Toggle cloud upload for this streamer">Upload to Cloud</label>
<InputCheckbox Value="overrideUpload" ValueChanged="@( (bool v) => overrideUpload = v )" ValueExpression="() => overrideUpload" title="Override upload-to-cloud setting" /> Override <InputCheckbox Value="uploadToCloudVal" ValueChanged="@( (bool v) => uploadToCloudVal = v )" ValueExpression="() => uploadToCloudVal" title="Upload to configured cloud destination" />
<InputCheckbox Value="uploadToCloudVal" ValueChanged="@( (bool v) => uploadToCloudVal = v )" ValueExpression="() => uploadToCloudVal" disabled="@(!overrideUpload)" title="Upload to configured cloud destination" />
</div> </div>
<div class="card"> <div class="card">
<label title="Cloud destination (e.g., rclone remote) for this streamer">Upload Destination</label> <label title="Cloud destination (e.g., rclone remote) for this streamer">Upload Destination</label>
<InputText @bind="model.UploadDestination" title="Cloud destination (e.g., rclone remote)" /> <InputText @bind-Value="model.UploadDestination" title="Cloud destination (e.g., rclone remote)" />
</div> </div>
<div class="card"> @* Streamlink path is global-only; not configurable per-streamer *@
<label title="Optional: override system Streamlink path for this streamer">Streamlink Path (override)</label>
<InputCheckbox Value="overrideStreamlink" ValueChanged="@( (bool v) => overrideStreamlink = v )" ValueExpression="() => overrideStreamlink" title="Override streamlink path" /> Override
<InputText @bind="model.StreamlinkPath" disabled="@(!overrideStreamlink)" placeholder="@(global?.StreamlinkPath ?? "")" title="Full path to streamlink executable" />
</div>
<div class="card"> <div class="card">
<h4>Per-streamer overrides</h4> <h4>Per-streamer settings</h4>
<div> <div>
<label>Download VOD</label> <label>Download VOD</label>
<InputCheckbox Value="overrideDownloadVOD" ValueChanged="@( (bool v) => overrideDownloadVOD = v )" ValueExpression="() => overrideDownloadVOD" /> Override <InputCheckbox Value="downloadVODVal" ValueChanged="@( (bool v) => downloadVODVal = v )" ValueExpression="() => downloadVODVal" />
<InputCheckbox Value="downloadVODVal" ValueChanged="@( (bool v) => downloadVODVal = v )" ValueExpression="() => downloadVODVal" disabled="@(!overrideDownloadVOD)" />
</div> </div>
<div> <div>
<label>Download CHAT</label> <label>Download CHAT</label>
<InputCheckbox Value="overrideDownloadCHAT" ValueChanged="@( (bool v) => overrideDownloadCHAT = v )" ValueExpression="() => overrideDownloadCHAT" /> Override <InputCheckbox Value="downloadCHATVal" ValueChanged="@( (bool v) => downloadCHATVal = v )" ValueExpression="() => downloadCHATVal" />
<InputCheckbox Value="downloadCHATVal" ValueChanged="@( (bool v) => downloadCHATVal = v )" ValueExpression="() => downloadCHATVal" disabled="@(!overrideDownloadCHAT)" />
</div> </div>
<div> <div>
<label>Merge Video & Chat</label> <label>Merge Video & Chat</label>
<InputCheckbox Value="overrideMergeVideoChat" ValueChanged="@( (bool v) => overrideMergeVideoChat = v )" ValueExpression="() => overrideMergeVideoChat" /> Override <InputCheckbox Value="mergeVideoChatVal" ValueChanged="@( (bool v) => mergeVideoChatVal = v )" ValueExpression="() => mergeVideoChatVal" />
<InputCheckbox Value="mergeVideoChatVal" ValueChanged="@( (bool v) => mergeVideoChatVal = v )" ValueExpression="() => mergeVideoChatVal" disabled="@(!overrideMergeVideoChat)" />
</div> </div>
<div> <div>
<label>Merge Chat Layout</label> <label>Merge Chat Layout</label>
<InputCheckbox Value="overrideMergeChatLayout" ValueChanged="@( (bool v) => overrideMergeChatLayout = v )" ValueExpression="() => overrideMergeChatLayout" /> Override <InputSelect @bind-Value="mergeChatLayoutVal">
<InputSelect @bind-Value="mergeChatLayoutVal" disabled="@(!overrideMergeChatLayout)">
<option value="side-by-side">Side by side</option> <option value="side-by-side">Side by side</option>
<option value="stacked">Stacked</option> <option value="stacked">Stacked</option>
<option value="overlay">Overlay</option> <option value="overlay">Overlay</option>
@ -67,105 +57,35 @@
</div> </div>
<div> <div>
<label>VOD Timeout (sec)</label> <label>VOD Timeout (sec)</label>
<InputCheckbox Value="overrideVodTimeout" ValueChanged="@( (bool v) => overrideVodTimeout = v )" ValueExpression="() => overrideVodTimeout" /> Override <InputNumber @bind-Value="vodTimeoutVal" />
<InputNumber @bind-Value="vodTimeoutVal" disabled="@(!overrideVodTimeout)" />
</div> </div>
<div> <div>
<label>Delete Files</label> <label>Delete Files</label>
<InputCheckbox Value="overrideDeleteFiles" ValueChanged="@( (bool v) => overrideDeleteFiles = v )" ValueExpression="() => overrideDeleteFiles" /> Override <InputCheckbox Value="deleteFilesVal" ValueChanged="@( (bool v) => deleteFilesVal = v )" ValueExpression="() => deleteFilesVal" />
<InputCheckbox Value="deleteFilesVal" ValueChanged="@( (bool v) => deleteFilesVal = v )" ValueExpression="() => deleteFilesVal" disabled="@(!overrideDeleteFiles)" />
</div>
<div>
<label>HLS Segments (live)</label>
<InputCheckbox Value="overrideHlsSegments" ValueChanged="@( (bool v) => overrideHlsSegments = v )" ValueExpression="() => overrideHlsSegments" /> Override
<InputNumber @bind-Value="hlsSegmentsVal" disabled="@(!overrideHlsSegments)" />
</div>
<div>
<label>FFmpeg HW Accel</label>
<InputCheckbox Value="overrideFfmpegHwaccel" ValueChanged="@( (bool v) => overrideFfmpegHwaccel = v )" ValueExpression="() => overrideFfmpegHwaccel" /> Override
<InputSelect @bind-Value="ffmpegHwaccelVal" disabled="@(!overrideFfmpegHwaccel)">
<option value="auto">Auto</option>
<option value="none">None</option>
<option value="vaapi">VAAPI</option>
<option value="dxva2">DXVA2</option>
<option value="qsv">QSV</option>
<option value="cuda">CUDA</option>
</InputSelect>
</div>
<div>
<label>FFmpeg Threads</label>
<InputCheckbox Value="overrideFfmpegThreads" ValueChanged="@( (bool v) => overrideFfmpegThreads = v )" ValueExpression="() => overrideFfmpegThreads" /> Override
<InputNumber @bind-Value="ffmpegThreadsVal" disabled="@(!overrideFfmpegThreads)" />
</div>
<div>
<label>FFmpeg Audio Bitrate</label>
<InputCheckbox Value="overrideFfmpegAudioBitrate" ValueChanged="@( (bool v) => overrideFfmpegAudioBitrate = v )" ValueExpression="() => overrideFfmpegAudioBitrate" /> Override
<InputText @bind-Value="ffmpegAudioBitrateVal" disabled="@(!overrideFfmpegAudioBitrate)" />
</div> </div>
<div> <div>
<label>Download Live CHAT</label> <label>Download Live CHAT</label>
<InputCheckbox Value="overrideDownloadLiveCHAT" ValueChanged="@( (bool v) => overrideDownloadLiveCHAT = v )" ValueExpression="() => overrideDownloadLiveCHAT" /> Override <InputCheckbox Value="downloadLiveCHATVal" ValueChanged="@( (bool v) => downloadLiveCHATVal = v )" ValueExpression="() => downloadLiveCHATVal" />
<InputCheckbox Value="downloadLiveCHATVal" ValueChanged="@( (bool v) => downloadLiveCHATVal = v )" ValueExpression="() => downloadLiveCHATVal" disabled="@(!overrideDownloadLiveCHAT)" />
</div> </div>
<div> <div>
<label>Upload Pre-Merge Video</label> <label>Upload Pre-Merge Video</label>
<InputCheckbox Value="overrideUploadPreMergeVideo" ValueChanged="@( (bool v) => overrideUploadPreMergeVideo = v )" ValueExpression="() => overrideUploadPreMergeVideo" /> Override <InputCheckbox Value="uploadPreMergeVideoVal" ValueChanged="@( (bool v) => uploadPreMergeVideoVal = v )" ValueExpression="() => uploadPreMergeVideoVal" />
<InputCheckbox Value="uploadPreMergeVideoVal" ValueChanged="@( (bool v) => uploadPreMergeVideoVal = v )" ValueExpression="() => uploadPreMergeVideoVal" disabled="@(!overrideUploadPreMergeVideo)" />
</div> </div>
<div> <div>
<label>Upload Merged Video</label> <label>Upload Merged Video</label>
<InputCheckbox Value="overrideUploadMergedVideo" ValueChanged="@( (bool v) => overrideUploadMergedVideo = v )" ValueExpression="() => overrideUploadMergedVideo" /> Override <InputCheckbox Value="uploadMergedVideoVal" ValueChanged="@( (bool v) => uploadMergedVideoVal = v )" ValueExpression="() => uploadMergedVideoVal" />
<InputCheckbox Value="uploadMergedVideoVal" ValueChanged="@( (bool v) => uploadMergedVideoVal = v )" ValueExpression="() => uploadMergedVideoVal" disabled="@(!overrideUploadMergedVideo)" />
</div> </div>
<div> <div>
<label>Upload Chat Video</label> <label>Upload Chat Video</label>
<InputCheckbox Value="overrideUploadChatVideo" ValueChanged="@( (bool v) => overrideUploadChatVideo = v )" ValueExpression="() => overrideUploadChatVideo" /> Override <InputCheckbox Value="uploadChatVideoVal" ValueChanged="@( (bool v) => uploadChatVideoVal = v )" ValueExpression="() => uploadChatVideoVal" />
<InputCheckbox Value="uploadChatVideoVal" ValueChanged="@( (bool v) => uploadChatVideoVal = v )" ValueExpression="() => uploadChatVideoVal" disabled="@(!overrideUploadChatVideo)" />
</div> </div>
<div> <div>
<label>Only Raw</label> <label>Only Raw</label>
<InputCheckbox Value="overrideOnlyRaw" ValueChanged="@( (bool v) => overrideOnlyRaw = v )" ValueExpression="() => overrideOnlyRaw" /> Override <InputCheckbox Value="onlyRawVal" ValueChanged="@( (bool v) => onlyRawVal = v )" ValueExpression="() => onlyRawVal" />
<InputCheckbox Value="onlyRawVal" ValueChanged="@( (bool v) => onlyRawVal = v )" ValueExpression="() => onlyRawVal" disabled="@(!overrideOnlyRaw)" />
</div> </div>
<div> <div>
<label>Clean Raw</label> <label>Clean Raw</label>
<InputCheckbox Value="overrideCleanRaw" ValueChanged="@( (bool v) => overrideCleanRaw = v )" ValueExpression="() => overrideCleanRaw" /> Override <InputCheckbox Value="cleanRawVal" ValueChanged="@( (bool v) => cleanRawVal = v )" ValueExpression="() => cleanRawVal" />
<InputCheckbox Value="cleanRawVal" ValueChanged="@( (bool v) => cleanRawVal = v )" ValueExpression="() => cleanRawVal" disabled="@(!overrideCleanRaw)" />
</div>
<div>
<label>HLS Segments (VOD)</label>
<InputCheckbox Value="overrideHlsSegmentsVOD" ValueChanged="@( (bool v) => overrideHlsSegmentsVOD = v )" ValueExpression="() => overrideHlsSegmentsVOD" /> Override
<InputNumber @bind-Value="hlsSegmentsVODVal" disabled="@(!overrideHlsSegmentsVOD)" />
</div>
<div>
<label>Streamlink ttvlol</label>
<InputCheckbox Value="overrideStreamlinkTtvlol" ValueChanged="@( (bool v) => overrideStreamlinkTtvlol = v )" ValueExpression="() => overrideStreamlinkTtvlol" /> Override
<InputCheckbox Value="streamlinkTtvlolVal" ValueChanged="@( (bool v) => streamlinkTtvlolVal = v )" ValueExpression="() => streamlinkTtvlolVal" disabled="@(!overrideStreamlinkTtvlol)" />
</div>
<div>
<label>FFmpeg Audio Codec</label>
<InputCheckbox Value="overrideFfmpegAudioCodec" ValueChanged="@( (bool v) => overrideFfmpegAudioCodec = v )" ValueExpression="() => overrideFfmpegAudioCodec" /> Override
<InputText @bind-Value="ffmpegAudioCodecVal" disabled="@(!overrideFfmpegAudioCodec)" />
</div>
<div>
<label>FFmpeg Audio Sample Rate</label>
<InputCheckbox Value="overrideFfmpegAudioSamplerate" ValueChanged="@( (bool v) => overrideFfmpegAudioSamplerate = v )" ValueExpression="() => overrideFfmpegAudioSamplerate" /> Override
<InputNumber @bind-Value="ffmpegAudioSamplerateVal" disabled="@(!overrideFfmpegAudioSamplerate)" />
</div>
<div>
<label>FFmpeg Error Recovery</label>
<InputCheckbox Value="overrideFfmpegErrorRecovery" ValueChanged="@( (bool v) => overrideFfmpegErrorRecovery = v )" ValueExpression="() => overrideFfmpegErrorRecovery" /> Override
<InputCheckbox Value="ffmpegErrorRecoveryVal" ValueChanged="@( (bool v) => ffmpegErrorRecoveryVal = v )" ValueExpression="() => ffmpegErrorRecoveryVal" disabled="@(!overrideFfmpegErrorRecovery)" />
</div>
<div>
<label>FFmpeg Faststart</label>
<InputCheckbox Value="overrideFfmpegFaststart" ValueChanged="@( (bool v) => overrideFfmpegFaststart = v )" ValueExpression="() => overrideFfmpegFaststart" /> Override
<InputCheckbox Value="ffmpegFaststartVal" ValueChanged="@( (bool v) => ffmpegFaststartVal = v )" ValueExpression="() => ffmpegFaststartVal" disabled="@(!overrideFfmpegFaststart)" />
</div>
<div>
<label>FFmpeg Progress</label>
<InputCheckbox Value="overrideFfmpegProgress" ValueChanged="@( (bool v) => overrideFfmpegProgress = v )" ValueExpression="() => overrideFfmpegProgress" /> Override
<InputCheckbox Value="ffmpegProgressVal" ValueChanged="@( (bool v) => ffmpegProgressVal = v )" ValueExpression="() => ffmpegProgressVal" disabled="@(!overrideFfmpegProgress)" />
</div> </div>
</div> </div>
@ -193,37 +113,10 @@
@code { @code {
[Parameter] [Parameter]
public string Username { get; set; } = string.Empty; public string Username { get; set; } = string.Empty;
private TwitchArchive.Core.Config.StreamerConfig model = new(); private TwitchArchive.Core.Config.StreamerConfig model = new();
private TwitchArchive.Core.Config.GlobalConfig? global; private TwitchArchive.Core.Config.GlobalConfig? global;
private bool saved = false; private bool saved = false;
private bool showConfirm = false; private bool showConfirm = false;
private bool overrideQuality = false;
private bool overrideUpload = false;
private bool overrideStreamlink = false;
private bool overrideDownloadVOD = false;
private bool overrideDownloadCHAT = false;
private bool overrideMergeVideoChat = false;
private bool overrideMergeChatLayout = false;
private bool overrideVodTimeout = false;
private bool overrideDeleteFiles = false;
private bool overrideHlsSegments = false;
private bool overrideFfmpegHwaccel = false;
private bool overrideFfmpegThreads = false;
private bool overrideFfmpegAudioBitrate = false;
private bool overrideDownloadLiveCHAT = false;
private bool overrideUploadPreMergeVideo = false;
private bool overrideUploadMergedVideo = false;
private bool overrideUploadChatVideo = false;
private bool overrideOnlyRaw = false;
private bool overrideCleanRaw = false;
private bool overrideHlsSegmentsVOD = false;
private bool overrideStreamlinkTtvlol = false;
private bool overrideFfmpegAudioCodec = false;
private bool overrideFfmpegAudioSamplerate = false;
private bool overrideFfmpegErrorRecovery = false;
private bool overrideFfmpegFaststart = false;
private bool overrideFfmpegProgress = false;
// local values for nullable per-streamer settings (bind safely) // local values for nullable per-streamer settings (bind safely)
private bool downloadVODVal; private bool downloadVODVal;
@ -233,28 +126,18 @@
private string mergeChatLayoutVal = "side-by-side"; private string mergeChatLayoutVal = "side-by-side";
private int? vodTimeoutVal; private int? vodTimeoutVal;
private bool deleteFilesVal; private bool deleteFilesVal;
private int? hlsSegmentsVal;
private string ffmpegHwaccelVal = "auto";
private int? ffmpegThreadsVal;
private string? ffmpegAudioBitrateVal;
private bool uploadPreMergeVideoVal; private bool uploadPreMergeVideoVal;
private bool uploadMergedVideoVal; private bool uploadMergedVideoVal;
private bool uploadChatVideoVal; private bool uploadChatVideoVal;
private bool onlyRawVal; private bool onlyRawVal;
private bool cleanRawVal; private bool cleanRawVal;
private int? hlsSegmentsVODVal;
private bool streamlinkTtvlolVal;
private string? ffmpegAudioCodecVal;
private int? ffmpegAudioSamplerateVal;
private bool ffmpegErrorRecoveryVal;
private bool ffmpegFaststartVal;
private bool ffmpegProgressVal;
private bool uploadToCloudVal; private bool uploadToCloudVal;
protected override void OnInitialized() protected override void OnInitialized()
{ {
global = ConfigService.LoadGlobal(); global = ConfigService.LoadGlobal();
var s = ConfigService.LoadStreamer(Username); var s = ConfigService.LoadStreamer(Username);
var isNew = s == null;
if (s != null) model = s; if (s != null) model = s;
// initialize local values from model or global defaults // initialize local values from model or global defaults
downloadVODVal = model.DownloadVOD ?? global?.Defaults.DownloadVOD ?? true; downloadVODVal = model.DownloadVOD ?? global?.Defaults.DownloadVOD ?? true;
@ -264,57 +147,67 @@
mergeChatLayoutVal = model.MergeChatLayout ?? global?.Defaults.MergeChatLayout ?? "side-by-side"; mergeChatLayoutVal = model.MergeChatLayout ?? global?.Defaults.MergeChatLayout ?? "side-by-side";
vodTimeoutVal = model.VodTimeout ?? global?.Defaults.VodTimeout ?? 300; vodTimeoutVal = model.VodTimeout ?? global?.Defaults.VodTimeout ?? 300;
deleteFilesVal = model.DeleteFiles ?? global?.Defaults.DeleteFiles ?? false; deleteFilesVal = model.DeleteFiles ?? global?.Defaults.DeleteFiles ?? false;
hlsSegmentsVal = model.HlsSegments ?? global?.Defaults.HlsSegments ?? 3;
ffmpegHwaccelVal = model.FfmpegHwaccel ?? global?.Defaults.FfmpegHwaccel ?? "auto";
ffmpegThreadsVal = model.FfmpegThreads ?? global?.Defaults.FfmpegThreads ?? 0;
ffmpegAudioBitrateVal = model.FfmpegAudioBitrate ?? global?.Defaults.FfmpegAudioBitrate ?? "192k";
uploadPreMergeVideoVal = model.UploadPreMergeVideo ?? global?.Defaults.UploadPreMergeVideo ?? true; uploadPreMergeVideoVal = model.UploadPreMergeVideo ?? global?.Defaults.UploadPreMergeVideo ?? true;
uploadMergedVideoVal = model.UploadMergedVideo ?? global?.Defaults.UploadMergedVideo ?? true; uploadMergedVideoVal = model.UploadMergedVideo ?? global?.Defaults.UploadMergedVideo ?? true;
uploadChatVideoVal = model.UploadChatVideo ?? global?.Defaults.UploadChatVideo ?? false; uploadChatVideoVal = model.UploadChatVideo ?? global?.Defaults.UploadChatVideo ?? false;
onlyRawVal = model.OnlyRaw ?? global?.Defaults.OnlyRaw ?? false; onlyRawVal = model.OnlyRaw ?? global?.Defaults.OnlyRaw ?? false;
cleanRawVal = model.CleanRaw ?? global?.Defaults.CleanRaw ?? true; cleanRawVal = model.CleanRaw ?? global?.Defaults.CleanRaw ?? true;
hlsSegmentsVODVal = model.HlsSegmentsVOD ?? global?.Defaults.HlsSegmentsVOD ?? 10;
streamlinkTtvlolVal = model.StreamlinkTtvlol ?? global?.Defaults.StreamlinkTtvlol ?? false;
ffmpegAudioCodecVal = model.FfmpegAudioCodec ?? global?.Defaults.FfmpegAudioCodec ?? "aac";
ffmpegAudioSamplerateVal = model.FfmpegAudioSamplerate ?? global?.Defaults.FfmpegAudioSamplerate ?? 48000;
ffmpegErrorRecoveryVal = model.FfmpegErrorRecovery ?? global?.Defaults.FfmpegErrorRecovery ?? true;
ffmpegFaststartVal = model.FfmpegFaststart ?? global?.Defaults.FfmpegFaststart ?? true;
ffmpegProgressVal = model.FfmpegProgress ?? global?.Defaults.FfmpegProgress ?? false;
uploadToCloudVal = model.UploadToCloud ?? global?.UploadToCloud ?? false; uploadToCloudVal = model.UploadToCloud ?? global?.UploadToCloud ?? false;
// when creating a new streamer config, populate model with global defaults so
// the streamer config stores initial values and subsequent edits use streamer values
if (isNew)
{
model.Quality = model.Quality ?? global?.DefaultQuality;
model.UploadToCloud = uploadToCloudVal;
model.UploadDestination = model.UploadDestination ?? global?.UploadDestination;
model.DownloadVOD = downloadVODVal;
model.DownloadCHAT = downloadCHATVal;
model.DownloadLiveCHAT = downloadLiveCHATVal;
model.MergeVideoChat = mergeVideoChatVal;
model.MergeChatLayout = mergeChatLayoutVal;
model.VodTimeout = vodTimeoutVal;
model.DeleteFiles = deleteFilesVal;
model.UploadPreMergeVideo = uploadPreMergeVideoVal;
model.UploadMergedVideo = uploadMergedVideoVal;
model.UploadChatVideo = uploadChatVideoVal;
model.OnlyRaw = onlyRawVal;
model.CleanRaw = cleanRawVal;
}
} }
private void Save() private void Save()
{ {
model.Username = Username; model.Username = Username;
if (!overrideQuality) model.Quality = null; if (string.IsNullOrWhiteSpace(model.Quality)) model.Quality = null;
// Upload to cloud // Upload to cloud
model.UploadToCloud = overrideUpload ? uploadToCloudVal : (bool?)null; model.UploadToCloud = uploadToCloudVal;
// Streamlink path // Ensure global-only settings are not stored per-streamer
model.StreamlinkPath = overrideStreamlink ? model.StreamlinkPath : null; model.StreamlinkPath = null;
// Per-streamer values: map local values when overridden, otherwise clear model.HlsSegments = null;
model.DownloadVOD = overrideDownloadVOD ? downloadVODVal : (bool?)null; model.HlsSegmentsVOD = null;
model.DownloadCHAT = overrideDownloadCHAT ? downloadCHATVal : (bool?)null; model.StreamlinkTtvlol = null;
model.DownloadLiveCHAT = overrideDownloadLiveCHAT ? downloadLiveCHATVal : (bool?)null; model.FfmpegHwaccel = null;
model.MergeVideoChat = overrideMergeVideoChat ? mergeVideoChatVal : (bool?)null; model.FfmpegThreads = null;
model.MergeChatLayout = overrideMergeChatLayout ? mergeChatLayoutVal : null; model.FfmpegAudioBitrate = null;
model.VodTimeout = overrideVodTimeout ? vodTimeoutVal : (int?)null; model.FfmpegAudioCodec = null;
model.DeleteFiles = overrideDeleteFiles ? deleteFilesVal : (bool?)null; model.FfmpegAudioSamplerate = null;
model.HlsSegments = overrideHlsSegments ? hlsSegmentsVal : (int?)null; model.FfmpegErrorRecovery = null;
model.FfmpegHwaccel = overrideFfmpegHwaccel ? ffmpegHwaccelVal : null; model.FfmpegFaststart = null;
model.FfmpegThreads = overrideFfmpegThreads ? ffmpegThreadsVal : (int?)null; model.FfmpegProgress = null;
model.FfmpegAudioBitrate = overrideFfmpegAudioBitrate ? ffmpegAudioBitrateVal : null; // Per-streamer values: always map local values into the model
model.UploadPreMergeVideo = overrideUploadPreMergeVideo ? uploadPreMergeVideoVal : (bool?)null; model.DownloadVOD = downloadVODVal;
model.UploadMergedVideo = overrideUploadMergedVideo ? uploadMergedVideoVal : (bool?)null; model.DownloadCHAT = downloadCHATVal;
model.UploadChatVideo = overrideUploadChatVideo ? uploadChatVideoVal : (bool?)null; model.DownloadLiveCHAT = downloadLiveCHATVal;
model.OnlyRaw = overrideOnlyRaw ? onlyRawVal : (bool?)null; model.MergeVideoChat = mergeVideoChatVal;
model.CleanRaw = overrideCleanRaw ? cleanRawVal : (bool?)null; model.MergeChatLayout = mergeChatLayoutVal;
model.HlsSegmentsVOD = overrideHlsSegmentsVOD ? hlsSegmentsVODVal : (int?)null; model.VodTimeout = vodTimeoutVal;
model.StreamlinkTtvlol = overrideStreamlinkTtvlol ? streamlinkTtvlolVal : (bool?)null; model.DeleteFiles = deleteFilesVal;
model.FfmpegAudioCodec = overrideFfmpegAudioCodec ? ffmpegAudioCodecVal : null; model.UploadPreMergeVideo = uploadPreMergeVideoVal;
model.FfmpegAudioSamplerate = overrideFfmpegAudioSamplerate ? ffmpegAudioSamplerateVal : (int?)null; model.UploadMergedVideo = uploadMergedVideoVal;
model.FfmpegErrorRecovery = overrideFfmpegErrorRecovery ? ffmpegErrorRecoveryVal : (bool?)null; model.UploadChatVideo = uploadChatVideoVal;
model.FfmpegFaststart = overrideFfmpegFaststart ? ffmpegFaststartVal : (bool?)null; model.OnlyRaw = onlyRawVal;
model.FfmpegProgress = overrideFfmpegProgress ? ffmpegProgressVal : (bool?)null; model.CleanRaw = cleanRawVal;
ConfigService.SaveStreamer(model); ConfigService.SaveStreamer(model);
saved = true; saved = true;
} }

View file

@ -40,9 +40,9 @@ class ContentDownloader:
self.ffmpeg_path = ffmpeg_path self.ffmpeg_path = ffmpeg_path
self.quality = config.get('quality', 'best') self.quality = config.get('quality', 'best')
self.hls_segments_vod = config.get('hls_segmentsVOD', 10) self.hls_segments_vod = config.get('hls_segmentsVOD', 10)
self.download_vod = config.get('downloadVOD', True) self.download_vod_enabled = config.get('downloadVOD', True)
self.download_chat = config.get('downloadCHAT', True) self.download_chat_enabled = config.get('downloadCHAT', True)
self.download_live_chat = config.get('downloadLiveCHAT', True) self.download_live_chat_enabled = config.get('downloadLiveCHAT', True)
self.use_chat_downloader_primary = config.get('useChatDownloaderPrimary', False) self.use_chat_downloader_primary = config.get('useChatDownloaderPrimary', False)
self.use_chat_downloader_fallback = config.get('useChatDownloaderFallback', True) self.use_chat_downloader_fallback = config.get('useChatDownloaderFallback', True)
@ -73,7 +73,7 @@ class ContentDownloader:
Returns: Returns:
bool: True if download succeeded, False otherwise bool: True if download succeeded, False otherwise
""" """
if not self.download_vod: if not self.download_vod_enabled:
return False return False
print(f'\n{Fore.CYAN}Downloading VOD: {vod_info["title"]}{Style.RESET_ALL}') print(f'\n{Fore.CYAN}Downloading VOD: {vod_info["title"]}{Style.RESET_ALL}')
@ -272,7 +272,7 @@ class ContentDownloader:
Returns: Returns:
bool: True if succeeded, False otherwise bool: True if succeeded, False otherwise
""" """
if not self.download_chat: if not self.download_chat_enabled:
return False return False
print(f'\n{Fore.CYAN}Downloading chat: {vod_info["title"]}{Style.RESET_ALL}') print(f'\n{Fore.CYAN}Downloading chat: {vod_info["title"]}{Style.RESET_ALL}')
@ -298,7 +298,7 @@ class ContentDownloader:
Returns: Returns:
subprocess.Popen: The process handle, or None if failed to start subprocess.Popen: The process handle, or None if failed to start
""" """
if not self.download_live_chat: if not self.download_live_chat_enabled:
return None return None
print(f'\n{Fore.CYAN}Starting live chat download...{Style.RESET_ALL}') print(f'\n{Fore.CYAN}Starting live chat download...{Style.RESET_ALL}')
@ -502,7 +502,7 @@ class ContentDownloader:
print(f'{Fore.YELLOW} Install with: pip install chat-downloader{Style.RESET_ALL}') print(f'{Fore.YELLOW} Install with: pip install chat-downloader{Style.RESET_ALL}')
return False return False
if not self.download_live_chat: if not self.download_live_chat_enabled:
print(f'{Fore.YELLOW}⚠ downloadLiveCHAT is disabled in config{Style.RESET_ALL}') print(f'{Fore.YELLOW}⚠ downloadLiveCHAT is disabled in config{Style.RESET_ALL}')
return False return False

View file

@ -46,6 +46,128 @@ class FileManager:
self.metadata_path = self.root_path / username / "metadata" self.metadata_path = self.root_path / username / "metadata"
self.log_file = self.root_path / ".log" self.log_file = self.root_path / ".log"
def _to_rclone_relative_path(self, *parts: str) -> str:
"""Build a POSIX-style relative path for rclone --files-from."""
return pathlib.PurePosixPath(*parts).as_posix()
def _build_upload_relative_paths(self, filename_base: str) -> List[str]:
"""Build the candidate upload list relative to root_path for rclone."""
files_to_upload: List[str] = [
self._to_rclone_relative_path(self.username, 'metadata', f"{PREFIX_METADATA}{filename_base}.json"),
self._to_rclone_relative_path(self.username, 'chat', 'json', f"{PREFIX_CHAT}{filename_base}.json")
]
if self.upload_pre_merge_video:
files_to_upload.extend([
self._to_rclone_relative_path(self.username, 'video', 'raw', f"{PREFIX_LIVE}{filename_base}.ts"),
self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_LIVE}{filename_base}.mp4"),
self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_LIVE}{filename_base}.mp3"),
self._to_rclone_relative_path(self.username, 'video', 'raw', f"{PREFIX_VOD}{filename_base}.ts"),
self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_VOD}{filename_base}.mp4"),
self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_VOD}{filename_base}.mp3")
])
if self.upload_merged_video:
files_to_upload.extend([
self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_MERGED}{filename_base}.mp4"),
self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_MERGED}{filename_base}.mp3"),
self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}.mp4"),
self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}.mp3")
])
if self.upload_chat_video:
files_to_upload.append(self._to_rclone_relative_path(self.username, 'chat', f"{PREFIX_CHAT}{filename_base}.mp4"))
return files_to_upload
def _get_existing_upload_relative_paths(self, relative_paths: List[str]) -> List[str]:
"""Filter candidate upload paths to the files that actually exist."""
existing_paths: List[str] = []
for relative_path in relative_paths:
if (self.root_path / pathlib.PurePosixPath(relative_path)).exists():
existing_paths.append(relative_path)
return existing_paths
def _run_rclone_copy(self, relative_paths: List[str], description: str) -> bool:
"""Run rclone copy for a set of paths relative to root_path."""
existing_paths = self._get_existing_upload_relative_paths(relative_paths)
missing_paths = [path for path in relative_paths if path not in existing_paths]
if not existing_paths:
print(f'{Fore.RED}✗ Upload skipped: no matching files found for {description}{Style.RESET_ALL}')
for missing_path in missing_paths:
print(f'{Fore.YELLOW} Missing: {missing_path}{Style.RESET_ALL}')
return False
if missing_paths:
print(f'{Fore.YELLOW}⚠ Some configured upload files were not found and will be skipped{Style.RESET_ALL}')
for missing_path in missing_paths:
print(f'{Fore.YELLOW} Missing: {missing_path}{Style.RESET_ALL}')
print(f'{Fore.CYAN}rclone source: {self.root_path.resolve()}{Style.RESET_ALL}')
print(f'{Fore.CYAN}rclone destination: {self.rclone_path}{Style.RESET_ALL}')
print(f'{Fore.CYAN}Files queued for upload: {len(existing_paths)}{Style.RESET_ALL}')
bin_path = get_bin_path()
upload_list_path = os.path.join(bin_path, 'temp', 'upload.txt')
os.makedirs(os.path.dirname(upload_list_path), exist_ok=True)
with open(upload_list_path, 'w', encoding='utf-8', newline='\n') as f:
f.write('\n'.join(existing_paths))
f.write('\n')
try:
cmd = [
'rclone', 'copy',
str(self.root_path.resolve()),
self.rclone_path,
'--files-from', upload_list_path,
'--progress'
]
print(f'{Fore.CYAN}Running: {' '.join(cmd)}{Style.RESET_ALL}')
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
if proc.stdout:
for line in proc.stdout:
print(line, end='')
proc.wait()
return proc.returncode == 0
finally:
if os.path.exists(upload_list_path):
os.remove(upload_list_path)
def run_rclone_smoke_test(self) -> bool:
"""Create and upload a tiny metadata file to verify rclone output and configuration."""
smoke_name = 'RCLONE_SMOKE_TEST'
smoke_relative_path = self._to_rclone_relative_path(
self.username,
'metadata',
f"{PREFIX_METADATA}{smoke_name}.json"
)
smoke_file_path = self.root_path / pathlib.PurePosixPath(smoke_relative_path)
smoke_payload = {
'type': 'rclone_smoke_test',
'username': self.username
}
smoke_file_path.parent.mkdir(parents=True, exist_ok=True)
with open(smoke_file_path, 'w', encoding='utf-8') as f:
json.dump(smoke_payload, f, indent=2)
print(f'{Fore.CYAN}Created smoke-test file: {smoke_file_path}{Style.RESET_ALL}')
try:
result = self._run_rclone_copy([smoke_relative_path], 'rclone smoke test')
if result:
print(f'{Fore.GREEN}✓ Rclone smoke test completed{Style.RESET_ALL}')
else:
print(f'{Fore.RED}✗ Rclone smoke test failed{Style.RESET_ALL}')
return result
finally:
if smoke_file_path.exists():
smoke_file_path.unlink()
def initialize_directories(self) -> None: def initialize_directories(self) -> None:
"""Create all necessary directory structures.""" """Create all necessary directory structures."""
for path in [self.raw_path, self.video_path, self.chat_json_path, for path in [self.raw_path, self.video_path, self.chat_json_path,
@ -121,80 +243,23 @@ class FileManager:
if notification_callback: if notification_callback:
notification_callback(f'☁ Uploading - {filename_base}', 'Uploading files to cloud storage') notification_callback(f'☁ Uploading - {filename_base}', 'Uploading files to cloud storage')
# Create list of files to upload files_to_upload = self._build_upload_relative_paths(filename_base)
bin_path = get_bin_path()
upload_list_path = os.path.join(bin_path, 'temp', 'upload.txt')
# Ensure temp directory exists
os.makedirs(os.path.dirname(upload_list_path), exist_ok=True)
files_to_upload = []
# Build files list relative to root_path so rclone can read them with --files-from
# Metadata and chat JSON
files_to_upload.append(os.path.join(self.username, 'metadata', f"{PREFIX_METADATA}{filename_base}.json"))
files_to_upload.append(os.path.join(self.username, 'chat', 'json', f"{PREFIX_CHAT}{filename_base}.json"))
# Pre-merge videos (raw .ts in video/raw, mp4/mp3 in video)
if self.upload_pre_merge_video:
files_to_upload.extend([
os.path.join(self.username, 'video', 'raw', f"{PREFIX_LIVE}{filename_base}.ts"),
os.path.join(self.username, 'video', f"{PREFIX_LIVE}{filename_base}.mp4"),
os.path.join(self.username, 'video', f"{PREFIX_LIVE}{filename_base}.mp3"),
os.path.join(self.username, 'video', 'raw', f"{PREFIX_VOD}{filename_base}.ts"),
os.path.join(self.username, 'video', f"{PREFIX_VOD}{filename_base}.mp4"),
os.path.join(self.username, 'video', f"{PREFIX_VOD}{filename_base}.mp3")
])
# Merged videos (in video folder)
if self.upload_merged_video:
files_to_upload.extend([
os.path.join(self.username, 'video', f"{PREFIX_MERGED}{filename_base}.mp4"),
os.path.join(self.username, 'video', f"{PREFIX_MERGED}{filename_base}.mp3"),
os.path.join(self.username, 'video', f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}.mp4"),
os.path.join(self.username, 'video', f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}.mp3")
])
# Standalone chat video (in chat folder)
if self.upload_chat_video:
files_to_upload.append(os.path.join(self.username, 'chat', f"{PREFIX_CHAT}{filename_base}.mp4"))
with open(upload_list_path, 'w') as f:
f.write('\n'.join(files_to_upload))
# Run rclone using --files-from so the listed paths (relative to root_path) are uploaded.
try: try:
cmd = [ result = self._run_rclone_copy(files_to_upload, f'archive batch {filename_base}')
'rclone', 'copy',
str(self.root_path.resolve()),
self.rclone_path,
'--files-from', upload_list_path
]
# Stream rclone output to console so user can see progress/errors if result:
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
if proc.stdout:
for line in proc.stdout:
print(line, end='')
proc.wait()
result = proc.returncode
# Clean up upload list
if os.path.exists(upload_list_path):
os.remove(upload_list_path)
if result == 0:
print(f'{Fore.GREEN}✓ Upload complete{Style.RESET_ALL}') print(f'{Fore.GREEN}✓ Upload complete{Style.RESET_ALL}')
if notification_callback: if notification_callback:
notification_callback(f'✓ Upload Success - {filename_base}', 'All files uploaded successfully') notification_callback(f'✓ Upload Success - {filename_base}', 'All files uploaded successfully')
return True return True
else:
print(f'{Fore.RED}✗ Upload failed (exit code: {result}){Style.RESET_ALL}') print(f'{Fore.RED}✗ Upload failed{Style.RESET_ALL}')
print(f'{Fore.YELLOW}Files preserved locally due to upload failure{Style.RESET_ALL}') print(f'{Fore.YELLOW}Files preserved locally due to upload failure{Style.RESET_ALL}')
if notification_callback: if notification_callback:
notification_callback(f'✗ Upload Failed - {filename_base}', notification_callback(f'✗ Upload Failed - {filename_base}',
f'Upload failed with code {result}. Files preserved locally.') 'Upload failed. Files preserved locally. Check rclone output above.')
return False return False
except Exception as e: except Exception as e:
print(f'{Fore.RED}✗ Upload error: {str(e)}{Style.RESET_ALL}') print(f'{Fore.RED}✗ Upload error: {str(e)}{Style.RESET_ALL}')

View file

@ -37,37 +37,78 @@ class StreamProcessor:
os_type os_type
) )
def process_raw_stream(self, raw_path: str, output_path: str) -> None: def process_raw_stream(self, raw_path: str, output_path: str) -> bool:
""" """
Process raw .ts file into mp4/mp3 using ffmpeg. Process raw .ts file into mp4/mp3 using ffmpeg.
Args: Args:
raw_path: Path to the raw .ts file raw_path: Path to the raw .ts file
output_path: Path for the processed output file output_path: Path for the processed output file
Returns:
bool: True when conversion succeeded, False otherwise
""" """
if not os.path.exists(raw_path): if not os.path.exists(raw_path):
print(f'{Fore.YELLOW}⚠ Raw file not found, skipping processing{Style.RESET_ALL}') print(f'{Fore.YELLOW}⚠ Raw file not found, skipping processing{Style.RESET_ALL}')
return return False
if self.only_raw: if self.only_raw:
print(f'{Fore.CYAN}Keeping raw .ts file (onlyRaw mode){Style.RESET_ALL}') print(f'{Fore.CYAN}Keeping raw .ts file (onlyRaw mode){Style.RESET_ALL}')
return return False
print(f'{Fore.YELLOW}Processing raw stream file...{Style.RESET_ALL}') print(f'{Fore.YELLOW}Processing raw stream file...{Style.RESET_ALL}')
# Build ffmpeg command based on quality # Build ffmpeg command based on quality
if self.quality == 'audio_only': if self.quality == 'audio_only':
self._process_audio(raw_path, output_path) result = self._process_audio(raw_path, output_path)
else: else:
self._process_video(raw_path, output_path) result = self._process_video(raw_path, output_path)
print(f'{Fore.GREEN}✓ Stream processed successfully{Style.RESET_ALL}') if result:
print(f'{Fore.GREEN}✓ Stream processed successfully{Style.RESET_ALL}')
else:
print(f'{Fore.RED}✗ Stream processing failed{Style.RESET_ALL}')
def _process_audio(self, raw_path: str, output_path: str) -> None: return result
def _run_ffmpeg_command(self, cmd: list, output_path: str) -> bool:
"""Run FFmpeg while streaming its output to the terminal."""
print(f'{Fore.CYAN}Running FFmpeg: {' '.join(cmd)}{Style.RESET_ALL}')
process = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
encoding='utf-8',
errors='replace'
)
if process.stdout:
for line in process.stdout:
print(line, end='')
result = process.wait()
if result != 0:
print(f'{Fore.RED}✗ FFmpeg exited with code: {result}{Style.RESET_ALL}')
return False
if not os.path.exists(output_path):
print(f'{Fore.RED}✗ FFmpeg did not create output: {output_path}{Style.RESET_ALL}')
return False
if os.path.getsize(output_path) == 0:
print(f'{Fore.RED}✗ FFmpeg created an empty output file: {output_path}{Style.RESET_ALL}')
return False
return True
def _process_audio(self, raw_path: str, output_path: str) -> bool:
"""Process audio-only stream.""" """Process audio-only stream."""
# Audio-only conversion with modern AAC encoding # Audio-only conversion with modern AAC encoding
cmd = [ cmd = [
self.ffmpeg_path, self.ffmpeg_path,
'-y',
'-i', raw_path, '-i', raw_path,
'-vn', # No video '-vn', # No video
'-c:a', self.ffmpeg_audio_codec, '-c:a', self.ffmpeg_audio_codec,
@ -85,14 +126,9 @@ class StreamProcessor:
cmd.extend(['-movflags', '+faststart']) cmd.extend(['-movflags', '+faststart'])
cmd.append(output_path) cmd.append(output_path)
return self._run_ffmpeg_command(cmd, output_path)
# Run FFmpeg def _process_video(self, raw_path: str, output_path: str) -> bool:
if self.ffmpeg_progress:
subprocess.call(cmd)
else:
subprocess.call(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
def _process_video(self, raw_path: str, output_path: str) -> None:
"""Process video stream.""" """Process video stream."""
cmd = [ cmd = [
self.ffmpeg_path, self.ffmpeg_path,
@ -135,12 +171,7 @@ class StreamProcessor:
cmd.extend(['-movflags', '+faststart']) cmd.extend(['-movflags', '+faststart'])
cmd.append(output_path) cmd.append(output_path)
return self._run_ffmpeg_command(cmd, output_path)
# Run FFmpeg
if self.ffmpeg_progress:
subprocess.call(cmd)
else:
subprocess.call(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
def build_chat_output_args(self) -> str: def build_chat_output_args(self) -> str:
""" """

View file

@ -7,6 +7,8 @@ import subprocess
from typing import Dict, Any, Optional from typing import Dict, Any, Optional
from colorama import Fore, Style from colorama import Fore, Style
from .utils import get_env_value
class StreamRecorder: class StreamRecorder:
"""Handles live stream recording using streamlink.""" """Handles live stream recording using streamlink."""
@ -68,7 +70,7 @@ class StreamRecorder:
print(f'{Fore.YELLOW} Consider disabling streamlink_ttvlol in config or using alternative methods{Style.RESET_ALL}') print(f'{Fore.YELLOW} Consider disabling streamlink_ttvlol in config or using alternative methods{Style.RESET_ALL}')
# Add authentication if available # Add authentication if available
oauth_token = os.getenv("OAUTH-PRIVATE-TOKEN", "") oauth_token = get_env_value("OAUTH-PRIVATE-TOKEN", "OAUTH_PRIVATE_TOKEN", default="")
if oauth_token and oauth_token != "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx": if oauth_token and oauth_token != "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx":
cmd.extend(['--twitch-api-header', f'Authorization=OAuth {oauth_token}']) cmd.extend(['--twitch-api-header', f'Authorization=OAuth {oauth_token}'])

View file

@ -9,6 +9,7 @@ import requests
from colorama import Fore, Style from colorama import Fore, Style
from .constants import TWITCH_OAUTH_URL, TWITCH_API_URL, TWITCH_GQL_URL, TWITCH_GQL_CLIENT_ID from .constants import TWITCH_OAUTH_URL, TWITCH_API_URL, TWITCH_GQL_URL, TWITCH_GQL_CLIENT_ID
from .utils import get_env_value
class StreamMonitor: class StreamMonitor:
@ -40,7 +41,9 @@ class StreamMonitor:
return self._oauth_token return self._oauth_token
try: try:
url = f"{TWITCH_OAUTH_URL}?client_id={os.getenv('CLIENT-ID')}&client_secret={os.getenv('CLIENT-SECRET')}&grant_type=client_credentials" client_id = get_env_value('CLIENT-ID', 'CLIENT_ID')
client_secret = get_env_value('CLIENT-SECRET', 'CLIENT_SECRET')
url = f"{TWITCH_OAUTH_URL}?client_id={client_id}&client_secret={client_secret}&grant_type=client_credentials"
response = requests.post(url, timeout=15) response = requests.post(url, timeout=15)
response.raise_for_status() response.raise_for_status()
self._oauth_token = response.json()['access_token'] self._oauth_token = response.json()['access_token']
@ -69,7 +72,7 @@ class StreamMonitor:
url = f'{TWITCH_API_URL}/users?login={self.username}' url = f'{TWITCH_API_URL}/users?login={self.username}'
headers = { headers = {
"Authorization": f"Bearer {self.get_oauth_token()}", "Authorization": f"Bearer {self.get_oauth_token()}",
"Client-ID": os.getenv('CLIENT-ID') "Client-ID": get_env_value('CLIENT-ID', 'CLIENT_ID')
} }
response = requests.get(url, headers=headers, timeout=15) response = requests.get(url, headers=headers, timeout=15)
response.raise_for_status() response.raise_for_status()

View file

@ -4,6 +4,7 @@ Utility functions and helpers for Twitch Archive.
import os import os
import sys import sys
import shutil
import pathlib import pathlib
import subprocess import subprocess
from typing import Optional from typing import Optional
@ -35,6 +36,15 @@ def get_bin_path() -> str:
return str(pathlib.Path(__file__).parent.parent.resolve() / "bin") return str(pathlib.Path(__file__).parent.parent.resolve() / "bin")
def get_env_value(*names: str, default: Optional[str] = None) -> Optional[str]:
"""Return the first non-empty environment variable from the provided names."""
for name in names:
value = os.getenv(name)
if value not in (None, ""):
return value
return default
def get_ffmpeg_executable(os_type: str) -> str: def get_ffmpeg_executable(os_type: str) -> str:
""" """
Get the platform-specific ffmpeg executable path. Get the platform-specific ffmpeg executable path.
@ -48,6 +58,11 @@ def get_ffmpeg_executable(os_type: str) -> str:
bin_path = get_bin_path() bin_path = get_bin_path()
if os_type == 'windows': if os_type == 'windows':
return os.path.join(bin_path, 'ffmpeg.exe') return os.path.join(bin_path, 'ffmpeg.exe')
system_ffmpeg = shutil.which('ffmpeg')
if system_ffmpeg:
return system_ffmpeg
return os.path.join(bin_path, 'ffmpeg') return os.path.join(bin_path, 'ffmpeg')
@ -64,6 +79,11 @@ def get_twitch_downloader_executable(os_type: str) -> str:
bin_path = get_bin_path() bin_path = get_bin_path()
if os_type == 'windows': if os_type == 'windows':
return os.path.join(bin_path, 'TwitchDownloaderCLI.exe') return os.path.join(bin_path, 'TwitchDownloaderCLI.exe')
system_twitch_downloader = shutil.which('TwitchDownloaderCLI')
if system_twitch_downloader:
return system_twitch_downloader
return os.path.join(bin_path, 'TwitchDownloaderCLI') return os.path.join(bin_path, 'TwitchDownloaderCLI')
@ -164,6 +184,24 @@ def verify_streamlink() -> bool:
return False return False
def verify_rclone() -> bool:
"""Verify that rclone is available on PATH."""
try:
result = subprocess.run(['rclone', 'version'],
capture_output=True,
text=True,
timeout=5)
if result.returncode == 0:
version_line = result.stdout.strip().splitlines()[0] if result.stdout.strip() else 'unknown'
print(f'{Fore.GREEN}✓ Rclone found ({version_line}){Style.RESET_ALL}')
return True
raise FileNotFoundError()
except (FileNotFoundError, subprocess.TimeoutExpired, IndexError):
print(f'{Fore.RED}✗ ERROR: rclone not found{Style.RESET_ALL}')
print(f'{Fore.CYAN} → Install rclone and ensure it is on PATH{Style.RESET_ALL}')
return False
def verify_ffmpeg(os_type: str) -> bool: def verify_ffmpeg(os_type: str) -> bool:
""" """
Verify that ffmpeg is available. Verify that ffmpeg is available.

View file

@ -20,12 +20,27 @@ import sys
import os import os
import json import json
import getopt import getopt
import tempfile
import importlib.util
from pathlib import Path
from unittest.mock import patch, MagicMock, Mock from unittest.mock import patch, MagicMock, Mock
# Add parent directory to path for imports # Add parent directory to path for imports
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from modules.constants import DEFAULT_CONFIG from modules.constants import DEFAULT_CONFIG
from modules.file_manager import FileManager
from modules.downloader import ContentDownloader
from modules.utils import get_ffmpeg_executable, get_twitch_downloader_executable
def load_twitch_archive_module():
"""Load the main script module for targeted regression tests."""
module_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'twitch-archive.py')
spec = importlib.util.spec_from_file_location('twitch_archive_main', module_path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
class TestCommandLineArgumentParsing(unittest.TestCase): class TestCommandLineArgumentParsing(unittest.TestCase):
@ -116,6 +131,34 @@ class TestCommandLineArgumentParsing(unittest.TestCase):
self.assertEqual(len(opts), 1) self.assertEqual(len(opts), 1)
self.assertEqual(opts[0], ('--chat-only', '')) self.assertEqual(opts[0], ('--chat-only', ''))
def test_rclone_smoke_test_option(self):
"""Test --rclone-smoke-test option parsing."""
argv = ['--rclone-smoke-test']
opts, args = getopt.getopt(
argv,
"hu:q:a:v:c:m:r:d:n:",
["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=",
"metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose",
"chat-only", "healthcheck", "rclone-smoke-test", "use-chat-downloader-primary", "no-chat-downloader-fallback"]
)
self.assertEqual(len(opts), 1)
self.assertEqual(opts[0], ('--rclone-smoke-test', ''))
def test_healthcheck_option(self):
"""Test --healthcheck option parsing."""
argv = ['--healthcheck']
opts, args = getopt.getopt(
argv,
"hu:q:a:v:c:m:r:d:n:",
["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=",
"metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose",
"chat-only", "healthcheck", "rclone-smoke-test", "use-chat-downloader-primary", "no-chat-downloader-fallback"]
)
self.assertEqual(len(opts), 1)
self.assertEqual(opts[0], ('--healthcheck', ''))
def test_legacy_option(self): def test_legacy_option(self):
"""Test --legacy option parsing.""" """Test --legacy option parsing."""
argv = ['--legacy'] argv = ['--legacy']
@ -439,6 +482,161 @@ class TestConfigLogic(unittest.TestCase):
self.assertIn('$schema', default_config) self.assertIn('$schema', default_config)
class TestFileManagerUploadPaths(unittest.TestCase):
"""Test rclone upload path preparation."""
def test_build_upload_relative_paths_uses_forward_slashes(self):
"""Rclone --files-from entries must use POSIX separators on Windows."""
with tempfile.TemporaryDirectory() as temp_dir:
manager = FileManager(
root_path=temp_dir,
username='testuser',
config={
'uploadCloud': True,
'uploadPreMergeVideo': True,
'uploadMergedVideo': True,
'uploadChatVideo': True
}
)
relative_paths = manager._build_upload_relative_paths('20260424_12h00m00s')
self.assertTrue(relative_paths)
self.assertTrue(all('\\' not in path for path in relative_paths))
self.assertIn('testuser/metadata/METADA_20260424_12h00m00s.json', relative_paths)
self.assertIn('testuser/chat/json/CHAT_20260424_12h00m00s.json', relative_paths)
class TestDownloaderConfiguration(unittest.TestCase):
"""Regression tests for downloader config wiring."""
def test_download_vod_method_not_shadowed_by_boolean_flag(self):
"""Config booleans must not overwrite callable downloader methods."""
downloader = ContentDownloader(
twitch_downloader_path='TwitchDownloaderCLI',
ffmpeg_path='ffmpeg',
config={
'downloadVOD': True,
'downloadCHAT': True,
'downloadLiveCHAT': True
}
)
self.assertTrue(callable(downloader.download_vod))
self.assertTrue(downloader.download_vod_enabled)
class TestLinuxToolResolution(unittest.TestCase):
"""Ensure Linux containers prefer their own installed toolchain."""
@patch('modules.utils.shutil.which')
def test_linux_prefers_system_ffmpeg(self, mock_which):
mock_which.return_value = '/usr/bin/ffmpeg'
self.assertEqual(get_ffmpeg_executable('linux'), '/usr/bin/ffmpeg')
@patch('modules.utils.shutil.which')
def test_linux_prefers_system_twitch_downloader(self, mock_which):
mock_which.return_value = '/usr/local/bin/TwitchDownloaderCLI'
self.assertEqual(get_twitch_downloader_executable('linux'), '/usr/local/bin/TwitchDownloaderCLI')
class TestMultiStreamerCleanupRegression(unittest.TestCase):
"""Regression tests for multi-streamer conversion and cleanup behavior."""
def setUp(self):
self.module = load_twitch_archive_module()
def _build_archiver(self, temp_dir: str, upload_cloud: bool = True):
archiver = MagicMock()
archiver.username = 'maddoscientist0'
archiver.os_type = 'linux'
archiver.quality = 'best'
archiver.downloadLiveCHAT = False
archiver.downloadCHAT = False
archiver.downloadVOD = False
archiver.downloadMETADATA = False
archiver.mergeVideoChat = False
archiver.mergeChatLayout = 'side-by-side'
archiver.onlyRaw = False
archiver.vodTimeout = 0
archiver.shutdown_requested = False
archiver.deleteFiles = True
archiver.uploadCloud = upload_cloud
archiver.notification_manager = MagicMock()
archiver.recorder = MagicMock()
archiver.processor = MagicMock()
archiver.downloader = MagicMock()
archiver.stream_monitor = MagicMock()
archiver.file_manager = MagicMock()
archiver.file_manager.raw_path = Path(temp_dir) / 'raw'
archiver.file_manager.video_path = Path(temp_dir) / 'video'
archiver.file_manager.chat_json_path = Path(temp_dir) / 'chat_json'
archiver.file_manager.chat_mp4_path = Path(temp_dir) / 'chat'
os.makedirs(archiver.file_manager.raw_path, exist_ok=True)
os.makedirs(archiver.file_manager.video_path, exist_ok=True)
os.makedirs(archiver.file_manager.chat_json_path, exist_ok=True)
os.makedirs(archiver.file_manager.chat_mp4_path, exist_ok=True)
return archiver
def test_process_stream_keeps_raw_when_conversion_fails(self):
manager = self.module.TwitchArchiveManager(specific_streamer='maddoscientist0')
with tempfile.TemporaryDirectory() as temp_dir:
archiver = self._build_archiver(temp_dir)
archiver.processor.process_raw_stream.return_value = False
archiver.file_manager.upload_to_cloud.return_value = True
def write_raw_file(_stream_info, raw_path):
with open(raw_path, 'wb') as handle:
handle.write(b'x' * 4096)
return True
archiver.recorder.record.side_effect = write_raw_file
stream_info = {
'title': 'Test',
'createdAt': '2026-04-25T09:14:01Z'
}
manager._process_stream(archiver, stream_info, 'stream-id')
archiver.file_manager.clean_raw_file.assert_not_called()
archiver.file_manager.delete_local_files.assert_called_once()
def test_process_stream_only_deletes_rendered_files_after_real_upload(self):
manager = self.module.TwitchArchiveManager(specific_streamer='maddoscientist0')
with tempfile.TemporaryDirectory() as temp_dir:
archiver = self._build_archiver(temp_dir, upload_cloud=False)
archiver.processor.process_raw_stream.return_value = True
archiver.file_manager.upload_to_cloud.return_value = True
def write_raw_file(_stream_info, raw_path):
with open(raw_path, 'wb') as handle:
handle.write(b'x' * 4096)
return True
archiver.recorder.record.side_effect = write_raw_file
stream_info = {
'title': 'Test',
'createdAt': '2026-04-25T09:14:01Z'
}
manager._process_stream(archiver, stream_info, 'stream-id')
archiver.file_manager.clean_raw_file.assert_called_once()
archiver.file_manager.delete_local_files.assert_not_called()
upload_filename_base = archiver.file_manager.upload_to_cloud.call_args.args[0]
self.assertFalse(upload_filename_base.startswith('LIVE_'))
if __name__ == '__main__': if __name__ == '__main__':
# Run tests with verbose output # Run tests with verbose output
print("="*70) print("="*70)

View file

@ -34,6 +34,8 @@ import time
import json import json
import signal import signal
import getopt import getopt
import pathlib
import subprocess
from typing import Dict, Optional, Any from typing import Dict, Optional, Any
from datetime import datetime, timedelta from datetime import datetime, timedelta
@ -48,7 +50,8 @@ from modules.config import ConfigManager
from modules.notifications import NotificationManager from modules.notifications import NotificationManager
from modules.utils import ( from modules.utils import (
detect_operating_system, get_ffmpeg_executable, get_twitch_downloader_executable, detect_operating_system, get_ffmpeg_executable, get_twitch_downloader_executable,
get_unique_filename, get_video_duration, verify_streamlink, verify_ffmpeg, verify_twitch_downloader get_unique_filename, get_video_duration, verify_streamlink, verify_ffmpeg, verify_twitch_downloader,
verify_rclone, get_env_value
) )
from modules.stream_monitor import StreamMonitor from modules.stream_monitor import StreamMonitor
from modules.recorder import StreamRecorder from modules.recorder import StreamRecorder
@ -147,10 +150,25 @@ class TwitchArchive:
Raises: Raises:
SystemExit: If .env file is not found SystemExit: If .env file is not found
""" """
if not load_dotenv(find_dotenv()): dotenv_loaded = load_dotenv(find_dotenv())
has_required_env = bool(
get_env_value('CLIENT-ID', 'CLIENT_ID') and
get_env_value('CLIENT-SECRET', 'CLIENT_SECRET')
)
if not dotenv_loaded and has_required_env:
print(f'{Fore.GREEN}✓ Twitch API credentials loaded from process environment{Style.RESET_ALL}')
return
if not dotenv_loaded and not has_required_env:
print(f'{Fore.RED}✗ ERROR: .env file not found{Style.RESET_ALL}') print(f'{Fore.RED}✗ ERROR: .env file not found{Style.RESET_ALL}')
print(f'{Fore.CYAN} → Create a .env file with your Twitch API credentials{Style.RESET_ALL}') print(f'{Fore.CYAN} → Create a .env file with your Twitch API credentials or pass them via environment variables{Style.RESET_ALL}')
print(f'{Fore.CYAN} → Required: CLIENT-ID, CLIENT-SECRET{Style.RESET_ALL}') print(f'{Fore.CYAN} → Required: CLIENT-ID/CLIENT_ID and CLIENT-SECRET/CLIENT_SECRET{Style.RESET_ALL}')
sys.exit(1)
if not has_required_env:
print(f'{Fore.RED}✗ ERROR: Twitch API credentials are missing{Style.RESET_ALL}')
print(f'{Fore.CYAN} → Required: CLIENT-ID/CLIENT_ID and CLIENT-SECRET/CLIENT_SECRET{Style.RESET_ALL}')
sys.exit(1) sys.exit(1)
def _initialize_components(self) -> None: def _initialize_components(self) -> None:
@ -259,6 +277,8 @@ class TwitchArchive:
verify_ffmpeg(self.os_type) verify_ffmpeg(self.os_type)
if self.downloadVOD or self.downloadCHAT: if self.downloadVOD or self.downloadCHAT:
verify_twitch_downloader(self.os_type) verify_twitch_downloader(self.os_type)
if self.uploadCloud and not verify_rclone():
sys.exit(1)
# Print configuration summary # Print configuration summary
self._print_configuration_summary() self._print_configuration_summary()
@ -404,7 +424,7 @@ class TwitchArchive:
print(f'{Fore.YELLOW}Attempting to process any recorded content...{Style.RESET_ALL}') print(f'{Fore.YELLOW}Attempting to process any recorded content...{Style.RESET_ALL}')
# Process the raw stream file # Process the raw stream file
self.processor.process_raw_stream(live_raw_path, live_proc_path) processing_succeeded = self.processor.process_raw_stream(live_raw_path, live_proc_path)
# Wait for live chat download if it was started # Wait for live chat download if it was started
live_chat_downloaded = False live_chat_downloaded = False
@ -426,7 +446,11 @@ class TwitchArchive:
else: else:
# Get video duration first (needed for chat conversion and trimming) # Get video duration first (needed for chat conversion and trimming)
ffmpeg_path = get_ffmpeg_executable(self.os_type) ffmpeg_path = get_ffmpeg_executable(self.os_type)
video_duration = get_video_duration(live_proc_path, ffmpeg_path) if not processing_succeeded or not os.path.exists(live_proc_path):
print(f'{Fore.YELLOW}⚠ Processed video file is unavailable, skipping chat render{Style.RESET_ALL}')
video_duration = None
else:
video_duration = get_video_duration(live_proc_path, ffmpeg_path)
print(f'{Fore.CYAN}Video duration for chat rendering: {video_duration}s{Style.RESET_ALL}') print(f'{Fore.CYAN}Video duration for chat rendering: {video_duration}s{Style.RESET_ALL}')
# Convert chat format if needed (chat_downloader uses different JSON structure) # Convert chat format if needed (chat_downloader uses different JSON structure)
@ -561,7 +585,10 @@ class TwitchArchive:
print(f'{Fore.YELLOW}⚠ No matching VOD found for this stream{Style.RESET_ALL}') print(f'{Fore.YELLOW}⚠ No matching VOD found for this stream{Style.RESET_ALL}')
# Clean up raw files if configured # Clean up raw files if configured
self.file_manager.clean_raw_file(live_raw_path) if processing_succeeded:
self.file_manager.clean_raw_file(live_raw_path)
elif os.path.exists(live_raw_path):
print(f'{Fore.YELLOW}⚠ Keeping raw file because conversion did not complete successfully{Style.RESET_ALL}')
# Upload to cloud if configured # Upload to cloud if configured
upload_success = self.file_manager.upload_to_cloud( upload_success = self.file_manager.upload_to_cloud(
@ -570,7 +597,7 @@ class TwitchArchive:
) )
# Delete local files if configured and upload succeeded # Delete local files if configured and upload succeeded
if self.deleteFiles and upload_success: if self.deleteFiles and self.uploadCloud and upload_success:
self.file_manager.delete_local_files( self.file_manager.delete_local_files(
filename_base, filename_base,
live_raw_path, live_raw_path,
@ -890,6 +917,8 @@ class TwitchArchiveManager:
verify_ffmpeg(first_archiver.os_type) verify_ffmpeg(first_archiver.os_type)
if first_archiver.downloadVOD or first_archiver.downloadCHAT: if first_archiver.downloadVOD or first_archiver.downloadCHAT:
verify_twitch_downloader(first_archiver.os_type) verify_twitch_downloader(first_archiver.os_type)
if any(archiver.uploadCloud for archiver in self.archivers.values()) and not verify_rclone():
sys.exit(1)
# Print configuration summary for each streamer # Print configuration summary for each streamer
for username, archiver in self.archivers.items(): for username, archiver in self.archivers.items():
@ -1018,7 +1047,7 @@ class TwitchArchiveManager:
# Generate timestamp and filename # Generate timestamp and filename
timestamp = datetime.now(timezone('UTC')).strftime("%Y%m%d_%Hh%Mm%Ss") timestamp = datetime.now(timezone('UTC')).strftime("%Y%m%d_%Hh%Mm%Ss")
filename_base = f"{PREFIX_LIVE}{archiver.username}_{timestamp}" filename_base = f"{archiver.username}_{timestamp}"
# Parse stream start time # Parse stream start time
live_date = datetime.strptime( live_date = datetime.strptime(
@ -1029,8 +1058,8 @@ class TwitchArchiveManager:
raw_extension = '.ts' raw_extension = '.ts'
proc_extension = '.mp3' if archiver.quality == 'audio_only' else '.mp4' proc_extension = '.mp3' if archiver.quality == 'audio_only' else '.mp4'
live_raw_path = str(archiver.file_manager.raw_path / f"{filename_base}{raw_extension}") live_raw_path = str(archiver.file_manager.raw_path / f"{PREFIX_LIVE}{filename_base}{raw_extension}")
live_proc_path = str(archiver.file_manager.video_path / f"{filename_base}{proc_extension}") live_proc_path = str(archiver.file_manager.video_path / f"{PREFIX_LIVE}{filename_base}{proc_extension}")
chat_json_path = str(archiver.file_manager.chat_json_path / f"{PREFIX_CHAT}{filename_base}.json") chat_json_path = str(archiver.file_manager.chat_json_path / f"{PREFIX_CHAT}{filename_base}.json")
# Send notification # Send notification
@ -1172,8 +1201,9 @@ class TwitchArchiveManager:
print(f'{Fore.CYAN}Processing recorded content ({file_size / (1024*1024):.2f} MB)...{Style.RESET_ALL}') print(f'{Fore.CYAN}Processing recorded content ({file_size / (1024*1024):.2f} MB)...{Style.RESET_ALL}')
# Process raw stream # Process raw stream
processing_succeeded = False
if not archiver.onlyRaw: if not archiver.onlyRaw:
archiver.processor.process_raw_stream(live_raw_path, live_proc_path) processing_succeeded = archiver.processor.process_raw_stream(live_raw_path, live_proc_path)
# Wait for live chat download if it was started # Wait for live chat download if it was started
live_chat_downloaded = False live_chat_downloaded = False
@ -1212,8 +1242,12 @@ class TwitchArchiveManager:
chat_rendered_successfully = False chat_rendered_successfully = False
else: else:
# Get video duration first # Get video duration first
ffmpeg_path = get_ffmpeg_executable(archiver.os_type) if not processing_succeeded or not os.path.exists(live_proc_path):
video_duration = get_video_duration(live_proc_path, ffmpeg_path) print(f'{Fore.YELLOW}⚠ Processed video file is unavailable, skipping chat render{Style.RESET_ALL}')
video_duration = None
else:
ffmpeg_path = get_ffmpeg_executable(archiver.os_type)
video_duration = get_video_duration(live_proc_path, ffmpeg_path)
if video_duration is None: if video_duration is None:
print(f'{Fore.YELLOW}⚠ Could not detect video duration from {live_proc_path}{Style.RESET_ALL}') print(f'{Fore.YELLOW}⚠ Could not detect video duration from {live_proc_path}{Style.RESET_ALL}')
@ -1362,7 +1396,10 @@ class TwitchArchiveManager:
archiver.file_manager.save_metadata(stream_info, filename_base) archiver.file_manager.save_metadata(stream_info, filename_base)
# Clean up raw file if configured # Clean up raw file if configured
archiver.file_manager.clean_raw_file(live_raw_path) if processing_succeeded:
archiver.file_manager.clean_raw_file(live_raw_path)
elif os.path.exists(live_raw_path):
print(f'{Fore.YELLOW}⚠ Keeping raw file because conversion did not complete successfully{Style.RESET_ALL}')
# Upload to cloud if configured # Upload to cloud if configured
upload_success = archiver.file_manager.upload_to_cloud( upload_success = archiver.file_manager.upload_to_cloud(
@ -1371,7 +1408,7 @@ class TwitchArchiveManager:
) )
# Delete files if configured # Delete files if configured
if archiver.deleteFiles and upload_success: if archiver.deleteFiles and archiver.uploadCloud and upload_success:
archiver.file_manager.delete_local_files( archiver.file_manager.delete_local_files(
filename_base, filename_base,
live_raw_path, live_raw_path,
@ -1386,6 +1423,79 @@ class TwitchArchiveManager:
) )
def run_rclone_smoke_test(specific_streamer: Optional[str] = None) -> int:
"""Run a one-off rclone smoke test using the configured upload destination."""
config_manager = ConfigManager()
if specific_streamer:
username = specific_streamer
else:
enabled_streamers = config_manager.get_all_enabled_streamers()
if not enabled_streamers:
print(f'{Fore.RED}✗ No enabled streamers available for smoke test{Style.RESET_ALL}')
print(f'{Fore.CYAN}→ Use -u <username> or enable a streamer config{Style.RESET_ALL}')
return 1
username = enabled_streamers[0]
config = config_manager.load_streamer_config(username)
file_manager = FileManager(
root_path=config.get('root_path', 'archive'),
username=username,
config=config
)
file_manager.initialize_directories()
print(f'\n{Fore.CYAN}{"=" * 70}{Style.RESET_ALL}')
print(f'{Fore.CYAN}TWITCH ARCHIVE - Rclone Smoke Test{Style.RESET_ALL}')
print(f'{Fore.CYAN}{"=" * 70}{Style.RESET_ALL}')
print(f'{Fore.GREEN}Streamer: {username}{Style.RESET_ALL}')
print(f'{Fore.GREEN}Remote: {config.get("rclone_path", "<not configured>")}{Style.RESET_ALL}\n')
return 0 if file_manager.run_rclone_smoke_test() else 1
def run_healthcheck(specific_streamer: Optional[str] = None) -> int:
"""Run a local readiness check suitable for Docker health checks."""
config_manager = ConfigManager()
if specific_streamer:
username = specific_streamer
else:
enabled_streamers = config_manager.get_all_enabled_streamers()
username = enabled_streamers[0] if enabled_streamers else 'vinesauce'
config = config_manager.load_streamer_config(username)
archive = TwitchArchive(config)
try:
archive._load_environment_variables()
except SystemExit:
return 1
archive._initialize_components()
checks_ok = True
if not verify_streamlink():
checks_ok = False
if not verify_ffmpeg(archive.os_type):
checks_ok = False
if (archive.downloadVOD or archive.downloadCHAT) and not verify_twitch_downloader(archive.os_type):
checks_ok = False
if archive.uploadCloud:
if not verify_rclone():
checks_ok = False
rclone_config_path = os.getenv('RCLONE_CONFIG')
if rclone_config_path and not os.path.exists(rclone_config_path):
print(f'{Fore.RED}✗ ERROR: RCLONE_CONFIG points to a missing file: {rclone_config_path}{Style.RESET_ALL}')
checks_ok = False
if not checks_ok:
return 1
print(f'{Fore.GREEN}✓ Healthcheck OK for {username}{Style.RESET_ALL}')
return 0
# ============================================================================ # ============================================================================
# COMMAND-LINE INTERFACE # COMMAND-LINE INTERFACE
# ============================================================================ # ============================================================================
@ -1401,6 +1511,8 @@ def main(argv: list) -> None:
""" """
specific_streamer = None specific_streamer = None
use_legacy_mode = False use_legacy_mode = False
rclone_smoke_test_mode = False
healthcheck_mode = False
help_msg = f''' help_msg = f'''
{Fore.CYAN}{"=" * 70} {Fore.CYAN}{"=" * 70}
@ -1427,6 +1539,8 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving
--legacy Force legacy mode (use config.json) --legacy Force legacy mode (use config.json)
--chat-only Test mode: Only download chat (skip video recording) --chat-only Test mode: Only download chat (skip video recording)
Automatically enables verbose logging Automatically enables verbose logging
--healthcheck Validate config and tool availability, then exit
--rclone-smoke-test Create a small test file and upload it with rclone
--use-chat-downloader-primary Use chat_downloader as primary chat source (for testing) --use-chat-downloader-primary Use chat_downloader as primary chat source (for testing)
--no-chat-downloader-fallback Disable chat_downloader fallback --no-chat-downloader-fallback Disable chat_downloader fallback
@ -1464,7 +1578,7 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving
"h:u:q:a:v:c:m:r:d:n:", "h:u:q:a:v:c:m:r:d:n:",
["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=",
"metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose",
"chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] "chat-only", "healthcheck", "rclone-smoke-test", "use-chat-downloader-primary", "no-chat-downloader-fallback"]
) )
except getopt.GetoptError as e: except getopt.GetoptError as e:
print(f'{Fore.RED}Error: {e}{Style.RESET_ALL}\n') print(f'{Fore.RED}Error: {e}{Style.RESET_ALL}\n')
@ -1491,6 +1605,10 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving
elif opt == "--chat-only": elif opt == "--chat-only":
chat_only_mode = True chat_only_mode = True
verbose_mode = True # Auto-enable verbose for chat-only mode verbose_mode = True # Auto-enable verbose for chat-only mode
elif opt == "--healthcheck":
healthcheck_mode = True
elif opt == "--rclone-smoke-test":
rclone_smoke_test_mode = True
elif opt == "--legacy": elif opt == "--legacy":
use_legacy_mode = True use_legacy_mode = True
elif opt == "--use-chat-downloader-primary": elif opt == "--use-chat-downloader-primary":
@ -1524,6 +1642,12 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving
elif opt in ("-n", "--notifications"): elif opt in ("-n", "--notifications"):
legacy_overrides['notifications'] = bool(int(arg)) legacy_overrides['notifications'] = bool(int(arg))
if rclone_smoke_test_mode:
sys.exit(run_rclone_smoke_test(specific_streamer))
if healthcheck_mode:
sys.exit(run_healthcheck(specific_streamer))
# Determine which mode to use # Determine which mode to use
if use_legacy_mode or (legacy_config_exists and not specific_streamer and not os.path.exists('config/global.json')): if use_legacy_mode or (legacy_config_exists and not specific_streamer and not os.path.exists('config/global.json')):
# Legacy mode: single streamer using config.json # Legacy mode: single streamer using config.json