Refactor downloader and file manager for improved rclone integration and add healthcheck and smoke test options

- Renamed download flags in ContentDownloader for clarity.
- Enhanced FileManager with methods to build upload paths and verify existing files for rclone uploads.
- Updated StreamProcessor to return success status for stream processing.
- Added rclone smoke test and healthcheck functions to validate configuration and tool availability.
- Improved environment variable handling with a utility function.
- Updated TwitchArchive to incorporate new rclone verification and processing logic.
- Added unit tests for new functionality and refactored existing tests for clarity and coverage.

Co-authored-by: Copilot <copilot@github.com>
This commit is contained in:
MaddoScientisto 2026-04-25 11:54:03 +02:00
commit f97e0200d6
23 changed files with 1013 additions and 289 deletions

18
.dockerignore Normal file
View file

@ -0,0 +1,18 @@
.git
.github
.forgejo
.vs
.vscode
venv*/
__pycache__/
*.pyc
archive/
dotnet/
tests/
.pytest_cache/
bin/temp/
bin/ffmpeg
bin/ffmpeg.exe
bin/ffprobe
bin/TwitchDownloaderCLI
bin/TwitchDownloaderCLI.exe

16
.env.development Normal file
View file

@ -0,0 +1,16 @@
TWITCH_ARCHIVE_DEV_IMAGE=twitch-archive-local
TWITCH_ARCHIVE_CONTAINER_NAME=twitch-archive-dev
TWITCH_ARCHIVE_APP_ENV_FILE=./.env.development
TWITCH_ARCHIVE_ARCHIVE_BIND=./archive
TWITCH_ARCHIVE_CONFIG_BIND=./config
TWITCH_ARCHIVE_ARGS=-u vinesauce --verbose
TWITCH_ARCHIVE_HEALTHCHECK_STREAMER=vinesauce
TWITCH_ARCHIVE_RCLONE_CONFIG=/app/config/rclone.conf
PYTHONUNBUFFERED=1
TZ=UTC
CLIENT-ID=vdyevjvllziylzwsm3y925p79pwtua
CLIENT-SECRET=y906xadsmf22q54suuzsmfnfav3jc7
OAUTH-PRIVATE-TOKEN=ll4kvlmxuajfgi9lgi5d8mkeglsyvm
SENDER=
RECEIVER=
PASSWD=

16
.env.production Normal file
View file

@ -0,0 +1,16 @@
TWITCH_ARCHIVE_IMAGE=forgejo.maddoscientisto.net/maddo/twitch-archive:latest
TWITCH_ARCHIVE_CONTAINER_NAME=twitch-archive
TWITCH_ARCHIVE_APP_ENV_FILE=./.env.production
TWITCH_ARCHIVE_ARCHIVE_BIND=./archive
TWITCH_ARCHIVE_CONFIG_BIND=./config
TWITCH_ARCHIVE_ARGS=-u vinesauce
TWITCH_ARCHIVE_HEALTHCHECK_STREAMER=vinesauce
TWITCH_ARCHIVE_RCLONE_CONFIG=/app/config/rclone.conf
PYTHONUNBUFFERED=1
TZ=UTC
CLIENT-ID=vdyevjvllziylzwsm3y925p79pwtua
CLIENT-SECRET=y906xadsmf22q54suuzsmfnfav3jc7
OAUTH-PRIVATE-TOKEN=ll4kvlmxuajfgi9lgi5d8mkeglsyvm
SENDER=
RECEIVER=
PASSWD=

View file

@ -0,0 +1,127 @@
name: Publish Twitch Archive Container
on:
push:
branches:
- master
- main
paths:
- docker/python.Dockerfile
- docker/entrypoint.sh
- docker-compose.yml
- docker-compose.override.yml
- requirements.txt
- twitch-archive.py
- run_chat_only.py
- modules/**
- .forgejo/workflows/publish-python-container.yml
workflow_dispatch:
env:
REGISTRY: ${{ vars.FORGEJO_REGISTRY }}
IMAGE_NAMESPACE: ${{ vars.IMAGE_NAMESPACE }}
IMAGE_NAME: ${{ vars.IMAGE_NAME != '' && vars.IMAGE_NAME || 'twitch-archive' }}
BUILD_CONTEXT: .
DOCKERFILE_PATH: docker/python.Dockerfile
jobs:
publish:
runs-on: docker
env:
DOCKER_HOST: ${{ vars.DOCKER_HOST != '' && vars.DOCKER_HOST || 'tcp://172.17.0.1:2375' }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Validate workflow variables
run: |
set -eu
if [ -z "${REGISTRY}" ]; then echo "vars.FORGEJO_REGISTRY is required"; exit 1; fi
if [ -z "${IMAGE_NAMESPACE}" ]; then echo "vars.IMAGE_NAMESPACE is required"; exit 1; fi
if [ ! -f "${DOCKERFILE_PATH}" ]; then echo "Dockerfile not found at ${DOCKERFILE_PATH}"; exit 1; fi
if [ ! -f "requirements.txt" ]; then echo "requirements.txt is missing"; exit 1; fi
- name: Ensure Docker CLI exists
run: |
set -eu
if command -v docker >/dev/null 2>&1; then
docker --version
exit 0
fi
ARCH="$(uname -m)"
case "${ARCH}" in
x86_64) DOCKER_ARCH="x86_64" ;;
aarch64|arm64) DOCKER_ARCH="aarch64" ;;
*) echo "Unsupported architecture for Docker CLI bootstrap: ${ARCH}"; exit 1 ;;
esac
DOCKER_CLI_VERSION="27.5.1"
curl -fsSL "https://download.docker.com/linux/static/stable/${DOCKER_ARCH}/docker-${DOCKER_CLI_VERSION}.tgz" -o docker.tgz
tar -xzf docker.tgz
mkdir -p "${HOME}/.local/bin"
mv docker/docker "${HOME}/.local/bin/docker"
chmod +x "${HOME}/.local/bin/docker"
echo "${HOME}/.local/bin" >> "${FORGEJO_PATH}"
"${HOME}/.local/bin/docker" --version
- name: Ensure Docker Buildx exists
run: |
set -eu
if docker buildx version >/dev/null 2>&1; then
docker buildx version
exit 0
fi
ARCH="$(uname -m)"
case "${ARCH}" in
x86_64) BUILDX_ARCH="amd64" ;;
aarch64|arm64) BUILDX_ARCH="arm64" ;;
*) echo "Unsupported architecture for Docker Buildx bootstrap: ${ARCH}"; exit 1 ;;
esac
BUILDX_VERSION="v0.21.1"
mkdir -p "${HOME}/.docker/cli-plugins"
curl -fsSL "https://github.com/docker/buildx/releases/download/${BUILDX_VERSION}/buildx-${BUILDX_VERSION}.linux-${BUILDX_ARCH}" -o "${HOME}/.docker/cli-plugins/docker-buildx"
chmod +x "${HOME}/.docker/cli-plugins/docker-buildx"
docker buildx version
- name: Check Docker daemon connectivity
run: |
set -eu
echo "Using DOCKER_HOST=${DOCKER_HOST}"
docker version
docker info >/dev/null
- name: Create Buildx builder
run: |
set -eu
docker buildx rm forgejo-builder >/dev/null 2>&1 || true
docker buildx create --name forgejo-builder --driver docker-container --use
docker buildx inspect --bootstrap
- name: Validate registry secrets
run: |
set -eu
if [ -z "${{ secrets.FORGEJO_REGISTRY_USERNAME }}" ]; then echo "secrets.FORGEJO_REGISTRY_USERNAME is required"; exit 1; fi
if [ -z "${{ secrets.FORGEJO_REGISTRY_TOKEN }}" ]; then echo "secrets.FORGEJO_REGISTRY_TOKEN is required"; exit 1; fi
- name: Login to registry
run: |
set -eu
echo "${{ secrets.FORGEJO_REGISTRY_TOKEN }}" | docker login "${REGISTRY}" -u "${{ secrets.FORGEJO_REGISTRY_USERNAME }}" --password-stdin
- name: Build and push image
run: |
set -eu
IMAGE_REF="${REGISTRY}/${IMAGE_NAMESPACE}/${IMAGE_NAME}"
SHA_TAG="${IMAGE_REF}:sha-${FORGEJO_SHA}"
BRANCH_TAG="${IMAGE_REF}:${FORGEJO_REF_NAME}"
docker buildx build \
--file "${DOCKERFILE_PATH}" \
--tag "${SHA_TAG}" \
--tag "${BRANCH_TAG}" \
--tag "${IMAGE_REF}:latest" \
--push \
"${BUILD_CONTEXT}"

2
.gitignore vendored
View file

@ -4,6 +4,8 @@ config/global.json
# Streamer-specific configurations (personal settings)
config/streamers/*.json
config/rclone.conf
config/rclone.conf.*
# Python cache
__pycache__/

View file

@ -26,6 +26,73 @@ Notes:
Python script to check, download live stream, VOD, chat and upload them to any cloud storage supported by rclone.
## Docker
This repository now includes a Python-only container setup for the archiver. The dotnet subapp is not part of this container flow.
Files:
- `docker/python.Dockerfile`: production image for the Python archiver
- `docker-compose.yml`: deployment-oriented compose file
- `docker-compose.override.yml`: local development and testing override
- `.env.production`: production container and app environment template
- `.env.development`: development container and app environment template
- `dockerstart.bat`: Windows helper to run the container like the old batch launcher
### Container layout
- Mount your external archive folder to `/app/archive`
- Mount your external config folder to `/app/config`
- Put your `rclone.conf` file at `/app/config/rclone.conf` on the mounted host path
- The container exports `RCLONE_CONFIG=/app/config/rclone.conf`, so rclone will use that file automatically
### Production deployment
1. Edit `.env.production` with your image name, Twitch credentials, bind paths, and default arguments.
2. Place your streamer JSON files and `rclone.conf` in the mounted config folder.
3. Start the container:
```powershell
docker compose --env-file .env.production up -d
```
4. Follow logs:
```powershell
docker compose --env-file .env.production logs -f twitch-archive
```
### Development and local testing
The override compose file builds the image locally and mounts the repository for faster iteration.
Start it with:
```powershell
docker compose --env-file .env.development -f docker-compose.yml -f docker-compose.override.yml up --build
```
Run a one-off manual test for another streamer:
```powershell
docker compose --env-file .env.development -f docker-compose.yml -f docker-compose.override.yml run --rm twitch-archive python twitch-archive.py -u hackerling --verbose
```
Or use the Windows helper:
```powershell
.\dockerstart.bat vinesauce --verbose
```
That batch launcher mirrors the old pattern and expands to a compose `run` command, so you can test any streamer manually.
### Healthcheck and smoke tests
- Container healthcheck command: `python twitch-archive.py --healthcheck -u vinesauce`
- Rclone smoke test command: `python twitch-archive.py -u vinesauce --rclone-smoke-test`
The healthcheck verifies config loading plus `streamlink`, `ffmpeg`, `TwitchDownloaderCLI`, and `rclone` availability. The smoke test writes a tiny file, uploads it with the configured rclone remote, and prints the live rclone output into the container logs.
## ⚡ FFmpeg 8.0 Enhanced
Now with FFmpeg 8.0+ support featuring hardware acceleration and performance improvements!
- **5-10x faster encoding** with NVIDIA, Intel, or AMD GPUs

View file

@ -0,0 +1,17 @@
services:
twitch-archive:
build:
context: .
dockerfile: docker/python.Dockerfile
image: ${TWITCH_ARCHIVE_DEV_IMAGE:-twitch-archive-local}
restart: "no"
env_file:
- ${TWITCH_ARCHIVE_APP_ENV_FILE:-./.env.development}
command:
- sh
- -lc
- python twitch-archive.py ${TWITCH_ARCHIVE_ARGS:--u vinesauce --verbose}
volumes:
- .:/app
- ${TWITCH_ARCHIVE_ARCHIVE_BIND:-./archive}:/app/archive
- ${TWITCH_ARCHIVE_CONFIG_BIND:-./config}:/app/config

28
docker-compose.yml Normal file
View file

@ -0,0 +1,28 @@
services:
twitch-archive:
image: ${TWITCH_ARCHIVE_IMAGE:-forgejo.maddoscientisto.net/maddo/twitch-archive:latest}
container_name: ${TWITCH_ARCHIVE_CONTAINER_NAME:-twitch-archive}
restart: unless-stopped
init: true
env_file:
- ${TWITCH_ARCHIVE_APP_ENV_FILE:-./.env.production}
environment:
PYTHONUNBUFFERED: ${PYTHONUNBUFFERED:-1}
TZ: ${TZ:-UTC}
RCLONE_CONFIG: ${TWITCH_ARCHIVE_RCLONE_CONFIG:-/app/config/rclone.conf}
TWITCH_ARCHIVE_HEALTHCHECK_STREAMER: ${TWITCH_ARCHIVE_HEALTHCHECK_STREAMER:-vinesauce}
command:
- sh
- -lc
- python twitch-archive.py ${TWITCH_ARCHIVE_ARGS:--u vinesauce}
volumes:
- ${TWITCH_ARCHIVE_ARCHIVE_BIND:-./archive}:/app/archive
- ${TWITCH_ARCHIVE_CONFIG_BIND:-./config}:/app/config
healthcheck:
test:
- CMD-SHELL
- python twitch-archive.py --healthcheck -u ${TWITCH_ARCHIVE_HEALTHCHECK_STREAMER:-vinesauce}
interval: 30s
timeout: 10s
retries: 3
start_period: 30s

6
docker/entrypoint.sh Normal file
View file

@ -0,0 +1,6 @@
#!/bin/sh
set -eu
mkdir -p /app/archive /app/config /app/bin/temp
exec "$@"

49
docker/python.Dockerfile Normal file
View file

@ -0,0 +1,49 @@
FROM python:3.12-slim
ARG TWITCH_DOWNLOADER_VERSION=1.56.4
ARG TARGETARCH
ENV PYTHONUNBUFFERED=1 \
PIP_NO_CACHE_DIR=1 \
RCLONE_CONFIG=/app/config/rclone.conf
WORKDIR /app
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
ca-certificates \
curl \
ffmpeg \
libicu-dev \
rclone \
unzip \
&& rm -rf /var/lib/apt/lists/*
RUN set -eux; \
case "${TARGETARCH}" in \
amd64) twitch_downloader_arch='Linux-x64' ;; \
arm64) twitch_downloader_arch='LinuxArm64' ;; \
arm) twitch_downloader_arch='LinuxArm' ;; \
*) echo "Unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; \
esac; \
curl -fsSL -o /tmp/TwitchDownloaderCLI.zip "https://github.com/lay295/TwitchDownloader/releases/download/${TWITCH_DOWNLOADER_VERSION}/TwitchDownloaderCLI-${TWITCH_DOWNLOADER_VERSION}-${twitch_downloader_arch}.zip"; \
mkdir -p /tmp/twitchdownloader; \
unzip -j /tmp/TwitchDownloaderCLI.zip TwitchDownloaderCLI -d /tmp/twitchdownloader; \
install -m 0755 /tmp/twitchdownloader/TwitchDownloaderCLI /usr/local/bin/TwitchDownloaderCLI; \
rm -rf /tmp/TwitchDownloaderCLI.zip /tmp/twitchdownloader
COPY requirements.txt ./requirements.txt
RUN python -m pip install --upgrade pip \
&& python -m pip install -r requirements.txt
COPY . /app
RUN mkdir -p /app/archive /app/config /app/bin/temp
COPY docker/entrypoint.sh /usr/local/bin/twitch-archive-entrypoint
RUN chmod +x /usr/local/bin/twitch-archive-entrypoint
ENTRYPOINT ["twitch-archive-entrypoint"]
CMD ["python", "twitch-archive.py", "-u", "vinesauce"]

4
dockerrebuild.bat Normal file
View file

@ -0,0 +1,4 @@
@echo off
setlocal
docker compose --env-file .env.development -f docker-compose.yml -f docker-compose.override.yml build --no-cache --pull twitch-archive

20
dockerstart.bat Normal file
View file

@ -0,0 +1,20 @@
@echo off
setlocal
if "%~1"=="" (
echo Usage: .\dockerstart.bat streamer [additional args]
exit /b 1
)
set STREAMER=%~1
shift
set EXTRA_ARGS=
:collect_args
if "%~1"=="" goto run_compose
set EXTRA_ARGS=%EXTRA_ARGS% %~1
shift
goto collect_args
:run_compose
docker compose --env-file .env.development -f docker-compose.yml -f docker-compose.override.yml run --rm twitch-archive python twitch-archive.py -u %STREAMER%%EXTRA_ARGS%

View file

@ -17,49 +17,39 @@
</div>
<div class="card">
<label title="Set a quality override for this streamer">Quality</label>
<InputCheckbox Value="overrideQuality" ValueChanged="@( (bool v) => overrideQuality = v )" ValueExpression="() => overrideQuality" title="Override default quality" /> Override
<InputText @bind="model.Quality" disabled="@(!overrideQuality)" placeholder="@(global?.DefaultQuality ?? "")" title="Enter a quality string (e.g., best, 720p)" />
<label title="Quality for this streamer">Quality</label>
<InputText @bind-Value="model.Quality" placeholder="@(global?.DefaultQuality ?? "")" title="Enter a quality string (e.g., best, 720p)" />
</div>
<div>
<label title="Toggle cloud upload for this streamer">Upload to Cloud</label>
<InputCheckbox Value="overrideUpload" ValueChanged="@( (bool v) => overrideUpload = v )" ValueExpression="() => overrideUpload" title="Override upload-to-cloud setting" /> Override
<InputCheckbox Value="uploadToCloudVal" ValueChanged="@( (bool v) => uploadToCloudVal = v )" ValueExpression="() => uploadToCloudVal" disabled="@(!overrideUpload)" title="Upload to configured cloud destination" />
<InputCheckbox Value="uploadToCloudVal" ValueChanged="@( (bool v) => uploadToCloudVal = v )" ValueExpression="() => uploadToCloudVal" title="Upload to configured cloud destination" />
</div>
<div class="card">
<label title="Cloud destination (e.g., rclone remote) for this streamer">Upload Destination</label>
<InputText @bind="model.UploadDestination" title="Cloud destination (e.g., rclone remote)" />
<InputText @bind-Value="model.UploadDestination" title="Cloud destination (e.g., rclone remote)" />
</div>
<div class="card">
<label title="Optional: override system Streamlink path for this streamer">Streamlink Path (override)</label>
<InputCheckbox Value="overrideStreamlink" ValueChanged="@( (bool v) => overrideStreamlink = v )" ValueExpression="() => overrideStreamlink" title="Override streamlink path" /> Override
<InputText @bind="model.StreamlinkPath" disabled="@(!overrideStreamlink)" placeholder="@(global?.StreamlinkPath ?? "")" title="Full path to streamlink executable" />
</div>
@* Streamlink path is global-only; not configurable per-streamer *@
<div class="card">
<h4>Per-streamer overrides</h4>
<h4>Per-streamer settings</h4>
<div>
<label>Download VOD</label>
<InputCheckbox Value="overrideDownloadVOD" ValueChanged="@( (bool v) => overrideDownloadVOD = v )" ValueExpression="() => overrideDownloadVOD" /> Override
<InputCheckbox Value="downloadVODVal" ValueChanged="@( (bool v) => downloadVODVal = v )" ValueExpression="() => downloadVODVal" disabled="@(!overrideDownloadVOD)" />
<InputCheckbox Value="downloadVODVal" ValueChanged="@( (bool v) => downloadVODVal = v )" ValueExpression="() => downloadVODVal" />
</div>
<div>
<label>Download CHAT</label>
<InputCheckbox Value="overrideDownloadCHAT" ValueChanged="@( (bool v) => overrideDownloadCHAT = v )" ValueExpression="() => overrideDownloadCHAT" /> Override
<InputCheckbox Value="downloadCHATVal" ValueChanged="@( (bool v) => downloadCHATVal = v )" ValueExpression="() => downloadCHATVal" disabled="@(!overrideDownloadCHAT)" />
<InputCheckbox Value="downloadCHATVal" ValueChanged="@( (bool v) => downloadCHATVal = v )" ValueExpression="() => downloadCHATVal" />
</div>
<div>
<label>Merge Video & Chat</label>
<InputCheckbox Value="overrideMergeVideoChat" ValueChanged="@( (bool v) => overrideMergeVideoChat = v )" ValueExpression="() => overrideMergeVideoChat" /> Override
<InputCheckbox Value="mergeVideoChatVal" ValueChanged="@( (bool v) => mergeVideoChatVal = v )" ValueExpression="() => mergeVideoChatVal" disabled="@(!overrideMergeVideoChat)" />
<InputCheckbox Value="mergeVideoChatVal" ValueChanged="@( (bool v) => mergeVideoChatVal = v )" ValueExpression="() => mergeVideoChatVal" />
</div>
<div>
<label>Merge Chat Layout</label>
<InputCheckbox Value="overrideMergeChatLayout" ValueChanged="@( (bool v) => overrideMergeChatLayout = v )" ValueExpression="() => overrideMergeChatLayout" /> Override
<InputSelect @bind-Value="mergeChatLayoutVal" disabled="@(!overrideMergeChatLayout)">
<InputSelect @bind-Value="mergeChatLayoutVal">
<option value="side-by-side">Side by side</option>
<option value="stacked">Stacked</option>
<option value="overlay">Overlay</option>
@ -67,105 +57,35 @@
</div>
<div>
<label>VOD Timeout (sec)</label>
<InputCheckbox Value="overrideVodTimeout" ValueChanged="@( (bool v) => overrideVodTimeout = v )" ValueExpression="() => overrideVodTimeout" /> Override
<InputNumber @bind-Value="vodTimeoutVal" disabled="@(!overrideVodTimeout)" />
<InputNumber @bind-Value="vodTimeoutVal" />
</div>
<div>
<label>Delete Files</label>
<InputCheckbox Value="overrideDeleteFiles" ValueChanged="@( (bool v) => overrideDeleteFiles = v )" ValueExpression="() => overrideDeleteFiles" /> Override
<InputCheckbox Value="deleteFilesVal" ValueChanged="@( (bool v) => deleteFilesVal = v )" ValueExpression="() => deleteFilesVal" disabled="@(!overrideDeleteFiles)" />
</div>
<div>
<label>HLS Segments (live)</label>
<InputCheckbox Value="overrideHlsSegments" ValueChanged="@( (bool v) => overrideHlsSegments = v )" ValueExpression="() => overrideHlsSegments" /> Override
<InputNumber @bind-Value="hlsSegmentsVal" disabled="@(!overrideHlsSegments)" />
</div>
<div>
<label>FFmpeg HW Accel</label>
<InputCheckbox Value="overrideFfmpegHwaccel" ValueChanged="@( (bool v) => overrideFfmpegHwaccel = v )" ValueExpression="() => overrideFfmpegHwaccel" /> Override
<InputSelect @bind-Value="ffmpegHwaccelVal" disabled="@(!overrideFfmpegHwaccel)">
<option value="auto">Auto</option>
<option value="none">None</option>
<option value="vaapi">VAAPI</option>
<option value="dxva2">DXVA2</option>
<option value="qsv">QSV</option>
<option value="cuda">CUDA</option>
</InputSelect>
</div>
<div>
<label>FFmpeg Threads</label>
<InputCheckbox Value="overrideFfmpegThreads" ValueChanged="@( (bool v) => overrideFfmpegThreads = v )" ValueExpression="() => overrideFfmpegThreads" /> Override
<InputNumber @bind-Value="ffmpegThreadsVal" disabled="@(!overrideFfmpegThreads)" />
</div>
<div>
<label>FFmpeg Audio Bitrate</label>
<InputCheckbox Value="overrideFfmpegAudioBitrate" ValueChanged="@( (bool v) => overrideFfmpegAudioBitrate = v )" ValueExpression="() => overrideFfmpegAudioBitrate" /> Override
<InputText @bind-Value="ffmpegAudioBitrateVal" disabled="@(!overrideFfmpegAudioBitrate)" />
<InputCheckbox Value="deleteFilesVal" ValueChanged="@( (bool v) => deleteFilesVal = v )" ValueExpression="() => deleteFilesVal" />
</div>
<div>
<label>Download Live CHAT</label>
<InputCheckbox Value="overrideDownloadLiveCHAT" ValueChanged="@( (bool v) => overrideDownloadLiveCHAT = v )" ValueExpression="() => overrideDownloadLiveCHAT" /> Override
<InputCheckbox Value="downloadLiveCHATVal" ValueChanged="@( (bool v) => downloadLiveCHATVal = v )" ValueExpression="() => downloadLiveCHATVal" disabled="@(!overrideDownloadLiveCHAT)" />
<InputCheckbox Value="downloadLiveCHATVal" ValueChanged="@( (bool v) => downloadLiveCHATVal = v )" ValueExpression="() => downloadLiveCHATVal" />
</div>
<div>
<label>Upload Pre-Merge Video</label>
<InputCheckbox Value="overrideUploadPreMergeVideo" ValueChanged="@( (bool v) => overrideUploadPreMergeVideo = v )" ValueExpression="() => overrideUploadPreMergeVideo" /> Override
<InputCheckbox Value="uploadPreMergeVideoVal" ValueChanged="@( (bool v) => uploadPreMergeVideoVal = v )" ValueExpression="() => uploadPreMergeVideoVal" disabled="@(!overrideUploadPreMergeVideo)" />
<InputCheckbox Value="uploadPreMergeVideoVal" ValueChanged="@( (bool v) => uploadPreMergeVideoVal = v )" ValueExpression="() => uploadPreMergeVideoVal" />
</div>
<div>
<label>Upload Merged Video</label>
<InputCheckbox Value="overrideUploadMergedVideo" ValueChanged="@( (bool v) => overrideUploadMergedVideo = v )" ValueExpression="() => overrideUploadMergedVideo" /> Override
<InputCheckbox Value="uploadMergedVideoVal" ValueChanged="@( (bool v) => uploadMergedVideoVal = v )" ValueExpression="() => uploadMergedVideoVal" disabled="@(!overrideUploadMergedVideo)" />
<InputCheckbox Value="uploadMergedVideoVal" ValueChanged="@( (bool v) => uploadMergedVideoVal = v )" ValueExpression="() => uploadMergedVideoVal" />
</div>
<div>
<label>Upload Chat Video</label>
<InputCheckbox Value="overrideUploadChatVideo" ValueChanged="@( (bool v) => overrideUploadChatVideo = v )" ValueExpression="() => overrideUploadChatVideo" /> Override
<InputCheckbox Value="uploadChatVideoVal" ValueChanged="@( (bool v) => uploadChatVideoVal = v )" ValueExpression="() => uploadChatVideoVal" disabled="@(!overrideUploadChatVideo)" />
<InputCheckbox Value="uploadChatVideoVal" ValueChanged="@( (bool v) => uploadChatVideoVal = v )" ValueExpression="() => uploadChatVideoVal" />
</div>
<div>
<label>Only Raw</label>
<InputCheckbox Value="overrideOnlyRaw" ValueChanged="@( (bool v) => overrideOnlyRaw = v )" ValueExpression="() => overrideOnlyRaw" /> Override
<InputCheckbox Value="onlyRawVal" ValueChanged="@( (bool v) => onlyRawVal = v )" ValueExpression="() => onlyRawVal" disabled="@(!overrideOnlyRaw)" />
<InputCheckbox Value="onlyRawVal" ValueChanged="@( (bool v) => onlyRawVal = v )" ValueExpression="() => onlyRawVal" />
</div>
<div>
<label>Clean Raw</label>
<InputCheckbox Value="overrideCleanRaw" ValueChanged="@( (bool v) => overrideCleanRaw = v )" ValueExpression="() => overrideCleanRaw" /> Override
<InputCheckbox Value="cleanRawVal" ValueChanged="@( (bool v) => cleanRawVal = v )" ValueExpression="() => cleanRawVal" disabled="@(!overrideCleanRaw)" />
</div>
<div>
<label>HLS Segments (VOD)</label>
<InputCheckbox Value="overrideHlsSegmentsVOD" ValueChanged="@( (bool v) => overrideHlsSegmentsVOD = v )" ValueExpression="() => overrideHlsSegmentsVOD" /> Override
<InputNumber @bind-Value="hlsSegmentsVODVal" disabled="@(!overrideHlsSegmentsVOD)" />
</div>
<div>
<label>Streamlink ttvlol</label>
<InputCheckbox Value="overrideStreamlinkTtvlol" ValueChanged="@( (bool v) => overrideStreamlinkTtvlol = v )" ValueExpression="() => overrideStreamlinkTtvlol" /> Override
<InputCheckbox Value="streamlinkTtvlolVal" ValueChanged="@( (bool v) => streamlinkTtvlolVal = v )" ValueExpression="() => streamlinkTtvlolVal" disabled="@(!overrideStreamlinkTtvlol)" />
</div>
<div>
<label>FFmpeg Audio Codec</label>
<InputCheckbox Value="overrideFfmpegAudioCodec" ValueChanged="@( (bool v) => overrideFfmpegAudioCodec = v )" ValueExpression="() => overrideFfmpegAudioCodec" /> Override
<InputText @bind-Value="ffmpegAudioCodecVal" disabled="@(!overrideFfmpegAudioCodec)" />
</div>
<div>
<label>FFmpeg Audio Sample Rate</label>
<InputCheckbox Value="overrideFfmpegAudioSamplerate" ValueChanged="@( (bool v) => overrideFfmpegAudioSamplerate = v )" ValueExpression="() => overrideFfmpegAudioSamplerate" /> Override
<InputNumber @bind-Value="ffmpegAudioSamplerateVal" disabled="@(!overrideFfmpegAudioSamplerate)" />
</div>
<div>
<label>FFmpeg Error Recovery</label>
<InputCheckbox Value="overrideFfmpegErrorRecovery" ValueChanged="@( (bool v) => overrideFfmpegErrorRecovery = v )" ValueExpression="() => overrideFfmpegErrorRecovery" /> Override
<InputCheckbox Value="ffmpegErrorRecoveryVal" ValueChanged="@( (bool v) => ffmpegErrorRecoveryVal = v )" ValueExpression="() => ffmpegErrorRecoveryVal" disabled="@(!overrideFfmpegErrorRecovery)" />
</div>
<div>
<label>FFmpeg Faststart</label>
<InputCheckbox Value="overrideFfmpegFaststart" ValueChanged="@( (bool v) => overrideFfmpegFaststart = v )" ValueExpression="() => overrideFfmpegFaststart" /> Override
<InputCheckbox Value="ffmpegFaststartVal" ValueChanged="@( (bool v) => ffmpegFaststartVal = v )" ValueExpression="() => ffmpegFaststartVal" disabled="@(!overrideFfmpegFaststart)" />
</div>
<div>
<label>FFmpeg Progress</label>
<InputCheckbox Value="overrideFfmpegProgress" ValueChanged="@( (bool v) => overrideFfmpegProgress = v )" ValueExpression="() => overrideFfmpegProgress" /> Override
<InputCheckbox Value="ffmpegProgressVal" ValueChanged="@( (bool v) => ffmpegProgressVal = v )" ValueExpression="() => ffmpegProgressVal" disabled="@(!overrideFfmpegProgress)" />
<InputCheckbox Value="cleanRawVal" ValueChanged="@( (bool v) => cleanRawVal = v )" ValueExpression="() => cleanRawVal" />
</div>
</div>
@ -193,37 +113,10 @@
@code {
[Parameter]
public string Username { get; set; } = string.Empty;
private TwitchArchive.Core.Config.StreamerConfig model = new();
private TwitchArchive.Core.Config.GlobalConfig? global;
private bool saved = false;
private bool showConfirm = false;
private bool overrideQuality = false;
private bool overrideUpload = false;
private bool overrideStreamlink = false;
private bool overrideDownloadVOD = false;
private bool overrideDownloadCHAT = false;
private bool overrideMergeVideoChat = false;
private bool overrideMergeChatLayout = false;
private bool overrideVodTimeout = false;
private bool overrideDeleteFiles = false;
private bool overrideHlsSegments = false;
private bool overrideFfmpegHwaccel = false;
private bool overrideFfmpegThreads = false;
private bool overrideFfmpegAudioBitrate = false;
private bool overrideDownloadLiveCHAT = false;
private bool overrideUploadPreMergeVideo = false;
private bool overrideUploadMergedVideo = false;
private bool overrideUploadChatVideo = false;
private bool overrideOnlyRaw = false;
private bool overrideCleanRaw = false;
private bool overrideHlsSegmentsVOD = false;
private bool overrideStreamlinkTtvlol = false;
private bool overrideFfmpegAudioCodec = false;
private bool overrideFfmpegAudioSamplerate = false;
private bool overrideFfmpegErrorRecovery = false;
private bool overrideFfmpegFaststart = false;
private bool overrideFfmpegProgress = false;
// local values for nullable per-streamer settings (bind safely)
private bool downloadVODVal;
@ -233,28 +126,18 @@
private string mergeChatLayoutVal = "side-by-side";
private int? vodTimeoutVal;
private bool deleteFilesVal;
private int? hlsSegmentsVal;
private string ffmpegHwaccelVal = "auto";
private int? ffmpegThreadsVal;
private string? ffmpegAudioBitrateVal;
private bool uploadPreMergeVideoVal;
private bool uploadMergedVideoVal;
private bool uploadChatVideoVal;
private bool onlyRawVal;
private bool cleanRawVal;
private int? hlsSegmentsVODVal;
private bool streamlinkTtvlolVal;
private string? ffmpegAudioCodecVal;
private int? ffmpegAudioSamplerateVal;
private bool ffmpegErrorRecoveryVal;
private bool ffmpegFaststartVal;
private bool ffmpegProgressVal;
private bool uploadToCloudVal;
protected override void OnInitialized()
{
global = ConfigService.LoadGlobal();
var s = ConfigService.LoadStreamer(Username);
var isNew = s == null;
if (s != null) model = s;
// initialize local values from model or global defaults
downloadVODVal = model.DownloadVOD ?? global?.Defaults.DownloadVOD ?? true;
@ -264,57 +147,67 @@
mergeChatLayoutVal = model.MergeChatLayout ?? global?.Defaults.MergeChatLayout ?? "side-by-side";
vodTimeoutVal = model.VodTimeout ?? global?.Defaults.VodTimeout ?? 300;
deleteFilesVal = model.DeleteFiles ?? global?.Defaults.DeleteFiles ?? false;
hlsSegmentsVal = model.HlsSegments ?? global?.Defaults.HlsSegments ?? 3;
ffmpegHwaccelVal = model.FfmpegHwaccel ?? global?.Defaults.FfmpegHwaccel ?? "auto";
ffmpegThreadsVal = model.FfmpegThreads ?? global?.Defaults.FfmpegThreads ?? 0;
ffmpegAudioBitrateVal = model.FfmpegAudioBitrate ?? global?.Defaults.FfmpegAudioBitrate ?? "192k";
uploadPreMergeVideoVal = model.UploadPreMergeVideo ?? global?.Defaults.UploadPreMergeVideo ?? true;
uploadMergedVideoVal = model.UploadMergedVideo ?? global?.Defaults.UploadMergedVideo ?? true;
uploadChatVideoVal = model.UploadChatVideo ?? global?.Defaults.UploadChatVideo ?? false;
onlyRawVal = model.OnlyRaw ?? global?.Defaults.OnlyRaw ?? false;
cleanRawVal = model.CleanRaw ?? global?.Defaults.CleanRaw ?? true;
hlsSegmentsVODVal = model.HlsSegmentsVOD ?? global?.Defaults.HlsSegmentsVOD ?? 10;
streamlinkTtvlolVal = model.StreamlinkTtvlol ?? global?.Defaults.StreamlinkTtvlol ?? false;
ffmpegAudioCodecVal = model.FfmpegAudioCodec ?? global?.Defaults.FfmpegAudioCodec ?? "aac";
ffmpegAudioSamplerateVal = model.FfmpegAudioSamplerate ?? global?.Defaults.FfmpegAudioSamplerate ?? 48000;
ffmpegErrorRecoveryVal = model.FfmpegErrorRecovery ?? global?.Defaults.FfmpegErrorRecovery ?? true;
ffmpegFaststartVal = model.FfmpegFaststart ?? global?.Defaults.FfmpegFaststart ?? true;
ffmpegProgressVal = model.FfmpegProgress ?? global?.Defaults.FfmpegProgress ?? false;
uploadToCloudVal = model.UploadToCloud ?? global?.UploadToCloud ?? false;
// when creating a new streamer config, populate model with global defaults so
// the streamer config stores initial values and subsequent edits use streamer values
if (isNew)
{
model.Quality = model.Quality ?? global?.DefaultQuality;
model.UploadToCloud = uploadToCloudVal;
model.UploadDestination = model.UploadDestination ?? global?.UploadDestination;
model.DownloadVOD = downloadVODVal;
model.DownloadCHAT = downloadCHATVal;
model.DownloadLiveCHAT = downloadLiveCHATVal;
model.MergeVideoChat = mergeVideoChatVal;
model.MergeChatLayout = mergeChatLayoutVal;
model.VodTimeout = vodTimeoutVal;
model.DeleteFiles = deleteFilesVal;
model.UploadPreMergeVideo = uploadPreMergeVideoVal;
model.UploadMergedVideo = uploadMergedVideoVal;
model.UploadChatVideo = uploadChatVideoVal;
model.OnlyRaw = onlyRawVal;
model.CleanRaw = cleanRawVal;
}
}
private void Save()
{
model.Username = Username;
if (!overrideQuality) model.Quality = null;
if (string.IsNullOrWhiteSpace(model.Quality)) model.Quality = null;
// Upload to cloud
model.UploadToCloud = overrideUpload ? uploadToCloudVal : (bool?)null;
// Streamlink path
model.StreamlinkPath = overrideStreamlink ? model.StreamlinkPath : null;
// Per-streamer values: map local values when overridden, otherwise clear
model.DownloadVOD = overrideDownloadVOD ? downloadVODVal : (bool?)null;
model.DownloadCHAT = overrideDownloadCHAT ? downloadCHATVal : (bool?)null;
model.DownloadLiveCHAT = overrideDownloadLiveCHAT ? downloadLiveCHATVal : (bool?)null;
model.MergeVideoChat = overrideMergeVideoChat ? mergeVideoChatVal : (bool?)null;
model.MergeChatLayout = overrideMergeChatLayout ? mergeChatLayoutVal : null;
model.VodTimeout = overrideVodTimeout ? vodTimeoutVal : (int?)null;
model.DeleteFiles = overrideDeleteFiles ? deleteFilesVal : (bool?)null;
model.HlsSegments = overrideHlsSegments ? hlsSegmentsVal : (int?)null;
model.FfmpegHwaccel = overrideFfmpegHwaccel ? ffmpegHwaccelVal : null;
model.FfmpegThreads = overrideFfmpegThreads ? ffmpegThreadsVal : (int?)null;
model.FfmpegAudioBitrate = overrideFfmpegAudioBitrate ? ffmpegAudioBitrateVal : null;
model.UploadPreMergeVideo = overrideUploadPreMergeVideo ? uploadPreMergeVideoVal : (bool?)null;
model.UploadMergedVideo = overrideUploadMergedVideo ? uploadMergedVideoVal : (bool?)null;
model.UploadChatVideo = overrideUploadChatVideo ? uploadChatVideoVal : (bool?)null;
model.OnlyRaw = overrideOnlyRaw ? onlyRawVal : (bool?)null;
model.CleanRaw = overrideCleanRaw ? cleanRawVal : (bool?)null;
model.HlsSegmentsVOD = overrideHlsSegmentsVOD ? hlsSegmentsVODVal : (int?)null;
model.StreamlinkTtvlol = overrideStreamlinkTtvlol ? streamlinkTtvlolVal : (bool?)null;
model.FfmpegAudioCodec = overrideFfmpegAudioCodec ? ffmpegAudioCodecVal : null;
model.FfmpegAudioSamplerate = overrideFfmpegAudioSamplerate ? ffmpegAudioSamplerateVal : (int?)null;
model.FfmpegErrorRecovery = overrideFfmpegErrorRecovery ? ffmpegErrorRecoveryVal : (bool?)null;
model.FfmpegFaststart = overrideFfmpegFaststart ? ffmpegFaststartVal : (bool?)null;
model.FfmpegProgress = overrideFfmpegProgress ? ffmpegProgressVal : (bool?)null;
model.UploadToCloud = uploadToCloudVal;
// Ensure global-only settings are not stored per-streamer
model.StreamlinkPath = null;
model.HlsSegments = null;
model.HlsSegmentsVOD = null;
model.StreamlinkTtvlol = null;
model.FfmpegHwaccel = null;
model.FfmpegThreads = null;
model.FfmpegAudioBitrate = null;
model.FfmpegAudioCodec = null;
model.FfmpegAudioSamplerate = null;
model.FfmpegErrorRecovery = null;
model.FfmpegFaststart = null;
model.FfmpegProgress = null;
// Per-streamer values: always map local values into the model
model.DownloadVOD = downloadVODVal;
model.DownloadCHAT = downloadCHATVal;
model.DownloadLiveCHAT = downloadLiveCHATVal;
model.MergeVideoChat = mergeVideoChatVal;
model.MergeChatLayout = mergeChatLayoutVal;
model.VodTimeout = vodTimeoutVal;
model.DeleteFiles = deleteFilesVal;
model.UploadPreMergeVideo = uploadPreMergeVideoVal;
model.UploadMergedVideo = uploadMergedVideoVal;
model.UploadChatVideo = uploadChatVideoVal;
model.OnlyRaw = onlyRawVal;
model.CleanRaw = cleanRawVal;
ConfigService.SaveStreamer(model);
saved = true;
}

View file

@ -40,9 +40,9 @@ class ContentDownloader:
self.ffmpeg_path = ffmpeg_path
self.quality = config.get('quality', 'best')
self.hls_segments_vod = config.get('hls_segmentsVOD', 10)
self.download_vod = config.get('downloadVOD', True)
self.download_chat = config.get('downloadCHAT', True)
self.download_live_chat = config.get('downloadLiveCHAT', True)
self.download_vod_enabled = config.get('downloadVOD', True)
self.download_chat_enabled = config.get('downloadCHAT', True)
self.download_live_chat_enabled = config.get('downloadLiveCHAT', True)
self.use_chat_downloader_primary = config.get('useChatDownloaderPrimary', False)
self.use_chat_downloader_fallback = config.get('useChatDownloaderFallback', True)
@ -73,7 +73,7 @@ class ContentDownloader:
Returns:
bool: True if download succeeded, False otherwise
"""
if not self.download_vod:
if not self.download_vod_enabled:
return False
print(f'\n{Fore.CYAN}Downloading VOD: {vod_info["title"]}{Style.RESET_ALL}')
@ -272,7 +272,7 @@ class ContentDownloader:
Returns:
bool: True if succeeded, False otherwise
"""
if not self.download_chat:
if not self.download_chat_enabled:
return False
print(f'\n{Fore.CYAN}Downloading chat: {vod_info["title"]}{Style.RESET_ALL}')
@ -298,7 +298,7 @@ class ContentDownloader:
Returns:
subprocess.Popen: The process handle, or None if failed to start
"""
if not self.download_live_chat:
if not self.download_live_chat_enabled:
return None
print(f'\n{Fore.CYAN}Starting live chat download...{Style.RESET_ALL}')
@ -502,7 +502,7 @@ class ContentDownloader:
print(f'{Fore.YELLOW} Install with: pip install chat-downloader{Style.RESET_ALL}')
return False
if not self.download_live_chat:
if not self.download_live_chat_enabled:
print(f'{Fore.YELLOW}⚠ downloadLiveCHAT is disabled in config{Style.RESET_ALL}')
return False

View file

@ -46,6 +46,128 @@ class FileManager:
self.metadata_path = self.root_path / username / "metadata"
self.log_file = self.root_path / ".log"
def _to_rclone_relative_path(self, *parts: str) -> str:
"""Build a POSIX-style relative path for rclone --files-from."""
return pathlib.PurePosixPath(*parts).as_posix()
def _build_upload_relative_paths(self, filename_base: str) -> List[str]:
"""Build the candidate upload list relative to root_path for rclone."""
files_to_upload: List[str] = [
self._to_rclone_relative_path(self.username, 'metadata', f"{PREFIX_METADATA}{filename_base}.json"),
self._to_rclone_relative_path(self.username, 'chat', 'json', f"{PREFIX_CHAT}{filename_base}.json")
]
if self.upload_pre_merge_video:
files_to_upload.extend([
self._to_rclone_relative_path(self.username, 'video', 'raw', f"{PREFIX_LIVE}{filename_base}.ts"),
self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_LIVE}{filename_base}.mp4"),
self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_LIVE}{filename_base}.mp3"),
self._to_rclone_relative_path(self.username, 'video', 'raw', f"{PREFIX_VOD}{filename_base}.ts"),
self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_VOD}{filename_base}.mp4"),
self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_VOD}{filename_base}.mp3")
])
if self.upload_merged_video:
files_to_upload.extend([
self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_MERGED}{filename_base}.mp4"),
self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_MERGED}{filename_base}.mp3"),
self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}.mp4"),
self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}.mp3")
])
if self.upload_chat_video:
files_to_upload.append(self._to_rclone_relative_path(self.username, 'chat', f"{PREFIX_CHAT}{filename_base}.mp4"))
return files_to_upload
def _get_existing_upload_relative_paths(self, relative_paths: List[str]) -> List[str]:
"""Filter candidate upload paths to the files that actually exist."""
existing_paths: List[str] = []
for relative_path in relative_paths:
if (self.root_path / pathlib.PurePosixPath(relative_path)).exists():
existing_paths.append(relative_path)
return existing_paths
def _run_rclone_copy(self, relative_paths: List[str], description: str) -> bool:
"""Run rclone copy for a set of paths relative to root_path."""
existing_paths = self._get_existing_upload_relative_paths(relative_paths)
missing_paths = [path for path in relative_paths if path not in existing_paths]
if not existing_paths:
print(f'{Fore.RED}✗ Upload skipped: no matching files found for {description}{Style.RESET_ALL}')
for missing_path in missing_paths:
print(f'{Fore.YELLOW} Missing: {missing_path}{Style.RESET_ALL}')
return False
if missing_paths:
print(f'{Fore.YELLOW}⚠ Some configured upload files were not found and will be skipped{Style.RESET_ALL}')
for missing_path in missing_paths:
print(f'{Fore.YELLOW} Missing: {missing_path}{Style.RESET_ALL}')
print(f'{Fore.CYAN}rclone source: {self.root_path.resolve()}{Style.RESET_ALL}')
print(f'{Fore.CYAN}rclone destination: {self.rclone_path}{Style.RESET_ALL}')
print(f'{Fore.CYAN}Files queued for upload: {len(existing_paths)}{Style.RESET_ALL}')
bin_path = get_bin_path()
upload_list_path = os.path.join(bin_path, 'temp', 'upload.txt')
os.makedirs(os.path.dirname(upload_list_path), exist_ok=True)
with open(upload_list_path, 'w', encoding='utf-8', newline='\n') as f:
f.write('\n'.join(existing_paths))
f.write('\n')
try:
cmd = [
'rclone', 'copy',
str(self.root_path.resolve()),
self.rclone_path,
'--files-from', upload_list_path,
'--progress'
]
print(f'{Fore.CYAN}Running: {' '.join(cmd)}{Style.RESET_ALL}')
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
if proc.stdout:
for line in proc.stdout:
print(line, end='')
proc.wait()
return proc.returncode == 0
finally:
if os.path.exists(upload_list_path):
os.remove(upload_list_path)
def run_rclone_smoke_test(self) -> bool:
"""Create and upload a tiny metadata file to verify rclone output and configuration."""
smoke_name = 'RCLONE_SMOKE_TEST'
smoke_relative_path = self._to_rclone_relative_path(
self.username,
'metadata',
f"{PREFIX_METADATA}{smoke_name}.json"
)
smoke_file_path = self.root_path / pathlib.PurePosixPath(smoke_relative_path)
smoke_payload = {
'type': 'rclone_smoke_test',
'username': self.username
}
smoke_file_path.parent.mkdir(parents=True, exist_ok=True)
with open(smoke_file_path, 'w', encoding='utf-8') as f:
json.dump(smoke_payload, f, indent=2)
print(f'{Fore.CYAN}Created smoke-test file: {smoke_file_path}{Style.RESET_ALL}')
try:
result = self._run_rclone_copy([smoke_relative_path], 'rclone smoke test')
if result:
print(f'{Fore.GREEN}✓ Rclone smoke test completed{Style.RESET_ALL}')
else:
print(f'{Fore.RED}✗ Rclone smoke test failed{Style.RESET_ALL}')
return result
finally:
if smoke_file_path.exists():
smoke_file_path.unlink()
def initialize_directories(self) -> None:
"""Create all necessary directory structures."""
for path in [self.raw_path, self.video_path, self.chat_json_path,
@ -121,80 +243,23 @@ class FileManager:
if notification_callback:
notification_callback(f'☁ Uploading - {filename_base}', 'Uploading files to cloud storage')
# Create list of files to upload
bin_path = get_bin_path()
upload_list_path = os.path.join(bin_path, 'temp', 'upload.txt')
files_to_upload = self._build_upload_relative_paths(filename_base)
# Ensure temp directory exists
os.makedirs(os.path.dirname(upload_list_path), exist_ok=True)
files_to_upload = []
# Build files list relative to root_path so rclone can read them with --files-from
# Metadata and chat JSON
files_to_upload.append(os.path.join(self.username, 'metadata', f"{PREFIX_METADATA}{filename_base}.json"))
files_to_upload.append(os.path.join(self.username, 'chat', 'json', f"{PREFIX_CHAT}{filename_base}.json"))
# Pre-merge videos (raw .ts in video/raw, mp4/mp3 in video)
if self.upload_pre_merge_video:
files_to_upload.extend([
os.path.join(self.username, 'video', 'raw', f"{PREFIX_LIVE}{filename_base}.ts"),
os.path.join(self.username, 'video', f"{PREFIX_LIVE}{filename_base}.mp4"),
os.path.join(self.username, 'video', f"{PREFIX_LIVE}{filename_base}.mp3"),
os.path.join(self.username, 'video', 'raw', f"{PREFIX_VOD}{filename_base}.ts"),
os.path.join(self.username, 'video', f"{PREFIX_VOD}{filename_base}.mp4"),
os.path.join(self.username, 'video', f"{PREFIX_VOD}{filename_base}.mp3")
])
# Merged videos (in video folder)
if self.upload_merged_video:
files_to_upload.extend([
os.path.join(self.username, 'video', f"{PREFIX_MERGED}{filename_base}.mp4"),
os.path.join(self.username, 'video', f"{PREFIX_MERGED}{filename_base}.mp3"),
os.path.join(self.username, 'video', f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}.mp4"),
os.path.join(self.username, 'video', f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}.mp3")
])
# Standalone chat video (in chat folder)
if self.upload_chat_video:
files_to_upload.append(os.path.join(self.username, 'chat', f"{PREFIX_CHAT}{filename_base}.mp4"))
with open(upload_list_path, 'w') as f:
f.write('\n'.join(files_to_upload))
# Run rclone using --files-from so the listed paths (relative to root_path) are uploaded.
try:
cmd = [
'rclone', 'copy',
str(self.root_path.resolve()),
self.rclone_path,
'--files-from', upload_list_path
]
result = self._run_rclone_copy(files_to_upload, f'archive batch {filename_base}')
# Stream rclone output to console so user can see progress/errors
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
if proc.stdout:
for line in proc.stdout:
print(line, end='')
proc.wait()
result = proc.returncode
# Clean up upload list
if os.path.exists(upload_list_path):
os.remove(upload_list_path)
if result == 0:
if result:
print(f'{Fore.GREEN}✓ Upload complete{Style.RESET_ALL}')
if notification_callback:
notification_callback(f'✓ Upload Success - {filename_base}', 'All files uploaded successfully')
return True
else:
print(f'{Fore.RED}✗ Upload failed (exit code: {result}){Style.RESET_ALL}')
print(f'{Fore.YELLOW}Files preserved locally due to upload failure{Style.RESET_ALL}')
if notification_callback:
notification_callback(f'✗ Upload Failed - {filename_base}',
f'Upload failed with code {result}. Files preserved locally.')
return False
print(f'{Fore.RED}✗ Upload failed{Style.RESET_ALL}')
print(f'{Fore.YELLOW}Files preserved locally due to upload failure{Style.RESET_ALL}')
if notification_callback:
notification_callback(f'✗ Upload Failed - {filename_base}',
'Upload failed. Files preserved locally. Check rclone output above.')
return False
except Exception as e:
print(f'{Fore.RED}✗ Upload error: {str(e)}{Style.RESET_ALL}')

View file

@ -37,37 +37,78 @@ class StreamProcessor:
os_type
)
def process_raw_stream(self, raw_path: str, output_path: str) -> None:
def process_raw_stream(self, raw_path: str, output_path: str) -> bool:
"""
Process raw .ts file into mp4/mp3 using ffmpeg.
Args:
raw_path: Path to the raw .ts file
output_path: Path for the processed output file
Returns:
bool: True when conversion succeeded, False otherwise
"""
if not os.path.exists(raw_path):
print(f'{Fore.YELLOW}⚠ Raw file not found, skipping processing{Style.RESET_ALL}')
return
return False
if self.only_raw:
print(f'{Fore.CYAN}Keeping raw .ts file (onlyRaw mode){Style.RESET_ALL}')
return
return False
print(f'{Fore.YELLOW}Processing raw stream file...{Style.RESET_ALL}')
# Build ffmpeg command based on quality
if self.quality == 'audio_only':
self._process_audio(raw_path, output_path)
result = self._process_audio(raw_path, output_path)
else:
self._process_video(raw_path, output_path)
result = self._process_video(raw_path, output_path)
print(f'{Fore.GREEN}✓ Stream processed successfully{Style.RESET_ALL}')
if result:
print(f'{Fore.GREEN}✓ Stream processed successfully{Style.RESET_ALL}')
else:
print(f'{Fore.RED}✗ Stream processing failed{Style.RESET_ALL}')
def _process_audio(self, raw_path: str, output_path: str) -> None:
return result
def _run_ffmpeg_command(self, cmd: list, output_path: str) -> bool:
"""Run FFmpeg while streaming its output to the terminal."""
print(f'{Fore.CYAN}Running FFmpeg: {' '.join(cmd)}{Style.RESET_ALL}')
process = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
encoding='utf-8',
errors='replace'
)
if process.stdout:
for line in process.stdout:
print(line, end='')
result = process.wait()
if result != 0:
print(f'{Fore.RED}✗ FFmpeg exited with code: {result}{Style.RESET_ALL}')
return False
if not os.path.exists(output_path):
print(f'{Fore.RED}✗ FFmpeg did not create output: {output_path}{Style.RESET_ALL}')
return False
if os.path.getsize(output_path) == 0:
print(f'{Fore.RED}✗ FFmpeg created an empty output file: {output_path}{Style.RESET_ALL}')
return False
return True
def _process_audio(self, raw_path: str, output_path: str) -> bool:
"""Process audio-only stream."""
# Audio-only conversion with modern AAC encoding
cmd = [
self.ffmpeg_path,
'-y',
'-i', raw_path,
'-vn', # No video
'-c:a', self.ffmpeg_audio_codec,
@ -85,14 +126,9 @@ class StreamProcessor:
cmd.extend(['-movflags', '+faststart'])
cmd.append(output_path)
return self._run_ffmpeg_command(cmd, output_path)
# Run FFmpeg
if self.ffmpeg_progress:
subprocess.call(cmd)
else:
subprocess.call(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
def _process_video(self, raw_path: str, output_path: str) -> None:
def _process_video(self, raw_path: str, output_path: str) -> bool:
"""Process video stream."""
cmd = [
self.ffmpeg_path,
@ -135,12 +171,7 @@ class StreamProcessor:
cmd.extend(['-movflags', '+faststart'])
cmd.append(output_path)
# Run FFmpeg
if self.ffmpeg_progress:
subprocess.call(cmd)
else:
subprocess.call(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
return self._run_ffmpeg_command(cmd, output_path)
def build_chat_output_args(self) -> str:
"""

View file

@ -7,6 +7,8 @@ import subprocess
from typing import Dict, Any, Optional
from colorama import Fore, Style
from .utils import get_env_value
class StreamRecorder:
"""Handles live stream recording using streamlink."""
@ -68,7 +70,7 @@ class StreamRecorder:
print(f'{Fore.YELLOW} Consider disabling streamlink_ttvlol in config or using alternative methods{Style.RESET_ALL}')
# Add authentication if available
oauth_token = os.getenv("OAUTH-PRIVATE-TOKEN", "")
oauth_token = get_env_value("OAUTH-PRIVATE-TOKEN", "OAUTH_PRIVATE_TOKEN", default="")
if oauth_token and oauth_token != "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx":
cmd.extend(['--twitch-api-header', f'Authorization=OAuth {oauth_token}'])

View file

@ -9,6 +9,7 @@ import requests
from colorama import Fore, Style
from .constants import TWITCH_OAUTH_URL, TWITCH_API_URL, TWITCH_GQL_URL, TWITCH_GQL_CLIENT_ID
from .utils import get_env_value
class StreamMonitor:
@ -40,7 +41,9 @@ class StreamMonitor:
return self._oauth_token
try:
url = f"{TWITCH_OAUTH_URL}?client_id={os.getenv('CLIENT-ID')}&client_secret={os.getenv('CLIENT-SECRET')}&grant_type=client_credentials"
client_id = get_env_value('CLIENT-ID', 'CLIENT_ID')
client_secret = get_env_value('CLIENT-SECRET', 'CLIENT_SECRET')
url = f"{TWITCH_OAUTH_URL}?client_id={client_id}&client_secret={client_secret}&grant_type=client_credentials"
response = requests.post(url, timeout=15)
response.raise_for_status()
self._oauth_token = response.json()['access_token']
@ -69,7 +72,7 @@ class StreamMonitor:
url = f'{TWITCH_API_URL}/users?login={self.username}'
headers = {
"Authorization": f"Bearer {self.get_oauth_token()}",
"Client-ID": os.getenv('CLIENT-ID')
"Client-ID": get_env_value('CLIENT-ID', 'CLIENT_ID')
}
response = requests.get(url, headers=headers, timeout=15)
response.raise_for_status()

View file

@ -4,6 +4,7 @@ Utility functions and helpers for Twitch Archive.
import os
import sys
import shutil
import pathlib
import subprocess
from typing import Optional
@ -35,6 +36,15 @@ def get_bin_path() -> str:
return str(pathlib.Path(__file__).parent.parent.resolve() / "bin")
def get_env_value(*names: str, default: Optional[str] = None) -> Optional[str]:
"""Return the first non-empty environment variable from the provided names."""
for name in names:
value = os.getenv(name)
if value not in (None, ""):
return value
return default
def get_ffmpeg_executable(os_type: str) -> str:
"""
Get the platform-specific ffmpeg executable path.
@ -48,6 +58,11 @@ def get_ffmpeg_executable(os_type: str) -> str:
bin_path = get_bin_path()
if os_type == 'windows':
return os.path.join(bin_path, 'ffmpeg.exe')
system_ffmpeg = shutil.which('ffmpeg')
if system_ffmpeg:
return system_ffmpeg
return os.path.join(bin_path, 'ffmpeg')
@ -64,6 +79,11 @@ def get_twitch_downloader_executable(os_type: str) -> str:
bin_path = get_bin_path()
if os_type == 'windows':
return os.path.join(bin_path, 'TwitchDownloaderCLI.exe')
system_twitch_downloader = shutil.which('TwitchDownloaderCLI')
if system_twitch_downloader:
return system_twitch_downloader
return os.path.join(bin_path, 'TwitchDownloaderCLI')
@ -164,6 +184,24 @@ def verify_streamlink() -> bool:
return False
def verify_rclone() -> bool:
"""Verify that rclone is available on PATH."""
try:
result = subprocess.run(['rclone', 'version'],
capture_output=True,
text=True,
timeout=5)
if result.returncode == 0:
version_line = result.stdout.strip().splitlines()[0] if result.stdout.strip() else 'unknown'
print(f'{Fore.GREEN}✓ Rclone found ({version_line}){Style.RESET_ALL}')
return True
raise FileNotFoundError()
except (FileNotFoundError, subprocess.TimeoutExpired, IndexError):
print(f'{Fore.RED}✗ ERROR: rclone not found{Style.RESET_ALL}')
print(f'{Fore.CYAN} → Install rclone and ensure it is on PATH{Style.RESET_ALL}')
return False
def verify_ffmpeg(os_type: str) -> bool:
"""
Verify that ffmpeg is available.

View file

@ -20,12 +20,27 @@ import sys
import os
import json
import getopt
import tempfile
import importlib.util
from pathlib import Path
from unittest.mock import patch, MagicMock, Mock
# Add parent directory to path for imports
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from modules.constants import DEFAULT_CONFIG
from modules.file_manager import FileManager
from modules.downloader import ContentDownloader
from modules.utils import get_ffmpeg_executable, get_twitch_downloader_executable
def load_twitch_archive_module():
"""Load the main script module for targeted regression tests."""
module_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'twitch-archive.py')
spec = importlib.util.spec_from_file_location('twitch_archive_main', module_path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
class TestCommandLineArgumentParsing(unittest.TestCase):
@ -116,6 +131,34 @@ class TestCommandLineArgumentParsing(unittest.TestCase):
self.assertEqual(len(opts), 1)
self.assertEqual(opts[0], ('--chat-only', ''))
def test_rclone_smoke_test_option(self):
"""Test --rclone-smoke-test option parsing."""
argv = ['--rclone-smoke-test']
opts, args = getopt.getopt(
argv,
"hu:q:a:v:c:m:r:d:n:",
["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=",
"metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose",
"chat-only", "healthcheck", "rclone-smoke-test", "use-chat-downloader-primary", "no-chat-downloader-fallback"]
)
self.assertEqual(len(opts), 1)
self.assertEqual(opts[0], ('--rclone-smoke-test', ''))
def test_healthcheck_option(self):
"""Test --healthcheck option parsing."""
argv = ['--healthcheck']
opts, args = getopt.getopt(
argv,
"hu:q:a:v:c:m:r:d:n:",
["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=",
"metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose",
"chat-only", "healthcheck", "rclone-smoke-test", "use-chat-downloader-primary", "no-chat-downloader-fallback"]
)
self.assertEqual(len(opts), 1)
self.assertEqual(opts[0], ('--healthcheck', ''))
def test_legacy_option(self):
"""Test --legacy option parsing."""
argv = ['--legacy']
@ -439,6 +482,161 @@ class TestConfigLogic(unittest.TestCase):
self.assertIn('$schema', default_config)
class TestFileManagerUploadPaths(unittest.TestCase):
"""Test rclone upload path preparation."""
def test_build_upload_relative_paths_uses_forward_slashes(self):
"""Rclone --files-from entries must use POSIX separators on Windows."""
with tempfile.TemporaryDirectory() as temp_dir:
manager = FileManager(
root_path=temp_dir,
username='testuser',
config={
'uploadCloud': True,
'uploadPreMergeVideo': True,
'uploadMergedVideo': True,
'uploadChatVideo': True
}
)
relative_paths = manager._build_upload_relative_paths('20260424_12h00m00s')
self.assertTrue(relative_paths)
self.assertTrue(all('\\' not in path for path in relative_paths))
self.assertIn('testuser/metadata/METADA_20260424_12h00m00s.json', relative_paths)
self.assertIn('testuser/chat/json/CHAT_20260424_12h00m00s.json', relative_paths)
class TestDownloaderConfiguration(unittest.TestCase):
"""Regression tests for downloader config wiring."""
def test_download_vod_method_not_shadowed_by_boolean_flag(self):
"""Config booleans must not overwrite callable downloader methods."""
downloader = ContentDownloader(
twitch_downloader_path='TwitchDownloaderCLI',
ffmpeg_path='ffmpeg',
config={
'downloadVOD': True,
'downloadCHAT': True,
'downloadLiveCHAT': True
}
)
self.assertTrue(callable(downloader.download_vod))
self.assertTrue(downloader.download_vod_enabled)
class TestLinuxToolResolution(unittest.TestCase):
"""Ensure Linux containers prefer their own installed toolchain."""
@patch('modules.utils.shutil.which')
def test_linux_prefers_system_ffmpeg(self, mock_which):
mock_which.return_value = '/usr/bin/ffmpeg'
self.assertEqual(get_ffmpeg_executable('linux'), '/usr/bin/ffmpeg')
@patch('modules.utils.shutil.which')
def test_linux_prefers_system_twitch_downloader(self, mock_which):
mock_which.return_value = '/usr/local/bin/TwitchDownloaderCLI'
self.assertEqual(get_twitch_downloader_executable('linux'), '/usr/local/bin/TwitchDownloaderCLI')
class TestMultiStreamerCleanupRegression(unittest.TestCase):
"""Regression tests for multi-streamer conversion and cleanup behavior."""
def setUp(self):
self.module = load_twitch_archive_module()
def _build_archiver(self, temp_dir: str, upload_cloud: bool = True):
archiver = MagicMock()
archiver.username = 'maddoscientist0'
archiver.os_type = 'linux'
archiver.quality = 'best'
archiver.downloadLiveCHAT = False
archiver.downloadCHAT = False
archiver.downloadVOD = False
archiver.downloadMETADATA = False
archiver.mergeVideoChat = False
archiver.mergeChatLayout = 'side-by-side'
archiver.onlyRaw = False
archiver.vodTimeout = 0
archiver.shutdown_requested = False
archiver.deleteFiles = True
archiver.uploadCloud = upload_cloud
archiver.notification_manager = MagicMock()
archiver.recorder = MagicMock()
archiver.processor = MagicMock()
archiver.downloader = MagicMock()
archiver.stream_monitor = MagicMock()
archiver.file_manager = MagicMock()
archiver.file_manager.raw_path = Path(temp_dir) / 'raw'
archiver.file_manager.video_path = Path(temp_dir) / 'video'
archiver.file_manager.chat_json_path = Path(temp_dir) / 'chat_json'
archiver.file_manager.chat_mp4_path = Path(temp_dir) / 'chat'
os.makedirs(archiver.file_manager.raw_path, exist_ok=True)
os.makedirs(archiver.file_manager.video_path, exist_ok=True)
os.makedirs(archiver.file_manager.chat_json_path, exist_ok=True)
os.makedirs(archiver.file_manager.chat_mp4_path, exist_ok=True)
return archiver
def test_process_stream_keeps_raw_when_conversion_fails(self):
manager = self.module.TwitchArchiveManager(specific_streamer='maddoscientist0')
with tempfile.TemporaryDirectory() as temp_dir:
archiver = self._build_archiver(temp_dir)
archiver.processor.process_raw_stream.return_value = False
archiver.file_manager.upload_to_cloud.return_value = True
def write_raw_file(_stream_info, raw_path):
with open(raw_path, 'wb') as handle:
handle.write(b'x' * 4096)
return True
archiver.recorder.record.side_effect = write_raw_file
stream_info = {
'title': 'Test',
'createdAt': '2026-04-25T09:14:01Z'
}
manager._process_stream(archiver, stream_info, 'stream-id')
archiver.file_manager.clean_raw_file.assert_not_called()
archiver.file_manager.delete_local_files.assert_called_once()
def test_process_stream_only_deletes_rendered_files_after_real_upload(self):
manager = self.module.TwitchArchiveManager(specific_streamer='maddoscientist0')
with tempfile.TemporaryDirectory() as temp_dir:
archiver = self._build_archiver(temp_dir, upload_cloud=False)
archiver.processor.process_raw_stream.return_value = True
archiver.file_manager.upload_to_cloud.return_value = True
def write_raw_file(_stream_info, raw_path):
with open(raw_path, 'wb') as handle:
handle.write(b'x' * 4096)
return True
archiver.recorder.record.side_effect = write_raw_file
stream_info = {
'title': 'Test',
'createdAt': '2026-04-25T09:14:01Z'
}
manager._process_stream(archiver, stream_info, 'stream-id')
archiver.file_manager.clean_raw_file.assert_called_once()
archiver.file_manager.delete_local_files.assert_not_called()
upload_filename_base = archiver.file_manager.upload_to_cloud.call_args.args[0]
self.assertFalse(upload_filename_base.startswith('LIVE_'))
if __name__ == '__main__':
# Run tests with verbose output
print("="*70)

View file

@ -34,6 +34,8 @@ import time
import json
import signal
import getopt
import pathlib
import subprocess
from typing import Dict, Optional, Any
from datetime import datetime, timedelta
@ -48,7 +50,8 @@ from modules.config import ConfigManager
from modules.notifications import NotificationManager
from modules.utils import (
detect_operating_system, get_ffmpeg_executable, get_twitch_downloader_executable,
get_unique_filename, get_video_duration, verify_streamlink, verify_ffmpeg, verify_twitch_downloader
get_unique_filename, get_video_duration, verify_streamlink, verify_ffmpeg, verify_twitch_downloader,
verify_rclone, get_env_value
)
from modules.stream_monitor import StreamMonitor
from modules.recorder import StreamRecorder
@ -147,10 +150,25 @@ class TwitchArchive:
Raises:
SystemExit: If .env file is not found
"""
if not load_dotenv(find_dotenv()):
dotenv_loaded = load_dotenv(find_dotenv())
has_required_env = bool(
get_env_value('CLIENT-ID', 'CLIENT_ID') and
get_env_value('CLIENT-SECRET', 'CLIENT_SECRET')
)
if not dotenv_loaded and has_required_env:
print(f'{Fore.GREEN}✓ Twitch API credentials loaded from process environment{Style.RESET_ALL}')
return
if not dotenv_loaded and not has_required_env:
print(f'{Fore.RED}✗ ERROR: .env file not found{Style.RESET_ALL}')
print(f'{Fore.CYAN} → Create a .env file with your Twitch API credentials{Style.RESET_ALL}')
print(f'{Fore.CYAN} → Required: CLIENT-ID, CLIENT-SECRET{Style.RESET_ALL}')
print(f'{Fore.CYAN} → Create a .env file with your Twitch API credentials or pass them via environment variables{Style.RESET_ALL}')
print(f'{Fore.CYAN} → Required: CLIENT-ID/CLIENT_ID and CLIENT-SECRET/CLIENT_SECRET{Style.RESET_ALL}')
sys.exit(1)
if not has_required_env:
print(f'{Fore.RED}✗ ERROR: Twitch API credentials are missing{Style.RESET_ALL}')
print(f'{Fore.CYAN} → Required: CLIENT-ID/CLIENT_ID and CLIENT-SECRET/CLIENT_SECRET{Style.RESET_ALL}')
sys.exit(1)
def _initialize_components(self) -> None:
@ -259,6 +277,8 @@ class TwitchArchive:
verify_ffmpeg(self.os_type)
if self.downloadVOD or self.downloadCHAT:
verify_twitch_downloader(self.os_type)
if self.uploadCloud and not verify_rclone():
sys.exit(1)
# Print configuration summary
self._print_configuration_summary()
@ -404,7 +424,7 @@ class TwitchArchive:
print(f'{Fore.YELLOW}Attempting to process any recorded content...{Style.RESET_ALL}')
# Process the raw stream file
self.processor.process_raw_stream(live_raw_path, live_proc_path)
processing_succeeded = self.processor.process_raw_stream(live_raw_path, live_proc_path)
# Wait for live chat download if it was started
live_chat_downloaded = False
@ -426,7 +446,11 @@ class TwitchArchive:
else:
# Get video duration first (needed for chat conversion and trimming)
ffmpeg_path = get_ffmpeg_executable(self.os_type)
video_duration = get_video_duration(live_proc_path, ffmpeg_path)
if not processing_succeeded or not os.path.exists(live_proc_path):
print(f'{Fore.YELLOW}⚠ Processed video file is unavailable, skipping chat render{Style.RESET_ALL}')
video_duration = None
else:
video_duration = get_video_duration(live_proc_path, ffmpeg_path)
print(f'{Fore.CYAN}Video duration for chat rendering: {video_duration}s{Style.RESET_ALL}')
# Convert chat format if needed (chat_downloader uses different JSON structure)
@ -561,7 +585,10 @@ class TwitchArchive:
print(f'{Fore.YELLOW}⚠ No matching VOD found for this stream{Style.RESET_ALL}')
# Clean up raw files if configured
self.file_manager.clean_raw_file(live_raw_path)
if processing_succeeded:
self.file_manager.clean_raw_file(live_raw_path)
elif os.path.exists(live_raw_path):
print(f'{Fore.YELLOW}⚠ Keeping raw file because conversion did not complete successfully{Style.RESET_ALL}')
# Upload to cloud if configured
upload_success = self.file_manager.upload_to_cloud(
@ -570,7 +597,7 @@ class TwitchArchive:
)
# Delete local files if configured and upload succeeded
if self.deleteFiles and upload_success:
if self.deleteFiles and self.uploadCloud and upload_success:
self.file_manager.delete_local_files(
filename_base,
live_raw_path,
@ -890,6 +917,8 @@ class TwitchArchiveManager:
verify_ffmpeg(first_archiver.os_type)
if first_archiver.downloadVOD or first_archiver.downloadCHAT:
verify_twitch_downloader(first_archiver.os_type)
if any(archiver.uploadCloud for archiver in self.archivers.values()) and not verify_rclone():
sys.exit(1)
# Print configuration summary for each streamer
for username, archiver in self.archivers.items():
@ -1018,7 +1047,7 @@ class TwitchArchiveManager:
# Generate timestamp and filename
timestamp = datetime.now(timezone('UTC')).strftime("%Y%m%d_%Hh%Mm%Ss")
filename_base = f"{PREFIX_LIVE}{archiver.username}_{timestamp}"
filename_base = f"{archiver.username}_{timestamp}"
# Parse stream start time
live_date = datetime.strptime(
@ -1029,8 +1058,8 @@ class TwitchArchiveManager:
raw_extension = '.ts'
proc_extension = '.mp3' if archiver.quality == 'audio_only' else '.mp4'
live_raw_path = str(archiver.file_manager.raw_path / f"{filename_base}{raw_extension}")
live_proc_path = str(archiver.file_manager.video_path / f"{filename_base}{proc_extension}")
live_raw_path = str(archiver.file_manager.raw_path / f"{PREFIX_LIVE}{filename_base}{raw_extension}")
live_proc_path = str(archiver.file_manager.video_path / f"{PREFIX_LIVE}{filename_base}{proc_extension}")
chat_json_path = str(archiver.file_manager.chat_json_path / f"{PREFIX_CHAT}{filename_base}.json")
# Send notification
@ -1172,8 +1201,9 @@ class TwitchArchiveManager:
print(f'{Fore.CYAN}Processing recorded content ({file_size / (1024*1024):.2f} MB)...{Style.RESET_ALL}')
# Process raw stream
processing_succeeded = False
if not archiver.onlyRaw:
archiver.processor.process_raw_stream(live_raw_path, live_proc_path)
processing_succeeded = archiver.processor.process_raw_stream(live_raw_path, live_proc_path)
# Wait for live chat download if it was started
live_chat_downloaded = False
@ -1212,8 +1242,12 @@ class TwitchArchiveManager:
chat_rendered_successfully = False
else:
# Get video duration first
ffmpeg_path = get_ffmpeg_executable(archiver.os_type)
video_duration = get_video_duration(live_proc_path, ffmpeg_path)
if not processing_succeeded or not os.path.exists(live_proc_path):
print(f'{Fore.YELLOW}⚠ Processed video file is unavailable, skipping chat render{Style.RESET_ALL}')
video_duration = None
else:
ffmpeg_path = get_ffmpeg_executable(archiver.os_type)
video_duration = get_video_duration(live_proc_path, ffmpeg_path)
if video_duration is None:
print(f'{Fore.YELLOW}⚠ Could not detect video duration from {live_proc_path}{Style.RESET_ALL}')
@ -1362,7 +1396,10 @@ class TwitchArchiveManager:
archiver.file_manager.save_metadata(stream_info, filename_base)
# Clean up raw file if configured
archiver.file_manager.clean_raw_file(live_raw_path)
if processing_succeeded:
archiver.file_manager.clean_raw_file(live_raw_path)
elif os.path.exists(live_raw_path):
print(f'{Fore.YELLOW}⚠ Keeping raw file because conversion did not complete successfully{Style.RESET_ALL}')
# Upload to cloud if configured
upload_success = archiver.file_manager.upload_to_cloud(
@ -1371,7 +1408,7 @@ class TwitchArchiveManager:
)
# Delete files if configured
if archiver.deleteFiles and upload_success:
if archiver.deleteFiles and archiver.uploadCloud and upload_success:
archiver.file_manager.delete_local_files(
filename_base,
live_raw_path,
@ -1386,6 +1423,79 @@ class TwitchArchiveManager:
)
def run_rclone_smoke_test(specific_streamer: Optional[str] = None) -> int:
"""Run a one-off rclone smoke test using the configured upload destination."""
config_manager = ConfigManager()
if specific_streamer:
username = specific_streamer
else:
enabled_streamers = config_manager.get_all_enabled_streamers()
if not enabled_streamers:
print(f'{Fore.RED}✗ No enabled streamers available for smoke test{Style.RESET_ALL}')
print(f'{Fore.CYAN}→ Use -u <username> or enable a streamer config{Style.RESET_ALL}')
return 1
username = enabled_streamers[0]
config = config_manager.load_streamer_config(username)
file_manager = FileManager(
root_path=config.get('root_path', 'archive'),
username=username,
config=config
)
file_manager.initialize_directories()
print(f'\n{Fore.CYAN}{"=" * 70}{Style.RESET_ALL}')
print(f'{Fore.CYAN}TWITCH ARCHIVE - Rclone Smoke Test{Style.RESET_ALL}')
print(f'{Fore.CYAN}{"=" * 70}{Style.RESET_ALL}')
print(f'{Fore.GREEN}Streamer: {username}{Style.RESET_ALL}')
print(f'{Fore.GREEN}Remote: {config.get("rclone_path", "<not configured>")}{Style.RESET_ALL}\n')
return 0 if file_manager.run_rclone_smoke_test() else 1
def run_healthcheck(specific_streamer: Optional[str] = None) -> int:
"""Run a local readiness check suitable for Docker health checks."""
config_manager = ConfigManager()
if specific_streamer:
username = specific_streamer
else:
enabled_streamers = config_manager.get_all_enabled_streamers()
username = enabled_streamers[0] if enabled_streamers else 'vinesauce'
config = config_manager.load_streamer_config(username)
archive = TwitchArchive(config)
try:
archive._load_environment_variables()
except SystemExit:
return 1
archive._initialize_components()
checks_ok = True
if not verify_streamlink():
checks_ok = False
if not verify_ffmpeg(archive.os_type):
checks_ok = False
if (archive.downloadVOD or archive.downloadCHAT) and not verify_twitch_downloader(archive.os_type):
checks_ok = False
if archive.uploadCloud:
if not verify_rclone():
checks_ok = False
rclone_config_path = os.getenv('RCLONE_CONFIG')
if rclone_config_path and not os.path.exists(rclone_config_path):
print(f'{Fore.RED}✗ ERROR: RCLONE_CONFIG points to a missing file: {rclone_config_path}{Style.RESET_ALL}')
checks_ok = False
if not checks_ok:
return 1
print(f'{Fore.GREEN}✓ Healthcheck OK for {username}{Style.RESET_ALL}')
return 0
# ============================================================================
# COMMAND-LINE INTERFACE
# ============================================================================
@ -1401,6 +1511,8 @@ def main(argv: list) -> None:
"""
specific_streamer = None
use_legacy_mode = False
rclone_smoke_test_mode = False
healthcheck_mode = False
help_msg = f'''
{Fore.CYAN}{"=" * 70}
@ -1427,6 +1539,8 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving
--legacy Force legacy mode (use config.json)
--chat-only Test mode: Only download chat (skip video recording)
Automatically enables verbose logging
--healthcheck Validate config and tool availability, then exit
--rclone-smoke-test Create a small test file and upload it with rclone
--use-chat-downloader-primary Use chat_downloader as primary chat source (for testing)
--no-chat-downloader-fallback Disable chat_downloader fallback
@ -1464,7 +1578,7 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving
"h:u:q:a:v:c:m:r:d:n:",
["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=",
"metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose",
"chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"]
"chat-only", "healthcheck", "rclone-smoke-test", "use-chat-downloader-primary", "no-chat-downloader-fallback"]
)
except getopt.GetoptError as e:
print(f'{Fore.RED}Error: {e}{Style.RESET_ALL}\n')
@ -1491,6 +1605,10 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving
elif opt == "--chat-only":
chat_only_mode = True
verbose_mode = True # Auto-enable verbose for chat-only mode
elif opt == "--healthcheck":
healthcheck_mode = True
elif opt == "--rclone-smoke-test":
rclone_smoke_test_mode = True
elif opt == "--legacy":
use_legacy_mode = True
elif opt == "--use-chat-downloader-primary":
@ -1524,6 +1642,12 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving
elif opt in ("-n", "--notifications"):
legacy_overrides['notifications'] = bool(int(arg))
if rclone_smoke_test_mode:
sys.exit(run_rclone_smoke_test(specific_streamer))
if healthcheck_mode:
sys.exit(run_healthcheck(specific_streamer))
# Determine which mode to use
if use_legacy_mode or (legacy_config_exists and not specific_streamer and not os.path.exists('config/global.json')):
# Legacy mode: single streamer using config.json