diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..49463af --- /dev/null +++ b/.dockerignore @@ -0,0 +1,18 @@ +.git +.github +.forgejo +.vs +.vscode +venv*/ +__pycache__/ +*.pyc +archive/ +dotnet/ +tests/ +.pytest_cache/ +bin/temp/ +bin/ffmpeg +bin/ffmpeg.exe +bin/ffprobe +bin/TwitchDownloaderCLI +bin/TwitchDownloaderCLI.exe \ No newline at end of file diff --git a/.env.development b/.env.development new file mode 100644 index 0000000..4ccc47f --- /dev/null +++ b/.env.development @@ -0,0 +1,16 @@ +TWITCH_ARCHIVE_DEV_IMAGE=twitch-archive-local +TWITCH_ARCHIVE_CONTAINER_NAME=twitch-archive-dev +TWITCH_ARCHIVE_APP_ENV_FILE=./.env.development +TWITCH_ARCHIVE_ARCHIVE_BIND=./archive +TWITCH_ARCHIVE_CONFIG_BIND=./config +TWITCH_ARCHIVE_ARGS=-u vinesauce --verbose +TWITCH_ARCHIVE_HEALTHCHECK_STREAMER=vinesauce +TWITCH_ARCHIVE_RCLONE_CONFIG=/app/config/rclone.conf +PYTHONUNBUFFERED=1 +TZ=UTC +CLIENT-ID=vdyevjvllziylzwsm3y925p79pwtua +CLIENT-SECRET=y906xadsmf22q54suuzsmfnfav3jc7 +OAUTH-PRIVATE-TOKEN=ll4kvlmxuajfgi9lgi5d8mkeglsyvm +SENDER= +RECEIVER= +PASSWD= \ No newline at end of file diff --git a/.env.production b/.env.production new file mode 100644 index 0000000..b1c4197 --- /dev/null +++ b/.env.production @@ -0,0 +1,16 @@ +TWITCH_ARCHIVE_IMAGE=forgejo.maddoscientisto.net/maddo/twitch-archive:latest +TWITCH_ARCHIVE_CONTAINER_NAME=twitch-archive +TWITCH_ARCHIVE_APP_ENV_FILE=./.env.production +TWITCH_ARCHIVE_ARCHIVE_BIND=./archive +TWITCH_ARCHIVE_CONFIG_BIND=./config +TWITCH_ARCHIVE_ARGS=-u vinesauce +TWITCH_ARCHIVE_HEALTHCHECK_STREAMER=vinesauce +TWITCH_ARCHIVE_RCLONE_CONFIG=/app/config/rclone.conf +PYTHONUNBUFFERED=1 +TZ=UTC +CLIENT-ID=vdyevjvllziylzwsm3y925p79pwtua +CLIENT-SECRET=y906xadsmf22q54suuzsmfnfav3jc7 +OAUTH-PRIVATE-TOKEN=ll4kvlmxuajfgi9lgi5d8mkeglsyvm +SENDER= +RECEIVER= +PASSWD= \ No newline at end of file diff --git a/.forgejo/workflows/publish-python-container.yml b/.forgejo/workflows/publish-python-container.yml new file mode 100644 index 0000000..831fa51 --- /dev/null +++ b/.forgejo/workflows/publish-python-container.yml @@ -0,0 +1,127 @@ +name: Publish Twitch Archive Container + +on: + push: + branches: + - master + - main + paths: + - docker/python.Dockerfile + - docker/entrypoint.sh + - docker-compose.yml + - docker-compose.override.yml + - requirements.txt + - twitch-archive.py + - run_chat_only.py + - modules/** + - .forgejo/workflows/publish-python-container.yml + workflow_dispatch: + +env: + REGISTRY: ${{ vars.FORGEJO_REGISTRY }} + IMAGE_NAMESPACE: ${{ vars.IMAGE_NAMESPACE }} + IMAGE_NAME: ${{ vars.IMAGE_NAME != '' && vars.IMAGE_NAME || 'twitch-archive' }} + BUILD_CONTEXT: . + DOCKERFILE_PATH: docker/python.Dockerfile + +jobs: + publish: + runs-on: docker + env: + DOCKER_HOST: ${{ vars.DOCKER_HOST != '' && vars.DOCKER_HOST || 'tcp://172.17.0.1:2375' }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Validate workflow variables + run: | + set -eu + if [ -z "${REGISTRY}" ]; then echo "vars.FORGEJO_REGISTRY is required"; exit 1; fi + if [ -z "${IMAGE_NAMESPACE}" ]; then echo "vars.IMAGE_NAMESPACE is required"; exit 1; fi + if [ ! -f "${DOCKERFILE_PATH}" ]; then echo "Dockerfile not found at ${DOCKERFILE_PATH}"; exit 1; fi + if [ ! -f "requirements.txt" ]; then echo "requirements.txt is missing"; exit 1; fi + + - name: Ensure Docker CLI exists + run: | + set -eu + if command -v docker >/dev/null 2>&1; then + docker --version + exit 0 + fi + + ARCH="$(uname -m)" + case "${ARCH}" in + x86_64) DOCKER_ARCH="x86_64" ;; + aarch64|arm64) DOCKER_ARCH="aarch64" ;; + *) echo "Unsupported architecture for Docker CLI bootstrap: ${ARCH}"; exit 1 ;; + esac + + DOCKER_CLI_VERSION="27.5.1" + curl -fsSL "https://download.docker.com/linux/static/stable/${DOCKER_ARCH}/docker-${DOCKER_CLI_VERSION}.tgz" -o docker.tgz + tar -xzf docker.tgz + mkdir -p "${HOME}/.local/bin" + mv docker/docker "${HOME}/.local/bin/docker" + chmod +x "${HOME}/.local/bin/docker" + echo "${HOME}/.local/bin" >> "${FORGEJO_PATH}" + "${HOME}/.local/bin/docker" --version + + - name: Ensure Docker Buildx exists + run: | + set -eu + if docker buildx version >/dev/null 2>&1; then + docker buildx version + exit 0 + fi + + ARCH="$(uname -m)" + case "${ARCH}" in + x86_64) BUILDX_ARCH="amd64" ;; + aarch64|arm64) BUILDX_ARCH="arm64" ;; + *) echo "Unsupported architecture for Docker Buildx bootstrap: ${ARCH}"; exit 1 ;; + esac + + BUILDX_VERSION="v0.21.1" + mkdir -p "${HOME}/.docker/cli-plugins" + curl -fsSL "https://github.com/docker/buildx/releases/download/${BUILDX_VERSION}/buildx-${BUILDX_VERSION}.linux-${BUILDX_ARCH}" -o "${HOME}/.docker/cli-plugins/docker-buildx" + chmod +x "${HOME}/.docker/cli-plugins/docker-buildx" + docker buildx version + + - name: Check Docker daemon connectivity + run: | + set -eu + echo "Using DOCKER_HOST=${DOCKER_HOST}" + docker version + docker info >/dev/null + + - name: Create Buildx builder + run: | + set -eu + docker buildx rm forgejo-builder >/dev/null 2>&1 || true + docker buildx create --name forgejo-builder --driver docker-container --use + docker buildx inspect --bootstrap + + - name: Validate registry secrets + run: | + set -eu + if [ -z "${{ secrets.FORGEJO_REGISTRY_USERNAME }}" ]; then echo "secrets.FORGEJO_REGISTRY_USERNAME is required"; exit 1; fi + if [ -z "${{ secrets.FORGEJO_REGISTRY_TOKEN }}" ]; then echo "secrets.FORGEJO_REGISTRY_TOKEN is required"; exit 1; fi + + - name: Login to registry + run: | + set -eu + echo "${{ secrets.FORGEJO_REGISTRY_TOKEN }}" | docker login "${REGISTRY}" -u "${{ secrets.FORGEJO_REGISTRY_USERNAME }}" --password-stdin + + - name: Build and push image + run: | + set -eu + IMAGE_REF="${REGISTRY}/${IMAGE_NAMESPACE}/${IMAGE_NAME}" + SHA_TAG="${IMAGE_REF}:sha-${FORGEJO_SHA}" + BRANCH_TAG="${IMAGE_REF}:${FORGEJO_REF_NAME}" + docker buildx build \ + --file "${DOCKERFILE_PATH}" \ + --tag "${SHA_TAG}" \ + --tag "${BRANCH_TAG}" \ + --tag "${IMAGE_REF}:latest" \ + --push \ + "${BUILD_CONTEXT}" \ No newline at end of file diff --git a/.gitignore b/.gitignore index ff0421f..49c4387 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ config/global.json # Streamer-specific configurations (personal settings) config/streamers/*.json +config/rclone.conf +config/rclone.conf.* # Python cache __pycache__/ diff --git a/README.md b/README.md index e2e2c1f..5005181 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,73 @@ Notes: Python script to check, download live stream, VOD, chat and upload them to any cloud storage supported by rclone. +## Docker + +This repository now includes a Python-only container setup for the archiver. The dotnet subapp is not part of this container flow. + +Files: + +- `docker/python.Dockerfile`: production image for the Python archiver +- `docker-compose.yml`: deployment-oriented compose file +- `docker-compose.override.yml`: local development and testing override +- `.env.production`: production container and app environment template +- `.env.development`: development container and app environment template +- `dockerstart.bat`: Windows helper to run the container like the old batch launcher + +### Container layout + +- Mount your external archive folder to `/app/archive` +- Mount your external config folder to `/app/config` +- Put your `rclone.conf` file at `/app/config/rclone.conf` on the mounted host path +- The container exports `RCLONE_CONFIG=/app/config/rclone.conf`, so rclone will use that file automatically + +### Production deployment + +1. Edit `.env.production` with your image name, Twitch credentials, bind paths, and default arguments. +2. Place your streamer JSON files and `rclone.conf` in the mounted config folder. +3. Start the container: + +```powershell +docker compose --env-file .env.production up -d +``` + +4. Follow logs: + +```powershell +docker compose --env-file .env.production logs -f twitch-archive +``` + +### Development and local testing + +The override compose file builds the image locally and mounts the repository for faster iteration. + +Start it with: + +```powershell +docker compose --env-file .env.development -f docker-compose.yml -f docker-compose.override.yml up --build +``` + +Run a one-off manual test for another streamer: + +```powershell +docker compose --env-file .env.development -f docker-compose.yml -f docker-compose.override.yml run --rm twitch-archive python twitch-archive.py -u hackerling --verbose +``` + +Or use the Windows helper: + +```powershell +.\dockerstart.bat vinesauce --verbose +``` + +That batch launcher mirrors the old pattern and expands to a compose `run` command, so you can test any streamer manually. + +### Healthcheck and smoke tests + +- Container healthcheck command: `python twitch-archive.py --healthcheck -u vinesauce` +- Rclone smoke test command: `python twitch-archive.py -u vinesauce --rclone-smoke-test` + +The healthcheck verifies config loading plus `streamlink`, `ffmpeg`, `TwitchDownloaderCLI`, and `rclone` availability. The smoke test writes a tiny file, uploads it with the configured rclone remote, and prints the live rclone output into the container logs. + ## ⚡ FFmpeg 8.0 Enhanced Now with FFmpeg 8.0+ support featuring hardware acceleration and performance improvements! - **5-10x faster encoding** with NVIDIA, Intel, or AMD GPUs diff --git a/docker-compose.override.yml b/docker-compose.override.yml new file mode 100644 index 0000000..c084df9 --- /dev/null +++ b/docker-compose.override.yml @@ -0,0 +1,17 @@ +services: + twitch-archive: + build: + context: . + dockerfile: docker/python.Dockerfile + image: ${TWITCH_ARCHIVE_DEV_IMAGE:-twitch-archive-local} + restart: "no" + env_file: + - ${TWITCH_ARCHIVE_APP_ENV_FILE:-./.env.development} + command: + - sh + - -lc + - python twitch-archive.py ${TWITCH_ARCHIVE_ARGS:--u vinesauce --verbose} + volumes: + - .:/app + - ${TWITCH_ARCHIVE_ARCHIVE_BIND:-./archive}:/app/archive + - ${TWITCH_ARCHIVE_CONFIG_BIND:-./config}:/app/config \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..68bc572 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,28 @@ +services: + twitch-archive: + image: ${TWITCH_ARCHIVE_IMAGE:-forgejo.maddoscientisto.net/maddo/twitch-archive:latest} + container_name: ${TWITCH_ARCHIVE_CONTAINER_NAME:-twitch-archive} + restart: unless-stopped + init: true + env_file: + - ${TWITCH_ARCHIVE_APP_ENV_FILE:-./.env.production} + environment: + PYTHONUNBUFFERED: ${PYTHONUNBUFFERED:-1} + TZ: ${TZ:-UTC} + RCLONE_CONFIG: ${TWITCH_ARCHIVE_RCLONE_CONFIG:-/app/config/rclone.conf} + TWITCH_ARCHIVE_HEALTHCHECK_STREAMER: ${TWITCH_ARCHIVE_HEALTHCHECK_STREAMER:-vinesauce} + command: + - sh + - -lc + - python twitch-archive.py ${TWITCH_ARCHIVE_ARGS:--u vinesauce} + volumes: + - ${TWITCH_ARCHIVE_ARCHIVE_BIND:-./archive}:/app/archive + - ${TWITCH_ARCHIVE_CONFIG_BIND:-./config}:/app/config + healthcheck: + test: + - CMD-SHELL + - python twitch-archive.py --healthcheck -u ${TWITCH_ARCHIVE_HEALTHCHECK_STREAMER:-vinesauce} + interval: 30s + timeout: 10s + retries: 3 + start_period: 30s \ No newline at end of file diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100644 index 0000000..d327347 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,6 @@ +#!/bin/sh +set -eu + +mkdir -p /app/archive /app/config /app/bin/temp + +exec "$@" \ No newline at end of file diff --git a/docker/python.Dockerfile b/docker/python.Dockerfile new file mode 100644 index 0000000..789f738 --- /dev/null +++ b/docker/python.Dockerfile @@ -0,0 +1,49 @@ +FROM python:3.12-slim + +ARG TWITCH_DOWNLOADER_VERSION=1.56.4 +ARG TARGETARCH + +ENV PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + RCLONE_CONFIG=/app/config/rclone.conf + +WORKDIR /app + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + ffmpeg \ + libicu-dev \ + rclone \ + unzip \ + && rm -rf /var/lib/apt/lists/* + +RUN set -eux; \ + case "${TARGETARCH}" in \ + amd64) twitch_downloader_arch='Linux-x64' ;; \ + arm64) twitch_downloader_arch='LinuxArm64' ;; \ + arm) twitch_downloader_arch='LinuxArm' ;; \ + *) echo "Unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; \ + esac; \ + curl -fsSL -o /tmp/TwitchDownloaderCLI.zip "https://github.com/lay295/TwitchDownloader/releases/download/${TWITCH_DOWNLOADER_VERSION}/TwitchDownloaderCLI-${TWITCH_DOWNLOADER_VERSION}-${twitch_downloader_arch}.zip"; \ + mkdir -p /tmp/twitchdownloader; \ + unzip -j /tmp/TwitchDownloaderCLI.zip TwitchDownloaderCLI -d /tmp/twitchdownloader; \ + install -m 0755 /tmp/twitchdownloader/TwitchDownloaderCLI /usr/local/bin/TwitchDownloaderCLI; \ + rm -rf /tmp/TwitchDownloaderCLI.zip /tmp/twitchdownloader + +COPY requirements.txt ./requirements.txt + +RUN python -m pip install --upgrade pip \ + && python -m pip install -r requirements.txt + +COPY . /app + +RUN mkdir -p /app/archive /app/config /app/bin/temp + +COPY docker/entrypoint.sh /usr/local/bin/twitch-archive-entrypoint + +RUN chmod +x /usr/local/bin/twitch-archive-entrypoint + +ENTRYPOINT ["twitch-archive-entrypoint"] +CMD ["python", "twitch-archive.py", "-u", "vinesauce"] \ No newline at end of file diff --git a/dockerrebuild.bat b/dockerrebuild.bat new file mode 100644 index 0000000..f333c1c --- /dev/null +++ b/dockerrebuild.bat @@ -0,0 +1,4 @@ +@echo off +setlocal + +docker compose --env-file .env.development -f docker-compose.yml -f docker-compose.override.yml build --no-cache --pull twitch-archive diff --git a/dockerstart.bat b/dockerstart.bat new file mode 100644 index 0000000..9d1e947 --- /dev/null +++ b/dockerstart.bat @@ -0,0 +1,20 @@ +@echo off +setlocal + +if "%~1"=="" ( + echo Usage: .\dockerstart.bat streamer [additional args] + exit /b 1 +) + +set STREAMER=%~1 +shift + +set EXTRA_ARGS= +:collect_args +if "%~1"=="" goto run_compose +set EXTRA_ARGS=%EXTRA_ARGS% %~1 +shift +goto collect_args + +:run_compose +docker compose --env-file .env.development -f docker-compose.yml -f docker-compose.override.yml run --rm twitch-archive python twitch-archive.py -u %STREAMER%%EXTRA_ARGS% \ No newline at end of file diff --git a/dotnet/src/TwitchArchive.Web/Pages/StreamerConfig.razor b/dotnet/src/TwitchArchive.Web/Pages/StreamerConfig.razor index 1b58f8a..7c9b082 100644 --- a/dotnet/src/TwitchArchive.Web/Pages/StreamerConfig.razor +++ b/dotnet/src/TwitchArchive.Web/Pages/StreamerConfig.razor @@ -17,49 +17,39 @@
- - Override - + +
- Override - +
- +
-
- - Override - -
+ @* Streamlink path is global-only; not configurable per-streamer *@
-

Per-streamer overrides

+

Per-streamer settings

- Override - +
- Override - +
- Override - +
- Override - + @@ -67,105 +57,35 @@
- Override - +
- Override - -
-
- - Override - -
-
- - Override - - - - - - - - -
-
- - Override - -
-
- - Override - +
- Override - +
- Override - +
- Override - +
- Override - +
- Override - +
- Override - -
-
- - Override - -
-
- - Override - -
-
- - Override - -
-
- - Override - -
-
- - Override - -
-
- - Override - -
-
- - Override - +
@@ -193,37 +113,10 @@ @code { [Parameter] public string Username { get; set; } = string.Empty; - private TwitchArchive.Core.Config.StreamerConfig model = new(); private TwitchArchive.Core.Config.GlobalConfig? global; private bool saved = false; private bool showConfirm = false; - private bool overrideQuality = false; - private bool overrideUpload = false; - private bool overrideStreamlink = false; - private bool overrideDownloadVOD = false; - private bool overrideDownloadCHAT = false; - private bool overrideMergeVideoChat = false; - private bool overrideMergeChatLayout = false; - private bool overrideVodTimeout = false; - private bool overrideDeleteFiles = false; - private bool overrideHlsSegments = false; - private bool overrideFfmpegHwaccel = false; - private bool overrideFfmpegThreads = false; - private bool overrideFfmpegAudioBitrate = false; - private bool overrideDownloadLiveCHAT = false; - private bool overrideUploadPreMergeVideo = false; - private bool overrideUploadMergedVideo = false; - private bool overrideUploadChatVideo = false; - private bool overrideOnlyRaw = false; - private bool overrideCleanRaw = false; - private bool overrideHlsSegmentsVOD = false; - private bool overrideStreamlinkTtvlol = false; - private bool overrideFfmpegAudioCodec = false; - private bool overrideFfmpegAudioSamplerate = false; - private bool overrideFfmpegErrorRecovery = false; - private bool overrideFfmpegFaststart = false; - private bool overrideFfmpegProgress = false; // local values for nullable per-streamer settings (bind safely) private bool downloadVODVal; @@ -233,28 +126,18 @@ private string mergeChatLayoutVal = "side-by-side"; private int? vodTimeoutVal; private bool deleteFilesVal; - private int? hlsSegmentsVal; - private string ffmpegHwaccelVal = "auto"; - private int? ffmpegThreadsVal; - private string? ffmpegAudioBitrateVal; private bool uploadPreMergeVideoVal; private bool uploadMergedVideoVal; private bool uploadChatVideoVal; private bool onlyRawVal; private bool cleanRawVal; - private int? hlsSegmentsVODVal; - private bool streamlinkTtvlolVal; - private string? ffmpegAudioCodecVal; - private int? ffmpegAudioSamplerateVal; - private bool ffmpegErrorRecoveryVal; - private bool ffmpegFaststartVal; - private bool ffmpegProgressVal; private bool uploadToCloudVal; protected override void OnInitialized() { global = ConfigService.LoadGlobal(); var s = ConfigService.LoadStreamer(Username); + var isNew = s == null; if (s != null) model = s; // initialize local values from model or global defaults downloadVODVal = model.DownloadVOD ?? global?.Defaults.DownloadVOD ?? true; @@ -264,57 +147,67 @@ mergeChatLayoutVal = model.MergeChatLayout ?? global?.Defaults.MergeChatLayout ?? "side-by-side"; vodTimeoutVal = model.VodTimeout ?? global?.Defaults.VodTimeout ?? 300; deleteFilesVal = model.DeleteFiles ?? global?.Defaults.DeleteFiles ?? false; - hlsSegmentsVal = model.HlsSegments ?? global?.Defaults.HlsSegments ?? 3; - ffmpegHwaccelVal = model.FfmpegHwaccel ?? global?.Defaults.FfmpegHwaccel ?? "auto"; - ffmpegThreadsVal = model.FfmpegThreads ?? global?.Defaults.FfmpegThreads ?? 0; - ffmpegAudioBitrateVal = model.FfmpegAudioBitrate ?? global?.Defaults.FfmpegAudioBitrate ?? "192k"; uploadPreMergeVideoVal = model.UploadPreMergeVideo ?? global?.Defaults.UploadPreMergeVideo ?? true; uploadMergedVideoVal = model.UploadMergedVideo ?? global?.Defaults.UploadMergedVideo ?? true; uploadChatVideoVal = model.UploadChatVideo ?? global?.Defaults.UploadChatVideo ?? false; onlyRawVal = model.OnlyRaw ?? global?.Defaults.OnlyRaw ?? false; cleanRawVal = model.CleanRaw ?? global?.Defaults.CleanRaw ?? true; - hlsSegmentsVODVal = model.HlsSegmentsVOD ?? global?.Defaults.HlsSegmentsVOD ?? 10; - streamlinkTtvlolVal = model.StreamlinkTtvlol ?? global?.Defaults.StreamlinkTtvlol ?? false; - ffmpegAudioCodecVal = model.FfmpegAudioCodec ?? global?.Defaults.FfmpegAudioCodec ?? "aac"; - ffmpegAudioSamplerateVal = model.FfmpegAudioSamplerate ?? global?.Defaults.FfmpegAudioSamplerate ?? 48000; - ffmpegErrorRecoveryVal = model.FfmpegErrorRecovery ?? global?.Defaults.FfmpegErrorRecovery ?? true; - ffmpegFaststartVal = model.FfmpegFaststart ?? global?.Defaults.FfmpegFaststart ?? true; - ffmpegProgressVal = model.FfmpegProgress ?? global?.Defaults.FfmpegProgress ?? false; uploadToCloudVal = model.UploadToCloud ?? global?.UploadToCloud ?? false; + + // when creating a new streamer config, populate model with global defaults so + // the streamer config stores initial values and subsequent edits use streamer values + if (isNew) + { + model.Quality = model.Quality ?? global?.DefaultQuality; + model.UploadToCloud = uploadToCloudVal; + model.UploadDestination = model.UploadDestination ?? global?.UploadDestination; + model.DownloadVOD = downloadVODVal; + model.DownloadCHAT = downloadCHATVal; + model.DownloadLiveCHAT = downloadLiveCHATVal; + model.MergeVideoChat = mergeVideoChatVal; + model.MergeChatLayout = mergeChatLayoutVal; + model.VodTimeout = vodTimeoutVal; + model.DeleteFiles = deleteFilesVal; + model.UploadPreMergeVideo = uploadPreMergeVideoVal; + model.UploadMergedVideo = uploadMergedVideoVal; + model.UploadChatVideo = uploadChatVideoVal; + model.OnlyRaw = onlyRawVal; + model.CleanRaw = cleanRawVal; + } } private void Save() { model.Username = Username; - if (!overrideQuality) model.Quality = null; + if (string.IsNullOrWhiteSpace(model.Quality)) model.Quality = null; // Upload to cloud - model.UploadToCloud = overrideUpload ? uploadToCloudVal : (bool?)null; - // Streamlink path - model.StreamlinkPath = overrideStreamlink ? model.StreamlinkPath : null; - // Per-streamer values: map local values when overridden, otherwise clear - model.DownloadVOD = overrideDownloadVOD ? downloadVODVal : (bool?)null; - model.DownloadCHAT = overrideDownloadCHAT ? downloadCHATVal : (bool?)null; - model.DownloadLiveCHAT = overrideDownloadLiveCHAT ? downloadLiveCHATVal : (bool?)null; - model.MergeVideoChat = overrideMergeVideoChat ? mergeVideoChatVal : (bool?)null; - model.MergeChatLayout = overrideMergeChatLayout ? mergeChatLayoutVal : null; - model.VodTimeout = overrideVodTimeout ? vodTimeoutVal : (int?)null; - model.DeleteFiles = overrideDeleteFiles ? deleteFilesVal : (bool?)null; - model.HlsSegments = overrideHlsSegments ? hlsSegmentsVal : (int?)null; - model.FfmpegHwaccel = overrideFfmpegHwaccel ? ffmpegHwaccelVal : null; - model.FfmpegThreads = overrideFfmpegThreads ? ffmpegThreadsVal : (int?)null; - model.FfmpegAudioBitrate = overrideFfmpegAudioBitrate ? ffmpegAudioBitrateVal : null; - model.UploadPreMergeVideo = overrideUploadPreMergeVideo ? uploadPreMergeVideoVal : (bool?)null; - model.UploadMergedVideo = overrideUploadMergedVideo ? uploadMergedVideoVal : (bool?)null; - model.UploadChatVideo = overrideUploadChatVideo ? uploadChatVideoVal : (bool?)null; - model.OnlyRaw = overrideOnlyRaw ? onlyRawVal : (bool?)null; - model.CleanRaw = overrideCleanRaw ? cleanRawVal : (bool?)null; - model.HlsSegmentsVOD = overrideHlsSegmentsVOD ? hlsSegmentsVODVal : (int?)null; - model.StreamlinkTtvlol = overrideStreamlinkTtvlol ? streamlinkTtvlolVal : (bool?)null; - model.FfmpegAudioCodec = overrideFfmpegAudioCodec ? ffmpegAudioCodecVal : null; - model.FfmpegAudioSamplerate = overrideFfmpegAudioSamplerate ? ffmpegAudioSamplerateVal : (int?)null; - model.FfmpegErrorRecovery = overrideFfmpegErrorRecovery ? ffmpegErrorRecoveryVal : (bool?)null; - model.FfmpegFaststart = overrideFfmpegFaststart ? ffmpegFaststartVal : (bool?)null; - model.FfmpegProgress = overrideFfmpegProgress ? ffmpegProgressVal : (bool?)null; + model.UploadToCloud = uploadToCloudVal; + // Ensure global-only settings are not stored per-streamer + model.StreamlinkPath = null; + model.HlsSegments = null; + model.HlsSegmentsVOD = null; + model.StreamlinkTtvlol = null; + model.FfmpegHwaccel = null; + model.FfmpegThreads = null; + model.FfmpegAudioBitrate = null; + model.FfmpegAudioCodec = null; + model.FfmpegAudioSamplerate = null; + model.FfmpegErrorRecovery = null; + model.FfmpegFaststart = null; + model.FfmpegProgress = null; + // Per-streamer values: always map local values into the model + model.DownloadVOD = downloadVODVal; + model.DownloadCHAT = downloadCHATVal; + model.DownloadLiveCHAT = downloadLiveCHATVal; + model.MergeVideoChat = mergeVideoChatVal; + model.MergeChatLayout = mergeChatLayoutVal; + model.VodTimeout = vodTimeoutVal; + model.DeleteFiles = deleteFilesVal; + model.UploadPreMergeVideo = uploadPreMergeVideoVal; + model.UploadMergedVideo = uploadMergedVideoVal; + model.UploadChatVideo = uploadChatVideoVal; + model.OnlyRaw = onlyRawVal; + model.CleanRaw = cleanRawVal; ConfigService.SaveStreamer(model); saved = true; } diff --git a/dotnet/src/TwitchArchive.Web/archive.db-shm b/dotnet/src/TwitchArchive.Web/archive.db-shm index b4164f6..fd7d358 100644 Binary files a/dotnet/src/TwitchArchive.Web/archive.db-shm and b/dotnet/src/TwitchArchive.Web/archive.db-shm differ diff --git a/dotnet/src/TwitchArchive.Web/archive.db-wal b/dotnet/src/TwitchArchive.Web/archive.db-wal index 4c417b6..4596f14 100644 Binary files a/dotnet/src/TwitchArchive.Web/archive.db-wal and b/dotnet/src/TwitchArchive.Web/archive.db-wal differ diff --git a/modules/downloader.py b/modules/downloader.py index a4ea645..9f9cc1f 100644 --- a/modules/downloader.py +++ b/modules/downloader.py @@ -40,9 +40,9 @@ class ContentDownloader: self.ffmpeg_path = ffmpeg_path self.quality = config.get('quality', 'best') self.hls_segments_vod = config.get('hls_segmentsVOD', 10) - self.download_vod = config.get('downloadVOD', True) - self.download_chat = config.get('downloadCHAT', True) - self.download_live_chat = config.get('downloadLiveCHAT', True) + self.download_vod_enabled = config.get('downloadVOD', True) + self.download_chat_enabled = config.get('downloadCHAT', True) + self.download_live_chat_enabled = config.get('downloadLiveCHAT', True) self.use_chat_downloader_primary = config.get('useChatDownloaderPrimary', False) self.use_chat_downloader_fallback = config.get('useChatDownloaderFallback', True) @@ -73,7 +73,7 @@ class ContentDownloader: Returns: bool: True if download succeeded, False otherwise """ - if not self.download_vod: + if not self.download_vod_enabled: return False print(f'\n{Fore.CYAN}Downloading VOD: {vod_info["title"]}{Style.RESET_ALL}') @@ -272,7 +272,7 @@ class ContentDownloader: Returns: bool: True if succeeded, False otherwise """ - if not self.download_chat: + if not self.download_chat_enabled: return False print(f'\n{Fore.CYAN}Downloading chat: {vod_info["title"]}{Style.RESET_ALL}') @@ -298,7 +298,7 @@ class ContentDownloader: Returns: subprocess.Popen: The process handle, or None if failed to start """ - if not self.download_live_chat: + if not self.download_live_chat_enabled: return None print(f'\n{Fore.CYAN}Starting live chat download...{Style.RESET_ALL}') @@ -502,7 +502,7 @@ class ContentDownloader: print(f'{Fore.YELLOW} Install with: pip install chat-downloader{Style.RESET_ALL}') return False - if not self.download_live_chat: + if not self.download_live_chat_enabled: print(f'{Fore.YELLOW}⚠ downloadLiveCHAT is disabled in config{Style.RESET_ALL}') return False diff --git a/modules/file_manager.py b/modules/file_manager.py index 4567cb6..b024e8f 100644 --- a/modules/file_manager.py +++ b/modules/file_manager.py @@ -45,6 +45,128 @@ class FileManager: self.chat_mp4_path = self.root_path / username / "chat" self.metadata_path = self.root_path / username / "metadata" self.log_file = self.root_path / ".log" + + def _to_rclone_relative_path(self, *parts: str) -> str: + """Build a POSIX-style relative path for rclone --files-from.""" + return pathlib.PurePosixPath(*parts).as_posix() + + def _build_upload_relative_paths(self, filename_base: str) -> List[str]: + """Build the candidate upload list relative to root_path for rclone.""" + files_to_upload: List[str] = [ + self._to_rclone_relative_path(self.username, 'metadata', f"{PREFIX_METADATA}{filename_base}.json"), + self._to_rclone_relative_path(self.username, 'chat', 'json', f"{PREFIX_CHAT}{filename_base}.json") + ] + + if self.upload_pre_merge_video: + files_to_upload.extend([ + self._to_rclone_relative_path(self.username, 'video', 'raw', f"{PREFIX_LIVE}{filename_base}.ts"), + self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_LIVE}{filename_base}.mp4"), + self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_LIVE}{filename_base}.mp3"), + self._to_rclone_relative_path(self.username, 'video', 'raw', f"{PREFIX_VOD}{filename_base}.ts"), + self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_VOD}{filename_base}.mp4"), + self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_VOD}{filename_base}.mp3") + ]) + + if self.upload_merged_video: + files_to_upload.extend([ + self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_MERGED}{filename_base}.mp4"), + self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_MERGED}{filename_base}.mp3"), + self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}.mp4"), + self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}.mp3") + ]) + + if self.upload_chat_video: + files_to_upload.append(self._to_rclone_relative_path(self.username, 'chat', f"{PREFIX_CHAT}{filename_base}.mp4")) + + return files_to_upload + + def _get_existing_upload_relative_paths(self, relative_paths: List[str]) -> List[str]: + """Filter candidate upload paths to the files that actually exist.""" + existing_paths: List[str] = [] + for relative_path in relative_paths: + if (self.root_path / pathlib.PurePosixPath(relative_path)).exists(): + existing_paths.append(relative_path) + return existing_paths + + def _run_rclone_copy(self, relative_paths: List[str], description: str) -> bool: + """Run rclone copy for a set of paths relative to root_path.""" + existing_paths = self._get_existing_upload_relative_paths(relative_paths) + missing_paths = [path for path in relative_paths if path not in existing_paths] + + if not existing_paths: + print(f'{Fore.RED}✗ Upload skipped: no matching files found for {description}{Style.RESET_ALL}') + for missing_path in missing_paths: + print(f'{Fore.YELLOW} Missing: {missing_path}{Style.RESET_ALL}') + return False + + if missing_paths: + print(f'{Fore.YELLOW}⚠ Some configured upload files were not found and will be skipped{Style.RESET_ALL}') + for missing_path in missing_paths: + print(f'{Fore.YELLOW} Missing: {missing_path}{Style.RESET_ALL}') + + print(f'{Fore.CYAN}rclone source: {self.root_path.resolve()}{Style.RESET_ALL}') + print(f'{Fore.CYAN}rclone destination: {self.rclone_path}{Style.RESET_ALL}') + print(f'{Fore.CYAN}Files queued for upload: {len(existing_paths)}{Style.RESET_ALL}') + + bin_path = get_bin_path() + upload_list_path = os.path.join(bin_path, 'temp', 'upload.txt') + os.makedirs(os.path.dirname(upload_list_path), exist_ok=True) + + with open(upload_list_path, 'w', encoding='utf-8', newline='\n') as f: + f.write('\n'.join(existing_paths)) + f.write('\n') + + try: + cmd = [ + 'rclone', 'copy', + str(self.root_path.resolve()), + self.rclone_path, + '--files-from', upload_list_path, + '--progress' + ] + + print(f'{Fore.CYAN}Running: {' '.join(cmd)}{Style.RESET_ALL}') + + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + if proc.stdout: + for line in proc.stdout: + print(line, end='') + proc.wait() + return proc.returncode == 0 + finally: + if os.path.exists(upload_list_path): + os.remove(upload_list_path) + + def run_rclone_smoke_test(self) -> bool: + """Create and upload a tiny metadata file to verify rclone output and configuration.""" + smoke_name = 'RCLONE_SMOKE_TEST' + smoke_relative_path = self._to_rclone_relative_path( + self.username, + 'metadata', + f"{PREFIX_METADATA}{smoke_name}.json" + ) + smoke_file_path = self.root_path / pathlib.PurePosixPath(smoke_relative_path) + + smoke_payload = { + 'type': 'rclone_smoke_test', + 'username': self.username + } + + smoke_file_path.parent.mkdir(parents=True, exist_ok=True) + with open(smoke_file_path, 'w', encoding='utf-8') as f: + json.dump(smoke_payload, f, indent=2) + + print(f'{Fore.CYAN}Created smoke-test file: {smoke_file_path}{Style.RESET_ALL}') + try: + result = self._run_rclone_copy([smoke_relative_path], 'rclone smoke test') + if result: + print(f'{Fore.GREEN}✓ Rclone smoke test completed{Style.RESET_ALL}') + else: + print(f'{Fore.RED}✗ Rclone smoke test failed{Style.RESET_ALL}') + return result + finally: + if smoke_file_path.exists(): + smoke_file_path.unlink() def initialize_directories(self) -> None: """Create all necessary directory structures.""" @@ -120,81 +242,24 @@ class FileManager: print(f'\n{Fore.CYAN}Uploading to cloud storage...{Style.RESET_ALL}') if notification_callback: notification_callback(f'☁ Uploading - {filename_base}', 'Uploading files to cloud storage') - - # Create list of files to upload - bin_path = get_bin_path() - upload_list_path = os.path.join(bin_path, 'temp', 'upload.txt') - - # Ensure temp directory exists - os.makedirs(os.path.dirname(upload_list_path), exist_ok=True) - - files_to_upload = [] - # Build files list relative to root_path so rclone can read them with --files-from - # Metadata and chat JSON - files_to_upload.append(os.path.join(self.username, 'metadata', f"{PREFIX_METADATA}{filename_base}.json")) - files_to_upload.append(os.path.join(self.username, 'chat', 'json', f"{PREFIX_CHAT}{filename_base}.json")) + files_to_upload = self._build_upload_relative_paths(filename_base) - # Pre-merge videos (raw .ts in video/raw, mp4/mp3 in video) - if self.upload_pre_merge_video: - files_to_upload.extend([ - os.path.join(self.username, 'video', 'raw', f"{PREFIX_LIVE}{filename_base}.ts"), - os.path.join(self.username, 'video', f"{PREFIX_LIVE}{filename_base}.mp4"), - os.path.join(self.username, 'video', f"{PREFIX_LIVE}{filename_base}.mp3"), - os.path.join(self.username, 'video', 'raw', f"{PREFIX_VOD}{filename_base}.ts"), - os.path.join(self.username, 'video', f"{PREFIX_VOD}{filename_base}.mp4"), - os.path.join(self.username, 'video', f"{PREFIX_VOD}{filename_base}.mp3") - ]) - - # Merged videos (in video folder) - if self.upload_merged_video: - files_to_upload.extend([ - os.path.join(self.username, 'video', f"{PREFIX_MERGED}{filename_base}.mp4"), - os.path.join(self.username, 'video', f"{PREFIX_MERGED}{filename_base}.mp3"), - os.path.join(self.username, 'video', f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}.mp4"), - os.path.join(self.username, 'video', f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}.mp3") - ]) - - # Standalone chat video (in chat folder) - if self.upload_chat_video: - files_to_upload.append(os.path.join(self.username, 'chat', f"{PREFIX_CHAT}{filename_base}.mp4")) - - with open(upload_list_path, 'w') as f: - f.write('\n'.join(files_to_upload)) - - # Run rclone using --files-from so the listed paths (relative to root_path) are uploaded. try: - cmd = [ - 'rclone', 'copy', - str(self.root_path.resolve()), - self.rclone_path, - '--files-from', upload_list_path - ] + result = self._run_rclone_copy(files_to_upload, f'archive batch {filename_base}') - # Stream rclone output to console so user can see progress/errors - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) - if proc.stdout: - for line in proc.stdout: - print(line, end='') - proc.wait() - result = proc.returncode - - # Clean up upload list - if os.path.exists(upload_list_path): - os.remove(upload_list_path) - - if result == 0: + if result: print(f'{Fore.GREEN}✓ Upload complete{Style.RESET_ALL}') if notification_callback: notification_callback(f'✓ Upload Success - {filename_base}', 'All files uploaded successfully') return True - else: - print(f'{Fore.RED}✗ Upload failed (exit code: {result}){Style.RESET_ALL}') - print(f'{Fore.YELLOW}Files preserved locally due to upload failure{Style.RESET_ALL}') - if notification_callback: - notification_callback(f'✗ Upload Failed - {filename_base}', - f'Upload failed with code {result}. Files preserved locally.') - return False + + print(f'{Fore.RED}✗ Upload failed{Style.RESET_ALL}') + print(f'{Fore.YELLOW}Files preserved locally due to upload failure{Style.RESET_ALL}') + if notification_callback: + notification_callback(f'✗ Upload Failed - {filename_base}', + 'Upload failed. Files preserved locally. Check rclone output above.') + return False except Exception as e: print(f'{Fore.RED}✗ Upload error: {str(e)}{Style.RESET_ALL}') diff --git a/modules/processor.py b/modules/processor.py index b1cae71..9e4518b 100644 --- a/modules/processor.py +++ b/modules/processor.py @@ -37,37 +37,78 @@ class StreamProcessor: os_type ) - def process_raw_stream(self, raw_path: str, output_path: str) -> None: + def process_raw_stream(self, raw_path: str, output_path: str) -> bool: """ Process raw .ts file into mp4/mp3 using ffmpeg. Args: raw_path: Path to the raw .ts file output_path: Path for the processed output file + + Returns: + bool: True when conversion succeeded, False otherwise """ if not os.path.exists(raw_path): print(f'{Fore.YELLOW}⚠ Raw file not found, skipping processing{Style.RESET_ALL}') - return + return False if self.only_raw: print(f'{Fore.CYAN}Keeping raw .ts file (onlyRaw mode){Style.RESET_ALL}') - return + return False print(f'{Fore.YELLOW}Processing raw stream file...{Style.RESET_ALL}') # Build ffmpeg command based on quality if self.quality == 'audio_only': - self._process_audio(raw_path, output_path) + result = self._process_audio(raw_path, output_path) else: - self._process_video(raw_path, output_path) + result = self._process_video(raw_path, output_path) - print(f'{Fore.GREEN}✓ Stream processed successfully{Style.RESET_ALL}') + if result: + print(f'{Fore.GREEN}✓ Stream processed successfully{Style.RESET_ALL}') + else: + print(f'{Fore.RED}✗ Stream processing failed{Style.RESET_ALL}') + + return result - def _process_audio(self, raw_path: str, output_path: str) -> None: + def _run_ffmpeg_command(self, cmd: list, output_path: str) -> bool: + """Run FFmpeg while streaming its output to the terminal.""" + print(f'{Fore.CYAN}Running FFmpeg: {' '.join(cmd)}{Style.RESET_ALL}') + + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + encoding='utf-8', + errors='replace' + ) + + if process.stdout: + for line in process.stdout: + print(line, end='') + + result = process.wait() + if result != 0: + print(f'{Fore.RED}✗ FFmpeg exited with code: {result}{Style.RESET_ALL}') + return False + + if not os.path.exists(output_path): + print(f'{Fore.RED}✗ FFmpeg did not create output: {output_path}{Style.RESET_ALL}') + return False + + if os.path.getsize(output_path) == 0: + print(f'{Fore.RED}✗ FFmpeg created an empty output file: {output_path}{Style.RESET_ALL}') + return False + + return True + + def _process_audio(self, raw_path: str, output_path: str) -> bool: """Process audio-only stream.""" # Audio-only conversion with modern AAC encoding cmd = [ self.ffmpeg_path, + '-y', '-i', raw_path, '-vn', # No video '-c:a', self.ffmpeg_audio_codec, @@ -85,14 +126,9 @@ class StreamProcessor: cmd.extend(['-movflags', '+faststart']) cmd.append(output_path) - - # Run FFmpeg - if self.ffmpeg_progress: - subprocess.call(cmd) - else: - subprocess.call(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) + return self._run_ffmpeg_command(cmd, output_path) - def _process_video(self, raw_path: str, output_path: str) -> None: + def _process_video(self, raw_path: str, output_path: str) -> bool: """Process video stream.""" cmd = [ self.ffmpeg_path, @@ -135,12 +171,7 @@ class StreamProcessor: cmd.extend(['-movflags', '+faststart']) cmd.append(output_path) - - # Run FFmpeg - if self.ffmpeg_progress: - subprocess.call(cmd) - else: - subprocess.call(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) + return self._run_ffmpeg_command(cmd, output_path) def build_chat_output_args(self) -> str: """ diff --git a/modules/recorder.py b/modules/recorder.py index fa9c9d1..3518591 100644 --- a/modules/recorder.py +++ b/modules/recorder.py @@ -7,6 +7,8 @@ import subprocess from typing import Dict, Any, Optional from colorama import Fore, Style +from .utils import get_env_value + class StreamRecorder: """Handles live stream recording using streamlink.""" @@ -68,7 +70,7 @@ class StreamRecorder: print(f'{Fore.YELLOW} Consider disabling streamlink_ttvlol in config or using alternative methods{Style.RESET_ALL}') # Add authentication if available - oauth_token = os.getenv("OAUTH-PRIVATE-TOKEN", "") + oauth_token = get_env_value("OAUTH-PRIVATE-TOKEN", "OAUTH_PRIVATE_TOKEN", default="") if oauth_token and oauth_token != "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx": cmd.extend(['--twitch-api-header', f'Authorization=OAuth {oauth_token}']) diff --git a/modules/stream_monitor.py b/modules/stream_monitor.py index bea3545..a6b89be 100644 --- a/modules/stream_monitor.py +++ b/modules/stream_monitor.py @@ -9,6 +9,7 @@ import requests from colorama import Fore, Style from .constants import TWITCH_OAUTH_URL, TWITCH_API_URL, TWITCH_GQL_URL, TWITCH_GQL_CLIENT_ID +from .utils import get_env_value class StreamMonitor: @@ -40,7 +41,9 @@ class StreamMonitor: return self._oauth_token try: - url = f"{TWITCH_OAUTH_URL}?client_id={os.getenv('CLIENT-ID')}&client_secret={os.getenv('CLIENT-SECRET')}&grant_type=client_credentials" + client_id = get_env_value('CLIENT-ID', 'CLIENT_ID') + client_secret = get_env_value('CLIENT-SECRET', 'CLIENT_SECRET') + url = f"{TWITCH_OAUTH_URL}?client_id={client_id}&client_secret={client_secret}&grant_type=client_credentials" response = requests.post(url, timeout=15) response.raise_for_status() self._oauth_token = response.json()['access_token'] @@ -69,7 +72,7 @@ class StreamMonitor: url = f'{TWITCH_API_URL}/users?login={self.username}' headers = { "Authorization": f"Bearer {self.get_oauth_token()}", - "Client-ID": os.getenv('CLIENT-ID') + "Client-ID": get_env_value('CLIENT-ID', 'CLIENT_ID') } response = requests.get(url, headers=headers, timeout=15) response.raise_for_status() diff --git a/modules/utils.py b/modules/utils.py index 280afa8..c8bc749 100644 --- a/modules/utils.py +++ b/modules/utils.py @@ -4,6 +4,7 @@ Utility functions and helpers for Twitch Archive. import os import sys +import shutil import pathlib import subprocess from typing import Optional @@ -35,6 +36,15 @@ def get_bin_path() -> str: return str(pathlib.Path(__file__).parent.parent.resolve() / "bin") +def get_env_value(*names: str, default: Optional[str] = None) -> Optional[str]: + """Return the first non-empty environment variable from the provided names.""" + for name in names: + value = os.getenv(name) + if value not in (None, ""): + return value + return default + + def get_ffmpeg_executable(os_type: str) -> str: """ Get the platform-specific ffmpeg executable path. @@ -48,6 +58,11 @@ def get_ffmpeg_executable(os_type: str) -> str: bin_path = get_bin_path() if os_type == 'windows': return os.path.join(bin_path, 'ffmpeg.exe') + + system_ffmpeg = shutil.which('ffmpeg') + if system_ffmpeg: + return system_ffmpeg + return os.path.join(bin_path, 'ffmpeg') @@ -64,6 +79,11 @@ def get_twitch_downloader_executable(os_type: str) -> str: bin_path = get_bin_path() if os_type == 'windows': return os.path.join(bin_path, 'TwitchDownloaderCLI.exe') + + system_twitch_downloader = shutil.which('TwitchDownloaderCLI') + if system_twitch_downloader: + return system_twitch_downloader + return os.path.join(bin_path, 'TwitchDownloaderCLI') @@ -164,6 +184,24 @@ def verify_streamlink() -> bool: return False +def verify_rclone() -> bool: + """Verify that rclone is available on PATH.""" + try: + result = subprocess.run(['rclone', 'version'], + capture_output=True, + text=True, + timeout=5) + if result.returncode == 0: + version_line = result.stdout.strip().splitlines()[0] if result.stdout.strip() else 'unknown' + print(f'{Fore.GREEN}✓ Rclone found ({version_line}){Style.RESET_ALL}') + return True + raise FileNotFoundError() + except (FileNotFoundError, subprocess.TimeoutExpired, IndexError): + print(f'{Fore.RED}✗ ERROR: rclone not found{Style.RESET_ALL}') + print(f'{Fore.CYAN} → Install rclone and ensure it is on PATH{Style.RESET_ALL}') + return False + + def verify_ffmpeg(os_type: str) -> bool: """ Verify that ffmpeg is available. diff --git a/test_twitch_archive_simple.py b/test_twitch_archive_simple.py index 0d49da2..96a5f26 100644 --- a/test_twitch_archive_simple.py +++ b/test_twitch_archive_simple.py @@ -20,12 +20,27 @@ import sys import os import json import getopt +import tempfile +import importlib.util +from pathlib import Path from unittest.mock import patch, MagicMock, Mock # Add parent directory to path for imports sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from modules.constants import DEFAULT_CONFIG +from modules.file_manager import FileManager +from modules.downloader import ContentDownloader +from modules.utils import get_ffmpeg_executable, get_twitch_downloader_executable + + +def load_twitch_archive_module(): + """Load the main script module for targeted regression tests.""" + module_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'twitch-archive.py') + spec = importlib.util.spec_from_file_location('twitch_archive_main', module_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module class TestCommandLineArgumentParsing(unittest.TestCase): @@ -115,6 +130,34 @@ class TestCommandLineArgumentParsing(unittest.TestCase): self.assertEqual(len(opts), 1) self.assertEqual(opts[0], ('--chat-only', '')) + + def test_rclone_smoke_test_option(self): + """Test --rclone-smoke-test option parsing.""" + argv = ['--rclone-smoke-test'] + opts, args = getopt.getopt( + argv, + "hu:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "healthcheck", "rclone-smoke-test", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 1) + self.assertEqual(opts[0], ('--rclone-smoke-test', '')) + + def test_healthcheck_option(self): + """Test --healthcheck option parsing.""" + argv = ['--healthcheck'] + opts, args = getopt.getopt( + argv, + "hu:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "healthcheck", "rclone-smoke-test", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 1) + self.assertEqual(opts[0], ('--healthcheck', '')) def test_legacy_option(self): """Test --legacy option parsing.""" @@ -439,6 +482,161 @@ class TestConfigLogic(unittest.TestCase): self.assertIn('$schema', default_config) +class TestFileManagerUploadPaths(unittest.TestCase): + """Test rclone upload path preparation.""" + + def test_build_upload_relative_paths_uses_forward_slashes(self): + """Rclone --files-from entries must use POSIX separators on Windows.""" + with tempfile.TemporaryDirectory() as temp_dir: + manager = FileManager( + root_path=temp_dir, + username='testuser', + config={ + 'uploadCloud': True, + 'uploadPreMergeVideo': True, + 'uploadMergedVideo': True, + 'uploadChatVideo': True + } + ) + + relative_paths = manager._build_upload_relative_paths('20260424_12h00m00s') + + self.assertTrue(relative_paths) + self.assertTrue(all('\\' not in path for path in relative_paths)) + self.assertIn('testuser/metadata/METADA_20260424_12h00m00s.json', relative_paths) + self.assertIn('testuser/chat/json/CHAT_20260424_12h00m00s.json', relative_paths) + + +class TestDownloaderConfiguration(unittest.TestCase): + """Regression tests for downloader config wiring.""" + + def test_download_vod_method_not_shadowed_by_boolean_flag(self): + """Config booleans must not overwrite callable downloader methods.""" + downloader = ContentDownloader( + twitch_downloader_path='TwitchDownloaderCLI', + ffmpeg_path='ffmpeg', + config={ + 'downloadVOD': True, + 'downloadCHAT': True, + 'downloadLiveCHAT': True + } + ) + + self.assertTrue(callable(downloader.download_vod)) + self.assertTrue(downloader.download_vod_enabled) + + +class TestLinuxToolResolution(unittest.TestCase): + """Ensure Linux containers prefer their own installed toolchain.""" + + @patch('modules.utils.shutil.which') + def test_linux_prefers_system_ffmpeg(self, mock_which): + mock_which.return_value = '/usr/bin/ffmpeg' + + self.assertEqual(get_ffmpeg_executable('linux'), '/usr/bin/ffmpeg') + + @patch('modules.utils.shutil.which') + def test_linux_prefers_system_twitch_downloader(self, mock_which): + mock_which.return_value = '/usr/local/bin/TwitchDownloaderCLI' + + self.assertEqual(get_twitch_downloader_executable('linux'), '/usr/local/bin/TwitchDownloaderCLI') + + +class TestMultiStreamerCleanupRegression(unittest.TestCase): + """Regression tests for multi-streamer conversion and cleanup behavior.""" + + def setUp(self): + self.module = load_twitch_archive_module() + + def _build_archiver(self, temp_dir: str, upload_cloud: bool = True): + archiver = MagicMock() + archiver.username = 'maddoscientist0' + archiver.os_type = 'linux' + archiver.quality = 'best' + archiver.downloadLiveCHAT = False + archiver.downloadCHAT = False + archiver.downloadVOD = False + archiver.downloadMETADATA = False + archiver.mergeVideoChat = False + archiver.mergeChatLayout = 'side-by-side' + archiver.onlyRaw = False + archiver.vodTimeout = 0 + archiver.shutdown_requested = False + archiver.deleteFiles = True + archiver.uploadCloud = upload_cloud + + archiver.notification_manager = MagicMock() + archiver.recorder = MagicMock() + archiver.processor = MagicMock() + archiver.downloader = MagicMock() + archiver.stream_monitor = MagicMock() + archiver.file_manager = MagicMock() + archiver.file_manager.raw_path = Path(temp_dir) / 'raw' + archiver.file_manager.video_path = Path(temp_dir) / 'video' + archiver.file_manager.chat_json_path = Path(temp_dir) / 'chat_json' + archiver.file_manager.chat_mp4_path = Path(temp_dir) / 'chat' + + os.makedirs(archiver.file_manager.raw_path, exist_ok=True) + os.makedirs(archiver.file_manager.video_path, exist_ok=True) + os.makedirs(archiver.file_manager.chat_json_path, exist_ok=True) + os.makedirs(archiver.file_manager.chat_mp4_path, exist_ok=True) + + return archiver + + def test_process_stream_keeps_raw_when_conversion_fails(self): + manager = self.module.TwitchArchiveManager(specific_streamer='maddoscientist0') + + with tempfile.TemporaryDirectory() as temp_dir: + archiver = self._build_archiver(temp_dir) + archiver.processor.process_raw_stream.return_value = False + archiver.file_manager.upload_to_cloud.return_value = True + + def write_raw_file(_stream_info, raw_path): + with open(raw_path, 'wb') as handle: + handle.write(b'x' * 4096) + return True + + archiver.recorder.record.side_effect = write_raw_file + + stream_info = { + 'title': 'Test', + 'createdAt': '2026-04-25T09:14:01Z' + } + + manager._process_stream(archiver, stream_info, 'stream-id') + + archiver.file_manager.clean_raw_file.assert_not_called() + archiver.file_manager.delete_local_files.assert_called_once() + + def test_process_stream_only_deletes_rendered_files_after_real_upload(self): + manager = self.module.TwitchArchiveManager(specific_streamer='maddoscientist0') + + with tempfile.TemporaryDirectory() as temp_dir: + archiver = self._build_archiver(temp_dir, upload_cloud=False) + archiver.processor.process_raw_stream.return_value = True + archiver.file_manager.upload_to_cloud.return_value = True + + def write_raw_file(_stream_info, raw_path): + with open(raw_path, 'wb') as handle: + handle.write(b'x' * 4096) + return True + + archiver.recorder.record.side_effect = write_raw_file + + stream_info = { + 'title': 'Test', + 'createdAt': '2026-04-25T09:14:01Z' + } + + manager._process_stream(archiver, stream_info, 'stream-id') + + archiver.file_manager.clean_raw_file.assert_called_once() + archiver.file_manager.delete_local_files.assert_not_called() + + upload_filename_base = archiver.file_manager.upload_to_cloud.call_args.args[0] + self.assertFalse(upload_filename_base.startswith('LIVE_')) + + if __name__ == '__main__': # Run tests with verbose output print("="*70) diff --git a/twitch-archive.py b/twitch-archive.py index 2fdcb4e..81f9526 100644 --- a/twitch-archive.py +++ b/twitch-archive.py @@ -34,6 +34,8 @@ import time import json import signal import getopt +import pathlib +import subprocess from typing import Dict, Optional, Any from datetime import datetime, timedelta @@ -48,7 +50,8 @@ from modules.config import ConfigManager from modules.notifications import NotificationManager from modules.utils import ( detect_operating_system, get_ffmpeg_executable, get_twitch_downloader_executable, - get_unique_filename, get_video_duration, verify_streamlink, verify_ffmpeg, verify_twitch_downloader + get_unique_filename, get_video_duration, verify_streamlink, verify_ffmpeg, verify_twitch_downloader, + verify_rclone, get_env_value ) from modules.stream_monitor import StreamMonitor from modules.recorder import StreamRecorder @@ -147,10 +150,25 @@ class TwitchArchive: Raises: SystemExit: If .env file is not found """ - if not load_dotenv(find_dotenv()): + dotenv_loaded = load_dotenv(find_dotenv()) + has_required_env = bool( + get_env_value('CLIENT-ID', 'CLIENT_ID') and + get_env_value('CLIENT-SECRET', 'CLIENT_SECRET') + ) + + if not dotenv_loaded and has_required_env: + print(f'{Fore.GREEN}✓ Twitch API credentials loaded from process environment{Style.RESET_ALL}') + return + + if not dotenv_loaded and not has_required_env: print(f'{Fore.RED}✗ ERROR: .env file not found{Style.RESET_ALL}') - print(f'{Fore.CYAN} → Create a .env file with your Twitch API credentials{Style.RESET_ALL}') - print(f'{Fore.CYAN} → Required: CLIENT-ID, CLIENT-SECRET{Style.RESET_ALL}') + print(f'{Fore.CYAN} → Create a .env file with your Twitch API credentials or pass them via environment variables{Style.RESET_ALL}') + print(f'{Fore.CYAN} → Required: CLIENT-ID/CLIENT_ID and CLIENT-SECRET/CLIENT_SECRET{Style.RESET_ALL}') + sys.exit(1) + + if not has_required_env: + print(f'{Fore.RED}✗ ERROR: Twitch API credentials are missing{Style.RESET_ALL}') + print(f'{Fore.CYAN} → Required: CLIENT-ID/CLIENT_ID and CLIENT-SECRET/CLIENT_SECRET{Style.RESET_ALL}') sys.exit(1) def _initialize_components(self) -> None: @@ -259,6 +277,8 @@ class TwitchArchive: verify_ffmpeg(self.os_type) if self.downloadVOD or self.downloadCHAT: verify_twitch_downloader(self.os_type) + if self.uploadCloud and not verify_rclone(): + sys.exit(1) # Print configuration summary self._print_configuration_summary() @@ -404,7 +424,7 @@ class TwitchArchive: print(f'{Fore.YELLOW}Attempting to process any recorded content...{Style.RESET_ALL}') # Process the raw stream file - self.processor.process_raw_stream(live_raw_path, live_proc_path) + processing_succeeded = self.processor.process_raw_stream(live_raw_path, live_proc_path) # Wait for live chat download if it was started live_chat_downloaded = False @@ -426,7 +446,11 @@ class TwitchArchive: else: # Get video duration first (needed for chat conversion and trimming) ffmpeg_path = get_ffmpeg_executable(self.os_type) - video_duration = get_video_duration(live_proc_path, ffmpeg_path) + if not processing_succeeded or not os.path.exists(live_proc_path): + print(f'{Fore.YELLOW}⚠ Processed video file is unavailable, skipping chat render{Style.RESET_ALL}') + video_duration = None + else: + video_duration = get_video_duration(live_proc_path, ffmpeg_path) print(f'{Fore.CYAN}Video duration for chat rendering: {video_duration}s{Style.RESET_ALL}') # Convert chat format if needed (chat_downloader uses different JSON structure) @@ -561,7 +585,10 @@ class TwitchArchive: print(f'{Fore.YELLOW}⚠ No matching VOD found for this stream{Style.RESET_ALL}') # Clean up raw files if configured - self.file_manager.clean_raw_file(live_raw_path) + if processing_succeeded: + self.file_manager.clean_raw_file(live_raw_path) + elif os.path.exists(live_raw_path): + print(f'{Fore.YELLOW}⚠ Keeping raw file because conversion did not complete successfully{Style.RESET_ALL}') # Upload to cloud if configured upload_success = self.file_manager.upload_to_cloud( @@ -570,7 +597,7 @@ class TwitchArchive: ) # Delete local files if configured and upload succeeded - if self.deleteFiles and upload_success: + if self.deleteFiles and self.uploadCloud and upload_success: self.file_manager.delete_local_files( filename_base, live_raw_path, @@ -890,6 +917,8 @@ class TwitchArchiveManager: verify_ffmpeg(first_archiver.os_type) if first_archiver.downloadVOD or first_archiver.downloadCHAT: verify_twitch_downloader(first_archiver.os_type) + if any(archiver.uploadCloud for archiver in self.archivers.values()) and not verify_rclone(): + sys.exit(1) # Print configuration summary for each streamer for username, archiver in self.archivers.items(): @@ -1018,7 +1047,7 @@ class TwitchArchiveManager: # Generate timestamp and filename timestamp = datetime.now(timezone('UTC')).strftime("%Y%m%d_%Hh%Mm%Ss") - filename_base = f"{PREFIX_LIVE}{archiver.username}_{timestamp}" + filename_base = f"{archiver.username}_{timestamp}" # Parse stream start time live_date = datetime.strptime( @@ -1029,8 +1058,8 @@ class TwitchArchiveManager: raw_extension = '.ts' proc_extension = '.mp3' if archiver.quality == 'audio_only' else '.mp4' - live_raw_path = str(archiver.file_manager.raw_path / f"{filename_base}{raw_extension}") - live_proc_path = str(archiver.file_manager.video_path / f"{filename_base}{proc_extension}") + live_raw_path = str(archiver.file_manager.raw_path / f"{PREFIX_LIVE}{filename_base}{raw_extension}") + live_proc_path = str(archiver.file_manager.video_path / f"{PREFIX_LIVE}{filename_base}{proc_extension}") chat_json_path = str(archiver.file_manager.chat_json_path / f"{PREFIX_CHAT}{filename_base}.json") # Send notification @@ -1172,8 +1201,9 @@ class TwitchArchiveManager: print(f'{Fore.CYAN}Processing recorded content ({file_size / (1024*1024):.2f} MB)...{Style.RESET_ALL}') # Process raw stream + processing_succeeded = False if not archiver.onlyRaw: - archiver.processor.process_raw_stream(live_raw_path, live_proc_path) + processing_succeeded = archiver.processor.process_raw_stream(live_raw_path, live_proc_path) # Wait for live chat download if it was started live_chat_downloaded = False @@ -1212,8 +1242,12 @@ class TwitchArchiveManager: chat_rendered_successfully = False else: # Get video duration first - ffmpeg_path = get_ffmpeg_executable(archiver.os_type) - video_duration = get_video_duration(live_proc_path, ffmpeg_path) + if not processing_succeeded or not os.path.exists(live_proc_path): + print(f'{Fore.YELLOW}⚠ Processed video file is unavailable, skipping chat render{Style.RESET_ALL}') + video_duration = None + else: + ffmpeg_path = get_ffmpeg_executable(archiver.os_type) + video_duration = get_video_duration(live_proc_path, ffmpeg_path) if video_duration is None: print(f'{Fore.YELLOW}⚠ Could not detect video duration from {live_proc_path}{Style.RESET_ALL}') @@ -1362,7 +1396,10 @@ class TwitchArchiveManager: archiver.file_manager.save_metadata(stream_info, filename_base) # Clean up raw file if configured - archiver.file_manager.clean_raw_file(live_raw_path) + if processing_succeeded: + archiver.file_manager.clean_raw_file(live_raw_path) + elif os.path.exists(live_raw_path): + print(f'{Fore.YELLOW}⚠ Keeping raw file because conversion did not complete successfully{Style.RESET_ALL}') # Upload to cloud if configured upload_success = archiver.file_manager.upload_to_cloud( @@ -1371,7 +1408,7 @@ class TwitchArchiveManager: ) # Delete files if configured - if archiver.deleteFiles and upload_success: + if archiver.deleteFiles and archiver.uploadCloud and upload_success: archiver.file_manager.delete_local_files( filename_base, live_raw_path, @@ -1386,6 +1423,79 @@ class TwitchArchiveManager: ) +def run_rclone_smoke_test(specific_streamer: Optional[str] = None) -> int: + """Run a one-off rclone smoke test using the configured upload destination.""" + config_manager = ConfigManager() + + if specific_streamer: + username = specific_streamer + else: + enabled_streamers = config_manager.get_all_enabled_streamers() + if not enabled_streamers: + print(f'{Fore.RED}✗ No enabled streamers available for smoke test{Style.RESET_ALL}') + print(f'{Fore.CYAN}→ Use -u or enable a streamer config{Style.RESET_ALL}') + return 1 + username = enabled_streamers[0] + + config = config_manager.load_streamer_config(username) + file_manager = FileManager( + root_path=config.get('root_path', 'archive'), + username=username, + config=config + ) + file_manager.initialize_directories() + + print(f'\n{Fore.CYAN}{"=" * 70}{Style.RESET_ALL}') + print(f'{Fore.CYAN}TWITCH ARCHIVE - Rclone Smoke Test{Style.RESET_ALL}') + print(f'{Fore.CYAN}{"=" * 70}{Style.RESET_ALL}') + print(f'{Fore.GREEN}Streamer: {username}{Style.RESET_ALL}') + print(f'{Fore.GREEN}Remote: {config.get("rclone_path", "")}{Style.RESET_ALL}\n') + + return 0 if file_manager.run_rclone_smoke_test() else 1 + + +def run_healthcheck(specific_streamer: Optional[str] = None) -> int: + """Run a local readiness check suitable for Docker health checks.""" + config_manager = ConfigManager() + + if specific_streamer: + username = specific_streamer + else: + enabled_streamers = config_manager.get_all_enabled_streamers() + username = enabled_streamers[0] if enabled_streamers else 'vinesauce' + + config = config_manager.load_streamer_config(username) + archive = TwitchArchive(config) + + try: + archive._load_environment_variables() + except SystemExit: + return 1 + + archive._initialize_components() + + checks_ok = True + if not verify_streamlink(): + checks_ok = False + if not verify_ffmpeg(archive.os_type): + checks_ok = False + if (archive.downloadVOD or archive.downloadCHAT) and not verify_twitch_downloader(archive.os_type): + checks_ok = False + if archive.uploadCloud: + if not verify_rclone(): + checks_ok = False + rclone_config_path = os.getenv('RCLONE_CONFIG') + if rclone_config_path and not os.path.exists(rclone_config_path): + print(f'{Fore.RED}✗ ERROR: RCLONE_CONFIG points to a missing file: {rclone_config_path}{Style.RESET_ALL}') + checks_ok = False + + if not checks_ok: + return 1 + + print(f'{Fore.GREEN}✓ Healthcheck OK for {username}{Style.RESET_ALL}') + return 0 + + # ============================================================================ # COMMAND-LINE INTERFACE # ============================================================================ @@ -1401,6 +1511,8 @@ def main(argv: list) -> None: """ specific_streamer = None use_legacy_mode = False + rclone_smoke_test_mode = False + healthcheck_mode = False help_msg = f''' {Fore.CYAN}{"=" * 70} @@ -1427,6 +1539,8 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving --legacy Force legacy mode (use config.json) --chat-only Test mode: Only download chat (skip video recording) Automatically enables verbose logging + --healthcheck Validate config and tool availability, then exit + --rclone-smoke-test Create a small test file and upload it with rclone --use-chat-downloader-primary Use chat_downloader as primary chat source (for testing) --no-chat-downloader-fallback Disable chat_downloader fallback @@ -1464,7 +1578,7 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving "h:u:q:a:v:c:m:r:d:n:", ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", - "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + "chat-only", "healthcheck", "rclone-smoke-test", "use-chat-downloader-primary", "no-chat-downloader-fallback"] ) except getopt.GetoptError as e: print(f'{Fore.RED}Error: {e}{Style.RESET_ALL}\n') @@ -1491,6 +1605,10 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving elif opt == "--chat-only": chat_only_mode = True verbose_mode = True # Auto-enable verbose for chat-only mode + elif opt == "--healthcheck": + healthcheck_mode = True + elif opt == "--rclone-smoke-test": + rclone_smoke_test_mode = True elif opt == "--legacy": use_legacy_mode = True elif opt == "--use-chat-downloader-primary": @@ -1523,6 +1641,12 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving legacy_overrides['deleteFiles'] = bool(int(arg)) elif opt in ("-n", "--notifications"): legacy_overrides['notifications'] = bool(int(arg)) + + if rclone_smoke_test_mode: + sys.exit(run_rclone_smoke_test(specific_streamer)) + + if healthcheck_mode: + sys.exit(run_healthcheck(specific_streamer)) # Determine which mode to use if use_legacy_mode or (legacy_config_exists and not specific_streamer and not os.path.exists('config/global.json')):